add gitignore
This commit is contained in:
160
.gitignore
vendored
Normal file
160
.gitignore
vendored
Normal file
@@ -0,0 +1,160 @@
|
||||
# Created by .ignore support plugin (hsz.mobi)
|
||||
### Python template
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# celery beat schedule file
|
||||
celerybeat-schedule
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
### JetBrains template
|
||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
|
||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||
|
||||
# User-specific stuff
|
||||
.idea/**/workspace.xml
|
||||
.idea/**/tasks.xml
|
||||
.idea/**/dictionaries
|
||||
.idea/**/shelf
|
||||
|
||||
# Sensitive or high-churn files
|
||||
.idea/**/dataSources/
|
||||
.idea/**/dataSources.ids
|
||||
.idea/**/dataSources.local.xml
|
||||
.idea/**/sqlDataSources.xml
|
||||
.idea/**/dynamic.xml
|
||||
.idea/**/uiDesigner.xml
|
||||
.idea/**/dbnavigator.xml
|
||||
|
||||
# Gradle
|
||||
.idea/**/gradle.xml
|
||||
.idea/**/libraries
|
||||
|
||||
# CMake
|
||||
cmake-build-debug/
|
||||
cmake-build-release/
|
||||
|
||||
# Mongo Explorer plugin
|
||||
.idea/**/mongoSettings.xml
|
||||
|
||||
# File-based project format
|
||||
*.iws
|
||||
|
||||
# IntelliJ
|
||||
out/
|
||||
|
||||
# mpeltonen/sbt-idea plugin
|
||||
.idea_modules/
|
||||
|
||||
# JIRA plugin
|
||||
atlassian-ide-plugin.xml
|
||||
|
||||
# Cursive Clojure plugin
|
||||
.idea/replstate.xml
|
||||
|
||||
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
|
||||
# Editor-based Rest Client
|
||||
.idea/httpRequests
|
||||
|
||||
6
.idea/vcs.xml
generated
Normal file
6
.idea/vcs.xml
generated
Normal file
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
@@ -1,76 +0,0 @@
|
||||
# This file must be used with "source bin/activate" *from bash*
|
||||
# you cannot run it directly
|
||||
|
||||
deactivate () {
|
||||
# reset old environment variables
|
||||
if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
|
||||
PATH="${_OLD_VIRTUAL_PATH:-}"
|
||||
export PATH
|
||||
unset _OLD_VIRTUAL_PATH
|
||||
fi
|
||||
if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
|
||||
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
|
||||
export PYTHONHOME
|
||||
unset _OLD_VIRTUAL_PYTHONHOME
|
||||
fi
|
||||
|
||||
# This should detect bash and zsh, which have a hash command that must
|
||||
# be called to get it to forget past commands. Without forgetting
|
||||
# past commands the $PATH changes we made may not be respected
|
||||
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
|
||||
hash -r
|
||||
fi
|
||||
|
||||
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
|
||||
PS1="${_OLD_VIRTUAL_PS1:-}"
|
||||
export PS1
|
||||
unset _OLD_VIRTUAL_PS1
|
||||
fi
|
||||
|
||||
unset VIRTUAL_ENV
|
||||
if [ ! "$1" = "nondestructive" ] ; then
|
||||
# Self destruct!
|
||||
unset -f deactivate
|
||||
fi
|
||||
}
|
||||
|
||||
# unset irrelevant variables
|
||||
deactivate nondestructive
|
||||
|
||||
VIRTUAL_ENV="/home/allem/PycharmProjects/thue/venv"
|
||||
export VIRTUAL_ENV
|
||||
|
||||
_OLD_VIRTUAL_PATH="$PATH"
|
||||
PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
export PATH
|
||||
|
||||
# unset PYTHONHOME if set
|
||||
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
|
||||
# could use `if (set -u; : $PYTHONHOME) ;` in bash
|
||||
if [ -n "${PYTHONHOME:-}" ] ; then
|
||||
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
|
||||
unset PYTHONHOME
|
||||
fi
|
||||
|
||||
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
|
||||
_OLD_VIRTUAL_PS1="${PS1:-}"
|
||||
if [ "x(venv) " != x ] ; then
|
||||
PS1="(venv) ${PS1:-}"
|
||||
else
|
||||
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
|
||||
# special case for Aspen magic directories
|
||||
# see http://www.zetadev.com/software/aspen/
|
||||
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
|
||||
else
|
||||
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
|
||||
fi
|
||||
fi
|
||||
export PS1
|
||||
fi
|
||||
|
||||
# This should detect bash and zsh, which have a hash command that must
|
||||
# be called to get it to forget past commands. Without forgetting
|
||||
# past commands the $PATH changes we made may not be respected
|
||||
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
|
||||
hash -r
|
||||
fi
|
||||
@@ -1,37 +0,0 @@
|
||||
# This file must be used with "source bin/activate.csh" *from csh*.
|
||||
# You cannot run it directly.
|
||||
# Created by Davide Di Blasi <davidedb@gmail.com>.
|
||||
# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com>
|
||||
|
||||
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate'
|
||||
|
||||
# Unset irrelevant variables.
|
||||
deactivate nondestructive
|
||||
|
||||
setenv VIRTUAL_ENV "/home/allem/PycharmProjects/thue/venv"
|
||||
|
||||
set _OLD_VIRTUAL_PATH="$PATH"
|
||||
setenv PATH "$VIRTUAL_ENV/bin:$PATH"
|
||||
|
||||
|
||||
set _OLD_VIRTUAL_PROMPT="$prompt"
|
||||
|
||||
if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
|
||||
if ("venv" != "") then
|
||||
set env_name = "venv"
|
||||
else
|
||||
if (`basename "VIRTUAL_ENV"` == "__") then
|
||||
# special case for Aspen magic directories
|
||||
# see http://www.zetadev.com/software/aspen/
|
||||
set env_name = `basename \`dirname "$VIRTUAL_ENV"\``
|
||||
else
|
||||
set env_name = `basename "$VIRTUAL_ENV"`
|
||||
endif
|
||||
endif
|
||||
set prompt = "[$env_name] $prompt"
|
||||
unset env_name
|
||||
endif
|
||||
|
||||
alias pydoc python -m pydoc
|
||||
|
||||
rehash
|
||||
@@ -1,75 +0,0 @@
|
||||
# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org)
|
||||
# you cannot run it directly
|
||||
|
||||
function deactivate -d "Exit virtualenv and return to normal shell environment"
|
||||
# reset old environment variables
|
||||
if test -n "$_OLD_VIRTUAL_PATH"
|
||||
set -gx PATH $_OLD_VIRTUAL_PATH
|
||||
set -e _OLD_VIRTUAL_PATH
|
||||
end
|
||||
if test -n "$_OLD_VIRTUAL_PYTHONHOME"
|
||||
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
|
||||
set -e _OLD_VIRTUAL_PYTHONHOME
|
||||
end
|
||||
|
||||
if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
|
||||
functions -e fish_prompt
|
||||
set -e _OLD_FISH_PROMPT_OVERRIDE
|
||||
functions -c _old_fish_prompt fish_prompt
|
||||
functions -e _old_fish_prompt
|
||||
end
|
||||
|
||||
set -e VIRTUAL_ENV
|
||||
if test "$argv[1]" != "nondestructive"
|
||||
# Self destruct!
|
||||
functions -e deactivate
|
||||
end
|
||||
end
|
||||
|
||||
# unset irrelevant variables
|
||||
deactivate nondestructive
|
||||
|
||||
set -gx VIRTUAL_ENV "/home/allem/PycharmProjects/thue/venv"
|
||||
|
||||
set -gx _OLD_VIRTUAL_PATH $PATH
|
||||
set -gx PATH "$VIRTUAL_ENV/bin" $PATH
|
||||
|
||||
# unset PYTHONHOME if set
|
||||
if set -q PYTHONHOME
|
||||
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
|
||||
set -e PYTHONHOME
|
||||
end
|
||||
|
||||
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
|
||||
# fish uses a function instead of an env var to generate the prompt.
|
||||
|
||||
# save the current fish_prompt function as the function _old_fish_prompt
|
||||
functions -c fish_prompt _old_fish_prompt
|
||||
|
||||
# with the original prompt function renamed, we can override with our own.
|
||||
function fish_prompt
|
||||
# Save the return status of the last command
|
||||
set -l old_status $status
|
||||
|
||||
# Prompt override?
|
||||
if test -n "(venv) "
|
||||
printf "%s%s" "(venv) " (set_color normal)
|
||||
else
|
||||
# ...Otherwise, prepend env
|
||||
set -l _checkbase (basename "$VIRTUAL_ENV")
|
||||
if test $_checkbase = "__"
|
||||
# special case for Aspen magic directories
|
||||
# see http://www.zetadev.com/software/aspen/
|
||||
printf "%s[%s]%s " (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal)
|
||||
else
|
||||
printf "%s(%s)%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal)
|
||||
end
|
||||
end
|
||||
|
||||
# Restore the return status of the previous command.
|
||||
echo "exit $old_status" | .
|
||||
_old_fish_prompt
|
||||
end
|
||||
|
||||
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
|
||||
end
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/home/allem/PycharmProjects/thue/venv/bin/python
|
||||
# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==39.1.0','console_scripts','easy_install'
|
||||
__requires__ = 'setuptools==39.1.0'
|
||||
import re
|
||||
import sys
|
||||
from pkg_resources import load_entry_point
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(
|
||||
load_entry_point('setuptools==39.1.0', 'console_scripts', 'easy_install')()
|
||||
)
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/home/allem/PycharmProjects/thue/venv/bin/python
|
||||
# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==39.1.0','console_scripts','easy_install-3.6'
|
||||
__requires__ = 'setuptools==39.1.0'
|
||||
import re
|
||||
import sys
|
||||
from pkg_resources import load_entry_point
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(
|
||||
load_entry_point('setuptools==39.1.0', 'console_scripts', 'easy_install-3.6')()
|
||||
)
|
||||
12
venv/bin/pip
12
venv/bin/pip
@@ -1,12 +0,0 @@
|
||||
#!/home/allem/PycharmProjects/thue/venv/bin/python
|
||||
# EASY-INSTALL-ENTRY-SCRIPT: 'pip==10.0.1','console_scripts','pip'
|
||||
__requires__ = 'pip==10.0.1'
|
||||
import re
|
||||
import sys
|
||||
from pkg_resources import load_entry_point
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(
|
||||
load_entry_point('pip==10.0.1', 'console_scripts', 'pip')()
|
||||
)
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/home/allem/PycharmProjects/thue/venv/bin/python
|
||||
# EASY-INSTALL-ENTRY-SCRIPT: 'pip==10.0.1','console_scripts','pip3'
|
||||
__requires__ = 'pip==10.0.1'
|
||||
import re
|
||||
import sys
|
||||
from pkg_resources import load_entry_point
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(
|
||||
load_entry_point('pip==10.0.1', 'console_scripts', 'pip3')()
|
||||
)
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/home/allem/PycharmProjects/thue/venv/bin/python
|
||||
# EASY-INSTALL-ENTRY-SCRIPT: 'pip==10.0.1','console_scripts','pip3.6'
|
||||
__requires__ = 'pip==10.0.1'
|
||||
import re
|
||||
import sys
|
||||
from pkg_resources import load_entry_point
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(
|
||||
load_entry_point('pip==10.0.1', 'console_scripts', 'pip3.6')()
|
||||
)
|
||||
BIN
venv/bin/python
BIN
venv/bin/python
Binary file not shown.
BIN
venv/bin/python3
BIN
venv/bin/python3
Binary file not shown.
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
./setuptools-39.1.0-py3.6.egg
|
||||
./pip-10.0.1-py3.6.egg
|
||||
@@ -1,8 +0,0 @@
|
||||
from .tree import Tree
|
||||
from .visitors import Transformer, Visitor, v_args, Discard
|
||||
from .visitors import InlineTransformer, inline_args # XXX Deprecated
|
||||
from .exceptions import ParseError, LexError, GrammarError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters
|
||||
from .lexer import Token
|
||||
from .lark import Lark
|
||||
|
||||
__version__ = "0.6.5"
|
||||
@@ -1,16 +0,0 @@
|
||||
|
||||
|
||||
class LexerConf:
|
||||
def __init__(self, tokens, ignore=(), postlex=None, callbacks=None):
|
||||
self.tokens = tokens
|
||||
self.ignore = ignore
|
||||
self.postlex = postlex
|
||||
self.callbacks = callbacks or {}
|
||||
|
||||
class ParserConf:
|
||||
def __init__(self, rules, callback, start):
|
||||
self.rules = rules
|
||||
self.callback = callback
|
||||
self.start = start
|
||||
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
from .utils import STRING_TYPE
|
||||
|
||||
###{standalone
|
||||
class LarkError(Exception):
|
||||
pass
|
||||
|
||||
class GrammarError(LarkError):
|
||||
pass
|
||||
|
||||
class ParseError(LarkError):
|
||||
pass
|
||||
|
||||
class LexError(LarkError):
|
||||
pass
|
||||
|
||||
class UnexpectedInput(LarkError):
|
||||
pos_in_stream = None
|
||||
|
||||
def get_context(self, text, span=40):
|
||||
pos = self.pos_in_stream
|
||||
start = max(pos - span, 0)
|
||||
end = pos + span
|
||||
before = text[start:pos].rsplit('\n', 1)[-1]
|
||||
after = text[pos:end].split('\n', 1)[0]
|
||||
return before + after + '\n' + ' ' * len(before) + '^\n'
|
||||
|
||||
def match_examples(self, parse_fn, examples):
|
||||
""" Given a parser instance and a dictionary mapping some label with
|
||||
some malformed syntax examples, it'll return the label for the
|
||||
example that bests matches the current error.
|
||||
"""
|
||||
assert self.state is not None, "Not supported for this exception"
|
||||
|
||||
candidate = None
|
||||
for label, example in examples.items():
|
||||
assert not isinstance(example, STRING_TYPE)
|
||||
|
||||
for malformed in example:
|
||||
try:
|
||||
parse_fn(malformed)
|
||||
except UnexpectedInput as ut:
|
||||
if ut.state == self.state:
|
||||
try:
|
||||
if ut.token == self.token: # Try exact match first
|
||||
return label
|
||||
except AttributeError:
|
||||
pass
|
||||
if not candidate:
|
||||
candidate = label
|
||||
|
||||
return candidate
|
||||
|
||||
|
||||
class UnexpectedCharacters(LexError, UnexpectedInput):
|
||||
def __init__(self, seq, lex_pos, line, column, allowed=None, considered_tokens=None, state=None):
|
||||
message = "No terminal defined for '%s' at line %d col %d" % (seq[lex_pos], line, column)
|
||||
|
||||
self.line = line
|
||||
self.column = column
|
||||
self.allowed = allowed
|
||||
self.considered_tokens = considered_tokens
|
||||
self.pos_in_stream = lex_pos
|
||||
self.state = state
|
||||
|
||||
message += '\n\n' + self.get_context(seq)
|
||||
if allowed:
|
||||
message += '\nExpecting: %s\n' % allowed
|
||||
|
||||
super(UnexpectedCharacters, self).__init__(message)
|
||||
|
||||
|
||||
|
||||
class UnexpectedToken(ParseError, UnexpectedInput):
|
||||
def __init__(self, token, expected, considered_rules=None, state=None):
|
||||
self.token = token
|
||||
self.expected = expected # XXX str shouldn't necessary
|
||||
self.line = getattr(token, 'line', '?')
|
||||
self.column = getattr(token, 'column', '?')
|
||||
self.considered_rules = considered_rules
|
||||
self.state = state
|
||||
self.pos_in_stream = getattr(token, 'pos_in_stream', None)
|
||||
|
||||
message = ("Unexpected token %r at line %s, column %s.\n"
|
||||
"Expected one of: \n\t* %s\n"
|
||||
% (token, self.line, self.column, '\n\t* '.join(self.expected)))
|
||||
|
||||
super(UnexpectedToken, self).__init__(message)
|
||||
|
||||
###}
|
||||
@@ -1,66 +0,0 @@
|
||||
class Symbol(object):
|
||||
is_term = NotImplemented
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __eq__(self, other):
|
||||
assert isinstance(other, Symbol), other
|
||||
return self.is_term == other.is_term and self.name == other.name
|
||||
|
||||
def __ne__(self, other):
|
||||
return not (self == other)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.name)
|
||||
|
||||
def __repr__(self):
|
||||
return '%s(%r)' % (type(self).__name__, self.name)
|
||||
|
||||
fullrepr = property(__repr__)
|
||||
|
||||
class Terminal(Symbol):
|
||||
is_term = True
|
||||
|
||||
def __init__(self, name, filter_out=False):
|
||||
self.name = name
|
||||
self.filter_out = filter_out
|
||||
|
||||
@property
|
||||
def fullrepr(self):
|
||||
return '%s(%r, %r)' % (type(self).__name__, self.name, self.filter_out)
|
||||
|
||||
|
||||
class NonTerminal(Symbol):
|
||||
is_term = False
|
||||
|
||||
class Rule(object):
|
||||
"""
|
||||
origin : a symbol
|
||||
expansion : a list of symbols
|
||||
"""
|
||||
def __init__(self, origin, expansion, alias=None, options=None):
|
||||
self.origin = origin
|
||||
self.expansion = expansion
|
||||
self.alias = alias
|
||||
self.options = options
|
||||
|
||||
def __str__(self):
|
||||
return '<%s : %s>' % (self.origin, ' '.join(map(str,self.expansion)))
|
||||
|
||||
def __repr__(self):
|
||||
return 'Rule(%r, %r, %r, %r)' % (self.origin, self.expansion, self.alias, self.options)
|
||||
|
||||
|
||||
class RuleOptions:
|
||||
def __init__(self, keep_all_tokens=False, expand1=False, priority=None):
|
||||
self.keep_all_tokens = keep_all_tokens
|
||||
self.expand1 = expand1
|
||||
self.priority = priority
|
||||
|
||||
def __repr__(self):
|
||||
return 'RuleOptions(%r, %r, %r)' % (
|
||||
self.keep_all_tokens,
|
||||
self.expand1,
|
||||
self.priority,
|
||||
)
|
||||
@@ -1,49 +0,0 @@
|
||||
//
|
||||
// Numbers
|
||||
//
|
||||
|
||||
DIGIT: "0".."9"
|
||||
HEXDIGIT: "a".."f"|"A".."F"|DIGIT
|
||||
|
||||
INT: DIGIT+
|
||||
SIGNED_INT: ["+"|"-"] INT
|
||||
DECIMAL: INT "." INT? | "." INT
|
||||
|
||||
// float = /-?\d+(\.\d+)?([eE][+-]?\d+)?/
|
||||
_EXP: ("e"|"E") SIGNED_INT
|
||||
FLOAT: INT _EXP | DECIMAL _EXP?
|
||||
SIGNED_FLOAT: ["+"|"-"] FLOAT
|
||||
|
||||
NUMBER: FLOAT | INT
|
||||
SIGNED_NUMBER: ["+"|"-"] NUMBER
|
||||
|
||||
//
|
||||
// Strings
|
||||
//
|
||||
//STRING: /"(\\\"|\\\\|[^"\n])*?"i?/
|
||||
STRING_INNER: ("\\\""|/[^"]/)
|
||||
ESCAPED_STRING: "\"" STRING_INNER* "\""
|
||||
|
||||
|
||||
//
|
||||
// Names (Variables)
|
||||
//
|
||||
LCASE_LETTER: "a".."z"
|
||||
UCASE_LETTER: "A".."Z"
|
||||
|
||||
LETTER: UCASE_LETTER | LCASE_LETTER
|
||||
WORD: LETTER+
|
||||
|
||||
CNAME: ("_"|LETTER) ("_"|LETTER|DIGIT)*
|
||||
|
||||
|
||||
//
|
||||
// Whitespace
|
||||
//
|
||||
WS_INLINE: (" "|/\t/)+
|
||||
WS: /[ \t\f\r\n]/+
|
||||
|
||||
CR : /\r/
|
||||
LF : /\n/
|
||||
NEWLINE: (CR? LF)+
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
"Provides Indentation services for languages with indentation similar to Python"
|
||||
|
||||
from .lexer import Token
|
||||
|
||||
###{standalone
|
||||
class Indenter:
|
||||
def __init__(self):
|
||||
self.paren_level = 0
|
||||
self.indent_level = [0]
|
||||
|
||||
def handle_NL(self, token):
|
||||
if self.paren_level > 0:
|
||||
return
|
||||
|
||||
yield token
|
||||
|
||||
indent_str = token.rsplit('\n', 1)[1] # Tabs and spaces
|
||||
indent = indent_str.count(' ') + indent_str.count('\t') * self.tab_len
|
||||
|
||||
if indent > self.indent_level[-1]:
|
||||
self.indent_level.append(indent)
|
||||
yield Token.new_borrow_pos(self.INDENT_type, indent_str, token)
|
||||
else:
|
||||
while indent < self.indent_level[-1]:
|
||||
self.indent_level.pop()
|
||||
yield Token.new_borrow_pos(self.DEDENT_type, indent_str, token)
|
||||
|
||||
assert indent == self.indent_level[-1], '%s != %s' % (indent, self.indent_level[-1])
|
||||
|
||||
def process(self, stream):
|
||||
for token in stream:
|
||||
if token.type == self.NL_type:
|
||||
for t in self.handle_NL(token):
|
||||
yield t
|
||||
else:
|
||||
yield token
|
||||
|
||||
if token.type in self.OPEN_PAREN_types:
|
||||
self.paren_level += 1
|
||||
elif token.type in self.CLOSE_PAREN_types:
|
||||
self.paren_level -= 1
|
||||
assert self.paren_level >= 0
|
||||
|
||||
while len(self.indent_level) > 1:
|
||||
self.indent_level.pop()
|
||||
yield Token(self.DEDENT_type, '')
|
||||
|
||||
assert self.indent_level == [0], self.indent_level
|
||||
|
||||
# XXX Hack for ContextualLexer. Maybe there's a more elegant solution?
|
||||
@property
|
||||
def always_accept(self):
|
||||
return (self.NL_type,)
|
||||
|
||||
###}
|
||||
@@ -1,235 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from io import open
|
||||
|
||||
from .utils import STRING_TYPE
|
||||
from .load_grammar import load_grammar
|
||||
from .tree import Tree
|
||||
from .common import LexerConf, ParserConf
|
||||
|
||||
from .lexer import Lexer, TraditionalLexer
|
||||
from .parse_tree_builder import ParseTreeBuilder
|
||||
from .parser_frontends import get_frontend
|
||||
|
||||
|
||||
class LarkOptions(object):
|
||||
"""Specifies the options for Lark
|
||||
|
||||
"""
|
||||
OPTIONS_DOC = """
|
||||
parser - Decides which parser engine to use, "earley" or "lalr". (Default: "earley")
|
||||
Note: "lalr" requires a lexer
|
||||
|
||||
lexer - Decides whether or not to use a lexer stage
|
||||
"standard": Use a standard lexer
|
||||
"contextual": Stronger lexer (only works with parser="lalr")
|
||||
"dynamic": Flexible and powerful (only with parser="earley")
|
||||
"dynamic_complete": Same as dynamic, but tries *every* variation
|
||||
of tokenizing possible. (only with parser="earley")
|
||||
"auto" (default): Choose for me based on grammar and parser
|
||||
|
||||
ambiguity - Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
|
||||
"resolve": The parser will automatically choose the simplest derivation
|
||||
(it chooses consistently: greedy for tokens, non-greedy for rules)
|
||||
"explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
|
||||
|
||||
transformer - Applies the transformer to every parse tree
|
||||
debug - Affects verbosity (default: False)
|
||||
keep_all_tokens - Don't automagically remove "punctuation" tokens (default: False)
|
||||
cache_grammar - Cache the Lark grammar (Default: False)
|
||||
postlex - Lexer post-processing (Default: None) Only works with the standard and contextual lexers.
|
||||
start - The start symbol (Default: start)
|
||||
profile - Measure run-time usage in Lark. Read results from the profiler proprety (Default: False)
|
||||
propagate_positions - Propagates [line, column, end_line, end_column] attributes into all tree branches.
|
||||
lexer_callbacks - Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
|
||||
"""
|
||||
__doc__ += OPTIONS_DOC
|
||||
def __init__(self, options_dict):
|
||||
o = dict(options_dict)
|
||||
|
||||
self.debug = bool(o.pop('debug', False))
|
||||
self.keep_all_tokens = bool(o.pop('keep_all_tokens', False))
|
||||
self.tree_class = o.pop('tree_class', Tree)
|
||||
self.cache_grammar = o.pop('cache_grammar', False)
|
||||
self.postlex = o.pop('postlex', None)
|
||||
self.parser = o.pop('parser', 'earley')
|
||||
self.lexer = o.pop('lexer', 'auto')
|
||||
self.transformer = o.pop('transformer', None)
|
||||
self.start = o.pop('start', 'start')
|
||||
self.profile = o.pop('profile', False)
|
||||
self.ambiguity = o.pop('ambiguity', 'auto')
|
||||
self.propagate_positions = o.pop('propagate_positions', False)
|
||||
self.earley__predict_all = o.pop('earley__predict_all', False)
|
||||
self.lexer_callbacks = o.pop('lexer_callbacks', {})
|
||||
|
||||
assert self.parser in ('earley', 'lalr', 'cyk', None)
|
||||
|
||||
if self.parser == 'earley' and self.transformer:
|
||||
raise ValueError('Cannot specify an embedded transformer when using the Earley algorithm.'
|
||||
'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. lalr)')
|
||||
|
||||
if o:
|
||||
raise ValueError("Unknown options: %s" % o.keys())
|
||||
|
||||
|
||||
class Profiler:
|
||||
def __init__(self):
|
||||
self.total_time = defaultdict(float)
|
||||
self.cur_section = '__init__'
|
||||
self.last_enter_time = time.time()
|
||||
|
||||
def enter_section(self, name):
|
||||
cur_time = time.time()
|
||||
self.total_time[self.cur_section] += cur_time - self.last_enter_time
|
||||
self.last_enter_time = cur_time
|
||||
self.cur_section = name
|
||||
|
||||
def make_wrapper(self, name, f):
|
||||
def wrapper(*args, **kwargs):
|
||||
last_section = self.cur_section
|
||||
self.enter_section(name)
|
||||
try:
|
||||
return f(*args, **kwargs)
|
||||
finally:
|
||||
self.enter_section(last_section)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class Lark:
|
||||
def __init__(self, grammar, **options):
|
||||
"""
|
||||
grammar : a string or file-object containing the grammar spec (using Lark's ebnf syntax)
|
||||
options : a dictionary controlling various aspects of Lark.
|
||||
"""
|
||||
self.options = LarkOptions(options)
|
||||
|
||||
# Some, but not all file-like objects have a 'name' attribute
|
||||
try:
|
||||
self.source = grammar.name
|
||||
except AttributeError:
|
||||
self.source = '<string>'
|
||||
cache_file = "larkcache_%s" % str(hash(grammar)%(2**32))
|
||||
else:
|
||||
cache_file = "larkcache_%s" % os.path.basename(self.source)
|
||||
|
||||
# Drain file-like objects to get their contents
|
||||
try:
|
||||
read = grammar.read
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
grammar = read()
|
||||
|
||||
assert isinstance(grammar, STRING_TYPE)
|
||||
|
||||
if self.options.cache_grammar:
|
||||
raise NotImplementedError("Not available yet")
|
||||
|
||||
assert not self.options.profile, "Feature temporarily disabled"
|
||||
self.profiler = Profiler() if self.options.profile else None
|
||||
|
||||
if self.options.lexer == 'auto':
|
||||
if self.options.parser == 'lalr':
|
||||
self.options.lexer = 'contextual'
|
||||
elif self.options.parser == 'earley':
|
||||
self.options.lexer = 'dynamic'
|
||||
elif self.options.parser == 'cyk':
|
||||
self.options.lexer = 'standard'
|
||||
else:
|
||||
assert False, self.options.parser
|
||||
lexer = self.options.lexer
|
||||
assert lexer in ('standard', 'contextual', 'dynamic', 'dynamic_complete') or issubclass(lexer, Lexer)
|
||||
|
||||
if self.options.ambiguity == 'auto':
|
||||
if self.options.parser == 'earley':
|
||||
self.options.ambiguity = 'resolve'
|
||||
else:
|
||||
disambig_parsers = ['earley', 'cyk']
|
||||
assert self.options.parser in disambig_parsers, (
|
||||
'Only %s supports disambiguation right now') % ', '.join(disambig_parsers)
|
||||
assert self.options.ambiguity in ('resolve', 'explicit', 'auto', 'resolve__antiscore_sum')
|
||||
|
||||
# Parse the grammar file and compose the grammars (TODO)
|
||||
self.grammar = load_grammar(grammar, self.source)
|
||||
|
||||
# Compile the EBNF grammar into BNF
|
||||
self.terminals, self.rules, self.ignore_tokens = self.grammar.compile()
|
||||
|
||||
self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, self.options.lexer_callbacks)
|
||||
|
||||
if self.options.parser:
|
||||
self.parser = self._build_parser()
|
||||
elif lexer:
|
||||
self.lexer = self._build_lexer()
|
||||
|
||||
if self.profiler: self.profiler.enter_section('outside_lark')
|
||||
|
||||
__init__.__doc__ += "\nOPTIONS:" + LarkOptions.OPTIONS_DOC
|
||||
|
||||
def _build_lexer(self):
|
||||
return TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks)
|
||||
|
||||
def _build_parser(self):
|
||||
self.parser_class = get_frontend(self.options.parser, self.options.lexer)
|
||||
|
||||
self._parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class, self.options.propagate_positions, self.options.keep_all_tokens, self.options.parser!='lalr')
|
||||
callback = self._parse_tree_builder.create_callback(self.options.transformer)
|
||||
if self.profiler:
|
||||
for f in dir(callback):
|
||||
if not (f.startswith('__') and f.endswith('__')):
|
||||
setattr(callback, f, self.profiler.make_wrapper('transformer', getattr(callback, f)))
|
||||
|
||||
parser_conf = ParserConf(self.rules, callback, self.options.start)
|
||||
|
||||
return self.parser_class(self.lexer_conf, parser_conf, options=self.options)
|
||||
|
||||
@classmethod
|
||||
def open(cls, grammar_filename, rel_to=None, **options):
|
||||
"""Create an instance of Lark with the grammar given by its filename
|
||||
|
||||
If rel_to is provided, the function will find the grammar filename in relation to it.
|
||||
|
||||
Example:
|
||||
|
||||
>>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr")
|
||||
Lark(...)
|
||||
|
||||
"""
|
||||
if rel_to:
|
||||
basepath = os.path.dirname(rel_to)
|
||||
grammar_filename = os.path.join(basepath, grammar_filename)
|
||||
with open(grammar_filename, encoding='utf8') as f:
|
||||
return cls(f, **options)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source, self.options.parser, self.options.lexer)
|
||||
|
||||
|
||||
def lex(self, text):
|
||||
"Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard'"
|
||||
if not hasattr(self, 'lexer'):
|
||||
self.lexer = self._build_lexer()
|
||||
stream = self.lexer.lex(text)
|
||||
if self.options.postlex:
|
||||
return self.options.postlex.process(stream)
|
||||
return stream
|
||||
|
||||
def parse(self, text):
|
||||
"Parse the given text, according to the options provided. Returns a tree, unless specified otherwise."
|
||||
return self.parser.parse(text)
|
||||
|
||||
# if self.profiler:
|
||||
# self.profiler.enter_section('lex')
|
||||
# l = list(self.lex(text))
|
||||
# self.profiler.enter_section('parse')
|
||||
# try:
|
||||
# return self.parser.parse(l)
|
||||
# finally:
|
||||
# self.profiler.enter_section('outside_lark')
|
||||
# else:
|
||||
# l = list(self.lex(text))
|
||||
# return self.parser.parse(l)
|
||||
@@ -1,324 +0,0 @@
|
||||
## Lexer Implementation
|
||||
|
||||
import re
|
||||
|
||||
from .utils import Str, classify, get_regexp_width, Py36
|
||||
from .exceptions import UnexpectedCharacters, LexError
|
||||
|
||||
class Pattern(object):
|
||||
def __init__(self, value, flags=()):
|
||||
self.value = value
|
||||
self.flags = frozenset(flags)
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.to_regexp())
|
||||
|
||||
# Pattern Hashing assumes all subclasses have a different priority!
|
||||
def __hash__(self):
|
||||
return hash((type(self), self.value, self.flags))
|
||||
def __eq__(self, other):
|
||||
return type(self) == type(other) and self.value == other.value and self.flags == other.flags
|
||||
|
||||
def to_regexp(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
if Py36:
|
||||
# Python 3.6 changed syntax for flags in regular expression
|
||||
def _get_flags(self, value):
|
||||
for f in self.flags:
|
||||
value = ('(?%s:%s)' % (f, value))
|
||||
return value
|
||||
|
||||
else:
|
||||
def _get_flags(self, value):
|
||||
for f in self.flags:
|
||||
value = ('(?%s)' % f) + value
|
||||
return value
|
||||
|
||||
class PatternStr(Pattern):
|
||||
def to_regexp(self):
|
||||
return self._get_flags(re.escape(self.value))
|
||||
|
||||
@property
|
||||
def min_width(self):
|
||||
return len(self.value)
|
||||
max_width = min_width
|
||||
|
||||
class PatternRE(Pattern):
|
||||
def to_regexp(self):
|
||||
return self._get_flags(self.value)
|
||||
|
||||
@property
|
||||
def min_width(self):
|
||||
return get_regexp_width(self.to_regexp())[0]
|
||||
@property
|
||||
def max_width(self):
|
||||
return get_regexp_width(self.to_regexp())[1]
|
||||
|
||||
class TerminalDef(object):
|
||||
def __init__(self, name, pattern, priority=1):
|
||||
assert isinstance(pattern, Pattern), pattern
|
||||
self.name = name
|
||||
self.pattern = pattern
|
||||
self.priority = priority
|
||||
|
||||
def __repr__(self):
|
||||
return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern)
|
||||
|
||||
|
||||
|
||||
###{standalone
|
||||
class Token(Str):
|
||||
__slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column')
|
||||
|
||||
def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None):
|
||||
self = super(Token, cls).__new__(cls, value)
|
||||
self.type = type_
|
||||
self.pos_in_stream = pos_in_stream
|
||||
self.value = value
|
||||
self.line = line
|
||||
self.column = column
|
||||
self.end_line = None
|
||||
self.end_column = None
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def new_borrow_pos(cls, type_, value, borrow_t):
|
||||
return cls(type_, value, borrow_t.pos_in_stream, line=borrow_t.line, column=borrow_t.column)
|
||||
|
||||
def __reduce__(self):
|
||||
return (self.__class__, (self.type, self.value, self.pos_in_stream, self.line, self.column, ))
|
||||
|
||||
def __repr__(self):
|
||||
return 'Token(%s, %r)' % (self.type, self.value)
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
return Token(self.type, self.value, self.pos_in_stream, self.line, self.column)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, Token) and self.type != other.type:
|
||||
return False
|
||||
|
||||
return Str.__eq__(self, other)
|
||||
|
||||
__hash__ = Str.__hash__
|
||||
|
||||
|
||||
class LineCounter:
|
||||
def __init__(self):
|
||||
self.newline_char = '\n'
|
||||
self.char_pos = 0
|
||||
self.line = 1
|
||||
self.column = 1
|
||||
self.line_start_pos = 0
|
||||
|
||||
def feed(self, token, test_newline=True):
|
||||
"""Consume a token and calculate the new line & column.
|
||||
|
||||
As an optional optimization, set test_newline=False is token doesn't contain a newline.
|
||||
"""
|
||||
if test_newline:
|
||||
newlines = token.count(self.newline_char)
|
||||
if newlines:
|
||||
self.line += newlines
|
||||
self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1
|
||||
|
||||
self.char_pos += len(token)
|
||||
self.column = self.char_pos - self.line_start_pos + 1
|
||||
|
||||
class _Lex:
|
||||
"Built to serve both Lexer and ContextualLexer"
|
||||
def __init__(self, lexer, state=None):
|
||||
self.lexer = lexer
|
||||
self.state = state
|
||||
|
||||
def lex(self, stream, newline_types, ignore_types):
|
||||
newline_types = frozenset(newline_types)
|
||||
ignore_types = frozenset(ignore_types)
|
||||
line_ctr = LineCounter()
|
||||
|
||||
while line_ctr.char_pos < len(stream):
|
||||
lexer = self.lexer
|
||||
for mre, type_from_index in lexer.mres:
|
||||
m = mre.match(stream, line_ctr.char_pos)
|
||||
if not m:
|
||||
continue
|
||||
|
||||
t = None
|
||||
value = m.group(0)
|
||||
type_ = type_from_index[m.lastindex]
|
||||
if type_ not in ignore_types:
|
||||
t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
|
||||
if t.type in lexer.callback:
|
||||
t = lexer.callback[t.type](t)
|
||||
yield t
|
||||
else:
|
||||
if type_ in lexer.callback:
|
||||
t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
|
||||
lexer.callback[type_](t)
|
||||
|
||||
line_ctr.feed(value, type_ in newline_types)
|
||||
if t:
|
||||
t.end_line = line_ctr.line
|
||||
t.end_column = line_ctr.column
|
||||
|
||||
break
|
||||
else:
|
||||
raise UnexpectedCharacters(stream, line_ctr.char_pos, line_ctr.line, line_ctr.column, state=self.state)
|
||||
|
||||
|
||||
class UnlessCallback:
|
||||
def __init__(self, mres):
|
||||
self.mres = mres
|
||||
|
||||
def __call__(self, t):
|
||||
for mre, type_from_index in self.mres:
|
||||
m = mre.match(t.value)
|
||||
if m:
|
||||
t.type = type_from_index[m.lastindex]
|
||||
break
|
||||
return t
|
||||
|
||||
###}
|
||||
|
||||
|
||||
|
||||
def _create_unless(terminals):
|
||||
tokens_by_type = classify(terminals, lambda t: type(t.pattern))
|
||||
assert len(tokens_by_type) <= 2, tokens_by_type.keys()
|
||||
embedded_strs = set()
|
||||
callback = {}
|
||||
for retok in tokens_by_type.get(PatternRE, []):
|
||||
unless = [] # {}
|
||||
for strtok in tokens_by_type.get(PatternStr, []):
|
||||
if strtok.priority > retok.priority:
|
||||
continue
|
||||
s = strtok.pattern.value
|
||||
m = re.match(retok.pattern.to_regexp(), s)
|
||||
if m and m.group(0) == s:
|
||||
unless.append(strtok)
|
||||
if strtok.pattern.flags <= retok.pattern.flags:
|
||||
embedded_strs.add(strtok)
|
||||
if unless:
|
||||
callback[retok.name] = UnlessCallback(build_mres(unless, match_whole=True))
|
||||
|
||||
terminals = [t for t in terminals if t not in embedded_strs]
|
||||
return terminals, callback
|
||||
|
||||
|
||||
def _build_mres(terminals, max_size, match_whole):
|
||||
# Python sets an unreasonable group limit (currently 100) in its re module
|
||||
# Worse, the only way to know we reached it is by catching an AssertionError!
|
||||
# This function recursively tries less and less groups until it's successful.
|
||||
postfix = '$' if match_whole else ''
|
||||
mres = []
|
||||
while terminals:
|
||||
try:
|
||||
mre = re.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in terminals[:max_size]))
|
||||
except AssertionError: # Yes, this is what Python provides us.. :/
|
||||
return _build_mres(terminals, max_size//2, match_whole)
|
||||
|
||||
# terms_from_name = {t.name: t for t in terminals[:max_size]}
|
||||
mres.append((mre, {i:n for n,i in mre.groupindex.items()} ))
|
||||
terminals = terminals[max_size:]
|
||||
return mres
|
||||
|
||||
def build_mres(terminals, match_whole=False):
|
||||
return _build_mres(terminals, len(terminals), match_whole)
|
||||
|
||||
def _regexp_has_newline(r):
|
||||
"""Expressions that may indicate newlines in a regexp:
|
||||
- newlines (\n)
|
||||
- escaped newline (\\n)
|
||||
- anything but ([^...])
|
||||
- any-char (.) when the flag (?s) exists
|
||||
"""
|
||||
return '\n' in r or '\\n' in r or '[^' in r or ('(?s' in r and '.' in r)
|
||||
|
||||
class Lexer:
|
||||
"""Lexer interface
|
||||
|
||||
Method Signatures:
|
||||
lex(self, stream) -> Iterator[Token]
|
||||
|
||||
set_parser_state(self, state) # Optional
|
||||
"""
|
||||
set_parser_state = NotImplemented
|
||||
lex = NotImplemented
|
||||
|
||||
class TraditionalLexer(Lexer):
|
||||
def __init__(self, terminals, ignore=(), user_callbacks={}):
|
||||
assert all(isinstance(t, TerminalDef) for t in terminals), terminals
|
||||
|
||||
terminals = list(terminals)
|
||||
|
||||
# Sanitization
|
||||
for t in terminals:
|
||||
try:
|
||||
re.compile(t.pattern.to_regexp())
|
||||
except:
|
||||
raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern))
|
||||
|
||||
if t.pattern.min_width == 0:
|
||||
raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern))
|
||||
|
||||
assert set(ignore) <= {t.name for t in terminals}
|
||||
|
||||
# Init
|
||||
self.newline_types = [t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp())]
|
||||
self.ignore_types = list(ignore)
|
||||
|
||||
terminals.sort(key=lambda x:(-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name))
|
||||
|
||||
terminals, self.callback = _create_unless(terminals)
|
||||
assert all(self.callback.values())
|
||||
|
||||
for type_, f in user_callbacks.items():
|
||||
assert type_ not in self.callback
|
||||
self.callback[type_] = f
|
||||
|
||||
self.terminals = terminals
|
||||
|
||||
self.mres = build_mres(terminals)
|
||||
|
||||
|
||||
def lex(self, stream):
|
||||
return _Lex(self).lex(stream, self.newline_types, self.ignore_types)
|
||||
|
||||
|
||||
class ContextualLexer(Lexer):
|
||||
def __init__(self, terminals, states, ignore=(), always_accept=(), user_callbacks={}):
|
||||
tokens_by_name = {}
|
||||
for t in terminals:
|
||||
assert t.name not in tokens_by_name, t
|
||||
tokens_by_name[t.name] = t
|
||||
|
||||
lexer_by_tokens = {}
|
||||
self.lexers = {}
|
||||
for state, accepts in states.items():
|
||||
key = frozenset(accepts)
|
||||
try:
|
||||
lexer = lexer_by_tokens[key]
|
||||
except KeyError:
|
||||
accepts = set(accepts) | set(ignore) | set(always_accept)
|
||||
state_tokens = [tokens_by_name[n] for n in accepts if n and n in tokens_by_name]
|
||||
lexer = TraditionalLexer(state_tokens, ignore=ignore, user_callbacks=user_callbacks)
|
||||
lexer_by_tokens[key] = lexer
|
||||
|
||||
self.lexers[state] = lexer
|
||||
|
||||
self.root_lexer = TraditionalLexer(terminals, ignore=ignore, user_callbacks=user_callbacks)
|
||||
|
||||
self.set_parser_state(None) # Needs to be set on the outside
|
||||
|
||||
def set_parser_state(self, state):
|
||||
self.parser_state = state
|
||||
|
||||
def lex(self, stream):
|
||||
l = _Lex(self.lexers[self.parser_state], self.parser_state)
|
||||
for x in l.lex(stream, self.root_lexer.newline_types, self.root_lexer.ignore_types):
|
||||
yield x
|
||||
l.lexer = self.lexers[self.parser_state]
|
||||
l.state = self.parser_state
|
||||
|
||||
|
||||
@@ -1,779 +0,0 @@
|
||||
"Parses and creates Grammar objects"
|
||||
|
||||
import os.path
|
||||
import sys
|
||||
from ast import literal_eval
|
||||
from copy import deepcopy
|
||||
|
||||
from .utils import bfs
|
||||
from .lexer import Token, TerminalDef, PatternStr, PatternRE
|
||||
|
||||
from .parse_tree_builder import ParseTreeBuilder
|
||||
from .parser_frontends import LALR_TraditionalLexer
|
||||
from .common import LexerConf, ParserConf
|
||||
from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
|
||||
from .utils import classify, suppress
|
||||
from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken
|
||||
|
||||
from .tree import Tree, SlottedTree as ST
|
||||
from .visitors import Transformer, Visitor, v_args, Transformer_InPlace
|
||||
inline_args = v_args(inline=True)
|
||||
|
||||
__path__ = os.path.dirname(__file__)
|
||||
IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
|
||||
|
||||
EXT = '.lark'
|
||||
|
||||
_RE_FLAGS = 'imslux'
|
||||
|
||||
_TERMINAL_NAMES = {
|
||||
'.' : 'DOT',
|
||||
',' : 'COMMA',
|
||||
':' : 'COLON',
|
||||
';' : 'SEMICOLON',
|
||||
'+' : 'PLUS',
|
||||
'-' : 'MINUS',
|
||||
'*' : 'STAR',
|
||||
'/' : 'SLASH',
|
||||
'\\' : 'BACKSLASH',
|
||||
'|' : 'VBAR',
|
||||
'?' : 'QMARK',
|
||||
'!' : 'BANG',
|
||||
'@' : 'AT',
|
||||
'#' : 'HASH',
|
||||
'$' : 'DOLLAR',
|
||||
'%' : 'PERCENT',
|
||||
'^' : 'CIRCUMFLEX',
|
||||
'&' : 'AMPERSAND',
|
||||
'_' : 'UNDERSCORE',
|
||||
'<' : 'LESSTHAN',
|
||||
'>' : 'MORETHAN',
|
||||
'=' : 'EQUAL',
|
||||
'"' : 'DBLQUOTE',
|
||||
'\'' : 'QUOTE',
|
||||
'`' : 'BACKQUOTE',
|
||||
'~' : 'TILDE',
|
||||
'(' : 'LPAR',
|
||||
')' : 'RPAR',
|
||||
'{' : 'LBRACE',
|
||||
'}' : 'RBRACE',
|
||||
'[' : 'LSQB',
|
||||
']' : 'RSQB',
|
||||
'\n' : 'NEWLINE',
|
||||
'\r\n' : 'CRLF',
|
||||
'\t' : 'TAB',
|
||||
' ' : 'SPACE',
|
||||
}
|
||||
|
||||
# Grammar Parser
|
||||
TERMINALS = {
|
||||
'_LPAR': r'\(',
|
||||
'_RPAR': r'\)',
|
||||
'_LBRA': r'\[',
|
||||
'_RBRA': r'\]',
|
||||
'OP': '[+*][?]?|[?](?![a-z])',
|
||||
'_COLON': ':',
|
||||
'_COMMA': ',',
|
||||
'_OR': r'\|',
|
||||
'_DOT': r'\.',
|
||||
'TILDE': '~',
|
||||
'RULE': '!?[_?]?[a-z][_a-z0-9]*',
|
||||
'TERMINAL': '_?[A-Z][_A-Z0-9]*',
|
||||
'STRING': r'"(\\"|\\\\|[^"\n])*?"i?',
|
||||
'REGEXP': r'/(?!/)(\\/|\\\\|[^/\n])*?/[%s]*' % _RE_FLAGS,
|
||||
'_NL': r'(\r?\n)+\s*',
|
||||
'WS': r'[ \t]+',
|
||||
'COMMENT': r'//[^\n]*',
|
||||
'_TO': '->',
|
||||
'_IGNORE': r'%ignore',
|
||||
'_DECLARE': r'%declare',
|
||||
'_IMPORT': r'%import',
|
||||
'NUMBER': r'\d+',
|
||||
}
|
||||
|
||||
RULES = {
|
||||
'start': ['_list'],
|
||||
'_list': ['_item', '_list _item'],
|
||||
'_item': ['rule', 'term', 'statement', '_NL'],
|
||||
|
||||
'rule': ['RULE _COLON expansions _NL',
|
||||
'RULE _DOT NUMBER _COLON expansions _NL'],
|
||||
'expansions': ['alias',
|
||||
'expansions _OR alias',
|
||||
'expansions _NL _OR alias'],
|
||||
|
||||
'?alias': ['expansion _TO RULE', 'expansion'],
|
||||
'expansion': ['_expansion'],
|
||||
|
||||
'_expansion': ['', '_expansion expr'],
|
||||
|
||||
'?expr': ['atom',
|
||||
'atom OP',
|
||||
'atom TILDE NUMBER',
|
||||
'atom TILDE NUMBER _DOT _DOT NUMBER',
|
||||
],
|
||||
|
||||
'?atom': ['_LPAR expansions _RPAR',
|
||||
'maybe',
|
||||
'value'],
|
||||
|
||||
'value': ['terminal',
|
||||
'nonterminal',
|
||||
'literal',
|
||||
'range'],
|
||||
|
||||
'terminal': ['TERMINAL'],
|
||||
'nonterminal': ['RULE'],
|
||||
|
||||
'?name': ['RULE', 'TERMINAL'],
|
||||
|
||||
'maybe': ['_LBRA expansions _RBRA'],
|
||||
'range': ['STRING _DOT _DOT STRING'],
|
||||
|
||||
'term': ['TERMINAL _COLON expansions _NL',
|
||||
'TERMINAL _DOT NUMBER _COLON expansions _NL'],
|
||||
'statement': ['ignore', 'import', 'declare'],
|
||||
'ignore': ['_IGNORE expansions _NL'],
|
||||
'declare': ['_DECLARE _declare_args _NL'],
|
||||
'import': ['_IMPORT _import_path _NL',
|
||||
'_IMPORT _import_path _LPAR name_list _RPAR _NL',
|
||||
'_IMPORT _import_path _TO TERMINAL _NL'],
|
||||
|
||||
'_import_path': ['import_lib', 'import_rel'],
|
||||
'import_lib': ['_import_args'],
|
||||
'import_rel': ['_DOT _import_args'],
|
||||
'_import_args': ['name', '_import_args _DOT name'],
|
||||
|
||||
'name_list': ['_name_list'],
|
||||
'_name_list': ['name', '_name_list _COMMA name'],
|
||||
|
||||
'_declare_args': ['name', '_declare_args name'],
|
||||
'literal': ['REGEXP', 'STRING'],
|
||||
}
|
||||
|
||||
|
||||
@inline_args
|
||||
class EBNF_to_BNF(Transformer_InPlace):
|
||||
def __init__(self):
|
||||
self.new_rules = []
|
||||
self.rules_by_expr = {}
|
||||
self.prefix = 'anon'
|
||||
self.i = 0
|
||||
self.rule_options = None
|
||||
|
||||
def _add_recurse_rule(self, type_, expr):
|
||||
if expr in self.rules_by_expr:
|
||||
return self.rules_by_expr[expr]
|
||||
|
||||
new_name = '__%s_%s_%d' % (self.prefix, type_, self.i)
|
||||
self.i += 1
|
||||
t = NonTerminal(new_name)
|
||||
tree = ST('expansions', [ST('expansion', [expr]), ST('expansion', [t, expr])])
|
||||
self.new_rules.append((new_name, tree, self.rule_options))
|
||||
self.rules_by_expr[expr] = t
|
||||
return t
|
||||
|
||||
def expr(self, rule, op, *args):
|
||||
if op.value == '?':
|
||||
return ST('expansions', [rule, ST('expansion', [])])
|
||||
elif op.value == '+':
|
||||
# a : b c+ d
|
||||
# -->
|
||||
# a : b _c d
|
||||
# _c : _c c | c;
|
||||
return self._add_recurse_rule('plus', rule)
|
||||
elif op.value == '*':
|
||||
# a : b c* d
|
||||
# -->
|
||||
# a : b _c? d
|
||||
# _c : _c c | c;
|
||||
new_name = self._add_recurse_rule('star', rule)
|
||||
return ST('expansions', [new_name, ST('expansion', [])])
|
||||
elif op.value == '~':
|
||||
if len(args) == 1:
|
||||
mn = mx = int(args[0])
|
||||
else:
|
||||
mn, mx = map(int, args)
|
||||
if mx < mn:
|
||||
raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (rule, mn, mx))
|
||||
return ST('expansions', [ST('expansion', [rule] * n) for n in range(mn, mx+1)])
|
||||
assert False, op
|
||||
|
||||
|
||||
class SimplifyRule_Visitor(Visitor):
|
||||
|
||||
@staticmethod
|
||||
def _flatten(tree):
|
||||
while True:
|
||||
to_expand = [i for i, child in enumerate(tree.children)
|
||||
if isinstance(child, Tree) and child.data == tree.data]
|
||||
if not to_expand:
|
||||
break
|
||||
tree.expand_kids_by_index(*to_expand)
|
||||
|
||||
def expansion(self, tree):
|
||||
# rules_list unpacking
|
||||
# a : b (c|d) e
|
||||
# -->
|
||||
# a : b c e | b d e
|
||||
#
|
||||
# In AST terms:
|
||||
# expansion(b, expansions(c, d), e)
|
||||
# -->
|
||||
# expansions( expansion(b, c, e), expansion(b, d, e) )
|
||||
|
||||
self._flatten(tree)
|
||||
|
||||
for i, child in enumerate(tree.children):
|
||||
if isinstance(child, Tree) and child.data == 'expansions':
|
||||
tree.data = 'expansions'
|
||||
tree.children = [self.visit(ST('expansion', [option if i==j else other
|
||||
for j, other in enumerate(tree.children)]))
|
||||
for option in set(child.children)]
|
||||
self._flatten(tree)
|
||||
break
|
||||
|
||||
def alias(self, tree):
|
||||
rule, alias_name = tree.children
|
||||
if rule.data == 'expansions':
|
||||
aliases = []
|
||||
for child in tree.children[0].children:
|
||||
aliases.append(ST('alias', [child, alias_name]))
|
||||
tree.data = 'expansions'
|
||||
tree.children = aliases
|
||||
|
||||
def expansions(self, tree):
|
||||
self._flatten(tree)
|
||||
tree.children = list(set(tree.children))
|
||||
|
||||
|
||||
class RuleTreeToText(Transformer):
|
||||
def expansions(self, x):
|
||||
return x
|
||||
def expansion(self, symbols):
|
||||
return symbols, None
|
||||
def alias(self, x):
|
||||
(expansion, _alias), alias = x
|
||||
assert _alias is None, (alias, expansion, '-', _alias) # Double alias not allowed
|
||||
return expansion, alias.value
|
||||
|
||||
|
||||
@inline_args
|
||||
class CanonizeTree(Transformer_InPlace):
|
||||
def maybe(self, expr):
|
||||
return ST('expr', [expr, Token('OP', '?', -1)])
|
||||
|
||||
def tokenmods(self, *args):
|
||||
if len(args) == 1:
|
||||
return list(args)
|
||||
tokenmods, value = args
|
||||
return tokenmods + [value]
|
||||
|
||||
class PrepareAnonTerminals(Transformer_InPlace):
|
||||
"Create a unique list of anonymous terminals. Attempt to give meaningful names to them when we add them"
|
||||
|
||||
def __init__(self, terminals):
|
||||
self.terminals = terminals
|
||||
self.term_set = {td.name for td in self.terminals}
|
||||
self.term_reverse = {td.pattern: td for td in terminals}
|
||||
self.i = 0
|
||||
|
||||
|
||||
@inline_args
|
||||
def pattern(self, p):
|
||||
value = p.value
|
||||
if p in self.term_reverse and p.flags != self.term_reverse[p].pattern.flags:
|
||||
raise GrammarError(u'Conflicting flags for the same terminal: %s' % p)
|
||||
|
||||
term_name = None
|
||||
|
||||
if isinstance(p, PatternStr):
|
||||
try:
|
||||
# If already defined, use the user-defined terminal name
|
||||
term_name = self.term_reverse[p].name
|
||||
except KeyError:
|
||||
# Try to assign an indicative anon-terminal name
|
||||
try:
|
||||
term_name = _TERMINAL_NAMES[value]
|
||||
except KeyError:
|
||||
if value.isalnum() and value[0].isalpha() and value.upper() not in self.term_set:
|
||||
with suppress(UnicodeEncodeError):
|
||||
value.upper().encode('ascii') # Make sure we don't have unicode in our terminal names
|
||||
term_name = value.upper()
|
||||
|
||||
if term_name in self.term_set:
|
||||
term_name = None
|
||||
|
||||
elif isinstance(p, PatternRE):
|
||||
if p in self.term_reverse: # Kind of a wierd placement.name
|
||||
term_name = self.term_reverse[p].name
|
||||
else:
|
||||
assert False, p
|
||||
|
||||
if term_name is None:
|
||||
term_name = '__ANON_%d' % self.i
|
||||
self.i += 1
|
||||
|
||||
if term_name not in self.term_set:
|
||||
assert p not in self.term_reverse
|
||||
self.term_set.add(term_name)
|
||||
termdef = TerminalDef(term_name, p)
|
||||
self.term_reverse[p] = termdef
|
||||
self.terminals.append(termdef)
|
||||
|
||||
return Terminal(term_name, filter_out=isinstance(p, PatternStr))
|
||||
|
||||
|
||||
def _rfind(s, choices):
|
||||
return max(s.rfind(c) for c in choices)
|
||||
|
||||
|
||||
|
||||
def _fix_escaping(s):
|
||||
w = ''
|
||||
i = iter(s)
|
||||
for n in i:
|
||||
w += n
|
||||
if n == '\\':
|
||||
n2 = next(i)
|
||||
if n2 == '\\':
|
||||
w += '\\\\'
|
||||
elif n2 not in 'unftr':
|
||||
w += '\\'
|
||||
w += n2
|
||||
w = w.replace('\\"', '"').replace("'", "\\'")
|
||||
|
||||
to_eval = "u'''%s'''" % w
|
||||
try:
|
||||
s = literal_eval(to_eval)
|
||||
except SyntaxError as e:
|
||||
raise ValueError(s, e)
|
||||
|
||||
return s
|
||||
|
||||
|
||||
def _literal_to_pattern(literal):
|
||||
v = literal.value
|
||||
flag_start = _rfind(v, '/"')+1
|
||||
assert flag_start > 0
|
||||
flags = v[flag_start:]
|
||||
assert all(f in _RE_FLAGS for f in flags), flags
|
||||
|
||||
v = v[:flag_start]
|
||||
assert v[0] == v[-1] and v[0] in '"/'
|
||||
x = v[1:-1]
|
||||
|
||||
s = _fix_escaping(x)
|
||||
|
||||
if literal.type == 'STRING':
|
||||
s = s.replace('\\\\', '\\')
|
||||
|
||||
return { 'STRING': PatternStr,
|
||||
'REGEXP': PatternRE }[literal.type](s, flags)
|
||||
|
||||
|
||||
@inline_args
|
||||
class PrepareLiterals(Transformer_InPlace):
|
||||
def literal(self, literal):
|
||||
return ST('pattern', [_literal_to_pattern(literal)])
|
||||
|
||||
def range(self, start, end):
|
||||
assert start.type == end.type == 'STRING'
|
||||
start = start.value[1:-1]
|
||||
end = end.value[1:-1]
|
||||
assert len(start) == len(end) == 1, (start, end, len(start), len(end))
|
||||
regexp = '[%s-%s]' % (start, end)
|
||||
return ST('pattern', [PatternRE(regexp)])
|
||||
|
||||
|
||||
class TerminalTreeToPattern(Transformer):
|
||||
def pattern(self, ps):
|
||||
p ,= ps
|
||||
return p
|
||||
|
||||
def expansion(self, items):
|
||||
assert items
|
||||
if len(items) == 1:
|
||||
return items[0]
|
||||
if len({i.flags for i in items}) > 1:
|
||||
raise GrammarError("Lark doesn't support joining terminals with conflicting flags!")
|
||||
return PatternRE(''.join(i.to_regexp() for i in items), items[0].flags if items else ())
|
||||
|
||||
def expansions(self, exps):
|
||||
if len(exps) == 1:
|
||||
return exps[0]
|
||||
if len({i.flags for i in exps}) > 1:
|
||||
raise GrammarError("Lark doesn't support joining terminals with conflicting flags!")
|
||||
return PatternRE('(?:%s)' % ('|'.join(i.to_regexp() for i in exps)), exps[0].flags)
|
||||
|
||||
def expr(self, args):
|
||||
inner, op = args[:2]
|
||||
if op == '~':
|
||||
if len(args) == 3:
|
||||
op = "{%d}" % int(args[2])
|
||||
else:
|
||||
mn, mx = map(int, args[2:])
|
||||
if mx < mn:
|
||||
raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (inner, mn, mx))
|
||||
op = "{%d,%d}" % (mn, mx)
|
||||
else:
|
||||
assert len(args) == 2
|
||||
return PatternRE('(?:%s)%s' % (inner.to_regexp(), op), inner.flags)
|
||||
|
||||
def alias(self, t):
|
||||
raise GrammarError("Aliasing not allowed in terminals (You used -> in the wrong place)")
|
||||
|
||||
def value(self, v):
|
||||
return v[0]
|
||||
|
||||
class PrepareSymbols(Transformer_InPlace):
|
||||
def value(self, v):
|
||||
v ,= v
|
||||
if isinstance(v, Tree):
|
||||
return v
|
||||
elif v.type == 'RULE':
|
||||
return NonTerminal(v.value)
|
||||
elif v.type == 'TERMINAL':
|
||||
return Terminal(v.value, filter_out=v.startswith('_'))
|
||||
assert False
|
||||
|
||||
def _choice_of_rules(rules):
|
||||
return ST('expansions', [ST('expansion', [Token('RULE', name)]) for name in rules])
|
||||
|
||||
class Grammar:
|
||||
def __init__(self, rule_defs, term_defs, ignore):
|
||||
self.term_defs = term_defs
|
||||
self.rule_defs = rule_defs
|
||||
self.ignore = ignore
|
||||
|
||||
def compile(self):
|
||||
# We change the trees in-place (to support huge grammars)
|
||||
# So deepcopy allows calling compile more than once.
|
||||
term_defs = deepcopy(list(self.term_defs))
|
||||
rule_defs = deepcopy(self.rule_defs)
|
||||
|
||||
# ===================
|
||||
# Compile Terminals
|
||||
# ===================
|
||||
|
||||
# Convert terminal-trees to strings/regexps
|
||||
transformer = PrepareLiterals() * TerminalTreeToPattern()
|
||||
for name, (term_tree, priority) in term_defs:
|
||||
if term_tree is None: # Terminal added through %declare
|
||||
continue
|
||||
expansions = list(term_tree.find_data('expansion'))
|
||||
if len(expansions) == 1 and not expansions[0].children:
|
||||
raise GrammarError("Terminals cannot be empty (%s)" % name)
|
||||
|
||||
terminals = [TerminalDef(name, transformer.transform(term_tree), priority)
|
||||
for name, (term_tree, priority) in term_defs if term_tree]
|
||||
|
||||
# =================
|
||||
# Compile Rules
|
||||
# =================
|
||||
|
||||
# 1. Pre-process terminals
|
||||
transformer = PrepareLiterals() * PrepareSymbols() * PrepareAnonTerminals(terminals) # Adds to terminals
|
||||
|
||||
# 2. Convert EBNF to BNF (and apply step 1)
|
||||
ebnf_to_bnf = EBNF_to_BNF()
|
||||
rules = []
|
||||
for name, rule_tree, options in rule_defs:
|
||||
ebnf_to_bnf.rule_options = RuleOptions(keep_all_tokens=True) if options and options.keep_all_tokens else None
|
||||
tree = transformer.transform(rule_tree)
|
||||
rules.append((name, ebnf_to_bnf.transform(tree), options))
|
||||
rules += ebnf_to_bnf.new_rules
|
||||
|
||||
assert len(rules) == len({name for name, _t, _o in rules}), "Whoops, name collision"
|
||||
|
||||
# 3. Compile tree to Rule objects
|
||||
rule_tree_to_text = RuleTreeToText()
|
||||
|
||||
simplify_rule = SimplifyRule_Visitor()
|
||||
compiled_rules = []
|
||||
for name, tree, options in rules:
|
||||
simplify_rule.visit(tree)
|
||||
expansions = rule_tree_to_text.transform(tree)
|
||||
|
||||
for expansion, alias in expansions:
|
||||
if alias and name.startswith('_'):
|
||||
raise GrammarError("Rule %s is marked for expansion (it starts with an underscore) and isn't allowed to have aliases (alias=%s)" % (name, alias))
|
||||
|
||||
assert all(isinstance(x, Symbol) for x in expansion), expansion
|
||||
|
||||
rule = Rule(NonTerminal(name), expansion, alias, options)
|
||||
compiled_rules.append(rule)
|
||||
|
||||
return terminals, compiled_rules, self.ignore
|
||||
|
||||
|
||||
|
||||
_imported_grammars = {}
|
||||
def import_grammar(grammar_path, base_paths=[]):
|
||||
if grammar_path not in _imported_grammars:
|
||||
import_paths = base_paths + IMPORT_PATHS
|
||||
for import_path in import_paths:
|
||||
with suppress(IOError):
|
||||
with open(os.path.join(import_path, grammar_path)) as f:
|
||||
text = f.read()
|
||||
grammar = load_grammar(text, grammar_path)
|
||||
_imported_grammars[grammar_path] = grammar
|
||||
break
|
||||
else:
|
||||
open(grammar_path)
|
||||
assert False
|
||||
|
||||
return _imported_grammars[grammar_path]
|
||||
|
||||
def import_from_grammar_into_namespace(grammar, namespace, aliases):
|
||||
imported_terms = dict(grammar.term_defs)
|
||||
imported_rules = {n:(n,t,o) for n,t,o in grammar.rule_defs}
|
||||
|
||||
term_defs = []
|
||||
rule_defs = []
|
||||
|
||||
def rule_dependencies(symbol):
|
||||
if symbol.type != 'RULE':
|
||||
return []
|
||||
_, tree, _ = imported_rules[symbol]
|
||||
return tree.scan_values(lambda x: x.type in ('RULE', 'TERMINAL'))
|
||||
|
||||
def get_namespace_name(name):
|
||||
try:
|
||||
return aliases[name].value
|
||||
except KeyError:
|
||||
return '%s.%s' % (namespace, name)
|
||||
|
||||
to_import = list(bfs(aliases, rule_dependencies))
|
||||
for symbol in to_import:
|
||||
if symbol.type == 'TERMINAL':
|
||||
term_defs.append([get_namespace_name(symbol), imported_terms[symbol]])
|
||||
else:
|
||||
assert symbol.type == 'RULE'
|
||||
rule = imported_rules[symbol]
|
||||
for t in rule[1].iter_subtrees():
|
||||
for i, c in enumerate(t.children):
|
||||
if isinstance(c, Token) and c.type in ('RULE', 'TERMINAL'):
|
||||
t.children[i] = Token(c.type, get_namespace_name(c))
|
||||
rule_defs.append((get_namespace_name(symbol), rule[1], rule[2]))
|
||||
|
||||
return term_defs, rule_defs
|
||||
|
||||
|
||||
|
||||
def resolve_term_references(term_defs):
|
||||
# TODO Cycles detection
|
||||
# TODO Solve with transitive closure (maybe)
|
||||
|
||||
token_dict = {k:t for k, (t,_p) in term_defs}
|
||||
assert len(token_dict) == len(term_defs), "Same name defined twice?"
|
||||
|
||||
while True:
|
||||
changed = False
|
||||
for name, (token_tree, _p) in term_defs:
|
||||
if token_tree is None: # Terminal added through %declare
|
||||
continue
|
||||
for exp in token_tree.find_data('value'):
|
||||
item ,= exp.children
|
||||
if isinstance(item, Token):
|
||||
if item.type == 'RULE':
|
||||
raise GrammarError("Rules aren't allowed inside terminals (%s in %s)" % (item, name))
|
||||
if item.type == 'TERMINAL':
|
||||
exp.children[0] = token_dict[item]
|
||||
changed = True
|
||||
if not changed:
|
||||
break
|
||||
|
||||
def options_from_rule(name, *x):
|
||||
if len(x) > 1:
|
||||
priority, expansions = x
|
||||
priority = int(priority)
|
||||
else:
|
||||
expansions ,= x
|
||||
priority = None
|
||||
|
||||
keep_all_tokens = name.startswith('!')
|
||||
name = name.lstrip('!')
|
||||
expand1 = name.startswith('?')
|
||||
name = name.lstrip('?')
|
||||
|
||||
return name, expansions, RuleOptions(keep_all_tokens, expand1, priority=priority)
|
||||
|
||||
|
||||
def symbols_from_strcase(expansion):
|
||||
return [Terminal(x, filter_out=x.startswith('_')) if x.isupper() else NonTerminal(x) for x in expansion]
|
||||
|
||||
@inline_args
|
||||
class PrepareGrammar(Transformer_InPlace):
|
||||
def terminal(self, name):
|
||||
return name
|
||||
def nonterminal(self, name):
|
||||
return name
|
||||
|
||||
|
||||
class GrammarLoader:
|
||||
def __init__(self):
|
||||
terminals = [TerminalDef(name, PatternRE(value)) for name, value in TERMINALS.items()]
|
||||
|
||||
rules = [options_from_rule(name, x) for name, x in RULES.items()]
|
||||
rules = [Rule(NonTerminal(r), symbols_from_strcase(x.split()), None, o) for r, xs, o in rules for x in xs]
|
||||
callback = ParseTreeBuilder(rules, ST).create_callback()
|
||||
lexer_conf = LexerConf(terminals, ['WS', 'COMMENT'])
|
||||
|
||||
parser_conf = ParserConf(rules, callback, 'start')
|
||||
self.parser = LALR_TraditionalLexer(lexer_conf, parser_conf)
|
||||
|
||||
self.canonize_tree = CanonizeTree()
|
||||
|
||||
def load_grammar(self, grammar_text, grammar_name='<?>'):
|
||||
"Parse grammar_text, verify, and create Grammar object. Display nice messages on error."
|
||||
|
||||
try:
|
||||
tree = self.canonize_tree.transform( self.parser.parse(grammar_text+'\n') )
|
||||
except UnexpectedCharacters as e:
|
||||
context = e.get_context(grammar_text)
|
||||
raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
|
||||
(e.line, e.column, grammar_name, context))
|
||||
except UnexpectedToken as e:
|
||||
context = e.get_context(grammar_text)
|
||||
error = e.match_examples(self.parser.parse, {
|
||||
'Unclosed parenthesis': ['a: (\n'],
|
||||
'Umatched closing parenthesis': ['a: )\n', 'a: [)\n', 'a: (]\n'],
|
||||
'Expecting rule or terminal definition (missing colon)': ['a\n', 'a->\n', 'A->\n', 'a A\n'],
|
||||
'Alias expects lowercase name': ['a: -> "a"\n'],
|
||||
'Unexpected colon': ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n'],
|
||||
'Misplaced operator': ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n'],
|
||||
'Expecting option ("|") or a new rule or terminal definition': ['a:a\n()\n'],
|
||||
'%import expects a name': ['%import "a"\n'],
|
||||
'%ignore expects a value': ['%ignore %import\n'],
|
||||
})
|
||||
if error:
|
||||
raise GrammarError("%s at line %s column %s\n\n%s" % (error, e.line, e.column, context))
|
||||
elif 'STRING' in e.expected:
|
||||
raise GrammarError("Expecting a value at line %s column %s\n\n%s" % (e.line, e.column, context))
|
||||
raise
|
||||
|
||||
tree = PrepareGrammar().transform(tree)
|
||||
|
||||
# Extract grammar items
|
||||
defs = classify(tree.children, lambda c: c.data, lambda c: c.children)
|
||||
term_defs = defs.pop('term', [])
|
||||
rule_defs = defs.pop('rule', [])
|
||||
statements = defs.pop('statement', [])
|
||||
assert not defs
|
||||
|
||||
term_defs = [td if len(td)==3 else (td[0], 1, td[1]) for td in term_defs]
|
||||
term_defs = [(name.value, (t, int(p))) for name, p, t in term_defs]
|
||||
rule_defs = [options_from_rule(*x) for x in rule_defs]
|
||||
|
||||
# Execute statements
|
||||
ignore = []
|
||||
for (stmt,) in statements:
|
||||
if stmt.data == 'ignore':
|
||||
t ,= stmt.children
|
||||
ignore.append(t)
|
||||
elif stmt.data == 'import':
|
||||
if len(stmt.children) > 1:
|
||||
path_node, arg1 = stmt.children
|
||||
else:
|
||||
path_node ,= stmt.children
|
||||
arg1 = None
|
||||
|
||||
if isinstance(arg1, Tree): # Multi import
|
||||
dotted_path = path_node.children
|
||||
names = arg1.children
|
||||
aliases = names # Can't have aliased multi import, so all aliases will be the same as names
|
||||
else: # Single import
|
||||
dotted_path = path_node.children[:-1]
|
||||
names = [path_node.children[-1]] # Get name from dotted path
|
||||
aliases = [arg1] if arg1 else names # Aliases if exist
|
||||
|
||||
grammar_path = os.path.join(*dotted_path) + EXT
|
||||
|
||||
if path_node.data == 'import_lib': # Import from library
|
||||
g = import_grammar(grammar_path)
|
||||
else: # Relative import
|
||||
if grammar_name == '<string>': # Import relative to script file path if grammar is coded in script
|
||||
base_file = os.path.abspath(sys.modules['__main__'].__file__)
|
||||
else:
|
||||
base_file = grammar_name # Import relative to grammar file path if external grammar file
|
||||
base_path = os.path.split(base_file)[0]
|
||||
g = import_grammar(grammar_path, base_paths=[base_path])
|
||||
|
||||
aliases_dict = dict(zip(names, aliases))
|
||||
new_td, new_rd = import_from_grammar_into_namespace(g, '.'.join(dotted_path), aliases_dict)
|
||||
|
||||
term_defs += new_td
|
||||
rule_defs += new_rd
|
||||
|
||||
elif stmt.data == 'declare':
|
||||
for t in stmt.children:
|
||||
term_defs.append([t.value, (None, None)])
|
||||
else:
|
||||
assert False, stmt
|
||||
|
||||
|
||||
# Verify correctness 1
|
||||
for name, _ in term_defs:
|
||||
if name.startswith('__'):
|
||||
raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
|
||||
|
||||
# Handle ignore tokens
|
||||
# XXX A slightly hacky solution. Recognition of %ignore TERMINAL as separate comes from the lexer's
|
||||
# inability to handle duplicate terminals (two names, one value)
|
||||
ignore_names = []
|
||||
for t in ignore:
|
||||
if t.data=='expansions' and len(t.children) == 1:
|
||||
t2 ,= t.children
|
||||
if t2.data=='expansion' and len(t2.children) == 1:
|
||||
item ,= t2.children
|
||||
if item.data == 'value':
|
||||
item ,= item.children
|
||||
if isinstance(item, Token) and item.type == 'TERMINAL':
|
||||
ignore_names.append(item.value)
|
||||
continue
|
||||
|
||||
name = '__IGNORE_%d'% len(ignore_names)
|
||||
ignore_names.append(name)
|
||||
term_defs.append((name, (t, 0)))
|
||||
|
||||
# Verify correctness 2
|
||||
terminal_names = set()
|
||||
for name, _ in term_defs:
|
||||
if name in terminal_names:
|
||||
raise GrammarError("Terminal '%s' defined more than once" % name)
|
||||
terminal_names.add(name)
|
||||
|
||||
if set(ignore_names) > terminal_names:
|
||||
raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(ignore_names) - terminal_names))
|
||||
|
||||
resolve_term_references(term_defs)
|
||||
|
||||
rules = rule_defs
|
||||
|
||||
rule_names = set()
|
||||
for name, _x, _o in rules:
|
||||
if name.startswith('__'):
|
||||
raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
|
||||
if name in rule_names:
|
||||
raise GrammarError("Rule '%s' defined more than once" % name)
|
||||
rule_names.add(name)
|
||||
|
||||
for name, expansions, _o in rules:
|
||||
used_symbols = {t for x in expansions.find_data('expansion')
|
||||
for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
|
||||
for sym in used_symbols:
|
||||
if sym.type == 'TERMINAL':
|
||||
if sym not in terminal_names:
|
||||
raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, name))
|
||||
else:
|
||||
if sym not in rule_names:
|
||||
raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, name))
|
||||
|
||||
# TODO don't include unused terminals, they can only cause trouble!
|
||||
|
||||
return Grammar(rules, term_defs, ignore_names)
|
||||
|
||||
|
||||
|
||||
load_grammar = GrammarLoader().load_grammar
|
||||
@@ -1,174 +0,0 @@
|
||||
from .exceptions import GrammarError
|
||||
from .utils import suppress
|
||||
from .lexer import Token
|
||||
from .grammar import Rule
|
||||
from .tree import Tree
|
||||
from .visitors import InlineTransformer # XXX Deprecated
|
||||
|
||||
###{standalone
|
||||
from functools import partial, wraps
|
||||
|
||||
|
||||
class ExpandSingleChild:
|
||||
def __init__(self, node_builder):
|
||||
self.node_builder = node_builder
|
||||
|
||||
def __call__(self, children):
|
||||
if len(children) == 1:
|
||||
return children[0]
|
||||
else:
|
||||
return self.node_builder(children)
|
||||
|
||||
|
||||
class PropagatePositions:
|
||||
def __init__(self, node_builder):
|
||||
self.node_builder = node_builder
|
||||
|
||||
def __call__(self, children):
|
||||
res = self.node_builder(children)
|
||||
|
||||
if isinstance(res, Tree):
|
||||
res.meta.empty = True
|
||||
|
||||
for c in children:
|
||||
if isinstance(c, Tree) and c.children and not c.meta.empty:
|
||||
res.meta.line = c.meta.line
|
||||
res.meta.column = c.meta.column
|
||||
res.meta.start_pos = c.meta.start_pos
|
||||
res.meta.empty = False
|
||||
break
|
||||
elif isinstance(c, Token):
|
||||
res.meta.line = c.line
|
||||
res.meta.column = c.column
|
||||
res.meta.start_pos = c.pos_in_stream
|
||||
res.meta.empty = False
|
||||
break
|
||||
|
||||
for c in reversed(children):
|
||||
if isinstance(c, Tree) and c.children and not c.meta.empty:
|
||||
res.meta.end_line = c.meta.end_line
|
||||
res.meta.end_column = c.meta.end_column
|
||||
res.meta.end_pos = c.meta.end_pos
|
||||
res.meta.empty = False
|
||||
break
|
||||
elif isinstance(c, Token):
|
||||
res.meta.end_line = c.end_line
|
||||
res.meta.end_column = c.end_column
|
||||
res.meta.end_pos = c.pos_in_stream + len(c.value)
|
||||
res.meta.empty = False
|
||||
break
|
||||
|
||||
return res
|
||||
|
||||
|
||||
class ChildFilter:
|
||||
def __init__(self, to_include, node_builder):
|
||||
self.node_builder = node_builder
|
||||
self.to_include = to_include
|
||||
|
||||
def __call__(self, children):
|
||||
filtered = []
|
||||
for i, to_expand in self.to_include:
|
||||
if to_expand:
|
||||
filtered += children[i].children
|
||||
else:
|
||||
filtered.append(children[i])
|
||||
|
||||
return self.node_builder(filtered)
|
||||
|
||||
class ChildFilterLALR(ChildFilter):
|
||||
"Optimized childfilter for LALR (assumes no duplication in parse tree, so it's safe to change it)"
|
||||
|
||||
def __call__(self, children):
|
||||
filtered = []
|
||||
for i, to_expand in self.to_include:
|
||||
if to_expand:
|
||||
if filtered:
|
||||
filtered += children[i].children
|
||||
else: # Optimize for left-recursion
|
||||
filtered = children[i].children
|
||||
else:
|
||||
filtered.append(children[i])
|
||||
|
||||
return self.node_builder(filtered)
|
||||
|
||||
def _should_expand(sym):
|
||||
return not sym.is_term and sym.name.startswith('_')
|
||||
|
||||
def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous):
|
||||
to_include = [(i, _should_expand(sym)) for i, sym in enumerate(expansion)
|
||||
if keep_all_tokens or not (sym.is_term and sym.filter_out)]
|
||||
|
||||
if len(to_include) < len(expansion) or any(to_expand for i, to_expand in to_include):
|
||||
return partial(ChildFilter if ambiguous else ChildFilterLALR, to_include)
|
||||
|
||||
|
||||
class Callback(object):
|
||||
pass
|
||||
|
||||
|
||||
def ptb_inline_args(func):
|
||||
@wraps(func)
|
||||
def f(children):
|
||||
return func(*children)
|
||||
return f
|
||||
|
||||
|
||||
|
||||
class ParseTreeBuilder:
|
||||
def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False):
|
||||
self.tree_class = tree_class
|
||||
self.propagate_positions = propagate_positions
|
||||
self.always_keep_all_tokens = keep_all_tokens
|
||||
self.ambiguous = ambiguous
|
||||
|
||||
self.rule_builders = list(self._init_builders(rules))
|
||||
|
||||
self.user_aliases = {}
|
||||
|
||||
def _init_builders(self, rules):
|
||||
for rule in rules:
|
||||
options = rule.options
|
||||
keep_all_tokens = self.always_keep_all_tokens or (options.keep_all_tokens if options else False)
|
||||
expand_single_child = options.expand1 if options else False
|
||||
|
||||
wrapper_chain = filter(None, [
|
||||
(expand_single_child and not rule.alias) and ExpandSingleChild,
|
||||
maybe_create_child_filter(rule.expansion, keep_all_tokens, self.ambiguous),
|
||||
self.propagate_positions and PropagatePositions,
|
||||
])
|
||||
|
||||
yield rule, wrapper_chain
|
||||
|
||||
|
||||
def create_callback(self, transformer=None):
|
||||
callback = Callback()
|
||||
|
||||
i = 0
|
||||
for rule, wrapper_chain in self.rule_builders:
|
||||
internal_callback_name = '_cb%d_%s' % (i, rule.origin)
|
||||
i += 1
|
||||
|
||||
user_callback_name = rule.alias or rule.origin.name
|
||||
try:
|
||||
f = getattr(transformer, user_callback_name)
|
||||
assert not getattr(f, 'meta', False), "Meta args not supported for internal transformer"
|
||||
# XXX InlineTransformer is deprecated!
|
||||
if getattr(f, 'inline', False) or isinstance(transformer, InlineTransformer):
|
||||
f = ptb_inline_args(f)
|
||||
except AttributeError:
|
||||
f = partial(self.tree_class, user_callback_name)
|
||||
|
||||
self.user_aliases[rule] = rule.alias
|
||||
rule.alias = internal_callback_name
|
||||
|
||||
for w in wrapper_chain:
|
||||
f = w(f)
|
||||
|
||||
if hasattr(callback, internal_callback_name):
|
||||
raise GrammarError("Rule '%s' already exists" % (rule,))
|
||||
setattr(callback, internal_callback_name, f)
|
||||
|
||||
return callback
|
||||
|
||||
###}
|
||||
@@ -1,192 +0,0 @@
|
||||
import re
|
||||
from functools import partial
|
||||
|
||||
from .utils import get_regexp_width
|
||||
from .parsers.grammar_analysis import GrammarAnalyzer
|
||||
from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token
|
||||
|
||||
from .parsers import lalr_parser, earley, xearley, resolve_ambig, cyk
|
||||
from .tree import Tree
|
||||
|
||||
class WithLexer:
|
||||
lexer = None
|
||||
parser = None
|
||||
lexer_conf = None
|
||||
|
||||
def init_traditional_lexer(self, lexer_conf):
|
||||
self.lexer_conf = lexer_conf
|
||||
self.lexer = TraditionalLexer(lexer_conf.tokens, ignore=lexer_conf.ignore, user_callbacks=lexer_conf.callbacks)
|
||||
|
||||
def init_contextual_lexer(self, lexer_conf):
|
||||
self.lexer_conf = lexer_conf
|
||||
states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()}
|
||||
always_accept = lexer_conf.postlex.always_accept if lexer_conf.postlex else ()
|
||||
self.lexer = ContextualLexer(lexer_conf.tokens, states,
|
||||
ignore=lexer_conf.ignore,
|
||||
always_accept=always_accept,
|
||||
user_callbacks=lexer_conf.callbacks)
|
||||
|
||||
def lex(self, text):
|
||||
stream = self.lexer.lex(text)
|
||||
if self.lexer_conf.postlex:
|
||||
return self.lexer_conf.postlex.process(stream)
|
||||
return stream
|
||||
|
||||
def parse(self, text):
|
||||
token_stream = self.lex(text)
|
||||
sps = self.lexer.set_parser_state
|
||||
return self.parser.parse(token_stream, *[sps] if sps is not NotImplemented else [])
|
||||
|
||||
class LALR_TraditionalLexer(WithLexer):
|
||||
def __init__(self, lexer_conf, parser_conf, options=None):
|
||||
self.parser = lalr_parser.Parser(parser_conf)
|
||||
self.init_traditional_lexer(lexer_conf)
|
||||
|
||||
class LALR_ContextualLexer(WithLexer):
|
||||
def __init__(self, lexer_conf, parser_conf, options=None):
|
||||
self.parser = lalr_parser.Parser(parser_conf)
|
||||
self.init_contextual_lexer(lexer_conf)
|
||||
|
||||
class LALR_CustomLexer(WithLexer):
|
||||
def __init__(self, lexer_cls, lexer_conf, parser_conf, options=None):
|
||||
self.parser = lalr_parser.Parser(parser_conf)
|
||||
self.lexer_conf = lexer_conf
|
||||
self.lexer = lexer_cls(lexer_conf)
|
||||
|
||||
|
||||
def get_ambiguity_resolver(options):
|
||||
if not options or options.ambiguity == 'resolve':
|
||||
return resolve_ambig.standard_resolve_ambig
|
||||
elif options.ambiguity == 'resolve__antiscore_sum':
|
||||
return resolve_ambig.antiscore_sum_resolve_ambig
|
||||
elif options.ambiguity == 'explicit':
|
||||
return None
|
||||
raise ValueError(options)
|
||||
|
||||
def tokenize_text(text):
|
||||
line = 1
|
||||
col_start_pos = 0
|
||||
for i, ch in enumerate(text):
|
||||
if '\n' in ch:
|
||||
line += ch.count('\n')
|
||||
col_start_pos = i + ch.rindex('\n')
|
||||
yield Token('CHAR', ch, line=line, column=i - col_start_pos)
|
||||
|
||||
class Earley(WithLexer):
|
||||
def __init__(self, lexer_conf, parser_conf, options=None):
|
||||
self.init_traditional_lexer(lexer_conf)
|
||||
|
||||
self.parser = earley.Parser(parser_conf, self.match,
|
||||
resolve_ambiguity=get_ambiguity_resolver(options))
|
||||
|
||||
def match(self, term, token):
|
||||
return term.name == token.type
|
||||
|
||||
|
||||
class XEarley:
|
||||
def __init__(self, lexer_conf, parser_conf, options=None, **kw):
|
||||
self.token_by_name = {t.name:t for t in lexer_conf.tokens}
|
||||
|
||||
self._prepare_match(lexer_conf)
|
||||
|
||||
self.parser = xearley.Parser(parser_conf,
|
||||
self.match,
|
||||
resolve_ambiguity=get_ambiguity_resolver(options),
|
||||
ignore=lexer_conf.ignore,
|
||||
predict_all=options.earley__predict_all,
|
||||
**kw
|
||||
)
|
||||
|
||||
def match(self, term, text, index=0):
|
||||
return self.regexps[term.name].match(text, index)
|
||||
|
||||
def _prepare_match(self, lexer_conf):
|
||||
self.regexps = {}
|
||||
for t in lexer_conf.tokens:
|
||||
regexp = t.pattern.to_regexp()
|
||||
try:
|
||||
width = get_regexp_width(regexp)[0]
|
||||
except ValueError:
|
||||
raise ValueError("Bad regexp in token %s: %s" % (t.name, regexp))
|
||||
else:
|
||||
if width == 0:
|
||||
raise ValueError("Dynamic Earley doesn't allow zero-width regexps", t)
|
||||
|
||||
self.regexps[t.name] = re.compile(regexp)
|
||||
|
||||
def parse(self, text):
|
||||
return self.parser.parse(text)
|
||||
|
||||
class XEarley_CompleteLex(XEarley):
|
||||
def __init__(self, *args, **kw):
|
||||
super(self).__init__(*args, complete_lex=True, **kw)
|
||||
|
||||
|
||||
|
||||
class CYK(WithLexer):
|
||||
|
||||
def __init__(self, lexer_conf, parser_conf, options=None):
|
||||
self.init_traditional_lexer(lexer_conf)
|
||||
|
||||
self._analysis = GrammarAnalyzer(parser_conf)
|
||||
self._parser = cyk.Parser(parser_conf.rules, parser_conf.start)
|
||||
|
||||
self._postprocess = {}
|
||||
for rule in parser_conf.rules:
|
||||
a = rule.alias
|
||||
self._postprocess[a] = a if callable(a) else (a and getattr(parser_conf.callback, a))
|
||||
|
||||
def parse(self, text):
|
||||
tokens = list(self.lex(text))
|
||||
parse = self._parser.parse(tokens)
|
||||
parse = self._transform(parse)
|
||||
return parse
|
||||
|
||||
def _transform(self, tree):
|
||||
subtrees = list(tree.iter_subtrees())
|
||||
for subtree in subtrees:
|
||||
subtree.children = [self._apply_callback(c) if isinstance(c, Tree) else c for c in subtree.children]
|
||||
|
||||
return self._apply_callback(tree)
|
||||
|
||||
def _apply_callback(self, tree):
|
||||
children = tree.children
|
||||
callback = self._postprocess[tree.rule.alias]
|
||||
assert callback, tree.rule.alias
|
||||
r = callback(children)
|
||||
return r
|
||||
|
||||
|
||||
def get_frontend(parser, lexer):
|
||||
if parser=='lalr':
|
||||
if lexer is None:
|
||||
raise ValueError('The LALR parser requires use of a lexer')
|
||||
elif lexer == 'standard':
|
||||
return LALR_TraditionalLexer
|
||||
elif lexer == 'contextual':
|
||||
return LALR_ContextualLexer
|
||||
elif issubclass(lexer, Lexer):
|
||||
return partial(LALR_CustomLexer, lexer)
|
||||
else:
|
||||
raise ValueError('Unknown lexer: %s' % lexer)
|
||||
elif parser=='earley':
|
||||
if lexer=='standard':
|
||||
return Earley
|
||||
elif lexer=='dynamic':
|
||||
return XEarley
|
||||
elif lexer=='dynamic_complete':
|
||||
return XEarley_CompleteLex
|
||||
elif lexer=='contextual':
|
||||
raise ValueError('The Earley parser does not support the contextual parser')
|
||||
else:
|
||||
raise ValueError('Unknown lexer: %s' % lexer)
|
||||
elif parser == 'cyk':
|
||||
if lexer == 'standard':
|
||||
return CYK
|
||||
else:
|
||||
raise ValueError('CYK parser requires using standard parser.')
|
||||
else:
|
||||
raise ValueError('Unknown parser: %s' % parser)
|
||||
|
||||
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
"""This module implements an ABLALR(1) Parser
|
||||
|
||||
Ambiguous Backtracking Look-Ahead(1) LR(0)
|
||||
"""
|
||||
# Author: Erez Shinan (2017)
|
||||
# Email : erezshin@gmail.com
|
||||
|
||||
from ..common import UnexpectedToken
|
||||
|
||||
from .lalr_analysis import LALR_Analyzer, Shift
|
||||
|
||||
class Parser:
|
||||
def __init__(self, parser_conf):
|
||||
assert all(r.options is None or r.options.priority is None
|
||||
for r in parser_conf.rules), "LALR doesn't yet support prioritization"
|
||||
self.analysis = analysis = LALR_Analyzer(parser_conf)
|
||||
analysis.compute_lookahead()
|
||||
callbacks = {rule: getattr(parser_conf.callback, rule.alias or rule.origin, None)
|
||||
for rule in analysis.rules}
|
||||
|
||||
self.parser_conf = parser_conf
|
||||
self.parser = _Parser(analysis.parse_table, callbacks)
|
||||
self.parse = self.parser.parse
|
||||
|
||||
###{standalone
|
||||
|
||||
class _Parser:
|
||||
def __init__(self, parse_table, callbacks):
|
||||
self.states = parse_table.states
|
||||
self.start_state = parse_table.start_state
|
||||
self.end_state = parse_table.end_state
|
||||
self.callbacks = callbacks
|
||||
|
||||
def parse(self, seq, set_state=None):
|
||||
i = 0
|
||||
token = None
|
||||
stream = iter(seq)
|
||||
states = self.states
|
||||
|
||||
state_stack = [self.start_state]
|
||||
value_stack = []
|
||||
|
||||
if set_state: set_state(self.start_state)
|
||||
|
||||
def get_action(key):
|
||||
state = state_stack[-1]
|
||||
try:
|
||||
return states[state][key]
|
||||
except KeyError:
|
||||
expected = states[state].keys()
|
||||
|
||||
raise UnexpectedToken(token, expected, seq, i)
|
||||
|
||||
def reduce(rule):
|
||||
size = len(rule.expansion)
|
||||
if size:
|
||||
s = value_stack[-size:]
|
||||
del state_stack[-size:]
|
||||
del value_stack[-size:]
|
||||
else:
|
||||
s = []
|
||||
|
||||
value = self.callbacks[rule](s)
|
||||
|
||||
_action, new_state = get_action(rule.origin)
|
||||
assert _action is Shift
|
||||
state_stack.append(new_state)
|
||||
value_stack.append(value)
|
||||
|
||||
# Main LALR-parser loop
|
||||
try:
|
||||
token = next(stream)
|
||||
i += 1
|
||||
while True:
|
||||
action, arg = get_action(token.type)
|
||||
assert arg != self.end_state
|
||||
|
||||
if action is Shift:
|
||||
state_stack.append(arg)
|
||||
value_stack.append(token)
|
||||
if set_state: set_state(arg)
|
||||
token = next(stream)
|
||||
i += 1
|
||||
else:
|
||||
reduce(arg)
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
while True:
|
||||
_action, arg = get_action('$END')
|
||||
if _action is Shift:
|
||||
assert arg == self.end_state
|
||||
val ,= value_stack
|
||||
return val
|
||||
else:
|
||||
reduce(arg)
|
||||
|
||||
###}
|
||||
@@ -1,343 +0,0 @@
|
||||
"""This module implements a CYK parser."""
|
||||
|
||||
# Author: https://github.com/ehudt (2018)
|
||||
#
|
||||
# Adapted by Erez
|
||||
|
||||
|
||||
from collections import defaultdict
|
||||
import itertools
|
||||
|
||||
from ..exceptions import ParseError
|
||||
from ..lexer import Token
|
||||
from ..tree import Tree
|
||||
from ..grammar import Terminal as T, NonTerminal as NT, Symbol
|
||||
|
||||
try:
|
||||
xrange
|
||||
except NameError:
|
||||
xrange = range
|
||||
|
||||
def match(t, s):
|
||||
assert isinstance(t, T)
|
||||
return t.name == s.type
|
||||
|
||||
|
||||
class Rule(object):
|
||||
"""Context-free grammar rule."""
|
||||
|
||||
def __init__(self, lhs, rhs, weight, alias):
|
||||
super(Rule, self).__init__()
|
||||
assert isinstance(lhs, NT), lhs
|
||||
assert all(isinstance(x, NT) or isinstance(x, T) for x in rhs), rhs
|
||||
self.lhs = lhs
|
||||
self.rhs = rhs
|
||||
self.weight = weight
|
||||
self.alias = alias
|
||||
|
||||
def __str__(self):
|
||||
return '%s -> %s' % (str(self.lhs), ' '.join(str(x) for x in self.rhs))
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.lhs, tuple(self.rhs)))
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.lhs == other.lhs and self.rhs == other.rhs
|
||||
|
||||
def __ne__(self, other):
|
||||
return not (self == other)
|
||||
|
||||
|
||||
class Grammar(object):
|
||||
"""Context-free grammar."""
|
||||
|
||||
def __init__(self, rules):
|
||||
self.rules = frozenset(rules)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.rules == other.rules
|
||||
|
||||
def __str__(self):
|
||||
return '\n' + '\n'.join(sorted(repr(x) for x in self.rules)) + '\n'
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
|
||||
|
||||
# Parse tree data structures
|
||||
class RuleNode(object):
|
||||
"""A node in the parse tree, which also contains the full rhs rule."""
|
||||
|
||||
def __init__(self, rule, children, weight=0):
|
||||
self.rule = rule
|
||||
self.children = children
|
||||
self.weight = weight
|
||||
|
||||
def __repr__(self):
|
||||
return 'RuleNode(%s, [%s])' % (repr(self.rule.lhs), ', '.join(str(x) for x in self.children))
|
||||
|
||||
|
||||
|
||||
class Parser(object):
|
||||
"""Parser wrapper."""
|
||||
|
||||
def __init__(self, rules, start):
|
||||
super(Parser, self).__init__()
|
||||
self.orig_rules = {rule.alias: rule for rule in rules}
|
||||
rules = [self._to_rule(rule) for rule in rules]
|
||||
self.grammar = to_cnf(Grammar(rules))
|
||||
self.start = NT(start)
|
||||
|
||||
def _to_rule(self, lark_rule):
|
||||
"""Converts a lark rule, (lhs, rhs, callback, options), to a Rule."""
|
||||
assert isinstance(lark_rule.origin, NT)
|
||||
assert all(isinstance(x, Symbol) for x in lark_rule.expansion)
|
||||
return Rule(
|
||||
lark_rule.origin, lark_rule.expansion,
|
||||
weight=lark_rule.options.priority if lark_rule.options and lark_rule.options.priority else 0,
|
||||
alias=lark_rule.alias)
|
||||
|
||||
def parse(self, tokenized): # pylint: disable=invalid-name
|
||||
"""Parses input, which is a list of tokens."""
|
||||
table, trees = _parse(tokenized, self.grammar)
|
||||
# Check if the parse succeeded.
|
||||
if all(r.lhs != self.start for r in table[(0, len(tokenized) - 1)]):
|
||||
raise ParseError('Parsing failed.')
|
||||
parse = trees[(0, len(tokenized) - 1)][self.start]
|
||||
return self._to_tree(revert_cnf(parse))
|
||||
|
||||
def _to_tree(self, rule_node):
|
||||
"""Converts a RuleNode parse tree to a lark Tree."""
|
||||
orig_rule = self.orig_rules[rule_node.rule.alias]
|
||||
children = []
|
||||
for child in rule_node.children:
|
||||
if isinstance(child, RuleNode):
|
||||
children.append(self._to_tree(child))
|
||||
else:
|
||||
assert isinstance(child.name, Token)
|
||||
children.append(child.name)
|
||||
t = Tree(orig_rule.origin, children)
|
||||
t.rule=orig_rule
|
||||
return t
|
||||
|
||||
|
||||
def print_parse(node, indent=0):
|
||||
if isinstance(node, RuleNode):
|
||||
print(' ' * (indent * 2) + str(node.rule.lhs))
|
||||
for child in node.children:
|
||||
print_parse(child, indent + 1)
|
||||
else:
|
||||
print(' ' * (indent * 2) + str(node.s))
|
||||
|
||||
|
||||
def _parse(s, g):
|
||||
"""Parses sentence 's' using CNF grammar 'g'."""
|
||||
# The CYK table. Indexed with a 2-tuple: (start pos, end pos)
|
||||
table = defaultdict(set)
|
||||
# Top-level structure is similar to the CYK table. Each cell is a dict from
|
||||
# rule name to the best (lightest) tree for that rule.
|
||||
trees = defaultdict(dict)
|
||||
# Populate base case with existing terminal production rules
|
||||
for i, w in enumerate(s):
|
||||
for terminal, rules in g.terminal_rules.items():
|
||||
if match(terminal, w):
|
||||
for rule in rules:
|
||||
table[(i, i)].add(rule)
|
||||
if (rule.lhs not in trees[(i, i)] or
|
||||
rule.weight < trees[(i, i)][rule.lhs].weight):
|
||||
trees[(i, i)][rule.lhs] = RuleNode(rule, [T(w)], weight=rule.weight)
|
||||
|
||||
# Iterate over lengths of sub-sentences
|
||||
for l in xrange(2, len(s) + 1):
|
||||
# Iterate over sub-sentences with the given length
|
||||
for i in xrange(len(s) - l + 1):
|
||||
# Choose partition of the sub-sentence in [1, l)
|
||||
for p in xrange(i + 1, i + l):
|
||||
span1 = (i, p - 1)
|
||||
span2 = (p, i + l - 1)
|
||||
for r1, r2 in itertools.product(table[span1], table[span2]):
|
||||
for rule in g.nonterminal_rules.get((r1.lhs, r2.lhs), []):
|
||||
table[(i, i + l - 1)].add(rule)
|
||||
r1_tree = trees[span1][r1.lhs]
|
||||
r2_tree = trees[span2][r2.lhs]
|
||||
rule_total_weight = rule.weight + r1_tree.weight + r2_tree.weight
|
||||
if (rule.lhs not in trees[(i, i + l - 1)]
|
||||
or rule_total_weight < trees[(i, i + l - 1)][rule.lhs].weight):
|
||||
trees[(i, i + l - 1)][rule.lhs] = RuleNode(rule, [r1_tree, r2_tree], weight=rule_total_weight)
|
||||
return table, trees
|
||||
|
||||
|
||||
# This section implements context-free grammar converter to Chomsky normal form.
|
||||
# It also implements a conversion of parse trees from its CNF to the original
|
||||
# grammar.
|
||||
# Overview:
|
||||
# Applies the following operations in this order:
|
||||
# * TERM: Eliminates non-solitary terminals from all rules
|
||||
# * BIN: Eliminates rules with more than 2 symbols on their right-hand-side.
|
||||
# * UNIT: Eliminates non-terminal unit rules
|
||||
#
|
||||
# The following grammar characteristics aren't featured:
|
||||
# * Start symbol appears on RHS
|
||||
# * Empty rules (epsilon rules)
|
||||
|
||||
|
||||
class CnfWrapper(object):
|
||||
"""CNF wrapper for grammar.
|
||||
|
||||
Validates that the input grammar is CNF and provides helper data structures.
|
||||
"""
|
||||
|
||||
def __init__(self, grammar):
|
||||
super(CnfWrapper, self).__init__()
|
||||
self.grammar = grammar
|
||||
self.rules = grammar.rules
|
||||
self.terminal_rules = defaultdict(list)
|
||||
self.nonterminal_rules = defaultdict(list)
|
||||
for r in self.rules:
|
||||
# Validate that the grammar is CNF and populate auxiliary data structures.
|
||||
assert isinstance(r.lhs, NT), r
|
||||
if len(r.rhs) not in [1, 2]:
|
||||
raise ParseError("CYK doesn't support empty rules")
|
||||
if len(r.rhs) == 1 and isinstance(r.rhs[0], T):
|
||||
self.terminal_rules[r.rhs[0]].append(r)
|
||||
elif len(r.rhs) == 2 and all(isinstance(x, NT) for x in r.rhs):
|
||||
self.nonterminal_rules[tuple(r.rhs)].append(r)
|
||||
else:
|
||||
assert False, r
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.grammar == other.grammar
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.grammar)
|
||||
|
||||
|
||||
class UnitSkipRule(Rule):
|
||||
"""A rule that records NTs that were skipped during transformation."""
|
||||
|
||||
def __init__(self, lhs, rhs, skipped_rules, weight, alias):
|
||||
super(UnitSkipRule, self).__init__(lhs, rhs, weight, alias)
|
||||
self.skipped_rules = skipped_rules
|
||||
|
||||
def __eq__(self, other):
|
||||
return isinstance(other, type(self)) and self.skipped_rules == other.skipped_rules
|
||||
|
||||
__hash__ = Rule.__hash__
|
||||
|
||||
|
||||
def build_unit_skiprule(unit_rule, target_rule):
|
||||
skipped_rules = []
|
||||
if isinstance(unit_rule, UnitSkipRule):
|
||||
skipped_rules += unit_rule.skipped_rules
|
||||
skipped_rules.append(target_rule)
|
||||
if isinstance(target_rule, UnitSkipRule):
|
||||
skipped_rules += target_rule.skipped_rules
|
||||
return UnitSkipRule(unit_rule.lhs, target_rule.rhs, skipped_rules,
|
||||
weight=unit_rule.weight + target_rule.weight, alias=unit_rule.alias)
|
||||
|
||||
|
||||
def get_any_nt_unit_rule(g):
|
||||
"""Returns a non-terminal unit rule from 'g', or None if there is none."""
|
||||
for rule in g.rules:
|
||||
if len(rule.rhs) == 1 and isinstance(rule.rhs[0], NT):
|
||||
return rule
|
||||
return None
|
||||
|
||||
|
||||
def _remove_unit_rule(g, rule):
|
||||
"""Removes 'rule' from 'g' without changing the langugage produced by 'g'."""
|
||||
new_rules = [x for x in g.rules if x != rule]
|
||||
refs = [x for x in g.rules if x.lhs == rule.rhs[0]]
|
||||
new_rules += [build_unit_skiprule(rule, ref) for ref in refs]
|
||||
return Grammar(new_rules)
|
||||
|
||||
|
||||
def _split(rule):
|
||||
"""Splits a rule whose len(rhs) > 2 into shorter rules."""
|
||||
rule_str = str(rule.lhs) + '__' + '_'.join(str(x) for x in rule.rhs)
|
||||
rule_name = '__SP_%s' % (rule_str) + '_%d'
|
||||
yield Rule(rule.lhs, [rule.rhs[0], NT(rule_name % 1)], weight=rule.weight, alias=rule.alias)
|
||||
for i in xrange(1, len(rule.rhs) - 2):
|
||||
yield Rule(NT(rule_name % i), [rule.rhs[i], NT(rule_name % (i + 1))], weight=0, alias='Split')
|
||||
yield Rule(NT(rule_name % (len(rule.rhs) - 2)), rule.rhs[-2:], weight=0, alias='Split')
|
||||
|
||||
|
||||
def _term(g):
|
||||
"""Applies the TERM rule on 'g' (see top comment)."""
|
||||
all_t = {x for rule in g.rules for x in rule.rhs if isinstance(x, T)}
|
||||
t_rules = {t: Rule(NT('__T_%s' % str(t)), [t], weight=0, alias='Term') for t in all_t}
|
||||
new_rules = []
|
||||
for rule in g.rules:
|
||||
if len(rule.rhs) > 1 and any(isinstance(x, T) for x in rule.rhs):
|
||||
new_rhs = [t_rules[x].lhs if isinstance(x, T) else x for x in rule.rhs]
|
||||
new_rules.append(Rule(rule.lhs, new_rhs, weight=rule.weight, alias=rule.alias))
|
||||
new_rules.extend(v for k, v in t_rules.items() if k in rule.rhs)
|
||||
else:
|
||||
new_rules.append(rule)
|
||||
return Grammar(new_rules)
|
||||
|
||||
|
||||
def _bin(g):
|
||||
"""Applies the BIN rule to 'g' (see top comment)."""
|
||||
new_rules = []
|
||||
for rule in g.rules:
|
||||
if len(rule.rhs) > 2:
|
||||
new_rules += _split(rule)
|
||||
else:
|
||||
new_rules.append(rule)
|
||||
return Grammar(new_rules)
|
||||
|
||||
|
||||
def _unit(g):
|
||||
"""Applies the UNIT rule to 'g' (see top comment)."""
|
||||
nt_unit_rule = get_any_nt_unit_rule(g)
|
||||
while nt_unit_rule:
|
||||
g = _remove_unit_rule(g, nt_unit_rule)
|
||||
nt_unit_rule = get_any_nt_unit_rule(g)
|
||||
return g
|
||||
|
||||
|
||||
def to_cnf(g):
|
||||
"""Creates a CNF grammar from a general context-free grammar 'g'."""
|
||||
g = _unit(_bin(_term(g)))
|
||||
return CnfWrapper(g)
|
||||
|
||||
|
||||
def unroll_unit_skiprule(lhs, orig_rhs, skipped_rules, children, weight, alias):
|
||||
if not skipped_rules:
|
||||
return RuleNode(Rule(lhs, orig_rhs, weight=weight, alias=alias), children, weight=weight)
|
||||
else:
|
||||
weight = weight - skipped_rules[0].weight
|
||||
return RuleNode(
|
||||
Rule(lhs, [skipped_rules[0].lhs], weight=weight, alias=alias), [
|
||||
unroll_unit_skiprule(skipped_rules[0].lhs, orig_rhs,
|
||||
skipped_rules[1:], children,
|
||||
skipped_rules[0].weight, skipped_rules[0].alias)
|
||||
], weight=weight)
|
||||
|
||||
|
||||
def revert_cnf(node):
|
||||
"""Reverts a parse tree (RuleNode) to its original non-CNF form (Node)."""
|
||||
if isinstance(node, T):
|
||||
return node
|
||||
# Reverts TERM rule.
|
||||
if node.rule.lhs.name.startswith('__T_'):
|
||||
return node.children[0]
|
||||
else:
|
||||
children = []
|
||||
for child in map(revert_cnf, node.children):
|
||||
# Reverts BIN rule.
|
||||
if isinstance(child, RuleNode) and child.rule.lhs.name.startswith('__SP_'):
|
||||
children += child.children
|
||||
else:
|
||||
children.append(child)
|
||||
# Reverts UNIT rule.
|
||||
if isinstance(node.rule, UnitSkipRule):
|
||||
return unroll_unit_skiprule(node.rule.lhs, node.rule.rhs,
|
||||
node.rule.skipped_rules, children,
|
||||
node.rule.weight, node.rule.alias)
|
||||
else:
|
||||
return RuleNode(node.rule, children)
|
||||
@@ -1,239 +0,0 @@
|
||||
"This module implements an Earley Parser"
|
||||
|
||||
# The parser uses a parse-forest to keep track of derivations and ambiguations.
|
||||
# When the parse ends successfully, a disambiguation stage resolves all ambiguity
|
||||
# (right now ambiguity resolution is not developed beyond the needs of lark)
|
||||
# Afterwards the parse tree is reduced (transformed) according to user callbacks.
|
||||
# I use the no-recursion version of Transformer, because the tree might be
|
||||
# deeper than Python's recursion limit (a bit absurd, but that's life)
|
||||
#
|
||||
# The algorithm keeps track of each state set, using a corresponding Column instance.
|
||||
# Column keeps track of new items using NewsList instances.
|
||||
#
|
||||
# Author: Erez Shinan (2017)
|
||||
# Email : erezshin@gmail.com
|
||||
|
||||
from ..tree import Tree
|
||||
from ..visitors import Transformer_InPlace, v_args
|
||||
from ..exceptions import ParseError, UnexpectedToken
|
||||
from .grammar_analysis import GrammarAnalyzer
|
||||
from ..grammar import NonTerminal
|
||||
|
||||
|
||||
class Derivation(Tree):
|
||||
def __init__(self, rule, items=None):
|
||||
Tree.__init__(self, 'drv', items or [])
|
||||
self.meta.rule = rule
|
||||
self._hash = None
|
||||
|
||||
def _pretty_label(self): # Nicer pretty for debugging the parser
|
||||
return self.meta.rule.origin.name if self.meta.rule else self.data
|
||||
|
||||
def __hash__(self):
|
||||
if self._hash is None:
|
||||
self._hash = Tree.__hash__(self)
|
||||
return self._hash
|
||||
|
||||
class Item(object):
|
||||
"An Earley Item, the atom of the algorithm."
|
||||
|
||||
def __init__(self, rule, ptr, start, tree):
|
||||
self.rule = rule
|
||||
self.ptr = ptr
|
||||
self.start = start
|
||||
self.tree = tree if tree is not None else Derivation(self.rule)
|
||||
|
||||
@property
|
||||
def expect(self):
|
||||
return self.rule.expansion[self.ptr]
|
||||
|
||||
@property
|
||||
def is_complete(self):
|
||||
return self.ptr == len(self.rule.expansion)
|
||||
|
||||
def advance(self, tree):
|
||||
assert self.tree.data == 'drv'
|
||||
new_tree = Derivation(self.rule, self.tree.children + [tree])
|
||||
return self.__class__(self.rule, self.ptr+1, self.start, new_tree)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.start is other.start and self.ptr == other.ptr and self.rule == other.rule
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.rule, self.ptr, id(self.start))) # Always runs Derivation.__hash__
|
||||
|
||||
def __repr__(self):
|
||||
before = list(map(str, self.rule.expansion[:self.ptr]))
|
||||
after = list(map(str, self.rule.expansion[self.ptr:]))
|
||||
return '<(%d) %s : %s * %s>' % (id(self.start), self.rule.origin, ' '.join(before), ' '.join(after))
|
||||
|
||||
class NewsList(list):
|
||||
"Keeps track of newly added items (append-only)"
|
||||
|
||||
def __init__(self, initial=None):
|
||||
list.__init__(self, initial or [])
|
||||
self.last_iter = 0
|
||||
|
||||
def get_news(self):
|
||||
i = self.last_iter
|
||||
self.last_iter = len(self)
|
||||
return self[i:]
|
||||
|
||||
|
||||
|
||||
class Column:
|
||||
"An entry in the table, aka Earley Chart. Contains lists of items."
|
||||
def __init__(self, i, FIRST, predict_all=False):
|
||||
self.i = i
|
||||
self.to_reduce = NewsList()
|
||||
self.to_predict = NewsList()
|
||||
self.to_scan = []
|
||||
self.item_count = 0
|
||||
self.FIRST = FIRST
|
||||
|
||||
self.predicted = set()
|
||||
self.completed = {}
|
||||
self.predict_all = predict_all
|
||||
|
||||
def add(self, items):
|
||||
"""Sort items into scan/predict/reduce newslists
|
||||
|
||||
Makes sure only unique items are added.
|
||||
"""
|
||||
for item in items:
|
||||
|
||||
item_key = item, item.tree # Elsewhere, tree is not part of the comparison
|
||||
if item.is_complete:
|
||||
# XXX Potential bug: What happens if there's ambiguity in an empty rule?
|
||||
if item.rule.expansion and item_key in self.completed:
|
||||
old_tree = self.completed[item_key].tree
|
||||
if old_tree == item.tree:
|
||||
is_empty = not self.FIRST[item.rule.origin]
|
||||
if not is_empty:
|
||||
continue
|
||||
|
||||
if old_tree.data != '_ambig':
|
||||
new_tree = old_tree.copy()
|
||||
new_tree.meta.rule = old_tree.meta.rule
|
||||
old_tree.set('_ambig', [new_tree])
|
||||
old_tree.meta.rule = None # No longer a 'drv' node
|
||||
|
||||
if item.tree.children[0] is old_tree: # XXX a little hacky!
|
||||
raise ParseError("Infinite recursion in grammar! (Rule %s)" % item.rule)
|
||||
|
||||
if item.tree not in old_tree.children:
|
||||
old_tree.children.append(item.tree)
|
||||
# old_tree.children.append(item.tree)
|
||||
else:
|
||||
self.completed[item_key] = item
|
||||
self.to_reduce.append(item)
|
||||
else:
|
||||
if item.expect.is_term:
|
||||
self.to_scan.append(item)
|
||||
else:
|
||||
k = item_key if self.predict_all else item
|
||||
if k in self.predicted:
|
||||
continue
|
||||
self.predicted.add(k)
|
||||
self.to_predict.append(item)
|
||||
|
||||
self.item_count += 1 # Only count if actually added
|
||||
|
||||
|
||||
def __bool__(self):
|
||||
return bool(self.item_count)
|
||||
__nonzero__ = __bool__ # Py2 backwards-compatibility
|
||||
|
||||
class Parser:
|
||||
def __init__(self, parser_conf, term_matcher, resolve_ambiguity=None):
|
||||
analysis = GrammarAnalyzer(parser_conf)
|
||||
self.parser_conf = parser_conf
|
||||
self.resolve_ambiguity = resolve_ambiguity
|
||||
|
||||
self.FIRST = analysis.FIRST
|
||||
self.postprocess = {}
|
||||
self.predictions = {}
|
||||
for rule in parser_conf.rules:
|
||||
self.postprocess[rule] = rule.alias if callable(rule.alias) else getattr(parser_conf.callback, rule.alias)
|
||||
self.predictions[rule.origin] = [x.rule for x in analysis.expand_rule(rule.origin)]
|
||||
|
||||
self.term_matcher = term_matcher
|
||||
|
||||
|
||||
def parse(self, stream, start_symbol=None):
|
||||
# Define parser functions
|
||||
start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
|
||||
|
||||
_Item = Item
|
||||
match = self.term_matcher
|
||||
|
||||
def predict(nonterm, column):
|
||||
assert not nonterm.is_term, nonterm
|
||||
return [_Item(rule, 0, column, None) for rule in self.predictions[nonterm]]
|
||||
|
||||
def complete(item):
|
||||
name = item.rule.origin
|
||||
return [i.advance(item.tree) for i in item.start.to_predict if i.expect == name]
|
||||
|
||||
def predict_and_complete(column):
|
||||
while True:
|
||||
to_predict = {x.expect for x in column.to_predict.get_news()
|
||||
if x.ptr} # if not part of an already predicted batch
|
||||
to_reduce = set(column.to_reduce.get_news())
|
||||
if not (to_predict or to_reduce):
|
||||
break
|
||||
|
||||
for nonterm in to_predict:
|
||||
column.add( predict(nonterm, column) )
|
||||
|
||||
for item in to_reduce:
|
||||
new_items = list(complete(item))
|
||||
if item in new_items:
|
||||
raise ParseError('Infinite recursion detected! (rule %s)' % item.rule)
|
||||
column.add(new_items)
|
||||
|
||||
def scan(i, token, column):
|
||||
next_set = Column(i, self.FIRST)
|
||||
next_set.add(item.advance(token) for item in column.to_scan if match(item.expect, token))
|
||||
|
||||
if not next_set:
|
||||
expect = {i.expect.name for i in column.to_scan}
|
||||
raise UnexpectedToken(token, expect, considered_rules=set(column.to_scan))
|
||||
|
||||
return next_set
|
||||
|
||||
# Main loop starts
|
||||
column0 = Column(0, self.FIRST)
|
||||
column0.add(predict(start_symbol, column0))
|
||||
|
||||
column = column0
|
||||
for i, token in enumerate(stream):
|
||||
predict_and_complete(column)
|
||||
column = scan(i, token, column)
|
||||
|
||||
predict_and_complete(column)
|
||||
|
||||
# Parse ended. Now build a parse tree
|
||||
solutions = [n.tree for n in column.to_reduce
|
||||
if n.rule.origin==start_symbol and n.start is column0]
|
||||
|
||||
if not solutions:
|
||||
raise ParseError('Incomplete parse: Could not find a solution to input')
|
||||
elif len(solutions) == 1:
|
||||
tree = solutions[0]
|
||||
else:
|
||||
tree = Tree('_ambig', solutions)
|
||||
|
||||
if self.resolve_ambiguity:
|
||||
tree = self.resolve_ambiguity(tree)
|
||||
|
||||
return ApplyCallbacks(self.postprocess).transform(tree)
|
||||
|
||||
|
||||
class ApplyCallbacks(Transformer_InPlace):
|
||||
def __init__(self, postprocess):
|
||||
self.postprocess = postprocess
|
||||
|
||||
@v_args(meta=True)
|
||||
def drv(self, children, meta):
|
||||
return self.postprocess[meta.rule](children)
|
||||
@@ -1,149 +0,0 @@
|
||||
|
||||
from ..utils import bfs, fzset, classify
|
||||
from ..exceptions import GrammarError
|
||||
from ..grammar import Rule, Terminal, NonTerminal
|
||||
|
||||
|
||||
class RulePtr(object):
|
||||
__slots__ = ('rule', 'index')
|
||||
|
||||
def __init__(self, rule, index):
|
||||
assert isinstance(rule, Rule)
|
||||
assert index <= len(rule.expansion)
|
||||
self.rule = rule
|
||||
self.index = index
|
||||
|
||||
def __repr__(self):
|
||||
before = self.rule.expansion[:self.index]
|
||||
after = self.rule.expansion[self.index:]
|
||||
return '<%s : %s * %s>' % (self.rule.origin, ' '.join(before), ' '.join(after))
|
||||
|
||||
@property
|
||||
def next(self):
|
||||
return self.rule.expansion[self.index]
|
||||
|
||||
def advance(self, sym):
|
||||
assert self.next == sym
|
||||
return RulePtr(self.rule, self.index+1)
|
||||
|
||||
@property
|
||||
def is_satisfied(self):
|
||||
return self.index == len(self.rule.expansion)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.rule == other.rule and self.index == other.index
|
||||
def __hash__(self):
|
||||
return hash((self.rule, self.index))
|
||||
|
||||
|
||||
def update_set(set1, set2):
|
||||
if not set2:
|
||||
return False
|
||||
|
||||
copy = set(set1)
|
||||
set1 |= set2
|
||||
return set1 != copy
|
||||
|
||||
def calculate_sets(rules):
|
||||
"""Calculate FOLLOW sets.
|
||||
|
||||
Adapted from: http://lara.epfl.ch/w/cc09:algorithm_for_first_and_follow_sets"""
|
||||
symbols = {sym for rule in rules for sym in rule.expansion} | {rule.origin for rule in rules}
|
||||
|
||||
# foreach grammar rule X ::= Y(1) ... Y(k)
|
||||
# if k=0 or {Y(1),...,Y(k)} subset of NULLABLE then
|
||||
# NULLABLE = NULLABLE union {X}
|
||||
# for i = 1 to k
|
||||
# if i=1 or {Y(1),...,Y(i-1)} subset of NULLABLE then
|
||||
# FIRST(X) = FIRST(X) union FIRST(Y(i))
|
||||
# for j = i+1 to k
|
||||
# if i=k or {Y(i+1),...Y(k)} subset of NULLABLE then
|
||||
# FOLLOW(Y(i)) = FOLLOW(Y(i)) union FOLLOW(X)
|
||||
# if i+1=j or {Y(i+1),...,Y(j-1)} subset of NULLABLE then
|
||||
# FOLLOW(Y(i)) = FOLLOW(Y(i)) union FIRST(Y(j))
|
||||
# until none of NULLABLE,FIRST,FOLLOW changed in last iteration
|
||||
|
||||
NULLABLE = set()
|
||||
FIRST = {}
|
||||
FOLLOW = {}
|
||||
for sym in symbols:
|
||||
FIRST[sym]={sym} if sym.is_term else set()
|
||||
FOLLOW[sym]=set()
|
||||
|
||||
# Calculate NULLABLE and FIRST
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
|
||||
for rule in rules:
|
||||
if set(rule.expansion) <= NULLABLE:
|
||||
if update_set(NULLABLE, {rule.origin}):
|
||||
changed = True
|
||||
|
||||
for i, sym in enumerate(rule.expansion):
|
||||
if set(rule.expansion[:i]) <= NULLABLE:
|
||||
if update_set(FIRST[rule.origin], FIRST[sym]):
|
||||
changed = True
|
||||
|
||||
# Calculate FOLLOW
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
|
||||
for rule in rules:
|
||||
for i, sym in enumerate(rule.expansion):
|
||||
if i==len(rule.expansion)-1 or set(rule.expansion[i+1:]) <= NULLABLE:
|
||||
if update_set(FOLLOW[sym], FOLLOW[rule.origin]):
|
||||
changed = True
|
||||
|
||||
for j in range(i+1, len(rule.expansion)):
|
||||
if set(rule.expansion[i+1:j]) <= NULLABLE:
|
||||
if update_set(FOLLOW[sym], FIRST[rule.expansion[j]]):
|
||||
changed = True
|
||||
|
||||
return FIRST, FOLLOW, NULLABLE
|
||||
|
||||
|
||||
class GrammarAnalyzer(object):
|
||||
def __init__(self, parser_conf, debug=False):
|
||||
self.debug = debug
|
||||
|
||||
rules = parser_conf.rules + [Rule(NonTerminal('$root'), [NonTerminal(parser_conf.start), Terminal('$END')])]
|
||||
self.rules_by_origin = classify(rules, lambda r: r.origin)
|
||||
|
||||
assert len(rules) == len(set(rules))
|
||||
for r in rules:
|
||||
for sym in r.expansion:
|
||||
if not (sym.is_term or sym in self.rules_by_origin):
|
||||
raise GrammarError("Using an undefined rule: %s" % sym) # TODO test validation
|
||||
|
||||
self.start_state = self.expand_rule(NonTerminal('$root'))
|
||||
|
||||
self.FIRST, self.FOLLOW, self.NULLABLE = calculate_sets(rules)
|
||||
|
||||
def expand_rule(self, rule):
|
||||
"Returns all init_ptrs accessible by rule (recursive)"
|
||||
init_ptrs = set()
|
||||
def _expand_rule(rule):
|
||||
assert not rule.is_term, rule
|
||||
|
||||
for r in self.rules_by_origin[rule]:
|
||||
init_ptr = RulePtr(r, 0)
|
||||
init_ptrs.add(init_ptr)
|
||||
|
||||
if r.expansion: # if not empty rule
|
||||
new_r = init_ptr.next
|
||||
if not new_r.is_term:
|
||||
yield new_r
|
||||
|
||||
for _ in bfs([rule], _expand_rule):
|
||||
pass
|
||||
|
||||
return fzset(init_ptrs)
|
||||
|
||||
def _first(self, r):
|
||||
if r.is_term:
|
||||
return {r}
|
||||
else:
|
||||
return {rp.next for rp in self.expand_rule(r) if rp.next.is_term}
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
"""This module builds a LALR(1) transition-table for lalr_parser.py
|
||||
|
||||
For now, shift/reduce conflicts are automatically resolved as shifts.
|
||||
"""
|
||||
|
||||
# Author: Erez Shinan (2017)
|
||||
# Email : erezshin@gmail.com
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from ..utils import classify, classify_bool, bfs, fzset
|
||||
from ..exceptions import GrammarError
|
||||
|
||||
from .grammar_analysis import GrammarAnalyzer, Terminal
|
||||
|
||||
class Action:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
def __str__(self):
|
||||
return self.name
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
|
||||
Shift = Action('Shift')
|
||||
Reduce = Action('Reduce')
|
||||
|
||||
class ParseTable:
|
||||
def __init__(self, states, start_state, end_state):
|
||||
self.states = states
|
||||
self.start_state = start_state
|
||||
self.end_state = end_state
|
||||
|
||||
class IntParseTable(ParseTable):
|
||||
|
||||
@classmethod
|
||||
def from_ParseTable(cls, parse_table):
|
||||
enum = list(parse_table.states)
|
||||
state_to_idx = {s:i for i,s in enumerate(enum)}
|
||||
int_states = {}
|
||||
|
||||
for s, la in parse_table.states.items():
|
||||
la = {k:(v[0], state_to_idx[v[1]]) if v[0] is Shift else v
|
||||
for k,v in la.items()}
|
||||
int_states[ state_to_idx[s] ] = la
|
||||
|
||||
|
||||
start_state = state_to_idx[parse_table.start_state]
|
||||
end_state = state_to_idx[parse_table.end_state]
|
||||
return cls(int_states, start_state, end_state)
|
||||
|
||||
|
||||
|
||||
|
||||
class LALR_Analyzer(GrammarAnalyzer):
|
||||
|
||||
def compute_lookahead(self):
|
||||
self.end_states = []
|
||||
|
||||
self.states = {}
|
||||
def step(state):
|
||||
lookahead = defaultdict(list)
|
||||
sat, unsat = classify_bool(state, lambda rp: rp.is_satisfied)
|
||||
for rp in sat:
|
||||
for term in self.FOLLOW.get(rp.rule.origin, ()):
|
||||
lookahead[term].append((Reduce, rp.rule))
|
||||
|
||||
d = classify(unsat, lambda rp: rp.next)
|
||||
for sym, rps in d.items():
|
||||
rps = {rp.advance(sym) for rp in rps}
|
||||
|
||||
for rp in set(rps):
|
||||
if not rp.is_satisfied and not rp.next.is_term:
|
||||
rps |= self.expand_rule(rp.next)
|
||||
|
||||
new_state = fzset(rps)
|
||||
lookahead[sym].append((Shift, new_state))
|
||||
if sym == Terminal('$END'):
|
||||
self.end_states.append( new_state )
|
||||
yield new_state
|
||||
|
||||
for k, v in lookahead.items():
|
||||
if len(v) > 1:
|
||||
if self.debug:
|
||||
logging.warn("Shift/reduce conflict for %s: %s. Resolving as shift.", k, v)
|
||||
for x in v:
|
||||
# XXX resolving shift/reduce into shift, like PLY
|
||||
# Give a proper warning
|
||||
if x[0] is Shift:
|
||||
lookahead[k] = [x]
|
||||
|
||||
for k, v in lookahead.items():
|
||||
if not len(v) == 1:
|
||||
raise GrammarError("Collision in %s: %s" %(k, ', '.join(['\n * %s: %s' % x for x in v])))
|
||||
|
||||
self.states[state] = {k.name:v[0] for k, v in lookahead.items()}
|
||||
|
||||
for _ in bfs([self.start_state], step):
|
||||
pass
|
||||
|
||||
self.end_state ,= self.end_states
|
||||
|
||||
self._parse_table = ParseTable(self.states, self.start_state, self.end_state)
|
||||
|
||||
if self.debug:
|
||||
self.parse_table = self._parse_table
|
||||
else:
|
||||
self.parse_table = IntParseTable.from_ParseTable(self._parse_table)
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
"""This module implements a LALR(1) Parser
|
||||
"""
|
||||
# Author: Erez Shinan (2017)
|
||||
# Email : erezshin@gmail.com
|
||||
from ..exceptions import UnexpectedToken
|
||||
from ..lexer import Token
|
||||
|
||||
from .lalr_analysis import LALR_Analyzer, Shift
|
||||
|
||||
class Parser:
|
||||
def __init__(self, parser_conf):
|
||||
assert all(r.options is None or r.options.priority is None
|
||||
for r in parser_conf.rules), "LALR doesn't yet support prioritization"
|
||||
analysis = LALR_Analyzer(parser_conf)
|
||||
analysis.compute_lookahead()
|
||||
callbacks = {rule: getattr(parser_conf.callback, rule.alias or rule.origin, None)
|
||||
for rule in parser_conf.rules}
|
||||
|
||||
self._parse_table = analysis.parse_table
|
||||
self.parser_conf = parser_conf
|
||||
self.parser = _Parser(analysis.parse_table, callbacks)
|
||||
self.parse = self.parser.parse
|
||||
|
||||
###{standalone
|
||||
|
||||
class _Parser:
|
||||
def __init__(self, parse_table, callbacks):
|
||||
self.states = parse_table.states
|
||||
self.start_state = parse_table.start_state
|
||||
self.end_state = parse_table.end_state
|
||||
self.callbacks = callbacks
|
||||
|
||||
def parse(self, seq, set_state=None):
|
||||
token = None
|
||||
stream = iter(seq)
|
||||
states = self.states
|
||||
|
||||
state_stack = [self.start_state]
|
||||
value_stack = []
|
||||
|
||||
if set_state: set_state(self.start_state)
|
||||
|
||||
def get_action(token):
|
||||
state = state_stack[-1]
|
||||
try:
|
||||
return states[state][token.type]
|
||||
except KeyError:
|
||||
expected = [s for s in states[state].keys() if s.isupper()]
|
||||
raise UnexpectedToken(token, expected, state=state)
|
||||
|
||||
def reduce(rule):
|
||||
size = len(rule.expansion)
|
||||
if size:
|
||||
s = value_stack[-size:]
|
||||
del state_stack[-size:]
|
||||
del value_stack[-size:]
|
||||
else:
|
||||
s = []
|
||||
|
||||
value = self.callbacks[rule](s)
|
||||
|
||||
_action, new_state = states[state_stack[-1]][rule.origin.name]
|
||||
assert _action is Shift
|
||||
state_stack.append(new_state)
|
||||
value_stack.append(value)
|
||||
|
||||
# Main LALR-parser loop
|
||||
for token in stream:
|
||||
while True:
|
||||
action, arg = get_action(token)
|
||||
assert arg != self.end_state
|
||||
|
||||
if action is Shift:
|
||||
state_stack.append(arg)
|
||||
value_stack.append(token)
|
||||
if set_state: set_state(arg)
|
||||
break # next token
|
||||
else:
|
||||
reduce(arg)
|
||||
|
||||
token = Token.new_borrow_pos('$END', '', token) if token else Token('$END', '', 0, 1, 1)
|
||||
while True:
|
||||
_action, arg = get_action(token)
|
||||
if _action is Shift:
|
||||
assert arg == self.end_state
|
||||
val ,= value_stack
|
||||
return val
|
||||
else:
|
||||
reduce(arg)
|
||||
|
||||
###}
|
||||
@@ -1,109 +0,0 @@
|
||||
from ..utils import compare
|
||||
from functools import cmp_to_key
|
||||
|
||||
from ..tree import Tree
|
||||
|
||||
|
||||
# Standard ambiguity resolver (uses comparison)
|
||||
#
|
||||
# Author: Erez Sh
|
||||
|
||||
def _compare_rules(rule1, rule2):
|
||||
return -compare( len(rule1.expansion), len(rule2.expansion))
|
||||
|
||||
def _sum_priority(tree):
|
||||
p = 0
|
||||
|
||||
for n in tree.iter_subtrees():
|
||||
try:
|
||||
p += n.meta.rule.options.priority or 0
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
return p
|
||||
|
||||
def _compare_priority(tree1, tree2):
|
||||
tree1.iter_subtrees()
|
||||
|
||||
def _compare_drv(tree1, tree2):
|
||||
try:
|
||||
rule1 = tree1.meta.rule
|
||||
except AttributeError:
|
||||
rule1 = None
|
||||
|
||||
try:
|
||||
rule2 = tree2.meta.rule
|
||||
except AttributeError:
|
||||
rule2 = None
|
||||
|
||||
if None == rule1 == rule2:
|
||||
return compare(tree1, tree2)
|
||||
elif rule1 is None:
|
||||
return -1
|
||||
elif rule2 is None:
|
||||
return 1
|
||||
|
||||
assert tree1.data != '_ambig'
|
||||
assert tree2.data != '_ambig'
|
||||
|
||||
p1 = _sum_priority(tree1)
|
||||
p2 = _sum_priority(tree2)
|
||||
c = (p1 or p2) and compare(p1, p2)
|
||||
if c:
|
||||
return c
|
||||
|
||||
c = _compare_rules(tree1.meta.rule, tree2.meta.rule)
|
||||
if c:
|
||||
return c
|
||||
|
||||
# rules are "equal", so compare trees
|
||||
if len(tree1.children) == len(tree2.children):
|
||||
for t1, t2 in zip(tree1.children, tree2.children):
|
||||
c = _compare_drv(t1, t2)
|
||||
if c:
|
||||
return c
|
||||
|
||||
return compare(len(tree1.children), len(tree2.children))
|
||||
|
||||
|
||||
def _standard_resolve_ambig(tree):
|
||||
assert tree.data == '_ambig'
|
||||
key_f = cmp_to_key(_compare_drv)
|
||||
best = max(tree.children, key=key_f)
|
||||
assert best.data == 'drv'
|
||||
tree.set('drv', best.children)
|
||||
tree.meta.rule = best.meta.rule # needed for applying callbacks
|
||||
|
||||
def standard_resolve_ambig(tree):
|
||||
for ambig in tree.find_data('_ambig'):
|
||||
_standard_resolve_ambig(ambig)
|
||||
|
||||
return tree
|
||||
|
||||
|
||||
|
||||
|
||||
# Anti-score Sum
|
||||
#
|
||||
# Author: Uriva (https://github.com/uriva)
|
||||
|
||||
def _antiscore_sum_drv(tree):
|
||||
if not isinstance(tree, Tree):
|
||||
return 0
|
||||
|
||||
assert tree.data != '_ambig'
|
||||
|
||||
return _sum_priority(tree)
|
||||
|
||||
def _antiscore_sum_resolve_ambig(tree):
|
||||
assert tree.data == '_ambig'
|
||||
best = min(tree.children, key=_antiscore_sum_drv)
|
||||
assert best.data == 'drv'
|
||||
tree.set('drv', best.children)
|
||||
tree.meta.rule = best.meta.rule # needed for applying callbacks
|
||||
|
||||
def antiscore_sum_resolve_ambig(tree):
|
||||
for ambig in tree.find_data('_ambig'):
|
||||
_antiscore_sum_resolve_ambig(ambig)
|
||||
|
||||
return tree
|
||||
@@ -1,158 +0,0 @@
|
||||
"This module implements an experimental Earley Parser with a dynamic lexer"
|
||||
|
||||
# The parser uses a parse-forest to keep track of derivations and ambiguations.
|
||||
# When the parse ends successfully, a disambiguation stage resolves all ambiguity
|
||||
# (right now ambiguity resolution is not developed beyond the needs of lark)
|
||||
# Afterwards the parse tree is reduced (transformed) according to user callbacks.
|
||||
# I use the no-recursion version of Transformer and Visitor, because the tree might be
|
||||
# deeper than Python's recursion limit (a bit absurd, but that's life)
|
||||
#
|
||||
# The algorithm keeps track of each state set, using a corresponding Column instance.
|
||||
# Column keeps track of new items using NewsList instances.
|
||||
#
|
||||
# Instead of running a lexer beforehand, or using a costy char-by-char method, this parser
|
||||
# uses regular expressions by necessity, achieving high-performance while maintaining all of
|
||||
# Earley's power in parsing any CFG.
|
||||
#
|
||||
#
|
||||
# Author: Erez Shinan (2017)
|
||||
# Email : erezshin@gmail.com
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
from ..exceptions import ParseError, UnexpectedCharacters
|
||||
from ..lexer import Token
|
||||
from ..tree import Tree
|
||||
from .grammar_analysis import GrammarAnalyzer
|
||||
from ..grammar import NonTerminal, Terminal
|
||||
|
||||
from .earley import ApplyCallbacks, Item, Column
|
||||
|
||||
|
||||
class Parser:
|
||||
def __init__(self, parser_conf, term_matcher, resolve_ambiguity=None, ignore=(), predict_all=False, complete_lex=False):
|
||||
self.analysis = GrammarAnalyzer(parser_conf)
|
||||
self.parser_conf = parser_conf
|
||||
self.resolve_ambiguity = resolve_ambiguity
|
||||
self.ignore = [Terminal(t) for t in ignore]
|
||||
self.predict_all = predict_all
|
||||
self.complete_lex = complete_lex
|
||||
|
||||
self.FIRST = self.analysis.FIRST
|
||||
self.postprocess = {}
|
||||
self.predictions = {}
|
||||
for rule in parser_conf.rules:
|
||||
self.postprocess[rule] = getattr(parser_conf.callback, rule.alias)
|
||||
self.predictions[rule.origin] = [x.rule for x in self.analysis.expand_rule(rule.origin)]
|
||||
|
||||
self.term_matcher = term_matcher
|
||||
|
||||
|
||||
def parse(self, stream, start_symbol=None):
|
||||
# Define parser functions
|
||||
start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
|
||||
delayed_matches = defaultdict(list)
|
||||
match = self.term_matcher
|
||||
|
||||
text_line = 1
|
||||
text_column = 1
|
||||
|
||||
def predict(nonterm, column):
|
||||
assert not nonterm.is_term, nonterm
|
||||
return [Item(rule, 0, column, None) for rule in self.predictions[nonterm]]
|
||||
|
||||
def complete(item):
|
||||
name = item.rule.origin
|
||||
return [i.advance(item.tree) for i in item.start.to_predict if i.expect == name]
|
||||
|
||||
def predict_and_complete(column):
|
||||
while True:
|
||||
to_predict = {x.expect for x in column.to_predict.get_news()
|
||||
if x.ptr} # if not part of an already predicted batch
|
||||
to_reduce = column.to_reduce.get_news()
|
||||
if not (to_predict or to_reduce):
|
||||
break
|
||||
|
||||
for nonterm in to_predict:
|
||||
column.add( predict(nonterm, column) )
|
||||
for item in to_reduce:
|
||||
new_items = list(complete(item))
|
||||
if item in new_items:
|
||||
raise ParseError('Infinite recursion detected! (rule %s)' % item.rule)
|
||||
column.add(new_items)
|
||||
|
||||
def scan(i, column):
|
||||
to_scan = column.to_scan
|
||||
|
||||
for x in self.ignore:
|
||||
m = match(x, stream, i)
|
||||
if m:
|
||||
delayed_matches[m.end()] += set(to_scan)
|
||||
delayed_matches[m.end()] += set(column.to_reduce)
|
||||
|
||||
# TODO add partial matches for ignore too?
|
||||
# s = m.group(0)
|
||||
# for j in range(1, len(s)):
|
||||
# m = x.match(s[:-j])
|
||||
# if m:
|
||||
# delayed_matches[m.end()] += to_scan
|
||||
|
||||
for item in to_scan:
|
||||
m = match(item.expect, stream, i)
|
||||
if m:
|
||||
t = Token(item.expect.name, m.group(0), i, text_line, text_column)
|
||||
delayed_matches[m.end()].append(item.advance(t))
|
||||
|
||||
if self.complete_lex:
|
||||
s = m.group(0)
|
||||
for j in range(1, len(s)):
|
||||
m = match(item.expect, s[:-j])
|
||||
if m:
|
||||
t = Token(item.expect.name, m.group(0), i, text_line, text_column)
|
||||
delayed_matches[i+m.end()].append(item.advance(t))
|
||||
|
||||
next_set = Column(i+1, self.FIRST, predict_all=self.predict_all)
|
||||
next_set.add(delayed_matches[i+1])
|
||||
del delayed_matches[i+1] # No longer needed, so unburden memory
|
||||
|
||||
if not next_set and not delayed_matches:
|
||||
raise UnexpectedCharacters(stream, i, text_line, text_column, {item.expect for item in to_scan}, set(to_scan))
|
||||
|
||||
return next_set
|
||||
|
||||
# Main loop starts
|
||||
column0 = Column(0, self.FIRST, predict_all=self.predict_all)
|
||||
column0.add(predict(start_symbol, column0))
|
||||
|
||||
column = column0
|
||||
for i, token in enumerate(stream):
|
||||
predict_and_complete(column)
|
||||
column = scan(i, column)
|
||||
|
||||
if token == '\n':
|
||||
text_line += 1
|
||||
text_column = 1
|
||||
else:
|
||||
text_column += 1
|
||||
|
||||
predict_and_complete(column)
|
||||
|
||||
# Parse ended. Now build a parse tree
|
||||
solutions = [n.tree for n in column.to_reduce
|
||||
if n.rule.origin==start_symbol and n.start is column0]
|
||||
|
||||
if not solutions:
|
||||
expected_tokens = [t.expect for t in column.to_scan]
|
||||
raise ParseError('Unexpected end of input! Expecting a terminal of: %s' % expected_tokens)
|
||||
|
||||
elif len(solutions) == 1:
|
||||
tree = solutions[0]
|
||||
else:
|
||||
tree = Tree('_ambig', solutions)
|
||||
|
||||
if self.resolve_ambiguity:
|
||||
tree = self.resolve_ambiguity(tree)
|
||||
|
||||
return ApplyCallbacks(self.postprocess).transform(tree)
|
||||
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
from collections import defaultdict
|
||||
|
||||
from .tree import Tree
|
||||
from .visitors import Transformer_InPlace
|
||||
from .common import ParserConf
|
||||
from .lexer import Token, PatternStr
|
||||
from .parsers import earley, resolve_ambig
|
||||
from .grammar import Rule, Terminal, NonTerminal
|
||||
|
||||
|
||||
|
||||
def is_discarded_terminal(t):
|
||||
return t.is_term and t.filter_out
|
||||
|
||||
def is_iter_empty(i):
|
||||
try:
|
||||
_ = next(i)
|
||||
return False
|
||||
except StopIteration:
|
||||
return True
|
||||
|
||||
class WriteTokensTransformer(Transformer_InPlace):
|
||||
def __init__(self, tokens):
|
||||
self.tokens = tokens
|
||||
|
||||
def __default__(self, data, children, meta):
|
||||
# if not isinstance(t, MatchTree):
|
||||
# return t
|
||||
if not getattr(meta, 'match_tree', False):
|
||||
return Tree(data, children)
|
||||
|
||||
iter_args = iter(children)
|
||||
to_write = []
|
||||
for sym in meta.orig_expansion:
|
||||
if is_discarded_terminal(sym):
|
||||
t = self.tokens[sym.name]
|
||||
assert isinstance(t.pattern, PatternStr)
|
||||
to_write.append(t.pattern.value)
|
||||
else:
|
||||
x = next(iter_args)
|
||||
if isinstance(x, list):
|
||||
to_write += x
|
||||
else:
|
||||
if isinstance(x, Token):
|
||||
assert Terminal(x.type) == sym, x
|
||||
else:
|
||||
assert NonTerminal(x.data) == sym, (sym, x)
|
||||
to_write.append(x)
|
||||
|
||||
assert is_iter_empty(iter_args)
|
||||
return to_write
|
||||
|
||||
|
||||
class MatchTree(Tree):
|
||||
pass
|
||||
|
||||
class MakeMatchTree:
|
||||
def __init__(self, name, expansion):
|
||||
self.name = name
|
||||
self.expansion = expansion
|
||||
|
||||
def __call__(self, args):
|
||||
t = MatchTree(self.name, args)
|
||||
t.meta.match_tree = True
|
||||
t.meta.orig_expansion = self.expansion
|
||||
return t
|
||||
|
||||
class Reconstructor:
|
||||
def __init__(self, parser):
|
||||
# XXX TODO calling compile twice returns different results!
|
||||
tokens, rules, _grammar_extra = parser.grammar.compile()
|
||||
|
||||
self.write_tokens = WriteTokensTransformer({t.name:t for t in tokens})
|
||||
self.rules = list(self._build_recons_rules(rules))
|
||||
|
||||
def _build_recons_rules(self, rules):
|
||||
expand1s = {r.origin for r in rules if r.options and r.options.expand1}
|
||||
|
||||
aliases = defaultdict(list)
|
||||
for r in rules:
|
||||
if r.alias:
|
||||
aliases[r.origin].append( r.alias )
|
||||
|
||||
rule_names = {r.origin for r in rules}
|
||||
nonterminals = {sym for sym in rule_names
|
||||
if sym.name.startswith('_') or sym in expand1s or sym in aliases }
|
||||
|
||||
for r in rules:
|
||||
recons_exp = [sym if sym in nonterminals else Terminal(sym.name)
|
||||
for sym in r.expansion if not is_discarded_terminal(sym)]
|
||||
|
||||
# Skip self-recursive constructs
|
||||
if recons_exp == [r.origin]:
|
||||
continue
|
||||
|
||||
sym = NonTerminal(r.alias) if r.alias else r.origin
|
||||
|
||||
yield Rule(sym, recons_exp, MakeMatchTree(sym.name, r.expansion))
|
||||
|
||||
for origin, rule_aliases in aliases.items():
|
||||
for alias in rule_aliases:
|
||||
yield Rule(origin, [Terminal(alias)], MakeMatchTree(origin.name, [NonTerminal(alias)]))
|
||||
|
||||
yield Rule(origin, [Terminal(origin.name)], MakeMatchTree(origin.name, [origin]))
|
||||
|
||||
|
||||
|
||||
def _match(self, term, token):
|
||||
if isinstance(token, Tree):
|
||||
return Terminal(token.data) == term
|
||||
elif isinstance(token, Token):
|
||||
return term == Terminal(token.type)
|
||||
assert False
|
||||
|
||||
def _reconstruct(self, tree):
|
||||
# TODO: ambiguity?
|
||||
parser = earley.Parser(ParserConf(self.rules, None, tree.data), self._match, resolve_ambiguity=resolve_ambig.standard_resolve_ambig)
|
||||
unreduced_tree = parser.parse(tree.children) # find a full derivation
|
||||
assert unreduced_tree.data == tree.data
|
||||
res = self.write_tokens.transform(unreduced_tree)
|
||||
for item in res:
|
||||
if isinstance(item, Tree):
|
||||
for x in self._reconstruct(item):
|
||||
yield x
|
||||
else:
|
||||
yield item
|
||||
|
||||
def reconstruct(self, tree):
|
||||
return ''.join(self._reconstruct(tree))
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from .tree import Tree
|
||||
from .common import is_terminal, ParserConf, PatternStr, Terminal
|
||||
from .lexer import Token
|
||||
from .parsers import earley
|
||||
|
||||
|
||||
|
||||
def is_discarded_terminal(t):
|
||||
return is_terminal(t) and t.startswith('_')
|
||||
|
||||
def is_iter_empty(i):
|
||||
try:
|
||||
_ = next(i)
|
||||
return False
|
||||
except StopIteration:
|
||||
return True
|
||||
|
||||
class Reconstructor:
|
||||
def __init__(self, parser):
|
||||
# Recreate the rules to assume a standard lexer
|
||||
_tokens, rules, _grammar_extra = parser.grammar.compile(lexer='standard', start='whatever')
|
||||
tokens = {t.name:t for t in _tokens}
|
||||
|
||||
token_res = {t.name:re.compile(t.pattern.to_regexp()) for t in _tokens}
|
||||
|
||||
class MatchTerminal(Terminal):
|
||||
def match(self, other):
|
||||
if isinstance(other, Tree):
|
||||
return False
|
||||
return token_res[self.data].match(other) is not None
|
||||
|
||||
class MatchTree(Terminal):
|
||||
def match(self, other):
|
||||
try:
|
||||
return self.data == other.data
|
||||
except AttributeError:
|
||||
return False
|
||||
|
||||
class WriteTokens:
|
||||
def __init__(self, name, expansion):
|
||||
self.name = name
|
||||
self.expansion = expansion
|
||||
|
||||
def f(self, args):
|
||||
args2 = iter(args)
|
||||
to_write = []
|
||||
for sym in self.expansion:
|
||||
if is_discarded_terminal(sym):
|
||||
t = tokens[sym]
|
||||
assert isinstance(t.pattern, PatternStr)
|
||||
to_write.append(t.pattern.value)
|
||||
else:
|
||||
x = next(args2)
|
||||
if isinstance(x, list):
|
||||
to_write += x
|
||||
else:
|
||||
if isinstance(x, Token):
|
||||
assert x.type == sym, x
|
||||
else:
|
||||
assert x.data == sym, x
|
||||
to_write.append(x)
|
||||
|
||||
assert is_iter_empty(args2)
|
||||
|
||||
return to_write
|
||||
|
||||
d = defaultdict(list)
|
||||
for name, (expansions, _o) in rules.items():
|
||||
for expansion, alias in expansions:
|
||||
if alias:
|
||||
d[alias].append(expansion)
|
||||
d[name].append([alias])
|
||||
else:
|
||||
d[name].append(expansion)
|
||||
|
||||
rules = []
|
||||
expand1s = {name for name, (_x, options) in parser.rules.items()
|
||||
if options and options.expand1}
|
||||
|
||||
for name, expansions in d.items():
|
||||
for expansion in expansions:
|
||||
reduced = [sym if sym.startswith('_') or sym in expand1s else
|
||||
MatchTerminal(sym) if is_terminal(sym) else MatchTree(sym)
|
||||
for sym in expansion if not is_discarded_terminal(sym)]
|
||||
|
||||
rules.append((name, reduced, WriteTokens(name, expansion).f, None))
|
||||
self.rules = rules
|
||||
|
||||
|
||||
def _reconstruct(self, tree):
|
||||
# TODO: ambiguity?
|
||||
parser = earley.Parser(self.rules, tree.data, {})
|
||||
res = parser.parse(tree.children)
|
||||
for item in res:
|
||||
if isinstance(item, Tree):
|
||||
for x in self._reconstruct(item):
|
||||
yield x
|
||||
else:
|
||||
yield item
|
||||
|
||||
def reconstruct(self, tree):
|
||||
return ''.join(self._reconstruct(tree))
|
||||
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
"Converts between Lark and Nearley grammars. Work in progress!"
|
||||
|
||||
import os.path
|
||||
import sys
|
||||
import codecs
|
||||
|
||||
|
||||
from lark import Lark, InlineTransformer
|
||||
|
||||
nearley_grammar = r"""
|
||||
start: (ruledef|directive)+
|
||||
|
||||
directive: "@" NAME (STRING|NAME)
|
||||
| "@" JS -> js_code
|
||||
ruledef: NAME "->" expansions
|
||||
| NAME REGEXP "->" expansions -> macro
|
||||
expansions: expansion ("|" expansion)*
|
||||
|
||||
expansion: expr+ js
|
||||
|
||||
?expr: item [":" /[+*?]/]
|
||||
|
||||
?item: rule|string|regexp
|
||||
| "(" expansions ")"
|
||||
|
||||
rule: NAME
|
||||
string: STRING
|
||||
regexp: REGEXP
|
||||
JS: /{%.*?%}/s
|
||||
js: JS?
|
||||
|
||||
NAME: /[a-zA-Z_$]\w*/
|
||||
COMMENT: /#[^\n]*/
|
||||
REGEXP: /\[.*?\]/
|
||||
STRING: /".*?"/
|
||||
|
||||
%import common.WS
|
||||
%ignore WS
|
||||
%ignore COMMENT
|
||||
|
||||
"""
|
||||
|
||||
nearley_grammar_parser = Lark(nearley_grammar, parser='earley', lexer='standard')
|
||||
|
||||
def _get_rulename(name):
|
||||
name = {'_': '_ws_maybe', '__':'_ws'}.get(name, name)
|
||||
return 'n_' + name.replace('$', '__DOLLAR__').lower()
|
||||
|
||||
class NearleyToLark(InlineTransformer):
|
||||
def __init__(self):
|
||||
self._count = 0
|
||||
self.extra_rules = {}
|
||||
self.extra_rules_rev = {}
|
||||
self.alias_js_code = {}
|
||||
|
||||
def _new_function(self, code):
|
||||
name = 'alias_%d' % self._count
|
||||
self._count += 1
|
||||
|
||||
self.alias_js_code[name] = code
|
||||
return name
|
||||
|
||||
def _extra_rule(self, rule):
|
||||
if rule in self.extra_rules_rev:
|
||||
return self.extra_rules_rev[rule]
|
||||
|
||||
name = 'xrule_%d' % len(self.extra_rules)
|
||||
assert name not in self.extra_rules
|
||||
self.extra_rules[name] = rule
|
||||
self.extra_rules_rev[rule] = name
|
||||
return name
|
||||
|
||||
def rule(self, name):
|
||||
return _get_rulename(name)
|
||||
|
||||
def ruledef(self, name, exps):
|
||||
return '!%s: %s' % (_get_rulename(name), exps)
|
||||
|
||||
def expr(self, item, op):
|
||||
rule = '(%s)%s' % (item, op)
|
||||
return self._extra_rule(rule)
|
||||
|
||||
def regexp(self, r):
|
||||
return '/%s/' % r
|
||||
|
||||
def string(self, s):
|
||||
return self._extra_rule(s)
|
||||
|
||||
def expansion(self, *x):
|
||||
x, js = x[:-1], x[-1]
|
||||
if js.children:
|
||||
js_code ,= js.children
|
||||
js_code = js_code[2:-2]
|
||||
alias = '-> ' + self._new_function(js_code)
|
||||
else:
|
||||
alias = ''
|
||||
return ' '.join(x) + alias
|
||||
|
||||
def expansions(self, *x):
|
||||
return '%s' % ('\n |'.join(x))
|
||||
|
||||
def start(self, *rules):
|
||||
return '\n'.join(filter(None, rules))
|
||||
|
||||
def _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, includes):
|
||||
rule_defs = []
|
||||
|
||||
tree = nearley_grammar_parser.parse(g)
|
||||
for statement in tree.children:
|
||||
if statement.data == 'directive':
|
||||
directive, arg = statement.children
|
||||
if directive in ('builtin', 'include'):
|
||||
folder = builtin_path if directive == 'builtin' else folder_path
|
||||
path = os.path.join(folder, arg[1:-1])
|
||||
if path not in includes:
|
||||
includes.add(path)
|
||||
with codecs.open(path, encoding='utf8') as f:
|
||||
text = f.read()
|
||||
rule_defs += _nearley_to_lark(text, builtin_path, n2l, js_code, os.path.abspath(os.path.dirname(path)), includes)
|
||||
else:
|
||||
assert False, directive
|
||||
elif statement.data == 'js_code':
|
||||
code ,= statement.children
|
||||
code = code[2:-2]
|
||||
js_code.append(code)
|
||||
elif statement.data == 'macro':
|
||||
pass # TODO Add support for macros!
|
||||
elif statement.data == 'ruledef':
|
||||
rule_defs.append( n2l.transform(statement) )
|
||||
else:
|
||||
raise Exception("Unknown statement: %s" % statement)
|
||||
|
||||
return rule_defs
|
||||
|
||||
|
||||
def create_code_for_nearley_grammar(g, start, builtin_path, folder_path):
|
||||
import js2py
|
||||
|
||||
emit_code = []
|
||||
def emit(x=None):
|
||||
if x:
|
||||
emit_code.append(x)
|
||||
emit_code.append('\n')
|
||||
|
||||
js_code = ['function id(x) {return x[0];}']
|
||||
n2l = NearleyToLark()
|
||||
rule_defs = _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, set())
|
||||
lark_g = '\n'.join(rule_defs)
|
||||
lark_g += '\n'+'\n'.join('!%s: %s' % item for item in n2l.extra_rules.items())
|
||||
|
||||
emit('from lark import Lark, Transformer')
|
||||
emit()
|
||||
emit('grammar = ' + repr(lark_g))
|
||||
emit()
|
||||
|
||||
for alias, code in n2l.alias_js_code.items():
|
||||
js_code.append('%s = (%s);' % (alias, code))
|
||||
|
||||
emit(js2py.translate_js('\n'.join(js_code)))
|
||||
emit('class TransformNearley(Transformer):')
|
||||
for alias in n2l.alias_js_code:
|
||||
emit(" %s = var.get('%s').to_python()" % (alias, alias))
|
||||
emit(" __default__ = lambda self, n, c, m: c if c else None")
|
||||
|
||||
emit()
|
||||
emit('parser = Lark(grammar, start="n_%s")' % start)
|
||||
emit('def parse(text):')
|
||||
emit(' return TransformNearley().transform(parser.parse(text))')
|
||||
|
||||
return ''.join(emit_code)
|
||||
|
||||
def main(fn, start, nearley_lib):
|
||||
with codecs.open(fn, encoding='utf8') as f:
|
||||
grammar = f.read()
|
||||
return create_code_for_nearley_grammar(grammar, start, os.path.join(nearley_lib, 'builtin'), os.path.abspath(os.path.dirname(fn)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 4:
|
||||
print("Reads Nearley grammar (with js functions) outputs an equivalent lark parser.")
|
||||
print("Usage: %s <nearley_grammar_path> <start_rule> <nearley_lib_path>" % sys.argv[0])
|
||||
sys.exit(1)
|
||||
|
||||
fn, start, nearley_lib = sys.argv[1:]
|
||||
|
||||
print(main(fn, start, nearley_lib))
|
||||
@@ -1,251 +0,0 @@
|
||||
###{standalone
|
||||
#
|
||||
#
|
||||
# Lark Stand-alone Generator Tool
|
||||
# ----------------------------------
|
||||
# Generates a stand-alone LALR(1) parser with a standard lexer
|
||||
#
|
||||
# Git: https://github.com/erezsh/lark
|
||||
# Author: Erez Shinan (erezshin@gmail.com)
|
||||
#
|
||||
#
|
||||
# >>> LICENSE
|
||||
#
|
||||
# This tool and its generated code use a separate license from Lark.
|
||||
#
|
||||
# It is licensed under GPLv2 or above.
|
||||
#
|
||||
# If you wish to purchase a commercial license for this tool and its
|
||||
# generated code, contact me via email.
|
||||
#
|
||||
# If GPL is incompatible with your free or open-source project,
|
||||
# contact me and we'll work it out (for free).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# See <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
#
|
||||
###}
|
||||
|
||||
import codecs
|
||||
import sys
|
||||
import os
|
||||
from pprint import pprint
|
||||
from os import path
|
||||
from collections import defaultdict
|
||||
|
||||
import lark
|
||||
from lark import Lark
|
||||
from lark.parsers.lalr_analysis import Shift, Reduce
|
||||
|
||||
from ..grammar import Rule
|
||||
|
||||
_dir = path.dirname(__file__)
|
||||
_larkdir = path.join(_dir, path.pardir)
|
||||
|
||||
|
||||
EXTRACT_STANDALONE_FILES = [
|
||||
'tools/standalone.py',
|
||||
'exceptions.py',
|
||||
'utils.py',
|
||||
'tree.py',
|
||||
'visitors.py',
|
||||
'indenter.py',
|
||||
'lexer.py',
|
||||
'parse_tree_builder.py',
|
||||
'parsers/lalr_parser.py',
|
||||
]
|
||||
|
||||
|
||||
def extract_sections(lines):
|
||||
section = None
|
||||
text = []
|
||||
sections = defaultdict(list)
|
||||
for l in lines:
|
||||
if l.startswith('###'):
|
||||
if l[3] == '{':
|
||||
section = l[4:].strip()
|
||||
elif l[3] == '}':
|
||||
sections[section] += text
|
||||
section = None
|
||||
text = []
|
||||
else:
|
||||
raise ValueError(l)
|
||||
elif section:
|
||||
text.append(l)
|
||||
|
||||
return {name:''.join(text) for name, text in sections.items()}
|
||||
|
||||
def _prepare_mres(mres):
|
||||
return [(p.pattern,{i: t for i, t in d.items()}) for p,d in mres]
|
||||
|
||||
class TraditionalLexerAtoms:
|
||||
def __init__(self, lexer):
|
||||
self.mres = _prepare_mres(lexer.mres)
|
||||
self.newline_types = lexer.newline_types
|
||||
self.ignore_types = lexer.ignore_types
|
||||
self.callback = {name:_prepare_mres(c.mres)
|
||||
for name, c in lexer.callback.items()}
|
||||
|
||||
def print_python(self):
|
||||
print('import re')
|
||||
print('class LexerRegexps: pass')
|
||||
print('NEWLINE_TYPES = %s' % self.newline_types)
|
||||
print('IGNORE_TYPES = %s' % self.ignore_types)
|
||||
self._print_python('lexer')
|
||||
|
||||
def _print_python(self, var_name):
|
||||
print('MRES = (')
|
||||
pprint(self.mres)
|
||||
print(')')
|
||||
print('LEXER_CALLBACK = (')
|
||||
pprint(self.callback)
|
||||
print(')')
|
||||
print('lexer_regexps = LexerRegexps()')
|
||||
print('lexer_regexps.mres = [(re.compile(p), d) for p, d in MRES]')
|
||||
print('lexer_regexps.callback = {n: UnlessCallback([(re.compile(p), d) for p, d in mres])')
|
||||
print(' for n, mres in LEXER_CALLBACK.items()}')
|
||||
print('%s = (lexer_regexps)' % var_name)
|
||||
|
||||
|
||||
class ContextualLexerAtoms:
|
||||
def __init__(self, lexer):
|
||||
self.lexer_atoms = {state: TraditionalLexerAtoms(lexer) for state, lexer in lexer.lexers.items()}
|
||||
self.root_lexer_atoms = TraditionalLexerAtoms(lexer.root_lexer)
|
||||
|
||||
def print_python(self):
|
||||
print('import re')
|
||||
print('class LexerRegexps: pass')
|
||||
print('NEWLINE_TYPES = %s' % self.root_lexer_atoms.newline_types)
|
||||
print('IGNORE_TYPES = %s' % self.root_lexer_atoms.ignore_types)
|
||||
|
||||
print('LEXERS = {}')
|
||||
for state, lexer_atoms in self.lexer_atoms.items():
|
||||
lexer_atoms._print_python('LEXERS[%d]' % state)
|
||||
|
||||
print('class ContextualLexer:')
|
||||
print(' def __init__(self):')
|
||||
print(' self.lexers = LEXERS')
|
||||
print(' self.set_parser_state(None)')
|
||||
print(' def set_parser_state(self, state):')
|
||||
print(' self.parser_state = state')
|
||||
print(' def lex(self, stream):')
|
||||
print(' newline_types = NEWLINE_TYPES')
|
||||
print(' ignore_types = IGNORE_TYPES')
|
||||
print(' lexers = LEXERS')
|
||||
print(' l = _Lex(lexers[self.parser_state], self.parser_state)')
|
||||
print(' for x in l.lex(stream, newline_types, ignore_types):')
|
||||
print(' yield x')
|
||||
print(' l.lexer = lexers[self.parser_state]')
|
||||
print(' l.state = self.parser_state')
|
||||
|
||||
print('CON_LEXER = ContextualLexer()')
|
||||
print('def lex(stream):')
|
||||
print(' return CON_LEXER.lex(stream)')
|
||||
|
||||
class GetRule:
|
||||
def __init__(self, rule_id):
|
||||
self.rule_id = rule_id
|
||||
|
||||
def __repr__(self):
|
||||
return 'RULES[%d]' % self.rule_id
|
||||
|
||||
rule_ids = {}
|
||||
token_types = {}
|
||||
|
||||
def _get_token_type(token_type):
|
||||
if token_type not in token_types:
|
||||
token_types[token_type] = len(token_types)
|
||||
return token_types[token_type]
|
||||
|
||||
class ParserAtoms:
|
||||
def __init__(self, parser):
|
||||
self.parse_table = parser._parse_table
|
||||
|
||||
def print_python(self):
|
||||
print('class ParseTable: pass')
|
||||
print('parse_table = ParseTable()')
|
||||
print('STATES = {')
|
||||
for state, actions in self.parse_table.states.items():
|
||||
print(' %r: %r,' % (state, {_get_token_type(token): ((1, rule_ids[arg]) if action is Reduce else (0, arg))
|
||||
for token, (action, arg) in actions.items()}))
|
||||
print('}')
|
||||
print('TOKEN_TYPES = (')
|
||||
pprint({v:k for k, v in token_types.items()})
|
||||
print(')')
|
||||
print('parse_table.states = {s: {TOKEN_TYPES[t]: (a, RULES[x] if a is Reduce else x) for t, (a, x) in acts.items()}')
|
||||
print(' for s, acts in STATES.items()}')
|
||||
print('parse_table.start_state = %s' % self.parse_table.start_state)
|
||||
print('parse_table.end_state = %s' % self.parse_table.end_state)
|
||||
print('class Lark_StandAlone:')
|
||||
print(' def __init__(self, transformer=None, postlex=None):')
|
||||
print(' callback = parse_tree_builder.create_callback(transformer=transformer)')
|
||||
print(' callbacks = {rule: getattr(callback, rule.alias or rule.origin, None) for rule in RULES.values()}')
|
||||
print(' self.parser = _Parser(parse_table, callbacks)')
|
||||
print(' self.postlex = postlex')
|
||||
print(' def parse(self, stream):')
|
||||
print(' tokens = lex(stream)')
|
||||
print(' sps = CON_LEXER.set_parser_state')
|
||||
print(' if self.postlex: tokens = self.postlex.process(tokens)')
|
||||
print(' return self.parser.parse(tokens, sps)')
|
||||
|
||||
class TreeBuilderAtoms:
|
||||
def __init__(self, lark):
|
||||
self.rules = lark.rules
|
||||
self.ptb = lark._parse_tree_builder
|
||||
|
||||
def print_python(self):
|
||||
# print('class InlineTransformer: pass')
|
||||
print('RULES = {')
|
||||
for i, r in enumerate(self.rules):
|
||||
rule_ids[r] = i
|
||||
print(' %d: Rule(%r, [%s], %r, %r),' % (i, r.origin, ', '.join(s.fullrepr for s in r.expansion), self.ptb.user_aliases[r], r.options ))
|
||||
print('}')
|
||||
print('parse_tree_builder = ParseTreeBuilder(RULES.values(), Tree)')
|
||||
|
||||
def main(fobj, start):
|
||||
lark_inst = Lark(fobj, parser="lalr", lexer="contextual", start=start)
|
||||
|
||||
lexer_atoms = ContextualLexerAtoms(lark_inst.parser.lexer)
|
||||
parser_atoms = ParserAtoms(lark_inst.parser.parser)
|
||||
tree_builder_atoms = TreeBuilderAtoms(lark_inst)
|
||||
|
||||
print('# The file was automatically generated by Lark v%s' % lark.__version__)
|
||||
|
||||
for pyfile in EXTRACT_STANDALONE_FILES:
|
||||
with open(os.path.join(_larkdir, pyfile)) as f:
|
||||
print (extract_sections(f)['standalone'])
|
||||
|
||||
with open(os.path.join(_larkdir, 'grammar.py')) as grammar_py:
|
||||
print(grammar_py.read())
|
||||
|
||||
print('Shift = 0')
|
||||
print('Reduce = 1')
|
||||
lexer_atoms.print_python()
|
||||
tree_builder_atoms.print_python()
|
||||
parser_atoms.print_python()
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 2:
|
||||
print("Lark Stand-alone Generator Tool")
|
||||
print("Usage: python -m lark.tools.standalone <grammar-file> [<start>]")
|
||||
sys.exit(1)
|
||||
|
||||
if len(sys.argv) == 3:
|
||||
fn, start = sys.argv[1:]
|
||||
elif len(sys.argv) == 2:
|
||||
fn, start = sys.argv[1], 'start'
|
||||
else:
|
||||
assert False, sys.argv
|
||||
|
||||
with codecs.open(fn, encoding='utf8') as f:
|
||||
main(f, start)
|
||||
@@ -1,168 +0,0 @@
|
||||
try:
|
||||
from future_builtins import filter
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from copy import deepcopy
|
||||
|
||||
###{standalone
|
||||
class Meta:
|
||||
pass
|
||||
|
||||
class Tree(object):
|
||||
def __init__(self, data, children, meta=None):
|
||||
self.data = data
|
||||
self.children = children
|
||||
self._meta = meta
|
||||
|
||||
@property
|
||||
def meta(self):
|
||||
if self._meta is None:
|
||||
self._meta = Meta()
|
||||
return self._meta
|
||||
|
||||
def __repr__(self):
|
||||
return 'Tree(%s, %s)' % (self.data, self.children)
|
||||
|
||||
def _pretty_label(self):
|
||||
return self.data
|
||||
|
||||
def _pretty(self, level, indent_str):
|
||||
if len(self.children) == 1 and not isinstance(self.children[0], Tree):
|
||||
return [ indent_str*level, self._pretty_label(), '\t', '%s' % (self.children[0],), '\n']
|
||||
|
||||
l = [ indent_str*level, self._pretty_label(), '\n' ]
|
||||
for n in self.children:
|
||||
if isinstance(n, Tree):
|
||||
l += n._pretty(level+1, indent_str)
|
||||
else:
|
||||
l += [ indent_str*(level+1), '%s' % (n,), '\n' ]
|
||||
|
||||
return l
|
||||
|
||||
def pretty(self, indent_str=' '):
|
||||
return ''.join(self._pretty(0, indent_str))
|
||||
def __eq__(self, other):
|
||||
try:
|
||||
return self.data == other.data and self.children == other.children
|
||||
except AttributeError:
|
||||
return False
|
||||
|
||||
def __ne__(self, other):
|
||||
return not (self == other)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.data, tuple(self.children)))
|
||||
###}
|
||||
|
||||
def expand_kids_by_index(self, *indices):
|
||||
"Expand (inline) children at the given indices"
|
||||
for i in sorted(indices, reverse=True): # reverse so that changing tail won't affect indices
|
||||
kid = self.children[i]
|
||||
self.children[i:i+1] = kid.children
|
||||
|
||||
def find_pred(self, pred):
|
||||
"Find all nodes where pred(tree) == True"
|
||||
return filter(pred, self.iter_subtrees())
|
||||
|
||||
def find_data(self, data):
|
||||
"Find all nodes where tree.data == data"
|
||||
return self.find_pred(lambda t: t.data == data)
|
||||
|
||||
def scan_values(self, pred):
|
||||
for c in self.children:
|
||||
if isinstance(c, Tree):
|
||||
for t in c.scan_values(pred):
|
||||
yield t
|
||||
else:
|
||||
if pred(c):
|
||||
yield c
|
||||
|
||||
def iter_subtrees(self):
|
||||
# TODO: Re-write as a more efficient version
|
||||
|
||||
visited = set()
|
||||
q = [self]
|
||||
|
||||
l = []
|
||||
while q:
|
||||
subtree = q.pop()
|
||||
l.append( subtree )
|
||||
if id(subtree) in visited:
|
||||
continue # already been here from another branch
|
||||
visited.add(id(subtree))
|
||||
q += [c for c in subtree.children if isinstance(c, Tree)]
|
||||
|
||||
seen = set()
|
||||
for x in reversed(l):
|
||||
if id(x) not in seen:
|
||||
yield x
|
||||
seen.add(id(x))
|
||||
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
return type(self)(self.data, deepcopy(self.children, memo))
|
||||
|
||||
def copy(self):
|
||||
return type(self)(self.data, self.children)
|
||||
def set(self, data, children):
|
||||
self.data = data
|
||||
self.children = children
|
||||
|
||||
# XXX Deprecated! Here for backwards compatibility <0.6.0
|
||||
@property
|
||||
def line(self):
|
||||
return self.meta.line
|
||||
@property
|
||||
def column(self):
|
||||
return self.meta.column
|
||||
@property
|
||||
def end_line(self):
|
||||
return self.meta.end_line
|
||||
@property
|
||||
def end_column(self):
|
||||
return self.meta.end_column
|
||||
|
||||
|
||||
class SlottedTree(Tree):
|
||||
__slots__ = 'data', 'children', 'rule', '_meta'
|
||||
|
||||
|
||||
def pydot__tree_to_png(tree, filename, rankdir="LR"):
|
||||
"""Creates a colorful image that represents the tree (data+children, without meta)
|
||||
|
||||
Possible values for `rankdir` are "TB", "LR", "BT", "RL", corresponding to
|
||||
directed graphs drawn from top to bottom, from left to right, from bottom to
|
||||
top, and from right to left, respectively. See:
|
||||
https://www.graphviz.org/doc/info/attrs.html#k:rankdir
|
||||
"""
|
||||
|
||||
import pydot
|
||||
graph = pydot.Dot(graph_type='digraph', rankdir=rankdir)
|
||||
|
||||
i = [0]
|
||||
|
||||
def new_leaf(leaf):
|
||||
node = pydot.Node(i[0], label=repr(leaf))
|
||||
i[0] += 1
|
||||
graph.add_node(node)
|
||||
return node
|
||||
|
||||
def _to_pydot(subtree):
|
||||
color = hash(subtree.data) & 0xffffff
|
||||
color |= 0x808080
|
||||
|
||||
subnodes = [_to_pydot(child) if isinstance(child, Tree) else new_leaf(child)
|
||||
for child in subtree.children]
|
||||
node = pydot.Node(i[0], style="filled", fillcolor="#%x"%color, label=subtree.data)
|
||||
i[0] += 1
|
||||
graph.add_node(node)
|
||||
|
||||
for subnode in subnodes:
|
||||
graph.add_edge(pydot.Edge(node, subnode))
|
||||
|
||||
return node
|
||||
|
||||
_to_pydot(tree)
|
||||
graph.write_png(filename)
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
import sys
|
||||
from collections import deque
|
||||
|
||||
Py36 = (sys.version_info[:2] >= (3, 6))
|
||||
|
||||
class fzset(frozenset):
|
||||
def __repr__(self):
|
||||
return '{%s}' % ', '.join(map(repr, self))
|
||||
|
||||
|
||||
def classify_bool(seq, pred):
|
||||
true_elems = []
|
||||
false_elems = []
|
||||
|
||||
for elem in seq:
|
||||
if pred(elem):
|
||||
true_elems.append(elem)
|
||||
else:
|
||||
false_elems.append(elem)
|
||||
|
||||
return true_elems, false_elems
|
||||
|
||||
def classify(seq, key=None, value=None):
|
||||
d = {}
|
||||
for item in seq:
|
||||
k = key(item) if (key is not None) else item
|
||||
v = value(item) if (value is not None) else item
|
||||
if k in d:
|
||||
d[k].append(v)
|
||||
else:
|
||||
d[k] = [v]
|
||||
return d
|
||||
|
||||
def bfs(initial, expand):
|
||||
open_q = deque(list(initial))
|
||||
visited = set(open_q)
|
||||
while open_q:
|
||||
node = open_q.popleft()
|
||||
yield node
|
||||
for next_node in expand(node):
|
||||
if next_node not in visited:
|
||||
visited.add(next_node)
|
||||
open_q.append(next_node)
|
||||
|
||||
|
||||
|
||||
|
||||
###{standalone
|
||||
try:
|
||||
STRING_TYPE = basestring
|
||||
except NameError: # Python 3
|
||||
STRING_TYPE = str
|
||||
|
||||
|
||||
import types
|
||||
from functools import wraps, partial
|
||||
from contextlib import contextmanager
|
||||
|
||||
Str = type(u'')
|
||||
|
||||
def smart_decorator(f, create_decorator):
|
||||
if isinstance(f, types.FunctionType):
|
||||
return wraps(f)(create_decorator(f, True))
|
||||
|
||||
elif isinstance(f, (type, types.BuiltinFunctionType)):
|
||||
return wraps(f)(create_decorator(f, False))
|
||||
|
||||
elif isinstance(f, types.MethodType):
|
||||
return wraps(f)(create_decorator(f.__func__, True))
|
||||
|
||||
elif isinstance(f, partial):
|
||||
# wraps does not work for partials in 2.7: https://bugs.python.org/issue3445
|
||||
return create_decorator(f.__func__, True)
|
||||
|
||||
else:
|
||||
return create_decorator(f.__func__.__call__, True)
|
||||
|
||||
|
||||
###}
|
||||
|
||||
|
||||
try:
|
||||
from contextlib import suppress # Python 3
|
||||
except ImportError:
|
||||
@contextmanager
|
||||
def suppress(*excs):
|
||||
'''Catch and dismiss the provided exception
|
||||
|
||||
>>> x = 'hello'
|
||||
>>> with suppress(IndexError):
|
||||
... x = x[10]
|
||||
>>> x
|
||||
'hello'
|
||||
'''
|
||||
try:
|
||||
yield
|
||||
except excs:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
|
||||
try:
|
||||
compare = cmp
|
||||
except NameError:
|
||||
def compare(a, b):
|
||||
if a == b:
|
||||
return 0
|
||||
elif a > b:
|
||||
return 1
|
||||
return -1
|
||||
|
||||
|
||||
import sre_parse
|
||||
import sre_constants
|
||||
def get_regexp_width(regexp):
|
||||
try:
|
||||
return sre_parse.parse(regexp).getwidth()
|
||||
except sre_constants.error:
|
||||
raise ValueError(regexp)
|
||||
@@ -1,259 +0,0 @@
|
||||
from functools import wraps
|
||||
|
||||
from .utils import smart_decorator
|
||||
from .tree import Tree
|
||||
|
||||
###{standalone
|
||||
from inspect import getmembers, getmro
|
||||
|
||||
class Discard(Exception):
|
||||
pass
|
||||
|
||||
# Transformers
|
||||
|
||||
class Transformer:
|
||||
"""Visits the tree recursively, starting with the leaves and finally the root (bottom-up)
|
||||
|
||||
Calls its methods (provided by user via inheritance) according to tree.data
|
||||
The returned value replaces the old one in the structure.
|
||||
|
||||
Can be used to implement map or reduce.
|
||||
"""
|
||||
|
||||
def _call_userfunc(self, tree, new_children=None):
|
||||
# Assumes tree is already transformed
|
||||
children = new_children if new_children is not None else tree.children
|
||||
try:
|
||||
f = getattr(self, tree.data)
|
||||
except AttributeError:
|
||||
return self.__default__(tree.data, children, tree.meta)
|
||||
else:
|
||||
if getattr(f, 'meta', False):
|
||||
return f(children, tree.meta)
|
||||
elif getattr(f, 'inline', False):
|
||||
return f(*children)
|
||||
elif getattr(f, 'whole_tree', False):
|
||||
if new_children is not None:
|
||||
raise NotImplementedError("Doesn't work with the base Transformer class")
|
||||
return f(tree)
|
||||
else:
|
||||
return f(children)
|
||||
|
||||
def _transform_children(self, children):
|
||||
for c in children:
|
||||
try:
|
||||
yield self._transform_tree(c) if isinstance(c, Tree) else c
|
||||
except Discard:
|
||||
pass
|
||||
|
||||
def _transform_tree(self, tree):
|
||||
children = list(self._transform_children(tree.children))
|
||||
return self._call_userfunc(tree, children)
|
||||
|
||||
def transform(self, tree):
|
||||
return self._transform_tree(tree)
|
||||
|
||||
def __mul__(self, other):
|
||||
return TransformerChain(self, other)
|
||||
|
||||
def __default__(self, data, children, meta):
|
||||
"Default operation on tree (for override)"
|
||||
return Tree(data, children, meta)
|
||||
|
||||
@classmethod
|
||||
def _apply_decorator(cls, decorator, **kwargs):
|
||||
mro = getmro(cls)
|
||||
assert mro[0] is cls
|
||||
libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)}
|
||||
for name, value in getmembers(cls):
|
||||
if name.startswith('_') or name in libmembers:
|
||||
continue
|
||||
|
||||
if isinstance(cls.__dict__[name], (staticmethod, classmethod)):
|
||||
kwargs['static'] = True
|
||||
setattr(cls, name, decorator(value, **kwargs))
|
||||
return cls
|
||||
|
||||
|
||||
class InlineTransformer(Transformer): # XXX Deprecated
|
||||
def _call_userfunc(self, tree, new_children=None):
|
||||
# Assumes tree is already transformed
|
||||
children = new_children if new_children is not None else tree.children
|
||||
try:
|
||||
f = getattr(self, tree.data)
|
||||
except AttributeError:
|
||||
return self.__default__(tree.data, children, tree.meta)
|
||||
else:
|
||||
return f(*children)
|
||||
|
||||
|
||||
class TransformerChain(object):
|
||||
def __init__(self, *transformers):
|
||||
self.transformers = transformers
|
||||
|
||||
def transform(self, tree):
|
||||
for t in self.transformers:
|
||||
tree = t.transform(tree)
|
||||
return tree
|
||||
|
||||
def __mul__(self, other):
|
||||
return TransformerChain(*self.transformers + (other,))
|
||||
|
||||
|
||||
class Transformer_InPlace(Transformer):
|
||||
"Non-recursive. Changes the tree in-place instead of returning new instances"
|
||||
def _transform_tree(self, tree): # Cancel recursion
|
||||
return self._call_userfunc(tree)
|
||||
|
||||
def transform(self, tree):
|
||||
for subtree in tree.iter_subtrees():
|
||||
subtree.children = list(self._transform_children(subtree.children))
|
||||
|
||||
return self._transform_tree(tree)
|
||||
|
||||
|
||||
class Transformer_InPlaceRecursive(Transformer):
|
||||
"Recursive. Changes the tree in-place instead of returning new instances"
|
||||
def _transform_tree(self, tree):
|
||||
tree.children = list(self._transform_children(tree.children))
|
||||
return self._call_userfunc(tree)
|
||||
|
||||
|
||||
|
||||
# Visitors
|
||||
|
||||
class VisitorBase:
|
||||
def _call_userfunc(self, tree):
|
||||
return getattr(self, tree.data, self.__default__)(tree)
|
||||
|
||||
def __default__(self, tree):
|
||||
"Default operation on tree (for override)"
|
||||
return tree
|
||||
|
||||
|
||||
class Visitor(VisitorBase):
|
||||
"""Bottom-up visitor, non-recursive
|
||||
|
||||
Visits the tree, starting with the leaves and finally the root (bottom-up)
|
||||
Calls its methods (provided by user via inheritance) according to tree.data
|
||||
"""
|
||||
|
||||
|
||||
def visit(self, tree):
|
||||
for subtree in tree.iter_subtrees():
|
||||
self._call_userfunc(subtree)
|
||||
return tree
|
||||
|
||||
class Visitor_Recursive(VisitorBase):
|
||||
"""Bottom-up visitor, recursive
|
||||
|
||||
Visits the tree, starting with the leaves and finally the root (bottom-up)
|
||||
Calls its methods (provided by user via inheritance) according to tree.data
|
||||
"""
|
||||
|
||||
def visit(self, tree):
|
||||
for child in tree.children:
|
||||
if isinstance(child, Tree):
|
||||
self.visit(child)
|
||||
|
||||
f = getattr(self, tree.data, self.__default__)
|
||||
f(tree)
|
||||
return tree
|
||||
|
||||
|
||||
|
||||
def visit_children_decor(func):
|
||||
"See Interpreter"
|
||||
@wraps(func)
|
||||
def inner(cls, tree):
|
||||
values = cls.visit_children(tree)
|
||||
return func(cls, values)
|
||||
return inner
|
||||
|
||||
|
||||
class Interpreter:
|
||||
"""Top-down visitor, recursive
|
||||
|
||||
Visits the tree, starting with the root and finally the leaves (top-down)
|
||||
Calls its methods (provided by user via inheritance) according to tree.data
|
||||
|
||||
Unlike Transformer and Visitor, the Interpreter doesn't automatically visit its sub-branches.
|
||||
The user has to explicitly call visit_children, or use the @visit_children_decor
|
||||
"""
|
||||
def visit(self, tree):
|
||||
return getattr(self, tree.data)(tree)
|
||||
|
||||
def visit_children(self, tree):
|
||||
return [self.visit(child) if isinstance(child, Tree) else child
|
||||
for child in tree.children]
|
||||
|
||||
def __getattr__(self, name):
|
||||
return self.__default__
|
||||
|
||||
def __default__(self, tree):
|
||||
return self.visit_children(tree)
|
||||
|
||||
|
||||
|
||||
|
||||
# Decorators
|
||||
|
||||
def _apply_decorator(obj, decorator, **kwargs):
|
||||
try:
|
||||
_apply = obj._apply_decorator
|
||||
except AttributeError:
|
||||
return decorator(obj, **kwargs)
|
||||
else:
|
||||
return _apply(decorator, **kwargs)
|
||||
|
||||
|
||||
|
||||
def _inline_args__func(func):
|
||||
@wraps(func)
|
||||
def create_decorator(_f, with_self):
|
||||
if with_self:
|
||||
def f(self, children):
|
||||
return _f(self, *children)
|
||||
else:
|
||||
def f(self, children):
|
||||
return _f(*children)
|
||||
return f
|
||||
|
||||
return smart_decorator(func, create_decorator)
|
||||
|
||||
|
||||
def inline_args(obj): # XXX Deprecated
|
||||
return _apply_decorator(obj, _inline_args__func)
|
||||
|
||||
|
||||
|
||||
def _visitor_args_func_dec(func, inline=False, meta=False, whole_tree=False, static=False):
|
||||
assert [whole_tree, meta, inline].count(True) <= 1
|
||||
def create_decorator(_f, with_self):
|
||||
if with_self:
|
||||
def f(self, *args, **kwargs):
|
||||
return _f(self, *args, **kwargs)
|
||||
else:
|
||||
def f(self, *args, **kwargs):
|
||||
return _f(*args, **kwargs)
|
||||
return f
|
||||
|
||||
if static:
|
||||
f = wraps(func)(create_decorator(func, False))
|
||||
else:
|
||||
f = smart_decorator(func, create_decorator)
|
||||
f.inline = inline
|
||||
f.meta = meta
|
||||
f.whole_tree = whole_tree
|
||||
return f
|
||||
|
||||
def v_args(inline=False, meta=False, tree=False):
|
||||
"A convenience decorator factory, for modifying the behavior of user-supplied visitor methods"
|
||||
if [tree, meta, inline].count(True) > 1:
|
||||
raise ValueError("Visitor functions can either accept tree, or meta, or be inlined. These cannot be combined.")
|
||||
def _visitor_args_dec(obj):
|
||||
return _apply_decorator(obj, _visitor_args_func_dec, inline=inline, meta=meta, whole_tree=tree)
|
||||
return _visitor_args_dec
|
||||
|
||||
|
||||
###}
|
||||
@@ -1,42 +0,0 @@
|
||||
Metadata-Version: 1.1
|
||||
Name: lark-parser
|
||||
Version: 0.6.5
|
||||
Summary: a modern parsing library
|
||||
Home-page: https://github.com/erezsh/lark
|
||||
Author: Erez Shinan
|
||||
Author-email: erezshin@gmail.com
|
||||
License: MIT
|
||||
Download-URL: https://github.com/erezsh/lark/tarball/master
|
||||
Description:
|
||||
Lark is a modern general-purpose parsing library for Python.
|
||||
|
||||
With Lark, you can parse any context-free grammar, efficiently, with very little code.
|
||||
|
||||
Main Features:
|
||||
- Builds a parse-tree (AST) automagically, based on the structure of the grammar
|
||||
- Earley parser
|
||||
- Can parse all context-free grammars
|
||||
- Full support for ambiguous grammars
|
||||
- LALR(1) parser
|
||||
- Fast and light, competitive with PLY
|
||||
- Can generate a stand-alone parser
|
||||
- CYK parser, for highly ambiguous grammars
|
||||
- EBNF grammar
|
||||
- Unicode fully supported
|
||||
- Python 2 & 3 compatible
|
||||
- Automatic line & column tracking
|
||||
- Standard library of terminals (strings, numbers, names, etc.)
|
||||
- Import grammars from Nearley.js
|
||||
- Extensive test suite
|
||||
- And much more!
|
||||
|
||||
Keywords: Earley LALR parser parsing ast
|
||||
Platform: UNKNOWN
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: Programming Language :: Python :: 2.7
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
||||
Classifier: Topic :: Text Processing :: General
|
||||
Classifier: Topic :: Text Processing :: Linguistic
|
||||
Classifier: License :: OSI Approved :: MIT License
|
||||
@@ -1,80 +0,0 @@
|
||||
LICENSE
|
||||
MANIFEST.in
|
||||
README.md
|
||||
setup.cfg
|
||||
setup.py
|
||||
docs/classes.md
|
||||
docs/comparison_memory.png
|
||||
docs/comparison_runtime.png
|
||||
docs/features.md
|
||||
docs/grammar.md
|
||||
docs/how_to_use.md
|
||||
docs/index.md
|
||||
docs/json_tutorial.md
|
||||
docs/lark_cheatsheet.pdf
|
||||
docs/philosophy.md
|
||||
docs/recipes.md
|
||||
docs/tree_construction.md
|
||||
examples/__init__.py
|
||||
examples/calc.py
|
||||
examples/comments.py
|
||||
examples/conf_earley.py
|
||||
examples/conf_lalr.py
|
||||
examples/custom_lexer.py
|
||||
examples/error_handling.py
|
||||
examples/error_reporting_lalr.py
|
||||
examples/fruitflies.png
|
||||
examples/fruitflies.py
|
||||
examples/indented_tree.py
|
||||
examples/json_parser.py
|
||||
examples/lark.lark
|
||||
examples/lark_grammar.py
|
||||
examples/python2.lark
|
||||
examples/python3.lark
|
||||
examples/python_parser.py
|
||||
examples/qscintilla_json.py
|
||||
examples/reconstruct_json.py
|
||||
examples/turtle_dsl.py
|
||||
lark/__init__.py
|
||||
lark/common.py
|
||||
lark/exceptions.py
|
||||
lark/grammar.py
|
||||
lark/indenter.py
|
||||
lark/lark.py
|
||||
lark/lexer.py
|
||||
lark/load_grammar.py
|
||||
lark/parse_tree_builder.py
|
||||
lark/parser_frontends.py
|
||||
lark/reconstruct.py
|
||||
lark/reconstruct2.py
|
||||
lark/tree.py
|
||||
lark/utils.py
|
||||
lark/visitors.py
|
||||
lark/grammars/common.lark
|
||||
lark/parsers/__init__.py
|
||||
lark/parsers/ablalr.py
|
||||
lark/parsers/cyk.py
|
||||
lark/parsers/earley.py
|
||||
lark/parsers/grammar_analysis.py
|
||||
lark/parsers/lalr_analysis.py
|
||||
lark/parsers/lalr_parser.py
|
||||
lark/parsers/resolve_ambig.py
|
||||
lark/parsers/xearley.py
|
||||
lark/tools/__init__.py
|
||||
lark/tools/nearley.py
|
||||
lark/tools/standalone.py
|
||||
lark_parser.egg-info/PKG-INFO
|
||||
lark_parser.egg-info/SOURCES.txt
|
||||
lark_parser.egg-info/dependency_links.txt
|
||||
lark_parser.egg-info/not-zip-safe
|
||||
lark_parser.egg-info/top_level.txt
|
||||
tests/__init__.py
|
||||
tests/__main__.py
|
||||
tests/test_parser.py
|
||||
tests/test_reconstructor.py
|
||||
tests/test_tools.py
|
||||
tests/test_trees.py
|
||||
tests/test_nearley/__init__.py
|
||||
tests/test_nearley/test_nearley.py
|
||||
tests/test_nearley/grammars/include_unicode.ne
|
||||
tests/test_nearley/grammars/unicode.ne
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
../lark/__init__.py
|
||||
../lark/__pycache__/__init__.cpython-36.pyc
|
||||
../lark/__pycache__/common.cpython-36.pyc
|
||||
../lark/__pycache__/exceptions.cpython-36.pyc
|
||||
../lark/__pycache__/grammar.cpython-36.pyc
|
||||
../lark/__pycache__/indenter.cpython-36.pyc
|
||||
../lark/__pycache__/lark.cpython-36.pyc
|
||||
../lark/__pycache__/lexer.cpython-36.pyc
|
||||
../lark/__pycache__/load_grammar.cpython-36.pyc
|
||||
../lark/__pycache__/parse_tree_builder.cpython-36.pyc
|
||||
../lark/__pycache__/parser_frontends.cpython-36.pyc
|
||||
../lark/__pycache__/reconstruct.cpython-36.pyc
|
||||
../lark/__pycache__/reconstruct2.cpython-36.pyc
|
||||
../lark/__pycache__/tree.cpython-36.pyc
|
||||
../lark/__pycache__/utils.cpython-36.pyc
|
||||
../lark/__pycache__/visitors.cpython-36.pyc
|
||||
../lark/common.py
|
||||
../lark/exceptions.py
|
||||
../lark/grammar.py
|
||||
../lark/grammars/common.lark
|
||||
../lark/indenter.py
|
||||
../lark/lark.py
|
||||
../lark/lexer.py
|
||||
../lark/load_grammar.py
|
||||
../lark/parse_tree_builder.py
|
||||
../lark/parser_frontends.py
|
||||
../lark/parsers/__init__.py
|
||||
../lark/parsers/__pycache__/__init__.cpython-36.pyc
|
||||
../lark/parsers/__pycache__/ablalr.cpython-36.pyc
|
||||
../lark/parsers/__pycache__/cyk.cpython-36.pyc
|
||||
../lark/parsers/__pycache__/earley.cpython-36.pyc
|
||||
../lark/parsers/__pycache__/grammar_analysis.cpython-36.pyc
|
||||
../lark/parsers/__pycache__/lalr_analysis.cpython-36.pyc
|
||||
../lark/parsers/__pycache__/lalr_parser.cpython-36.pyc
|
||||
../lark/parsers/__pycache__/resolve_ambig.cpython-36.pyc
|
||||
../lark/parsers/__pycache__/xearley.cpython-36.pyc
|
||||
../lark/parsers/ablalr.py
|
||||
../lark/parsers/cyk.py
|
||||
../lark/parsers/earley.py
|
||||
../lark/parsers/grammar_analysis.py
|
||||
../lark/parsers/lalr_analysis.py
|
||||
../lark/parsers/lalr_parser.py
|
||||
../lark/parsers/resolve_ambig.py
|
||||
../lark/parsers/xearley.py
|
||||
../lark/reconstruct.py
|
||||
../lark/reconstruct2.py
|
||||
../lark/tools/__init__.py
|
||||
../lark/tools/__pycache__/__init__.cpython-36.pyc
|
||||
../lark/tools/__pycache__/nearley.cpython-36.pyc
|
||||
../lark/tools/__pycache__/standalone.cpython-36.pyc
|
||||
../lark/tools/nearley.py
|
||||
../lark/tools/standalone.py
|
||||
../lark/tree.py
|
||||
../lark/utils.py
|
||||
../lark/visitors.py
|
||||
PKG-INFO
|
||||
SOURCES.txt
|
||||
dependency_links.txt
|
||||
not-zip-safe
|
||||
top_level.txt
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
lark
|
||||
@@ -1,69 +0,0 @@
|
||||
Metadata-Version: 2.1
|
||||
Name: pip
|
||||
Version: 10.0.1
|
||||
Summary: The PyPA recommended tool for installing Python packages.
|
||||
Home-page: https://pip.pypa.io/
|
||||
Author: The pip developers
|
||||
Author-email: python-virtualenv@groups.google.com
|
||||
License: MIT
|
||||
Description: pip
|
||||
===
|
||||
|
||||
The `PyPA recommended`_ tool for installing Python packages.
|
||||
|
||||
.. image:: https://img.shields.io/pypi/v/pip.svg
|
||||
:target: https://pypi.org/project/pip/
|
||||
|
||||
.. image:: https://img.shields.io/travis/pypa/pip/master.svg
|
||||
:target: http://travis-ci.org/pypa/pip
|
||||
|
||||
.. image:: https://img.shields.io/appveyor/ci/pypa/pip.svg
|
||||
:target: https://ci.appveyor.com/project/pypa/pip/history
|
||||
|
||||
.. image:: https://readthedocs.org/projects/pip/badge/?version=latest
|
||||
:target: https://pip.pypa.io/en/latest
|
||||
|
||||
* `Installation`_
|
||||
* `Documentation`_
|
||||
* `Changelog`_
|
||||
* `GitHub Page`_
|
||||
* `Issue Tracking`_
|
||||
* `User mailing list`_
|
||||
* `Dev mailing list`_
|
||||
* User IRC: #pypa on Freenode.
|
||||
* Dev IRC: #pypa-dev on Freenode.
|
||||
|
||||
Code of Conduct
|
||||
---------------
|
||||
|
||||
Everyone interacting in the pip project's codebases, issue trackers, chat
|
||||
rooms and mailing lists is expected to follow the `PyPA Code of Conduct`_.
|
||||
|
||||
.. _PyPA recommended: https://packaging.python.org/en/latest/current/
|
||||
.. _Installation: https://pip.pypa.io/en/stable/installing.html
|
||||
.. _Documentation: https://pip.pypa.io/en/stable/
|
||||
.. _Changelog: https://pip.pypa.io/en/stable/news.html
|
||||
.. _GitHub Page: https://github.com/pypa/pip
|
||||
.. _Issue Tracking: https://github.com/pypa/pip/issues
|
||||
.. _User mailing list: http://groups.google.com/group/python-virtualenv
|
||||
.. _Dev mailing list: http://groups.google.com/group/pypa-dev
|
||||
.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/
|
||||
|
||||
Keywords: easy_install distutils setuptools egg virtualenv
|
||||
Platform: UNKNOWN
|
||||
Classifier: Development Status :: 5 - Production/Stable
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: License :: OSI Approved :: MIT License
|
||||
Classifier: Topic :: Software Development :: Build Tools
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 2
|
||||
Classifier: Programming Language :: Python :: 2.7
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.3
|
||||
Classifier: Programming Language :: Python :: 3.4
|
||||
Classifier: Programming Language :: Python :: 3.5
|
||||
Classifier: Programming Language :: Python :: 3.6
|
||||
Classifier: Programming Language :: Python :: Implementation :: CPython
|
||||
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
||||
Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*
|
||||
Provides-Extra: testing
|
||||
@@ -1,347 +0,0 @@
|
||||
AUTHORS.txt
|
||||
LICENSE.txt
|
||||
MANIFEST.in
|
||||
NEWS.rst
|
||||
README.rst
|
||||
pyproject.toml
|
||||
setup.cfg
|
||||
setup.py
|
||||
docs/Makefile
|
||||
docs/__init__.py
|
||||
docs/conf.py
|
||||
docs/configuration.rst
|
||||
docs/cookbook.rst
|
||||
docs/development.rst
|
||||
docs/docutils.conf
|
||||
docs/index.rst
|
||||
docs/installing.rst
|
||||
docs/logic.rst
|
||||
docs/make.bat
|
||||
docs/news.rst
|
||||
docs/pipext.py
|
||||
docs/quickstart.rst
|
||||
docs/usage.rst
|
||||
docs/user_guide.rst
|
||||
docs/man/pip.rst
|
||||
docs/man/commands/check.rst
|
||||
docs/man/commands/config.rst
|
||||
docs/man/commands/download.rst
|
||||
docs/man/commands/freeze.rst
|
||||
docs/man/commands/hash.rst
|
||||
docs/man/commands/help.rst
|
||||
docs/man/commands/install.rst
|
||||
docs/man/commands/list.rst
|
||||
docs/man/commands/search.rst
|
||||
docs/man/commands/show.rst
|
||||
docs/man/commands/uninstall.rst
|
||||
docs/man/commands/wheel.rst
|
||||
docs/reference/index.rst
|
||||
docs/reference/pip.rst
|
||||
docs/reference/pip_check.rst
|
||||
docs/reference/pip_config.rst
|
||||
docs/reference/pip_download.rst
|
||||
docs/reference/pip_freeze.rst
|
||||
docs/reference/pip_hash.rst
|
||||
docs/reference/pip_install.rst
|
||||
docs/reference/pip_list.rst
|
||||
docs/reference/pip_search.rst
|
||||
docs/reference/pip_show.rst
|
||||
docs/reference/pip_uninstall.rst
|
||||
docs/reference/pip_wheel.rst
|
||||
src/pip/__init__.py
|
||||
src/pip/__main__.py
|
||||
src/pip.egg-info/PKG-INFO
|
||||
src/pip.egg-info/SOURCES.txt
|
||||
src/pip.egg-info/dependency_links.txt
|
||||
src/pip.egg-info/entry_points.txt
|
||||
src/pip.egg-info/not-zip-safe
|
||||
src/pip.egg-info/requires.txt
|
||||
src/pip.egg-info/top_level.txt
|
||||
src/pip/_internal/__init__.py
|
||||
src/pip/_internal/basecommand.py
|
||||
src/pip/_internal/baseparser.py
|
||||
src/pip/_internal/build_env.py
|
||||
src/pip/_internal/cache.py
|
||||
src/pip/_internal/cmdoptions.py
|
||||
src/pip/_internal/compat.py
|
||||
src/pip/_internal/configuration.py
|
||||
src/pip/_internal/download.py
|
||||
src/pip/_internal/exceptions.py
|
||||
src/pip/_internal/index.py
|
||||
src/pip/_internal/locations.py
|
||||
src/pip/_internal/pep425tags.py
|
||||
src/pip/_internal/resolve.py
|
||||
src/pip/_internal/status_codes.py
|
||||
src/pip/_internal/wheel.py
|
||||
src/pip/_internal/commands/__init__.py
|
||||
src/pip/_internal/commands/check.py
|
||||
src/pip/_internal/commands/completion.py
|
||||
src/pip/_internal/commands/configuration.py
|
||||
src/pip/_internal/commands/download.py
|
||||
src/pip/_internal/commands/freeze.py
|
||||
src/pip/_internal/commands/hash.py
|
||||
src/pip/_internal/commands/help.py
|
||||
src/pip/_internal/commands/install.py
|
||||
src/pip/_internal/commands/list.py
|
||||
src/pip/_internal/commands/search.py
|
||||
src/pip/_internal/commands/show.py
|
||||
src/pip/_internal/commands/uninstall.py
|
||||
src/pip/_internal/commands/wheel.py
|
||||
src/pip/_internal/models/__init__.py
|
||||
src/pip/_internal/models/index.py
|
||||
src/pip/_internal/operations/__init__.py
|
||||
src/pip/_internal/operations/check.py
|
||||
src/pip/_internal/operations/freeze.py
|
||||
src/pip/_internal/operations/prepare.py
|
||||
src/pip/_internal/req/__init__.py
|
||||
src/pip/_internal/req/req_file.py
|
||||
src/pip/_internal/req/req_install.py
|
||||
src/pip/_internal/req/req_set.py
|
||||
src/pip/_internal/req/req_uninstall.py
|
||||
src/pip/_internal/utils/__init__.py
|
||||
src/pip/_internal/utils/appdirs.py
|
||||
src/pip/_internal/utils/deprecation.py
|
||||
src/pip/_internal/utils/encoding.py
|
||||
src/pip/_internal/utils/filesystem.py
|
||||
src/pip/_internal/utils/glibc.py
|
||||
src/pip/_internal/utils/hashes.py
|
||||
src/pip/_internal/utils/logging.py
|
||||
src/pip/_internal/utils/misc.py
|
||||
src/pip/_internal/utils/outdated.py
|
||||
src/pip/_internal/utils/packaging.py
|
||||
src/pip/_internal/utils/setuptools_build.py
|
||||
src/pip/_internal/utils/temp_dir.py
|
||||
src/pip/_internal/utils/typing.py
|
||||
src/pip/_internal/utils/ui.py
|
||||
src/pip/_internal/vcs/__init__.py
|
||||
src/pip/_internal/vcs/bazaar.py
|
||||
src/pip/_internal/vcs/git.py
|
||||
src/pip/_internal/vcs/mercurial.py
|
||||
src/pip/_internal/vcs/subversion.py
|
||||
src/pip/_vendor/README.rst
|
||||
src/pip/_vendor/__init__.py
|
||||
src/pip/_vendor/appdirs.py
|
||||
src/pip/_vendor/distro.py
|
||||
src/pip/_vendor/ipaddress.py
|
||||
src/pip/_vendor/pyparsing.py
|
||||
src/pip/_vendor/retrying.py
|
||||
src/pip/_vendor/six.py
|
||||
src/pip/_vendor/vendor.txt
|
||||
src/pip/_vendor/cachecontrol/__init__.py
|
||||
src/pip/_vendor/cachecontrol/_cmd.py
|
||||
src/pip/_vendor/cachecontrol/adapter.py
|
||||
src/pip/_vendor/cachecontrol/cache.py
|
||||
src/pip/_vendor/cachecontrol/compat.py
|
||||
src/pip/_vendor/cachecontrol/controller.py
|
||||
src/pip/_vendor/cachecontrol/filewrapper.py
|
||||
src/pip/_vendor/cachecontrol/heuristics.py
|
||||
src/pip/_vendor/cachecontrol/serialize.py
|
||||
src/pip/_vendor/cachecontrol/wrapper.py
|
||||
src/pip/_vendor/cachecontrol/caches/__init__.py
|
||||
src/pip/_vendor/cachecontrol/caches/file_cache.py
|
||||
src/pip/_vendor/cachecontrol/caches/redis_cache.py
|
||||
src/pip/_vendor/certifi/__init__.py
|
||||
src/pip/_vendor/certifi/__main__.py
|
||||
src/pip/_vendor/certifi/cacert.pem
|
||||
src/pip/_vendor/certifi/core.py
|
||||
src/pip/_vendor/chardet/__init__.py
|
||||
src/pip/_vendor/chardet/big5freq.py
|
||||
src/pip/_vendor/chardet/big5prober.py
|
||||
src/pip/_vendor/chardet/chardistribution.py
|
||||
src/pip/_vendor/chardet/charsetgroupprober.py
|
||||
src/pip/_vendor/chardet/charsetprober.py
|
||||
src/pip/_vendor/chardet/codingstatemachine.py
|
||||
src/pip/_vendor/chardet/compat.py
|
||||
src/pip/_vendor/chardet/cp949prober.py
|
||||
src/pip/_vendor/chardet/enums.py
|
||||
src/pip/_vendor/chardet/escprober.py
|
||||
src/pip/_vendor/chardet/escsm.py
|
||||
src/pip/_vendor/chardet/eucjpprober.py
|
||||
src/pip/_vendor/chardet/euckrfreq.py
|
||||
src/pip/_vendor/chardet/euckrprober.py
|
||||
src/pip/_vendor/chardet/euctwfreq.py
|
||||
src/pip/_vendor/chardet/euctwprober.py
|
||||
src/pip/_vendor/chardet/gb2312freq.py
|
||||
src/pip/_vendor/chardet/gb2312prober.py
|
||||
src/pip/_vendor/chardet/hebrewprober.py
|
||||
src/pip/_vendor/chardet/jisfreq.py
|
||||
src/pip/_vendor/chardet/jpcntx.py
|
||||
src/pip/_vendor/chardet/langbulgarianmodel.py
|
||||
src/pip/_vendor/chardet/langcyrillicmodel.py
|
||||
src/pip/_vendor/chardet/langgreekmodel.py
|
||||
src/pip/_vendor/chardet/langhebrewmodel.py
|
||||
src/pip/_vendor/chardet/langhungarianmodel.py
|
||||
src/pip/_vendor/chardet/langthaimodel.py
|
||||
src/pip/_vendor/chardet/langturkishmodel.py
|
||||
src/pip/_vendor/chardet/latin1prober.py
|
||||
src/pip/_vendor/chardet/mbcharsetprober.py
|
||||
src/pip/_vendor/chardet/mbcsgroupprober.py
|
||||
src/pip/_vendor/chardet/mbcssm.py
|
||||
src/pip/_vendor/chardet/sbcharsetprober.py
|
||||
src/pip/_vendor/chardet/sbcsgroupprober.py
|
||||
src/pip/_vendor/chardet/sjisprober.py
|
||||
src/pip/_vendor/chardet/universaldetector.py
|
||||
src/pip/_vendor/chardet/utf8prober.py
|
||||
src/pip/_vendor/chardet/version.py
|
||||
src/pip/_vendor/chardet/cli/__init__.py
|
||||
src/pip/_vendor/chardet/cli/chardetect.py
|
||||
src/pip/_vendor/colorama/__init__.py
|
||||
src/pip/_vendor/colorama/ansi.py
|
||||
src/pip/_vendor/colorama/ansitowin32.py
|
||||
src/pip/_vendor/colorama/initialise.py
|
||||
src/pip/_vendor/colorama/win32.py
|
||||
src/pip/_vendor/colorama/winterm.py
|
||||
src/pip/_vendor/distlib/__init__.py
|
||||
src/pip/_vendor/distlib/compat.py
|
||||
src/pip/_vendor/distlib/database.py
|
||||
src/pip/_vendor/distlib/index.py
|
||||
src/pip/_vendor/distlib/locators.py
|
||||
src/pip/_vendor/distlib/manifest.py
|
||||
src/pip/_vendor/distlib/markers.py
|
||||
src/pip/_vendor/distlib/metadata.py
|
||||
src/pip/_vendor/distlib/resources.py
|
||||
src/pip/_vendor/distlib/scripts.py
|
||||
src/pip/_vendor/distlib/t32.exe
|
||||
src/pip/_vendor/distlib/t64.exe
|
||||
src/pip/_vendor/distlib/util.py
|
||||
src/pip/_vendor/distlib/version.py
|
||||
src/pip/_vendor/distlib/w32.exe
|
||||
src/pip/_vendor/distlib/w64.exe
|
||||
src/pip/_vendor/distlib/wheel.py
|
||||
src/pip/_vendor/distlib/_backport/__init__.py
|
||||
src/pip/_vendor/distlib/_backport/misc.py
|
||||
src/pip/_vendor/distlib/_backport/shutil.py
|
||||
src/pip/_vendor/distlib/_backport/sysconfig.cfg
|
||||
src/pip/_vendor/distlib/_backport/sysconfig.py
|
||||
src/pip/_vendor/distlib/_backport/tarfile.py
|
||||
src/pip/_vendor/html5lib/__init__.py
|
||||
src/pip/_vendor/html5lib/_ihatexml.py
|
||||
src/pip/_vendor/html5lib/_inputstream.py
|
||||
src/pip/_vendor/html5lib/_tokenizer.py
|
||||
src/pip/_vendor/html5lib/_utils.py
|
||||
src/pip/_vendor/html5lib/constants.py
|
||||
src/pip/_vendor/html5lib/html5parser.py
|
||||
src/pip/_vendor/html5lib/serializer.py
|
||||
src/pip/_vendor/html5lib/_trie/__init__.py
|
||||
src/pip/_vendor/html5lib/_trie/_base.py
|
||||
src/pip/_vendor/html5lib/_trie/datrie.py
|
||||
src/pip/_vendor/html5lib/_trie/py.py
|
||||
src/pip/_vendor/html5lib/filters/__init__.py
|
||||
src/pip/_vendor/html5lib/filters/alphabeticalattributes.py
|
||||
src/pip/_vendor/html5lib/filters/base.py
|
||||
src/pip/_vendor/html5lib/filters/inject_meta_charset.py
|
||||
src/pip/_vendor/html5lib/filters/lint.py
|
||||
src/pip/_vendor/html5lib/filters/optionaltags.py
|
||||
src/pip/_vendor/html5lib/filters/sanitizer.py
|
||||
src/pip/_vendor/html5lib/filters/whitespace.py
|
||||
src/pip/_vendor/html5lib/treeadapters/__init__.py
|
||||
src/pip/_vendor/html5lib/treeadapters/genshi.py
|
||||
src/pip/_vendor/html5lib/treeadapters/sax.py
|
||||
src/pip/_vendor/html5lib/treebuilders/__init__.py
|
||||
src/pip/_vendor/html5lib/treebuilders/base.py
|
||||
src/pip/_vendor/html5lib/treebuilders/dom.py
|
||||
src/pip/_vendor/html5lib/treebuilders/etree.py
|
||||
src/pip/_vendor/html5lib/treebuilders/etree_lxml.py
|
||||
src/pip/_vendor/html5lib/treewalkers/__init__.py
|
||||
src/pip/_vendor/html5lib/treewalkers/base.py
|
||||
src/pip/_vendor/html5lib/treewalkers/dom.py
|
||||
src/pip/_vendor/html5lib/treewalkers/etree.py
|
||||
src/pip/_vendor/html5lib/treewalkers/etree_lxml.py
|
||||
src/pip/_vendor/html5lib/treewalkers/genshi.py
|
||||
src/pip/_vendor/idna/__init__.py
|
||||
src/pip/_vendor/idna/codec.py
|
||||
src/pip/_vendor/idna/compat.py
|
||||
src/pip/_vendor/idna/core.py
|
||||
src/pip/_vendor/idna/idnadata.py
|
||||
src/pip/_vendor/idna/intranges.py
|
||||
src/pip/_vendor/idna/package_data.py
|
||||
src/pip/_vendor/idna/uts46data.py
|
||||
src/pip/_vendor/lockfile/__init__.py
|
||||
src/pip/_vendor/lockfile/linklockfile.py
|
||||
src/pip/_vendor/lockfile/mkdirlockfile.py
|
||||
src/pip/_vendor/lockfile/pidlockfile.py
|
||||
src/pip/_vendor/lockfile/sqlitelockfile.py
|
||||
src/pip/_vendor/lockfile/symlinklockfile.py
|
||||
src/pip/_vendor/msgpack/__init__.py
|
||||
src/pip/_vendor/msgpack/_version.py
|
||||
src/pip/_vendor/msgpack/exceptions.py
|
||||
src/pip/_vendor/msgpack/fallback.py
|
||||
src/pip/_vendor/packaging/__about__.py
|
||||
src/pip/_vendor/packaging/__init__.py
|
||||
src/pip/_vendor/packaging/_compat.py
|
||||
src/pip/_vendor/packaging/_structures.py
|
||||
src/pip/_vendor/packaging/markers.py
|
||||
src/pip/_vendor/packaging/requirements.py
|
||||
src/pip/_vendor/packaging/specifiers.py
|
||||
src/pip/_vendor/packaging/utils.py
|
||||
src/pip/_vendor/packaging/version.py
|
||||
src/pip/_vendor/pkg_resources/__init__.py
|
||||
src/pip/_vendor/pkg_resources/py31compat.py
|
||||
src/pip/_vendor/progress/__init__.py
|
||||
src/pip/_vendor/progress/bar.py
|
||||
src/pip/_vendor/progress/counter.py
|
||||
src/pip/_vendor/progress/helpers.py
|
||||
src/pip/_vendor/progress/spinner.py
|
||||
src/pip/_vendor/pytoml/__init__.py
|
||||
src/pip/_vendor/pytoml/core.py
|
||||
src/pip/_vendor/pytoml/parser.py
|
||||
src/pip/_vendor/pytoml/writer.py
|
||||
src/pip/_vendor/requests/__init__.py
|
||||
src/pip/_vendor/requests/__version__.py
|
||||
src/pip/_vendor/requests/_internal_utils.py
|
||||
src/pip/_vendor/requests/adapters.py
|
||||
src/pip/_vendor/requests/api.py
|
||||
src/pip/_vendor/requests/auth.py
|
||||
src/pip/_vendor/requests/certs.py
|
||||
src/pip/_vendor/requests/compat.py
|
||||
src/pip/_vendor/requests/cookies.py
|
||||
src/pip/_vendor/requests/exceptions.py
|
||||
src/pip/_vendor/requests/help.py
|
||||
src/pip/_vendor/requests/hooks.py
|
||||
src/pip/_vendor/requests/models.py
|
||||
src/pip/_vendor/requests/packages.py
|
||||
src/pip/_vendor/requests/sessions.py
|
||||
src/pip/_vendor/requests/status_codes.py
|
||||
src/pip/_vendor/requests/structures.py
|
||||
src/pip/_vendor/requests/utils.py
|
||||
src/pip/_vendor/urllib3/__init__.py
|
||||
src/pip/_vendor/urllib3/_collections.py
|
||||
src/pip/_vendor/urllib3/connection.py
|
||||
src/pip/_vendor/urllib3/connectionpool.py
|
||||
src/pip/_vendor/urllib3/exceptions.py
|
||||
src/pip/_vendor/urllib3/fields.py
|
||||
src/pip/_vendor/urllib3/filepost.py
|
||||
src/pip/_vendor/urllib3/poolmanager.py
|
||||
src/pip/_vendor/urllib3/request.py
|
||||
src/pip/_vendor/urllib3/response.py
|
||||
src/pip/_vendor/urllib3/contrib/__init__.py
|
||||
src/pip/_vendor/urllib3/contrib/appengine.py
|
||||
src/pip/_vendor/urllib3/contrib/ntlmpool.py
|
||||
src/pip/_vendor/urllib3/contrib/pyopenssl.py
|
||||
src/pip/_vendor/urllib3/contrib/securetransport.py
|
||||
src/pip/_vendor/urllib3/contrib/socks.py
|
||||
src/pip/_vendor/urllib3/contrib/_securetransport/__init__.py
|
||||
src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py
|
||||
src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py
|
||||
src/pip/_vendor/urllib3/packages/__init__.py
|
||||
src/pip/_vendor/urllib3/packages/ordered_dict.py
|
||||
src/pip/_vendor/urllib3/packages/six.py
|
||||
src/pip/_vendor/urllib3/packages/backports/__init__.py
|
||||
src/pip/_vendor/urllib3/packages/backports/makefile.py
|
||||
src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py
|
||||
src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py
|
||||
src/pip/_vendor/urllib3/util/__init__.py
|
||||
src/pip/_vendor/urllib3/util/connection.py
|
||||
src/pip/_vendor/urllib3/util/request.py
|
||||
src/pip/_vendor/urllib3/util/response.py
|
||||
src/pip/_vendor/urllib3/util/retry.py
|
||||
src/pip/_vendor/urllib3/util/selectors.py
|
||||
src/pip/_vendor/urllib3/util/ssl_.py
|
||||
src/pip/_vendor/urllib3/util/timeout.py
|
||||
src/pip/_vendor/urllib3/util/url.py
|
||||
src/pip/_vendor/urllib3/util/wait.py
|
||||
src/pip/_vendor/webencodings/__init__.py
|
||||
src/pip/_vendor/webencodings/labels.py
|
||||
src/pip/_vendor/webencodings/mklabels.py
|
||||
src/pip/_vendor/webencodings/tests.py
|
||||
src/pip/_vendor/webencodings/x_user_defined.py
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
[console_scripts]
|
||||
pip = pip._internal:main
|
||||
pip3 = pip._internal:main
|
||||
pip3.6 = pip._internal:main
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
|
||||
[testing]
|
||||
pytest
|
||||
mock
|
||||
pretend
|
||||
scripttest>=1.3
|
||||
virtualenv>=1.10
|
||||
freezegun
|
||||
@@ -1 +0,0 @@
|
||||
pip
|
||||
@@ -1 +0,0 @@
|
||||
__version__ = "10.0.1"
|
||||
@@ -1,19 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
# If we are running from a wheel, add the wheel to sys.path
|
||||
# This allows the usage python pip-*.whl/pip install pip-*.whl
|
||||
if __package__ == '':
|
||||
# __file__ is pip-*.whl/pip/__main__.py
|
||||
# first dirname call strips of '/__main__.py', second strips off '/pip'
|
||||
# Resulting path is the name of the wheel itself
|
||||
# Add that to sys.path so we can import pip
|
||||
path = os.path.dirname(os.path.dirname(__file__))
|
||||
sys.path.insert(0, path)
|
||||
|
||||
from pip._internal import main as _main # noqa
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(_main())
|
||||
@@ -1,246 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import absolute_import
|
||||
|
||||
import locale
|
||||
import logging
|
||||
import os
|
||||
import optparse
|
||||
import warnings
|
||||
|
||||
import sys
|
||||
|
||||
# 2016-06-17 barry@debian.org: urllib3 1.14 added optional support for socks,
|
||||
# but if invoked (i.e. imported), it will issue a warning to stderr if socks
|
||||
# isn't available. requests unconditionally imports urllib3's socks contrib
|
||||
# module, triggering this warning. The warning breaks DEP-8 tests (because of
|
||||
# the stderr output) and is just plain annoying in normal usage. I don't want
|
||||
# to add socks as yet another dependency for pip, nor do I want to allow-stder
|
||||
# in the DEP-8 tests, so just suppress the warning. pdb tells me this has to
|
||||
# be done before the import of pip.vcs.
|
||||
from pip._vendor.urllib3.exceptions import DependencyWarning
|
||||
warnings.filterwarnings("ignore", category=DependencyWarning) # noqa
|
||||
|
||||
# We want to inject the use of SecureTransport as early as possible so that any
|
||||
# references or sessions or what have you are ensured to have it, however we
|
||||
# only want to do this in the case that we're running on macOS and the linked
|
||||
# OpenSSL is too old to handle TLSv1.2
|
||||
try:
|
||||
import ssl
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
# Checks for OpenSSL 1.0.1 on MacOS
|
||||
if sys.platform == "darwin" and ssl.OPENSSL_VERSION_NUMBER < 0x1000100f:
|
||||
try:
|
||||
from pip._vendor.urllib3.contrib import securetransport
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
else:
|
||||
securetransport.inject_into_urllib3()
|
||||
|
||||
from pip import __version__
|
||||
from pip._internal import cmdoptions
|
||||
from pip._internal.exceptions import CommandError, PipError
|
||||
from pip._internal.utils.misc import get_installed_distributions, get_prog
|
||||
from pip._internal.utils import deprecation
|
||||
from pip._internal.vcs import git, mercurial, subversion, bazaar # noqa
|
||||
from pip._internal.baseparser import (
|
||||
ConfigOptionParser, UpdatingDefaultsHelpFormatter,
|
||||
)
|
||||
from pip._internal.commands import get_summaries, get_similar_commands
|
||||
from pip._internal.commands import commands_dict
|
||||
from pip._vendor.urllib3.exceptions import InsecureRequestWarning
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Hide the InsecureRequestWarning from urllib3
|
||||
warnings.filterwarnings("ignore", category=InsecureRequestWarning)
|
||||
|
||||
|
||||
def autocomplete():
|
||||
"""Command and option completion for the main option parser (and options)
|
||||
and its subcommands (and options).
|
||||
|
||||
Enable by sourcing one of the completion shell scripts (bash, zsh or fish).
|
||||
"""
|
||||
# Don't complete if user hasn't sourced bash_completion file.
|
||||
if 'PIP_AUTO_COMPLETE' not in os.environ:
|
||||
return
|
||||
cwords = os.environ['COMP_WORDS'].split()[1:]
|
||||
cword = int(os.environ['COMP_CWORD'])
|
||||
try:
|
||||
current = cwords[cword - 1]
|
||||
except IndexError:
|
||||
current = ''
|
||||
|
||||
subcommands = [cmd for cmd, summary in get_summaries()]
|
||||
options = []
|
||||
# subcommand
|
||||
try:
|
||||
subcommand_name = [w for w in cwords if w in subcommands][0]
|
||||
except IndexError:
|
||||
subcommand_name = None
|
||||
|
||||
parser = create_main_parser()
|
||||
# subcommand options
|
||||
if subcommand_name:
|
||||
# special case: 'help' subcommand has no options
|
||||
if subcommand_name == 'help':
|
||||
sys.exit(1)
|
||||
# special case: list locally installed dists for show and uninstall
|
||||
should_list_installed = (
|
||||
subcommand_name in ['show', 'uninstall'] and
|
||||
not current.startswith('-')
|
||||
)
|
||||
if should_list_installed:
|
||||
installed = []
|
||||
lc = current.lower()
|
||||
for dist in get_installed_distributions(local_only=True):
|
||||
if dist.key.startswith(lc) and dist.key not in cwords[1:]:
|
||||
installed.append(dist.key)
|
||||
# if there are no dists installed, fall back to option completion
|
||||
if installed:
|
||||
for dist in installed:
|
||||
print(dist)
|
||||
sys.exit(1)
|
||||
|
||||
subcommand = commands_dict[subcommand_name]()
|
||||
|
||||
for opt in subcommand.parser.option_list_all:
|
||||
if opt.help != optparse.SUPPRESS_HELP:
|
||||
for opt_str in opt._long_opts + opt._short_opts:
|
||||
options.append((opt_str, opt.nargs))
|
||||
|
||||
# filter out previously specified options from available options
|
||||
prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]]
|
||||
options = [(x, v) for (x, v) in options if x not in prev_opts]
|
||||
# filter options by current input
|
||||
options = [(k, v) for k, v in options if k.startswith(current)]
|
||||
for option in options:
|
||||
opt_label = option[0]
|
||||
# append '=' to options which require args
|
||||
if option[1] and option[0][:2] == "--":
|
||||
opt_label += '='
|
||||
print(opt_label)
|
||||
else:
|
||||
# show main parser options only when necessary
|
||||
if current.startswith('-') or current.startswith('--'):
|
||||
opts = [i.option_list for i in parser.option_groups]
|
||||
opts.append(parser.option_list)
|
||||
opts = (o for it in opts for o in it)
|
||||
|
||||
for opt in opts:
|
||||
if opt.help != optparse.SUPPRESS_HELP:
|
||||
subcommands += opt._long_opts + opt._short_opts
|
||||
|
||||
print(' '.join([x for x in subcommands if x.startswith(current)]))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def create_main_parser():
|
||||
parser_kw = {
|
||||
'usage': '\n%prog <command> [options]',
|
||||
'add_help_option': False,
|
||||
'formatter': UpdatingDefaultsHelpFormatter(),
|
||||
'name': 'global',
|
||||
'prog': get_prog(),
|
||||
}
|
||||
|
||||
parser = ConfigOptionParser(**parser_kw)
|
||||
parser.disable_interspersed_args()
|
||||
|
||||
pip_pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
parser.version = 'pip %s from %s (python %s)' % (
|
||||
__version__, pip_pkg_dir, sys.version[:3],
|
||||
)
|
||||
|
||||
# add the general options
|
||||
gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser)
|
||||
parser.add_option_group(gen_opts)
|
||||
|
||||
parser.main = True # so the help formatter knows
|
||||
|
||||
# create command listing for description
|
||||
command_summaries = get_summaries()
|
||||
description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries]
|
||||
parser.description = '\n'.join(description)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def parseopts(args):
|
||||
parser = create_main_parser()
|
||||
|
||||
# Note: parser calls disable_interspersed_args(), so the result of this
|
||||
# call is to split the initial args into the general options before the
|
||||
# subcommand and everything else.
|
||||
# For example:
|
||||
# args: ['--timeout=5', 'install', '--user', 'INITools']
|
||||
# general_options: ['--timeout==5']
|
||||
# args_else: ['install', '--user', 'INITools']
|
||||
general_options, args_else = parser.parse_args(args)
|
||||
|
||||
# --version
|
||||
if general_options.version:
|
||||
sys.stdout.write(parser.version)
|
||||
sys.stdout.write(os.linesep)
|
||||
sys.exit()
|
||||
|
||||
# pip || pip help -> print_help()
|
||||
if not args_else or (args_else[0] == 'help' and len(args_else) == 1):
|
||||
parser.print_help()
|
||||
sys.exit()
|
||||
|
||||
# the subcommand name
|
||||
cmd_name = args_else[0]
|
||||
|
||||
if cmd_name not in commands_dict:
|
||||
guess = get_similar_commands(cmd_name)
|
||||
|
||||
msg = ['unknown command "%s"' % cmd_name]
|
||||
if guess:
|
||||
msg.append('maybe you meant "%s"' % guess)
|
||||
|
||||
raise CommandError(' - '.join(msg))
|
||||
|
||||
# all the args without the subcommand
|
||||
cmd_args = args[:]
|
||||
cmd_args.remove(cmd_name)
|
||||
|
||||
return cmd_name, cmd_args
|
||||
|
||||
|
||||
def check_isolated(args):
|
||||
isolated = False
|
||||
|
||||
if "--isolated" in args:
|
||||
isolated = True
|
||||
|
||||
return isolated
|
||||
|
||||
|
||||
def main(args=None):
|
||||
if args is None:
|
||||
args = sys.argv[1:]
|
||||
|
||||
# Configure our deprecation warnings to be sent through loggers
|
||||
deprecation.install_warning_logger()
|
||||
|
||||
autocomplete()
|
||||
|
||||
try:
|
||||
cmd_name, cmd_args = parseopts(args)
|
||||
except PipError as exc:
|
||||
sys.stderr.write("ERROR: %s" % exc)
|
||||
sys.stderr.write(os.linesep)
|
||||
sys.exit(1)
|
||||
|
||||
# Needed for locale.getpreferredencoding(False) to work
|
||||
# in pip._internal.utils.encoding.auto_decode
|
||||
try:
|
||||
locale.setlocale(locale.LC_ALL, '')
|
||||
except locale.Error as e:
|
||||
# setlocale can apparently crash if locale are uninitialized
|
||||
logger.debug("Ignoring error %s when setting locale", e)
|
||||
command = commands_dict[cmd_name](isolated=check_isolated(cmd_args))
|
||||
return command.main(cmd_args)
|
||||
@@ -1,373 +0,0 @@
|
||||
"""Base Command class, and related routines"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
import logging.config
|
||||
import optparse
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
from pip._internal import cmdoptions
|
||||
from pip._internal.baseparser import (
|
||||
ConfigOptionParser, UpdatingDefaultsHelpFormatter,
|
||||
)
|
||||
from pip._internal.compat import WINDOWS
|
||||
from pip._internal.download import PipSession
|
||||
from pip._internal.exceptions import (
|
||||
BadCommand, CommandError, InstallationError, PreviousBuildDirError,
|
||||
UninstallationError,
|
||||
)
|
||||
from pip._internal.index import PackageFinder
|
||||
from pip._internal.locations import running_under_virtualenv
|
||||
from pip._internal.req.req_file import parse_requirements
|
||||
from pip._internal.req.req_install import InstallRequirement
|
||||
from pip._internal.status_codes import (
|
||||
ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR,
|
||||
VIRTUALENV_NOT_FOUND,
|
||||
)
|
||||
from pip._internal.utils import deprecation
|
||||
from pip._internal.utils.logging import IndentingFormatter
|
||||
from pip._internal.utils.misc import get_prog, normalize_path
|
||||
from pip._internal.utils.outdated import pip_version_check
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from typing import Optional
|
||||
|
||||
__all__ = ['Command']
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Command(object):
|
||||
name = None # type: Optional[str]
|
||||
usage = None # type: Optional[str]
|
||||
hidden = False # type: bool
|
||||
ignore_require_venv = False # type: bool
|
||||
log_streams = ("ext://sys.stdout", "ext://sys.stderr")
|
||||
|
||||
def __init__(self, isolated=False):
|
||||
parser_kw = {
|
||||
'usage': self.usage,
|
||||
'prog': '%s %s' % (get_prog(), self.name),
|
||||
'formatter': UpdatingDefaultsHelpFormatter(),
|
||||
'add_help_option': False,
|
||||
'name': self.name,
|
||||
'description': self.__doc__,
|
||||
'isolated': isolated,
|
||||
}
|
||||
|
||||
self.parser = ConfigOptionParser(**parser_kw)
|
||||
|
||||
# Commands should add options to this option group
|
||||
optgroup_name = '%s Options' % self.name.capitalize()
|
||||
self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name)
|
||||
|
||||
# Add the general options
|
||||
gen_opts = cmdoptions.make_option_group(
|
||||
cmdoptions.general_group,
|
||||
self.parser,
|
||||
)
|
||||
self.parser.add_option_group(gen_opts)
|
||||
|
||||
def _build_session(self, options, retries=None, timeout=None):
|
||||
session = PipSession(
|
||||
cache=(
|
||||
normalize_path(os.path.join(options.cache_dir, "http"))
|
||||
if options.cache_dir else None
|
||||
),
|
||||
retries=retries if retries is not None else options.retries,
|
||||
insecure_hosts=options.trusted_hosts,
|
||||
)
|
||||
|
||||
# Handle custom ca-bundles from the user
|
||||
if options.cert:
|
||||
session.verify = options.cert
|
||||
|
||||
# Handle SSL client certificate
|
||||
if options.client_cert:
|
||||
session.cert = options.client_cert
|
||||
|
||||
# Handle timeouts
|
||||
if options.timeout or timeout:
|
||||
session.timeout = (
|
||||
timeout if timeout is not None else options.timeout
|
||||
)
|
||||
|
||||
# Handle configured proxies
|
||||
if options.proxy:
|
||||
session.proxies = {
|
||||
"http": options.proxy,
|
||||
"https": options.proxy,
|
||||
}
|
||||
|
||||
# Determine if we can prompt the user for authentication or not
|
||||
session.auth.prompting = not options.no_input
|
||||
|
||||
return session
|
||||
|
||||
def parse_args(self, args):
|
||||
# factored out for testability
|
||||
return self.parser.parse_args(args)
|
||||
|
||||
def main(self, args):
|
||||
options, args = self.parse_args(args)
|
||||
|
||||
# Set verbosity so that it can be used elsewhere.
|
||||
self.verbosity = options.verbose - options.quiet
|
||||
|
||||
if self.verbosity >= 1:
|
||||
level = "DEBUG"
|
||||
elif self.verbosity == -1:
|
||||
level = "WARNING"
|
||||
elif self.verbosity == -2:
|
||||
level = "ERROR"
|
||||
elif self.verbosity <= -3:
|
||||
level = "CRITICAL"
|
||||
else:
|
||||
level = "INFO"
|
||||
|
||||
# The root logger should match the "console" level *unless* we
|
||||
# specified "--log" to send debug logs to a file.
|
||||
root_level = level
|
||||
if options.log:
|
||||
root_level = "DEBUG"
|
||||
|
||||
logger_class = "pip._internal.utils.logging.ColorizedStreamHandler"
|
||||
handler_class = "pip._internal.utils.logging.BetterRotatingFileHandler"
|
||||
|
||||
logging.config.dictConfig({
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"filters": {
|
||||
"exclude_warnings": {
|
||||
"()": "pip._internal.utils.logging.MaxLevelFilter",
|
||||
"level": logging.WARNING,
|
||||
},
|
||||
},
|
||||
"formatters": {
|
||||
"indent": {
|
||||
"()": IndentingFormatter,
|
||||
"format": "%(message)s",
|
||||
},
|
||||
},
|
||||
"handlers": {
|
||||
"console": {
|
||||
"level": level,
|
||||
"class": logger_class,
|
||||
"no_color": options.no_color,
|
||||
"stream": self.log_streams[0],
|
||||
"filters": ["exclude_warnings"],
|
||||
"formatter": "indent",
|
||||
},
|
||||
"console_errors": {
|
||||
"level": "WARNING",
|
||||
"class": logger_class,
|
||||
"no_color": options.no_color,
|
||||
"stream": self.log_streams[1],
|
||||
"formatter": "indent",
|
||||
},
|
||||
"user_log": {
|
||||
"level": "DEBUG",
|
||||
"class": handler_class,
|
||||
"filename": options.log or "/dev/null",
|
||||
"delay": True,
|
||||
"formatter": "indent",
|
||||
},
|
||||
},
|
||||
"root": {
|
||||
"level": root_level,
|
||||
"handlers": list(filter(None, [
|
||||
"console",
|
||||
"console_errors",
|
||||
"user_log" if options.log else None,
|
||||
])),
|
||||
},
|
||||
# Disable any logging besides WARNING unless we have DEBUG level
|
||||
# logging enabled. These use both pip._vendor and the bare names
|
||||
# for the case where someone unbundles our libraries.
|
||||
"loggers": {
|
||||
name: {
|
||||
"level": (
|
||||
"WARNING" if level in ["INFO", "ERROR"] else "DEBUG"
|
||||
)
|
||||
} for name in [
|
||||
"pip._vendor", "distlib", "requests", "urllib3"
|
||||
]
|
||||
},
|
||||
})
|
||||
|
||||
if sys.version_info[:2] == (3, 3):
|
||||
warnings.warn(
|
||||
"Python 3.3 supported has been deprecated and support for it "
|
||||
"will be dropped in the future. Please upgrade your Python.",
|
||||
deprecation.RemovedInPip11Warning,
|
||||
)
|
||||
|
||||
# TODO: try to get these passing down from the command?
|
||||
# without resorting to os.environ to hold these.
|
||||
|
||||
if options.no_input:
|
||||
os.environ['PIP_NO_INPUT'] = '1'
|
||||
|
||||
if options.exists_action:
|
||||
os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action)
|
||||
|
||||
if options.require_venv and not self.ignore_require_venv:
|
||||
# If a venv is required check if it can really be found
|
||||
if not running_under_virtualenv():
|
||||
logger.critical(
|
||||
'Could not find an activated virtualenv (required).'
|
||||
)
|
||||
sys.exit(VIRTUALENV_NOT_FOUND)
|
||||
|
||||
original_root_handlers = set(logging.root.handlers)
|
||||
|
||||
try:
|
||||
status = self.run(options, args)
|
||||
# FIXME: all commands should return an exit status
|
||||
# and when it is done, isinstance is not needed anymore
|
||||
if isinstance(status, int):
|
||||
return status
|
||||
except PreviousBuildDirError as exc:
|
||||
logger.critical(str(exc))
|
||||
logger.debug('Exception information:', exc_info=True)
|
||||
|
||||
return PREVIOUS_BUILD_DIR_ERROR
|
||||
except (InstallationError, UninstallationError, BadCommand) as exc:
|
||||
logger.critical(str(exc))
|
||||
logger.debug('Exception information:', exc_info=True)
|
||||
|
||||
return ERROR
|
||||
except CommandError as exc:
|
||||
logger.critical('ERROR: %s', exc)
|
||||
logger.debug('Exception information:', exc_info=True)
|
||||
|
||||
return ERROR
|
||||
except KeyboardInterrupt:
|
||||
logger.critical('Operation cancelled by user')
|
||||
logger.debug('Exception information:', exc_info=True)
|
||||
|
||||
return ERROR
|
||||
except:
|
||||
logger.critical('Exception:', exc_info=True)
|
||||
|
||||
return UNKNOWN_ERROR
|
||||
finally:
|
||||
# Check if we're using the latest version of pip available
|
||||
if (not options.disable_pip_version_check and not
|
||||
getattr(options, "no_index", False)):
|
||||
with self._build_session(
|
||||
options,
|
||||
retries=0,
|
||||
timeout=min(5, options.timeout)) as session:
|
||||
pip_version_check(session, options)
|
||||
# Avoid leaking loggers
|
||||
for handler in set(logging.root.handlers) - original_root_handlers:
|
||||
# this method benefit from the Logger class internal lock
|
||||
logging.root.removeHandler(handler)
|
||||
|
||||
return SUCCESS
|
||||
|
||||
|
||||
class RequirementCommand(Command):
|
||||
|
||||
@staticmethod
|
||||
def populate_requirement_set(requirement_set, args, options, finder,
|
||||
session, name, wheel_cache):
|
||||
"""
|
||||
Marshal cmd line args into a requirement set.
|
||||
"""
|
||||
# NOTE: As a side-effect, options.require_hashes and
|
||||
# requirement_set.require_hashes may be updated
|
||||
|
||||
for filename in options.constraints:
|
||||
for req_to_add in parse_requirements(
|
||||
filename,
|
||||
constraint=True, finder=finder, options=options,
|
||||
session=session, wheel_cache=wheel_cache):
|
||||
req_to_add.is_direct = True
|
||||
requirement_set.add_requirement(req_to_add)
|
||||
|
||||
for req in args:
|
||||
req_to_add = InstallRequirement.from_line(
|
||||
req, None, isolated=options.isolated_mode,
|
||||
wheel_cache=wheel_cache
|
||||
)
|
||||
req_to_add.is_direct = True
|
||||
requirement_set.add_requirement(req_to_add)
|
||||
|
||||
for req in options.editables:
|
||||
req_to_add = InstallRequirement.from_editable(
|
||||
req,
|
||||
isolated=options.isolated_mode,
|
||||
wheel_cache=wheel_cache
|
||||
)
|
||||
req_to_add.is_direct = True
|
||||
requirement_set.add_requirement(req_to_add)
|
||||
|
||||
for filename in options.requirements:
|
||||
for req_to_add in parse_requirements(
|
||||
filename,
|
||||
finder=finder, options=options, session=session,
|
||||
wheel_cache=wheel_cache):
|
||||
req_to_add.is_direct = True
|
||||
requirement_set.add_requirement(req_to_add)
|
||||
# If --require-hashes was a line in a requirements file, tell
|
||||
# RequirementSet about it:
|
||||
requirement_set.require_hashes = options.require_hashes
|
||||
|
||||
if not (args or options.editables or options.requirements):
|
||||
opts = {'name': name}
|
||||
if options.find_links:
|
||||
raise CommandError(
|
||||
'You must give at least one requirement to %(name)s '
|
||||
'(maybe you meant "pip %(name)s %(links)s"?)' %
|
||||
dict(opts, links=' '.join(options.find_links)))
|
||||
else:
|
||||
raise CommandError(
|
||||
'You must give at least one requirement to %(name)s '
|
||||
'(see "pip help %(name)s")' % opts)
|
||||
|
||||
# On Windows, any operation modifying pip should be run as:
|
||||
# python -m pip ...
|
||||
# See https://github.com/pypa/pip/issues/1299 for more discussion
|
||||
should_show_use_python_msg = (
|
||||
WINDOWS and
|
||||
requirement_set.has_requirement("pip") and
|
||||
os.path.basename(sys.argv[0]).startswith("pip")
|
||||
)
|
||||
if should_show_use_python_msg:
|
||||
new_command = [
|
||||
sys.executable, "-m", "pip"
|
||||
] + sys.argv[1:]
|
||||
raise CommandError(
|
||||
'To modify pip, please run the following command:\n{}'
|
||||
.format(" ".join(new_command))
|
||||
)
|
||||
|
||||
def _build_package_finder(self, options, session,
|
||||
platform=None, python_versions=None,
|
||||
abi=None, implementation=None):
|
||||
"""
|
||||
Create a package finder appropriate to this requirement command.
|
||||
"""
|
||||
index_urls = [options.index_url] + options.extra_index_urls
|
||||
if options.no_index:
|
||||
logger.debug('Ignoring indexes: %s', ','.join(index_urls))
|
||||
index_urls = []
|
||||
|
||||
return PackageFinder(
|
||||
find_links=options.find_links,
|
||||
format_control=options.format_control,
|
||||
index_urls=index_urls,
|
||||
trusted_hosts=options.trusted_hosts,
|
||||
allow_all_prereleases=options.pre,
|
||||
process_dependency_links=options.process_dependency_links,
|
||||
session=session,
|
||||
platform=platform,
|
||||
versions=python_versions,
|
||||
abi=abi,
|
||||
implementation=implementation,
|
||||
)
|
||||
@@ -1,240 +0,0 @@
|
||||
"""Base option parser setup"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
import optparse
|
||||
import sys
|
||||
import textwrap
|
||||
from distutils.util import strtobool
|
||||
|
||||
from pip._vendor.six import string_types
|
||||
|
||||
from pip._internal.compat import get_terminal_size
|
||||
from pip._internal.configuration import Configuration, ConfigurationError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PrettyHelpFormatter(optparse.IndentedHelpFormatter):
|
||||
"""A prettier/less verbose help formatter for optparse."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
# help position must be aligned with __init__.parseopts.description
|
||||
kwargs['max_help_position'] = 30
|
||||
kwargs['indent_increment'] = 1
|
||||
kwargs['width'] = get_terminal_size()[0] - 2
|
||||
optparse.IndentedHelpFormatter.__init__(self, *args, **kwargs)
|
||||
|
||||
def format_option_strings(self, option):
|
||||
return self._format_option_strings(option, ' <%s>', ', ')
|
||||
|
||||
def _format_option_strings(self, option, mvarfmt=' <%s>', optsep=', '):
|
||||
"""
|
||||
Return a comma-separated list of option strings and metavars.
|
||||
|
||||
:param option: tuple of (short opt, long opt), e.g: ('-f', '--format')
|
||||
:param mvarfmt: metavar format string - evaluated as mvarfmt % metavar
|
||||
:param optsep: separator
|
||||
"""
|
||||
opts = []
|
||||
|
||||
if option._short_opts:
|
||||
opts.append(option._short_opts[0])
|
||||
if option._long_opts:
|
||||
opts.append(option._long_opts[0])
|
||||
if len(opts) > 1:
|
||||
opts.insert(1, optsep)
|
||||
|
||||
if option.takes_value():
|
||||
metavar = option.metavar or option.dest.lower()
|
||||
opts.append(mvarfmt % metavar.lower())
|
||||
|
||||
return ''.join(opts)
|
||||
|
||||
def format_heading(self, heading):
|
||||
if heading == 'Options':
|
||||
return ''
|
||||
return heading + ':\n'
|
||||
|
||||
def format_usage(self, usage):
|
||||
"""
|
||||
Ensure there is only one newline between usage and the first heading
|
||||
if there is no description.
|
||||
"""
|
||||
msg = '\nUsage: %s\n' % self.indent_lines(textwrap.dedent(usage), " ")
|
||||
return msg
|
||||
|
||||
def format_description(self, description):
|
||||
# leave full control over description to us
|
||||
if description:
|
||||
if hasattr(self.parser, 'main'):
|
||||
label = 'Commands'
|
||||
else:
|
||||
label = 'Description'
|
||||
# some doc strings have initial newlines, some don't
|
||||
description = description.lstrip('\n')
|
||||
# some doc strings have final newlines and spaces, some don't
|
||||
description = description.rstrip()
|
||||
# dedent, then reindent
|
||||
description = self.indent_lines(textwrap.dedent(description), " ")
|
||||
description = '%s:\n%s\n' % (label, description)
|
||||
return description
|
||||
else:
|
||||
return ''
|
||||
|
||||
def format_epilog(self, epilog):
|
||||
# leave full control over epilog to us
|
||||
if epilog:
|
||||
return epilog
|
||||
else:
|
||||
return ''
|
||||
|
||||
def indent_lines(self, text, indent):
|
||||
new_lines = [indent + line for line in text.split('\n')]
|
||||
return "\n".join(new_lines)
|
||||
|
||||
|
||||
class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter):
|
||||
"""Custom help formatter for use in ConfigOptionParser.
|
||||
|
||||
This is updates the defaults before expanding them, allowing
|
||||
them to show up correctly in the help listing.
|
||||
"""
|
||||
|
||||
def expand_default(self, option):
|
||||
if self.parser is not None:
|
||||
self.parser._update_defaults(self.parser.defaults)
|
||||
return optparse.IndentedHelpFormatter.expand_default(self, option)
|
||||
|
||||
|
||||
class CustomOptionParser(optparse.OptionParser):
|
||||
|
||||
def insert_option_group(self, idx, *args, **kwargs):
|
||||
"""Insert an OptionGroup at a given position."""
|
||||
group = self.add_option_group(*args, **kwargs)
|
||||
|
||||
self.option_groups.pop()
|
||||
self.option_groups.insert(idx, group)
|
||||
|
||||
return group
|
||||
|
||||
@property
|
||||
def option_list_all(self):
|
||||
"""Get a list of all options, including those in option groups."""
|
||||
res = self.option_list[:]
|
||||
for i in self.option_groups:
|
||||
res.extend(i.option_list)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
class ConfigOptionParser(CustomOptionParser):
|
||||
"""Custom option parser which updates its defaults by checking the
|
||||
configuration files and environmental variables"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.name = kwargs.pop('name')
|
||||
|
||||
isolated = kwargs.pop("isolated", False)
|
||||
self.config = Configuration(isolated)
|
||||
|
||||
assert self.name
|
||||
optparse.OptionParser.__init__(self, *args, **kwargs)
|
||||
|
||||
def check_default(self, option, key, val):
|
||||
try:
|
||||
return option.check_value(key, val)
|
||||
except optparse.OptionValueError as exc:
|
||||
print("An error occurred during configuration: %s" % exc)
|
||||
sys.exit(3)
|
||||
|
||||
def _get_ordered_configuration_items(self):
|
||||
# Configuration gives keys in an unordered manner. Order them.
|
||||
override_order = ["global", self.name, ":env:"]
|
||||
|
||||
# Pool the options into different groups
|
||||
section_items = {name: [] for name in override_order}
|
||||
for section_key, val in self.config.items():
|
||||
# ignore empty values
|
||||
if not val:
|
||||
logger.debug(
|
||||
"Ignoring configuration key '%s' as it's value is empty.",
|
||||
section_key
|
||||
)
|
||||
continue
|
||||
|
||||
section, key = section_key.split(".", 1)
|
||||
if section in override_order:
|
||||
section_items[section].append((key, val))
|
||||
|
||||
# Yield each group in their override order
|
||||
for section in override_order:
|
||||
for key, val in section_items[section]:
|
||||
yield key, val
|
||||
|
||||
def _update_defaults(self, defaults):
|
||||
"""Updates the given defaults with values from the config files and
|
||||
the environ. Does a little special handling for certain types of
|
||||
options (lists)."""
|
||||
|
||||
# Accumulate complex default state.
|
||||
self.values = optparse.Values(self.defaults)
|
||||
late_eval = set()
|
||||
# Then set the options with those values
|
||||
for key, val in self._get_ordered_configuration_items():
|
||||
# '--' because configuration supports only long names
|
||||
option = self.get_option('--' + key)
|
||||
|
||||
# Ignore options not present in this parser. E.g. non-globals put
|
||||
# in [global] by users that want them to apply to all applicable
|
||||
# commands.
|
||||
if option is None:
|
||||
continue
|
||||
|
||||
if option.action in ('store_true', 'store_false', 'count'):
|
||||
val = strtobool(val)
|
||||
elif option.action == 'append':
|
||||
val = val.split()
|
||||
val = [self.check_default(option, key, v) for v in val]
|
||||
elif option.action == 'callback':
|
||||
late_eval.add(option.dest)
|
||||
opt_str = option.get_opt_string()
|
||||
val = option.convert_value(opt_str, val)
|
||||
# From take_action
|
||||
args = option.callback_args or ()
|
||||
kwargs = option.callback_kwargs or {}
|
||||
option.callback(option, opt_str, val, self, *args, **kwargs)
|
||||
else:
|
||||
val = self.check_default(option, key, val)
|
||||
|
||||
defaults[option.dest] = val
|
||||
|
||||
for key in late_eval:
|
||||
defaults[key] = getattr(self.values, key)
|
||||
self.values = None
|
||||
return defaults
|
||||
|
||||
def get_default_values(self):
|
||||
"""Overriding to make updating the defaults after instantiation of
|
||||
the option parser possible, _update_defaults() does the dirty work."""
|
||||
if not self.process_default_values:
|
||||
# Old, pre-Optik 1.5 behaviour.
|
||||
return optparse.Values(self.defaults)
|
||||
|
||||
# Load the configuration, or error out in case of an error
|
||||
try:
|
||||
self.config.load()
|
||||
except ConfigurationError as err:
|
||||
self.exit(2, err.args[0])
|
||||
|
||||
defaults = self._update_defaults(self.defaults.copy()) # ours
|
||||
for option in self._get_all_options():
|
||||
default = defaults.get(option.dest)
|
||||
if isinstance(default, string_types):
|
||||
opt_str = option.get_opt_string()
|
||||
defaults[option.dest] = option.check_value(opt_str, default)
|
||||
return optparse.Values(defaults)
|
||||
|
||||
def error(self, msg):
|
||||
self.print_usage(sys.stderr)
|
||||
self.exit(2, "%s\n" % msg)
|
||||
@@ -1,92 +0,0 @@
|
||||
"""Build Environment used for isolation during sdist building
|
||||
"""
|
||||
|
||||
import os
|
||||
from distutils.sysconfig import get_python_lib
|
||||
from sysconfig import get_paths
|
||||
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
|
||||
|
||||
class BuildEnvironment(object):
|
||||
"""Creates and manages an isolated environment to install build deps
|
||||
"""
|
||||
|
||||
def __init__(self, no_clean):
|
||||
self._temp_dir = TempDirectory(kind="build-env")
|
||||
self._no_clean = no_clean
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return self._temp_dir.path
|
||||
|
||||
def __enter__(self):
|
||||
self._temp_dir.create()
|
||||
|
||||
self.save_path = os.environ.get('PATH', None)
|
||||
self.save_pythonpath = os.environ.get('PYTHONPATH', None)
|
||||
self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None)
|
||||
|
||||
install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix'
|
||||
install_dirs = get_paths(install_scheme, vars={
|
||||
'base': self.path,
|
||||
'platbase': self.path,
|
||||
})
|
||||
|
||||
scripts = install_dirs['scripts']
|
||||
if self.save_path:
|
||||
os.environ['PATH'] = scripts + os.pathsep + self.save_path
|
||||
else:
|
||||
os.environ['PATH'] = scripts + os.pathsep + os.defpath
|
||||
|
||||
# Note: prefer distutils' sysconfig to get the
|
||||
# library paths so PyPy is correctly supported.
|
||||
purelib = get_python_lib(plat_specific=0, prefix=self.path)
|
||||
platlib = get_python_lib(plat_specific=1, prefix=self.path)
|
||||
if purelib == platlib:
|
||||
lib_dirs = purelib
|
||||
else:
|
||||
lib_dirs = purelib + os.pathsep + platlib
|
||||
if self.save_pythonpath:
|
||||
os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \
|
||||
self.save_pythonpath
|
||||
else:
|
||||
os.environ['PYTHONPATH'] = lib_dirs
|
||||
|
||||
os.environ['PYTHONNOUSERSITE'] = '1'
|
||||
|
||||
return self.path
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if not self._no_clean:
|
||||
self._temp_dir.cleanup()
|
||||
|
||||
def restore_var(varname, old_value):
|
||||
if old_value is None:
|
||||
os.environ.pop(varname, None)
|
||||
else:
|
||||
os.environ[varname] = old_value
|
||||
|
||||
restore_var('PATH', self.save_path)
|
||||
restore_var('PYTHONPATH', self.save_pythonpath)
|
||||
restore_var('PYTHONNOUSERSITE', self.save_nousersite)
|
||||
|
||||
def cleanup(self):
|
||||
self._temp_dir.cleanup()
|
||||
|
||||
|
||||
class NoOpBuildEnvironment(BuildEnvironment):
|
||||
"""A no-op drop-in replacement for BuildEnvironment
|
||||
"""
|
||||
|
||||
def __init__(self, no_clean):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
pass
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
pass
|
||||
|
||||
def cleanup(self):
|
||||
pass
|
||||
@@ -1,202 +0,0 @@
|
||||
"""Cache Management
|
||||
"""
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
|
||||
from pip._internal import index
|
||||
from pip._internal.compat import expanduser
|
||||
from pip._internal.download import path_to_url
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
from pip._internal.wheel import InvalidWheelFilename, Wheel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Cache(object):
|
||||
"""An abstract class - provides cache directories for data from links
|
||||
|
||||
|
||||
:param cache_dir: The root of the cache.
|
||||
:param format_control: A pip.index.FormatControl object to limit
|
||||
binaries being read from the cache.
|
||||
:param allowed_formats: which formats of files the cache should store.
|
||||
('binary' and 'source' are the only allowed values)
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir, format_control, allowed_formats):
|
||||
super(Cache, self).__init__()
|
||||
self.cache_dir = expanduser(cache_dir) if cache_dir else None
|
||||
self.format_control = format_control
|
||||
self.allowed_formats = allowed_formats
|
||||
|
||||
_valid_formats = {"source", "binary"}
|
||||
assert self.allowed_formats.union(_valid_formats) == _valid_formats
|
||||
|
||||
def _get_cache_path_parts(self, link):
|
||||
"""Get parts of part that must be os.path.joined with cache_dir
|
||||
"""
|
||||
|
||||
# We want to generate an url to use as our cache key, we don't want to
|
||||
# just re-use the URL because it might have other items in the fragment
|
||||
# and we don't care about those.
|
||||
key_parts = [link.url_without_fragment]
|
||||
if link.hash_name is not None and link.hash is not None:
|
||||
key_parts.append("=".join([link.hash_name, link.hash]))
|
||||
key_url = "#".join(key_parts)
|
||||
|
||||
# Encode our key url with sha224, we'll use this because it has similar
|
||||
# security properties to sha256, but with a shorter total output (and
|
||||
# thus less secure). However the differences don't make a lot of
|
||||
# difference for our use case here.
|
||||
hashed = hashlib.sha224(key_url.encode()).hexdigest()
|
||||
|
||||
# We want to nest the directories some to prevent having a ton of top
|
||||
# level directories where we might run out of sub directories on some
|
||||
# FS.
|
||||
parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]]
|
||||
|
||||
return parts
|
||||
|
||||
def _get_candidates(self, link, package_name):
|
||||
can_not_cache = (
|
||||
not self.cache_dir or
|
||||
not package_name or
|
||||
not link
|
||||
)
|
||||
if can_not_cache:
|
||||
return []
|
||||
|
||||
canonical_name = canonicalize_name(package_name)
|
||||
formats = index.fmt_ctl_formats(
|
||||
self.format_control, canonical_name
|
||||
)
|
||||
if not self.allowed_formats.intersection(formats):
|
||||
return []
|
||||
|
||||
root = self.get_path_for_link(link)
|
||||
try:
|
||||
return os.listdir(root)
|
||||
except OSError as err:
|
||||
if err.errno in {errno.ENOENT, errno.ENOTDIR}:
|
||||
return []
|
||||
raise
|
||||
|
||||
def get_path_for_link(self, link):
|
||||
"""Return a directory to store cached items in for link.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def get(self, link, package_name):
|
||||
"""Returns a link to a cached item if it exists, otherwise returns the
|
||||
passed link.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def _link_for_candidate(self, link, candidate):
|
||||
root = self.get_path_for_link(link)
|
||||
path = os.path.join(root, candidate)
|
||||
|
||||
return index.Link(path_to_url(path))
|
||||
|
||||
def cleanup(self):
|
||||
pass
|
||||
|
||||
|
||||
class SimpleWheelCache(Cache):
|
||||
"""A cache of wheels for future installs.
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir, format_control):
|
||||
super(SimpleWheelCache, self).__init__(
|
||||
cache_dir, format_control, {"binary"}
|
||||
)
|
||||
|
||||
def get_path_for_link(self, link):
|
||||
"""Return a directory to store cached wheels for link
|
||||
|
||||
Because there are M wheels for any one sdist, we provide a directory
|
||||
to cache them in, and then consult that directory when looking up
|
||||
cache hits.
|
||||
|
||||
We only insert things into the cache if they have plausible version
|
||||
numbers, so that we don't contaminate the cache with things that were
|
||||
not unique. E.g. ./package might have dozens of installs done for it
|
||||
and build a version of 0.0...and if we built and cached a wheel, we'd
|
||||
end up using the same wheel even if the source has been edited.
|
||||
|
||||
:param link: The link of the sdist for which this will cache wheels.
|
||||
"""
|
||||
parts = self._get_cache_path_parts(link)
|
||||
|
||||
# Store wheels within the root cache_dir
|
||||
return os.path.join(self.cache_dir, "wheels", *parts)
|
||||
|
||||
def get(self, link, package_name):
|
||||
candidates = []
|
||||
|
||||
for wheel_name in self._get_candidates(link, package_name):
|
||||
try:
|
||||
wheel = Wheel(wheel_name)
|
||||
except InvalidWheelFilename:
|
||||
continue
|
||||
if not wheel.supported():
|
||||
# Built for a different python/arch/etc
|
||||
continue
|
||||
candidates.append((wheel.support_index_min(), wheel_name))
|
||||
|
||||
if not candidates:
|
||||
return link
|
||||
|
||||
return self._link_for_candidate(link, min(candidates)[1])
|
||||
|
||||
|
||||
class EphemWheelCache(SimpleWheelCache):
|
||||
"""A SimpleWheelCache that creates it's own temporary cache directory
|
||||
"""
|
||||
|
||||
def __init__(self, format_control):
|
||||
self._temp_dir = TempDirectory(kind="ephem-wheel-cache")
|
||||
self._temp_dir.create()
|
||||
|
||||
super(EphemWheelCache, self).__init__(
|
||||
self._temp_dir.path, format_control
|
||||
)
|
||||
|
||||
def cleanup(self):
|
||||
self._temp_dir.cleanup()
|
||||
|
||||
|
||||
class WheelCache(Cache):
|
||||
"""Wraps EphemWheelCache and SimpleWheelCache into a single Cache
|
||||
|
||||
This Cache allows for gracefully degradation, using the ephem wheel cache
|
||||
when a certain link is not found in the simple wheel cache first.
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir, format_control):
|
||||
super(WheelCache, self).__init__(
|
||||
cache_dir, format_control, {'binary'}
|
||||
)
|
||||
self._wheel_cache = SimpleWheelCache(cache_dir, format_control)
|
||||
self._ephem_cache = EphemWheelCache(format_control)
|
||||
|
||||
def get_path_for_link(self, link):
|
||||
return self._wheel_cache.get_path_for_link(link)
|
||||
|
||||
def get_ephem_path_for_link(self, link):
|
||||
return self._ephem_cache.get_path_for_link(link)
|
||||
|
||||
def get(self, link, package_name):
|
||||
retval = self._wheel_cache.get(link, package_name)
|
||||
if retval is link:
|
||||
retval = self._ephem_cache.get(link, package_name)
|
||||
return retval
|
||||
|
||||
def cleanup(self):
|
||||
self._wheel_cache.cleanup()
|
||||
self._ephem_cache.cleanup()
|
||||
@@ -1,609 +0,0 @@
|
||||
"""
|
||||
shared options and groups
|
||||
|
||||
The principle here is to define options once, but *not* instantiate them
|
||||
globally. One reason being that options with action='append' can carry state
|
||||
between parses. pip parses general options twice internally, and shouldn't
|
||||
pass on state. To be consistent, all options will follow this design.
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import warnings
|
||||
from functools import partial
|
||||
from optparse import SUPPRESS_HELP, Option, OptionGroup
|
||||
|
||||
from pip._internal.index import (
|
||||
FormatControl, fmt_ctl_handle_mutual_exclude, fmt_ctl_no_binary,
|
||||
)
|
||||
from pip._internal.locations import USER_CACHE_DIR, src_prefix
|
||||
from pip._internal.models import PyPI
|
||||
from pip._internal.utils.hashes import STRONG_HASHES
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
from pip._internal.utils.ui import BAR_TYPES
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from typing import Any
|
||||
|
||||
|
||||
def make_option_group(group, parser):
|
||||
"""
|
||||
Return an OptionGroup object
|
||||
group -- assumed to be dict with 'name' and 'options' keys
|
||||
parser -- an optparse Parser
|
||||
"""
|
||||
option_group = OptionGroup(parser, group['name'])
|
||||
for option in group['options']:
|
||||
option_group.add_option(option())
|
||||
return option_group
|
||||
|
||||
|
||||
def check_install_build_global(options, check_options=None):
|
||||
"""Disable wheels if per-setup.py call options are set.
|
||||
|
||||
:param options: The OptionParser options to update.
|
||||
:param check_options: The options to check, if not supplied defaults to
|
||||
options.
|
||||
"""
|
||||
if check_options is None:
|
||||
check_options = options
|
||||
|
||||
def getname(n):
|
||||
return getattr(check_options, n, None)
|
||||
names = ["build_options", "global_options", "install_options"]
|
||||
if any(map(getname, names)):
|
||||
control = options.format_control
|
||||
fmt_ctl_no_binary(control)
|
||||
warnings.warn(
|
||||
'Disabling all use of wheels due to the use of --build-options '
|
||||
'/ --global-options / --install-options.', stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
###########
|
||||
# options #
|
||||
###########
|
||||
|
||||
help_ = partial(
|
||||
Option,
|
||||
'-h', '--help',
|
||||
dest='help',
|
||||
action='help',
|
||||
help='Show help.',
|
||||
) # type: Any
|
||||
|
||||
isolated_mode = partial(
|
||||
Option,
|
||||
"--isolated",
|
||||
dest="isolated_mode",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Run pip in an isolated mode, ignoring environment variables and user "
|
||||
"configuration."
|
||||
),
|
||||
)
|
||||
|
||||
require_virtualenv = partial(
|
||||
Option,
|
||||
# Run only if inside a virtualenv, bail if not.
|
||||
'--require-virtualenv', '--require-venv',
|
||||
dest='require_venv',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help=SUPPRESS_HELP
|
||||
) # type: Any
|
||||
|
||||
verbose = partial(
|
||||
Option,
|
||||
'-v', '--verbose',
|
||||
dest='verbose',
|
||||
action='count',
|
||||
default=0,
|
||||
help='Give more output. Option is additive, and can be used up to 3 times.'
|
||||
)
|
||||
|
||||
no_color = partial(
|
||||
Option,
|
||||
'--no-color',
|
||||
dest='no_color',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="Suppress colored output",
|
||||
)
|
||||
|
||||
version = partial(
|
||||
Option,
|
||||
'-V', '--version',
|
||||
dest='version',
|
||||
action='store_true',
|
||||
help='Show version and exit.',
|
||||
) # type: Any
|
||||
|
||||
quiet = partial(
|
||||
Option,
|
||||
'-q', '--quiet',
|
||||
dest='quiet',
|
||||
action='count',
|
||||
default=0,
|
||||
help=(
|
||||
'Give less output. Option is additive, and can be used up to 3'
|
||||
' times (corresponding to WARNING, ERROR, and CRITICAL logging'
|
||||
' levels).'
|
||||
),
|
||||
) # type: Any
|
||||
|
||||
progress_bar = partial(
|
||||
Option,
|
||||
'--progress-bar',
|
||||
dest='progress_bar',
|
||||
type='choice',
|
||||
choices=list(BAR_TYPES.keys()),
|
||||
default='on',
|
||||
help=(
|
||||
'Specify type of progress to be displayed [' +
|
||||
'|'.join(BAR_TYPES.keys()) + '] (default: %default)'
|
||||
),
|
||||
) # type: Any
|
||||
|
||||
log = partial(
|
||||
Option,
|
||||
"--log", "--log-file", "--local-log",
|
||||
dest="log",
|
||||
metavar="path",
|
||||
help="Path to a verbose appending log."
|
||||
) # type: Any
|
||||
|
||||
no_input = partial(
|
||||
Option,
|
||||
# Don't ask for input
|
||||
'--no-input',
|
||||
dest='no_input',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help=SUPPRESS_HELP
|
||||
) # type: Any
|
||||
|
||||
proxy = partial(
|
||||
Option,
|
||||
'--proxy',
|
||||
dest='proxy',
|
||||
type='str',
|
||||
default='',
|
||||
help="Specify a proxy in the form [user:passwd@]proxy.server:port."
|
||||
) # type: Any
|
||||
|
||||
retries = partial(
|
||||
Option,
|
||||
'--retries',
|
||||
dest='retries',
|
||||
type='int',
|
||||
default=5,
|
||||
help="Maximum number of retries each connection should attempt "
|
||||
"(default %default times).",
|
||||
) # type: Any
|
||||
|
||||
timeout = partial(
|
||||
Option,
|
||||
'--timeout', '--default-timeout',
|
||||
metavar='sec',
|
||||
dest='timeout',
|
||||
type='float',
|
||||
default=15,
|
||||
help='Set the socket timeout (default %default seconds).',
|
||||
) # type: Any
|
||||
|
||||
skip_requirements_regex = partial(
|
||||
Option,
|
||||
# A regex to be used to skip requirements
|
||||
'--skip-requirements-regex',
|
||||
dest='skip_requirements_regex',
|
||||
type='str',
|
||||
default='',
|
||||
help=SUPPRESS_HELP,
|
||||
) # type: Any
|
||||
|
||||
|
||||
def exists_action():
|
||||
return Option(
|
||||
# Option when path already exist
|
||||
'--exists-action',
|
||||
dest='exists_action',
|
||||
type='choice',
|
||||
choices=['s', 'i', 'w', 'b', 'a'],
|
||||
default=[],
|
||||
action='append',
|
||||
metavar='action',
|
||||
help="Default action when a path already exists: "
|
||||
"(s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort).",
|
||||
)
|
||||
|
||||
|
||||
cert = partial(
|
||||
Option,
|
||||
'--cert',
|
||||
dest='cert',
|
||||
type='str',
|
||||
metavar='path',
|
||||
help="Path to alternate CA bundle.",
|
||||
) # type: Any
|
||||
|
||||
client_cert = partial(
|
||||
Option,
|
||||
'--client-cert',
|
||||
dest='client_cert',
|
||||
type='str',
|
||||
default=None,
|
||||
metavar='path',
|
||||
help="Path to SSL client certificate, a single file containing the "
|
||||
"private key and the certificate in PEM format.",
|
||||
) # type: Any
|
||||
|
||||
index_url = partial(
|
||||
Option,
|
||||
'-i', '--index-url', '--pypi-url',
|
||||
dest='index_url',
|
||||
metavar='URL',
|
||||
default=PyPI.simple_url,
|
||||
help="Base URL of Python Package Index (default %default). "
|
||||
"This should point to a repository compliant with PEP 503 "
|
||||
"(the simple repository API) or a local directory laid out "
|
||||
"in the same format.",
|
||||
) # type: Any
|
||||
|
||||
|
||||
def extra_index_url():
|
||||
return Option(
|
||||
'--extra-index-url',
|
||||
dest='extra_index_urls',
|
||||
metavar='URL',
|
||||
action='append',
|
||||
default=[],
|
||||
help="Extra URLs of package indexes to use in addition to "
|
||||
"--index-url. Should follow the same rules as "
|
||||
"--index-url.",
|
||||
)
|
||||
|
||||
|
||||
no_index = partial(
|
||||
Option,
|
||||
'--no-index',
|
||||
dest='no_index',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Ignore package index (only looking at --find-links URLs instead).',
|
||||
) # type: Any
|
||||
|
||||
|
||||
def find_links():
|
||||
return Option(
|
||||
'-f', '--find-links',
|
||||
dest='find_links',
|
||||
action='append',
|
||||
default=[],
|
||||
metavar='url',
|
||||
help="If a url or path to an html file, then parse for links to "
|
||||
"archives. If a local path or file:// url that's a directory, "
|
||||
"then look for archives in the directory listing.",
|
||||
)
|
||||
|
||||
|
||||
def trusted_host():
|
||||
return Option(
|
||||
"--trusted-host",
|
||||
dest="trusted_hosts",
|
||||
action="append",
|
||||
metavar="HOSTNAME",
|
||||
default=[],
|
||||
help="Mark this host as trusted, even though it does not have valid "
|
||||
"or any HTTPS.",
|
||||
)
|
||||
|
||||
|
||||
# Remove after 1.5
|
||||
process_dependency_links = partial(
|
||||
Option,
|
||||
"--process-dependency-links",
|
||||
dest="process_dependency_links",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enable the processing of dependency links.",
|
||||
) # type: Any
|
||||
|
||||
|
||||
def constraints():
|
||||
return Option(
|
||||
'-c', '--constraint',
|
||||
dest='constraints',
|
||||
action='append',
|
||||
default=[],
|
||||
metavar='file',
|
||||
help='Constrain versions using the given constraints file. '
|
||||
'This option can be used multiple times.'
|
||||
)
|
||||
|
||||
|
||||
def requirements():
|
||||
return Option(
|
||||
'-r', '--requirement',
|
||||
dest='requirements',
|
||||
action='append',
|
||||
default=[],
|
||||
metavar='file',
|
||||
help='Install from the given requirements file. '
|
||||
'This option can be used multiple times.'
|
||||
)
|
||||
|
||||
|
||||
def editable():
|
||||
return Option(
|
||||
'-e', '--editable',
|
||||
dest='editables',
|
||||
action='append',
|
||||
default=[],
|
||||
metavar='path/url',
|
||||
help=('Install a project in editable mode (i.e. setuptools '
|
||||
'"develop mode") from a local project path or a VCS url.'),
|
||||
)
|
||||
|
||||
|
||||
src = partial(
|
||||
Option,
|
||||
'--src', '--source', '--source-dir', '--source-directory',
|
||||
dest='src_dir',
|
||||
metavar='dir',
|
||||
default=src_prefix,
|
||||
help='Directory to check out editable projects into. '
|
||||
'The default in a virtualenv is "<venv path>/src". '
|
||||
'The default for global installs is "<current dir>/src".'
|
||||
) # type: Any
|
||||
|
||||
|
||||
def _get_format_control(values, option):
|
||||
"""Get a format_control object."""
|
||||
return getattr(values, option.dest)
|
||||
|
||||
|
||||
def _handle_no_binary(option, opt_str, value, parser):
|
||||
existing = getattr(parser.values, option.dest)
|
||||
fmt_ctl_handle_mutual_exclude(
|
||||
value, existing.no_binary, existing.only_binary,
|
||||
)
|
||||
|
||||
|
||||
def _handle_only_binary(option, opt_str, value, parser):
|
||||
existing = getattr(parser.values, option.dest)
|
||||
fmt_ctl_handle_mutual_exclude(
|
||||
value, existing.only_binary, existing.no_binary,
|
||||
)
|
||||
|
||||
|
||||
def no_binary():
|
||||
return Option(
|
||||
"--no-binary", dest="format_control", action="callback",
|
||||
callback=_handle_no_binary, type="str",
|
||||
default=FormatControl(set(), set()),
|
||||
help="Do not use binary packages. Can be supplied multiple times, and "
|
||||
"each time adds to the existing value. Accepts either :all: to "
|
||||
"disable all binary packages, :none: to empty the set, or one or "
|
||||
"more package names with commas between them. Note that some "
|
||||
"packages are tricky to compile and may fail to install when "
|
||||
"this option is used on them.",
|
||||
)
|
||||
|
||||
|
||||
def only_binary():
|
||||
return Option(
|
||||
"--only-binary", dest="format_control", action="callback",
|
||||
callback=_handle_only_binary, type="str",
|
||||
default=FormatControl(set(), set()),
|
||||
help="Do not use source packages. Can be supplied multiple times, and "
|
||||
"each time adds to the existing value. Accepts either :all: to "
|
||||
"disable all source packages, :none: to empty the set, or one or "
|
||||
"more package names with commas between them. Packages without "
|
||||
"binary distributions will fail to install when this option is "
|
||||
"used on them.",
|
||||
)
|
||||
|
||||
|
||||
cache_dir = partial(
|
||||
Option,
|
||||
"--cache-dir",
|
||||
dest="cache_dir",
|
||||
default=USER_CACHE_DIR,
|
||||
metavar="dir",
|
||||
help="Store the cache data in <dir>."
|
||||
)
|
||||
|
||||
no_cache = partial(
|
||||
Option,
|
||||
"--no-cache-dir",
|
||||
dest="cache_dir",
|
||||
action="store_false",
|
||||
help="Disable the cache.",
|
||||
)
|
||||
|
||||
no_deps = partial(
|
||||
Option,
|
||||
'--no-deps', '--no-dependencies',
|
||||
dest='ignore_dependencies',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="Don't install package dependencies.",
|
||||
) # type: Any
|
||||
|
||||
build_dir = partial(
|
||||
Option,
|
||||
'-b', '--build', '--build-dir', '--build-directory',
|
||||
dest='build_dir',
|
||||
metavar='dir',
|
||||
help='Directory to unpack packages into and build in. Note that '
|
||||
'an initial build still takes place in a temporary directory. '
|
||||
'The location of temporary directories can be controlled by setting '
|
||||
'the TMPDIR environment variable (TEMP on Windows) appropriately. '
|
||||
'When passed, build directories are not cleaned in case of failures.'
|
||||
) # type: Any
|
||||
|
||||
ignore_requires_python = partial(
|
||||
Option,
|
||||
'--ignore-requires-python',
|
||||
dest='ignore_requires_python',
|
||||
action='store_true',
|
||||
help='Ignore the Requires-Python information.'
|
||||
) # type: Any
|
||||
|
||||
no_build_isolation = partial(
|
||||
Option,
|
||||
'--no-build-isolation',
|
||||
dest='build_isolation',
|
||||
action='store_false',
|
||||
default=True,
|
||||
help='Disable isolation when building a modern source distribution. '
|
||||
'Build dependencies specified by PEP 518 must be already installed '
|
||||
'if this option is used.'
|
||||
) # type: Any
|
||||
|
||||
install_options = partial(
|
||||
Option,
|
||||
'--install-option',
|
||||
dest='install_options',
|
||||
action='append',
|
||||
metavar='options',
|
||||
help="Extra arguments to be supplied to the setup.py install "
|
||||
"command (use like --install-option=\"--install-scripts=/usr/local/"
|
||||
"bin\"). Use multiple --install-option options to pass multiple "
|
||||
"options to setup.py install. If you are using an option with a "
|
||||
"directory path, be sure to use absolute path.",
|
||||
) # type: Any
|
||||
|
||||
global_options = partial(
|
||||
Option,
|
||||
'--global-option',
|
||||
dest='global_options',
|
||||
action='append',
|
||||
metavar='options',
|
||||
help="Extra global options to be supplied to the setup.py "
|
||||
"call before the install command.",
|
||||
) # type: Any
|
||||
|
||||
no_clean = partial(
|
||||
Option,
|
||||
'--no-clean',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="Don't clean up build directories)."
|
||||
) # type: Any
|
||||
|
||||
pre = partial(
|
||||
Option,
|
||||
'--pre',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="Include pre-release and development versions. By default, "
|
||||
"pip only finds stable versions.",
|
||||
) # type: Any
|
||||
|
||||
disable_pip_version_check = partial(
|
||||
Option,
|
||||
"--disable-pip-version-check",
|
||||
dest="disable_pip_version_check",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't periodically check PyPI to determine whether a new version "
|
||||
"of pip is available for download. Implied with --no-index.",
|
||||
) # type: Any
|
||||
|
||||
|
||||
# Deprecated, Remove later
|
||||
always_unzip = partial(
|
||||
Option,
|
||||
'-Z', '--always-unzip',
|
||||
dest='always_unzip',
|
||||
action='store_true',
|
||||
help=SUPPRESS_HELP,
|
||||
) # type: Any
|
||||
|
||||
|
||||
def _merge_hash(option, opt_str, value, parser):
|
||||
"""Given a value spelled "algo:digest", append the digest to a list
|
||||
pointed to in a dict by the algo name."""
|
||||
if not parser.values.hashes:
|
||||
parser.values.hashes = {}
|
||||
try:
|
||||
algo, digest = value.split(':', 1)
|
||||
except ValueError:
|
||||
parser.error('Arguments to %s must be a hash name '
|
||||
'followed by a value, like --hash=sha256:abcde...' %
|
||||
opt_str)
|
||||
if algo not in STRONG_HASHES:
|
||||
parser.error('Allowed hash algorithms for %s are %s.' %
|
||||
(opt_str, ', '.join(STRONG_HASHES)))
|
||||
parser.values.hashes.setdefault(algo, []).append(digest)
|
||||
|
||||
|
||||
hash = partial(
|
||||
Option,
|
||||
'--hash',
|
||||
# Hash values eventually end up in InstallRequirement.hashes due to
|
||||
# __dict__ copying in process_line().
|
||||
dest='hashes',
|
||||
action='callback',
|
||||
callback=_merge_hash,
|
||||
type='string',
|
||||
help="Verify that the package's archive matches this "
|
||||
'hash before installing. Example: --hash=sha256:abcdef...',
|
||||
) # type: Any
|
||||
|
||||
|
||||
require_hashes = partial(
|
||||
Option,
|
||||
'--require-hashes',
|
||||
dest='require_hashes',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Require a hash to check each requirement against, for '
|
||||
'repeatable installs. This option is implied when any package in a '
|
||||
'requirements file has a --hash option.',
|
||||
) # type: Any
|
||||
|
||||
|
||||
##########
|
||||
# groups #
|
||||
##########
|
||||
|
||||
general_group = {
|
||||
'name': 'General Options',
|
||||
'options': [
|
||||
help_,
|
||||
isolated_mode,
|
||||
require_virtualenv,
|
||||
verbose,
|
||||
version,
|
||||
quiet,
|
||||
log,
|
||||
no_input,
|
||||
proxy,
|
||||
retries,
|
||||
timeout,
|
||||
skip_requirements_regex,
|
||||
exists_action,
|
||||
trusted_host,
|
||||
cert,
|
||||
client_cert,
|
||||
cache_dir,
|
||||
no_cache,
|
||||
disable_pip_version_check,
|
||||
no_color,
|
||||
]
|
||||
}
|
||||
|
||||
index_group = {
|
||||
'name': 'Package Index Options',
|
||||
'options': [
|
||||
index_url,
|
||||
extra_index_url,
|
||||
no_index,
|
||||
find_links,
|
||||
process_dependency_links,
|
||||
]
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
"""
|
||||
Package containing all pip commands
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
from pip._internal.commands.completion import CompletionCommand
|
||||
from pip._internal.commands.configuration import ConfigurationCommand
|
||||
from pip._internal.commands.download import DownloadCommand
|
||||
from pip._internal.commands.freeze import FreezeCommand
|
||||
from pip._internal.commands.hash import HashCommand
|
||||
from pip._internal.commands.help import HelpCommand
|
||||
from pip._internal.commands.list import ListCommand
|
||||
from pip._internal.commands.check import CheckCommand
|
||||
from pip._internal.commands.search import SearchCommand
|
||||
from pip._internal.commands.show import ShowCommand
|
||||
from pip._internal.commands.install import InstallCommand
|
||||
from pip._internal.commands.uninstall import UninstallCommand
|
||||
from pip._internal.commands.wheel import WheelCommand
|
||||
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from typing import List, Type
|
||||
from pip._internal.basecommand import Command
|
||||
|
||||
commands_order = [
|
||||
InstallCommand,
|
||||
DownloadCommand,
|
||||
UninstallCommand,
|
||||
FreezeCommand,
|
||||
ListCommand,
|
||||
ShowCommand,
|
||||
CheckCommand,
|
||||
ConfigurationCommand,
|
||||
SearchCommand,
|
||||
WheelCommand,
|
||||
HashCommand,
|
||||
CompletionCommand,
|
||||
HelpCommand,
|
||||
] # type: List[Type[Command]]
|
||||
|
||||
commands_dict = {c.name: c for c in commands_order}
|
||||
|
||||
|
||||
def get_summaries(ordered=True):
|
||||
"""Yields sorted (command name, command summary) tuples."""
|
||||
|
||||
if ordered:
|
||||
cmditems = _sort_commands(commands_dict, commands_order)
|
||||
else:
|
||||
cmditems = commands_dict.items()
|
||||
|
||||
for name, command_class in cmditems:
|
||||
yield (name, command_class.summary)
|
||||
|
||||
|
||||
def get_similar_commands(name):
|
||||
"""Command name auto-correct."""
|
||||
from difflib import get_close_matches
|
||||
|
||||
name = name.lower()
|
||||
|
||||
close_commands = get_close_matches(name, commands_dict.keys())
|
||||
|
||||
if close_commands:
|
||||
return close_commands[0]
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def _sort_commands(cmddict, order):
|
||||
def keyfn(key):
|
||||
try:
|
||||
return order.index(key[1])
|
||||
except ValueError:
|
||||
# unordered items should come last
|
||||
return 0xff
|
||||
|
||||
return sorted(cmddict.items(), key=keyfn)
|
||||
@@ -1,42 +0,0 @@
|
||||
import logging
|
||||
|
||||
from pip._internal.basecommand import Command
|
||||
from pip._internal.operations.check import (
|
||||
check_package_set, create_package_set_from_installed,
|
||||
)
|
||||
from pip._internal.utils.misc import get_installed_distributions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CheckCommand(Command):
|
||||
"""Verify installed packages have compatible dependencies."""
|
||||
name = 'check'
|
||||
usage = """
|
||||
%prog [options]"""
|
||||
summary = 'Verify installed packages have compatible dependencies.'
|
||||
|
||||
def run(self, options, args):
|
||||
package_set = create_package_set_from_installed()
|
||||
missing, conflicting = check_package_set(package_set)
|
||||
|
||||
for project_name in missing:
|
||||
version = package_set[project_name].version
|
||||
for dependency in missing[project_name]:
|
||||
logger.info(
|
||||
"%s %s requires %s, which is not installed.",
|
||||
project_name, version, dependency[0],
|
||||
)
|
||||
|
||||
for project_name in conflicting:
|
||||
version = package_set[project_name].version
|
||||
for dep_name, dep_version, req in conflicting[project_name]:
|
||||
logger.info(
|
||||
"%s %s has requirement %s, but you have %s %s.",
|
||||
project_name, version, req, dep_name, dep_version,
|
||||
)
|
||||
|
||||
if missing or conflicting:
|
||||
return 1
|
||||
else:
|
||||
logger.info("No broken requirements found.")
|
||||
@@ -1,94 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
from pip._internal.basecommand import Command
|
||||
from pip._internal.utils.misc import get_prog
|
||||
|
||||
BASE_COMPLETION = """
|
||||
# pip %(shell)s completion start%(script)s# pip %(shell)s completion end
|
||||
"""
|
||||
|
||||
COMPLETION_SCRIPTS = {
|
||||
'bash': """
|
||||
_pip_completion()
|
||||
{
|
||||
COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\
|
||||
COMP_CWORD=$COMP_CWORD \\
|
||||
PIP_AUTO_COMPLETE=1 $1 ) )
|
||||
}
|
||||
complete -o default -F _pip_completion %(prog)s
|
||||
""",
|
||||
'zsh': """
|
||||
function _pip_completion {
|
||||
local words cword
|
||||
read -Ac words
|
||||
read -cn cword
|
||||
reply=( $( COMP_WORDS="$words[*]" \\
|
||||
COMP_CWORD=$(( cword-1 )) \\
|
||||
PIP_AUTO_COMPLETE=1 $words[1] ) )
|
||||
}
|
||||
compctl -K _pip_completion %(prog)s
|
||||
""",
|
||||
'fish': """
|
||||
function __fish_complete_pip
|
||||
set -lx COMP_WORDS (commandline -o) ""
|
||||
set -lx COMP_CWORD ( \\
|
||||
math (contains -i -- (commandline -t) $COMP_WORDS)-1 \\
|
||||
)
|
||||
set -lx PIP_AUTO_COMPLETE 1
|
||||
string split \\ -- (eval $COMP_WORDS[1])
|
||||
end
|
||||
complete -fa "(__fish_complete_pip)" -c %(prog)s
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
class CompletionCommand(Command):
|
||||
"""A helper command to be used for command completion."""
|
||||
name = 'completion'
|
||||
summary = 'A helper command used for command completion.'
|
||||
ignore_require_venv = True
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(CompletionCommand, self).__init__(*args, **kw)
|
||||
|
||||
cmd_opts = self.cmd_opts
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--bash', '-b',
|
||||
action='store_const',
|
||||
const='bash',
|
||||
dest='shell',
|
||||
help='Emit completion code for bash')
|
||||
cmd_opts.add_option(
|
||||
'--zsh', '-z',
|
||||
action='store_const',
|
||||
const='zsh',
|
||||
dest='shell',
|
||||
help='Emit completion code for zsh')
|
||||
cmd_opts.add_option(
|
||||
'--fish', '-f',
|
||||
action='store_const',
|
||||
const='fish',
|
||||
dest='shell',
|
||||
help='Emit completion code for fish')
|
||||
|
||||
self.parser.insert_option_group(0, cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
"""Prints the completion code of the given shell"""
|
||||
shells = COMPLETION_SCRIPTS.keys()
|
||||
shell_options = ['--' + shell for shell in sorted(shells)]
|
||||
if options.shell in shells:
|
||||
script = textwrap.dedent(
|
||||
COMPLETION_SCRIPTS.get(options.shell, '') % {
|
||||
'prog': get_prog(),
|
||||
}
|
||||
)
|
||||
print(BASE_COMPLETION % {'script': script, 'shell': options.shell})
|
||||
else:
|
||||
sys.stderr.write(
|
||||
'ERROR: You must pass %s\n' % ' or '.join(shell_options)
|
||||
)
|
||||
@@ -1,227 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from pip._internal.basecommand import Command
|
||||
from pip._internal.configuration import Configuration, kinds
|
||||
from pip._internal.exceptions import PipError
|
||||
from pip._internal.locations import venv_config_file
|
||||
from pip._internal.status_codes import ERROR, SUCCESS
|
||||
from pip._internal.utils.misc import get_prog
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConfigurationCommand(Command):
|
||||
"""Manage local and global configuration.
|
||||
|
||||
Subcommands:
|
||||
|
||||
list: List the active configuration (or from the file specified)
|
||||
edit: Edit the configuration file in an editor
|
||||
get: Get the value associated with name
|
||||
set: Set the name=value
|
||||
unset: Unset the value associated with name
|
||||
|
||||
If none of --user, --global and --venv are passed, a virtual
|
||||
environment configuration file is used if one is active and the file
|
||||
exists. Otherwise, all modifications happen on the to the user file by
|
||||
default.
|
||||
"""
|
||||
|
||||
name = 'config'
|
||||
usage = """
|
||||
%prog [<file-option>] list
|
||||
%prog [<file-option>] [--editor <editor-path>] edit
|
||||
|
||||
%prog [<file-option>] get name
|
||||
%prog [<file-option>] set name value
|
||||
%prog [<file-option>] unset name
|
||||
"""
|
||||
|
||||
summary = "Manage local and global configuration."
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(ConfigurationCommand, self).__init__(*args, **kwargs)
|
||||
|
||||
self.configuration = None
|
||||
|
||||
self.cmd_opts.add_option(
|
||||
'--editor',
|
||||
dest='editor',
|
||||
action='store',
|
||||
default=None,
|
||||
help=(
|
||||
'Editor to use to edit the file. Uses VISUAL or EDITOR '
|
||||
'environment variables if not provided.'
|
||||
)
|
||||
)
|
||||
|
||||
self.cmd_opts.add_option(
|
||||
'--global',
|
||||
dest='global_file',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use the system-wide configuration file only'
|
||||
)
|
||||
|
||||
self.cmd_opts.add_option(
|
||||
'--user',
|
||||
dest='user_file',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use the user configuration file only'
|
||||
)
|
||||
|
||||
self.cmd_opts.add_option(
|
||||
'--venv',
|
||||
dest='venv_file',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use the virtualenv configuration file only'
|
||||
)
|
||||
|
||||
self.parser.insert_option_group(0, self.cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
handlers = {
|
||||
"list": self.list_values,
|
||||
"edit": self.open_in_editor,
|
||||
"get": self.get_name,
|
||||
"set": self.set_name_value,
|
||||
"unset": self.unset_name
|
||||
}
|
||||
|
||||
# Determine action
|
||||
if not args or args[0] not in handlers:
|
||||
logger.error("Need an action ({}) to perform.".format(
|
||||
", ".join(sorted(handlers)))
|
||||
)
|
||||
return ERROR
|
||||
|
||||
action = args[0]
|
||||
|
||||
# Determine which configuration files are to be loaded
|
||||
# Depends on whether the command is modifying.
|
||||
try:
|
||||
load_only = self._determine_file(
|
||||
options, need_value=(action in ["get", "set", "unset", "edit"])
|
||||
)
|
||||
except PipError as e:
|
||||
logger.error(e.args[0])
|
||||
return ERROR
|
||||
|
||||
# Load a new configuration
|
||||
self.configuration = Configuration(
|
||||
isolated=options.isolated_mode, load_only=load_only
|
||||
)
|
||||
self.configuration.load()
|
||||
|
||||
# Error handling happens here, not in the action-handlers.
|
||||
try:
|
||||
handlers[action](options, args[1:])
|
||||
except PipError as e:
|
||||
logger.error(e.args[0])
|
||||
return ERROR
|
||||
|
||||
return SUCCESS
|
||||
|
||||
def _determine_file(self, options, need_value):
|
||||
file_options = {
|
||||
kinds.USER: options.user_file,
|
||||
kinds.GLOBAL: options.global_file,
|
||||
kinds.VENV: options.venv_file
|
||||
}
|
||||
|
||||
if sum(file_options.values()) == 0:
|
||||
if not need_value:
|
||||
return None
|
||||
# Default to user, unless there's a virtualenv file.
|
||||
elif os.path.exists(venv_config_file):
|
||||
return kinds.VENV
|
||||
else:
|
||||
return kinds.USER
|
||||
elif sum(file_options.values()) == 1:
|
||||
# There's probably a better expression for this.
|
||||
return [key for key in file_options if file_options[key]][0]
|
||||
|
||||
raise PipError(
|
||||
"Need exactly one file to operate upon "
|
||||
"(--user, --venv, --global) to perform."
|
||||
)
|
||||
|
||||
def list_values(self, options, args):
|
||||
self._get_n_args(args, "list", n=0)
|
||||
|
||||
for key, value in sorted(self.configuration.items()):
|
||||
logger.info("%s=%r", key, value)
|
||||
|
||||
def get_name(self, options, args):
|
||||
key = self._get_n_args(args, "get [name]", n=1)
|
||||
value = self.configuration.get_value(key)
|
||||
|
||||
logger.info("%s", value)
|
||||
|
||||
def set_name_value(self, options, args):
|
||||
key, value = self._get_n_args(args, "set [name] [value]", n=2)
|
||||
self.configuration.set_value(key, value)
|
||||
|
||||
self._save_configuration()
|
||||
|
||||
def unset_name(self, options, args):
|
||||
key = self._get_n_args(args, "unset [name]", n=1)
|
||||
self.configuration.unset_value(key)
|
||||
|
||||
self._save_configuration()
|
||||
|
||||
def open_in_editor(self, options, args):
|
||||
editor = self._determine_editor(options)
|
||||
|
||||
fname = self.configuration.get_file_to_edit()
|
||||
if fname is None:
|
||||
raise PipError("Could not determine appropriate file.")
|
||||
|
||||
try:
|
||||
subprocess.check_call([editor, fname])
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise PipError(
|
||||
"Editor Subprocess exited with exit code {}"
|
||||
.format(e.returncode)
|
||||
)
|
||||
|
||||
def _get_n_args(self, args, example, n):
|
||||
"""Helper to make sure the command got the right number of arguments
|
||||
"""
|
||||
if len(args) != n:
|
||||
msg = (
|
||||
'Got unexpected number of arguments, expected {}. '
|
||||
'(example: "{} config {}")'
|
||||
).format(n, get_prog(), example)
|
||||
raise PipError(msg)
|
||||
|
||||
if n == 1:
|
||||
return args[0]
|
||||
else:
|
||||
return args
|
||||
|
||||
def _save_configuration(self):
|
||||
# We successfully ran a modifying command. Need to save the
|
||||
# configuration.
|
||||
try:
|
||||
self.configuration.save()
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Unable to save configuration. Please report this as a bug.",
|
||||
exc_info=1
|
||||
)
|
||||
raise PipError("Internal Error.")
|
||||
|
||||
def _determine_editor(self, options):
|
||||
if options.editor is not None:
|
||||
return options.editor
|
||||
elif "VISUAL" in os.environ:
|
||||
return os.environ["VISUAL"]
|
||||
elif "EDITOR" in os.environ:
|
||||
return os.environ["EDITOR"]
|
||||
else:
|
||||
raise PipError("Could not determine editor to use.")
|
||||
@@ -1,233 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from pip._internal import cmdoptions
|
||||
from pip._internal.basecommand import RequirementCommand
|
||||
from pip._internal.exceptions import CommandError
|
||||
from pip._internal.index import FormatControl
|
||||
from pip._internal.operations.prepare import RequirementPreparer
|
||||
from pip._internal.req import RequirementSet
|
||||
from pip._internal.resolve import Resolver
|
||||
from pip._internal.utils.filesystem import check_path_owner
|
||||
from pip._internal.utils.misc import ensure_dir, normalize_path
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadCommand(RequirementCommand):
|
||||
"""
|
||||
Download packages from:
|
||||
|
||||
- PyPI (and other indexes) using requirement specifiers.
|
||||
- VCS project urls.
|
||||
- Local project directories.
|
||||
- Local or remote source archives.
|
||||
|
||||
pip also supports downloading from "requirements files", which provide
|
||||
an easy way to specify a whole environment to be downloaded.
|
||||
"""
|
||||
name = 'download'
|
||||
|
||||
usage = """
|
||||
%prog [options] <requirement specifier> [package-index-options] ...
|
||||
%prog [options] -r <requirements file> [package-index-options] ...
|
||||
%prog [options] <vcs project url> ...
|
||||
%prog [options] <local project path> ...
|
||||
%prog [options] <archive url/path> ..."""
|
||||
|
||||
summary = 'Download packages.'
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(DownloadCommand, self).__init__(*args, **kw)
|
||||
|
||||
cmd_opts = self.cmd_opts
|
||||
|
||||
cmd_opts.add_option(cmdoptions.constraints())
|
||||
cmd_opts.add_option(cmdoptions.requirements())
|
||||
cmd_opts.add_option(cmdoptions.build_dir())
|
||||
cmd_opts.add_option(cmdoptions.no_deps())
|
||||
cmd_opts.add_option(cmdoptions.global_options())
|
||||
cmd_opts.add_option(cmdoptions.no_binary())
|
||||
cmd_opts.add_option(cmdoptions.only_binary())
|
||||
cmd_opts.add_option(cmdoptions.src())
|
||||
cmd_opts.add_option(cmdoptions.pre())
|
||||
cmd_opts.add_option(cmdoptions.no_clean())
|
||||
cmd_opts.add_option(cmdoptions.require_hashes())
|
||||
cmd_opts.add_option(cmdoptions.progress_bar())
|
||||
cmd_opts.add_option(cmdoptions.no_build_isolation())
|
||||
|
||||
cmd_opts.add_option(
|
||||
'-d', '--dest', '--destination-dir', '--destination-directory',
|
||||
dest='download_dir',
|
||||
metavar='dir',
|
||||
default=os.curdir,
|
||||
help=("Download packages into <dir>."),
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--platform',
|
||||
dest='platform',
|
||||
metavar='platform',
|
||||
default=None,
|
||||
help=("Only download wheels compatible with <platform>. "
|
||||
"Defaults to the platform of the running system."),
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--python-version',
|
||||
dest='python_version',
|
||||
metavar='python_version',
|
||||
default=None,
|
||||
help=("Only download wheels compatible with Python "
|
||||
"interpreter version <version>. If not specified, then the "
|
||||
"current system interpreter minor version is used. A major "
|
||||
"version (e.g. '2') can be specified to match all "
|
||||
"minor revs of that major version. A minor version "
|
||||
"(e.g. '34') can also be specified."),
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--implementation',
|
||||
dest='implementation',
|
||||
metavar='implementation',
|
||||
default=None,
|
||||
help=("Only download wheels compatible with Python "
|
||||
"implementation <implementation>, e.g. 'pp', 'jy', 'cp', "
|
||||
" or 'ip'. If not specified, then the current "
|
||||
"interpreter implementation is used. Use 'py' to force "
|
||||
"implementation-agnostic wheels."),
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--abi',
|
||||
dest='abi',
|
||||
metavar='abi',
|
||||
default=None,
|
||||
help=("Only download wheels compatible with Python "
|
||||
"abi <abi>, e.g. 'pypy_41'. If not specified, then the "
|
||||
"current interpreter abi tag is used. Generally "
|
||||
"you will need to specify --implementation, "
|
||||
"--platform, and --python-version when using "
|
||||
"this option."),
|
||||
)
|
||||
|
||||
index_opts = cmdoptions.make_option_group(
|
||||
cmdoptions.index_group,
|
||||
self.parser,
|
||||
)
|
||||
|
||||
self.parser.insert_option_group(0, index_opts)
|
||||
self.parser.insert_option_group(0, cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
options.ignore_installed = True
|
||||
# editable doesn't really make sense for `pip download`, but the bowels
|
||||
# of the RequirementSet code require that property.
|
||||
options.editables = []
|
||||
|
||||
if options.python_version:
|
||||
python_versions = [options.python_version]
|
||||
else:
|
||||
python_versions = None
|
||||
|
||||
dist_restriction_set = any([
|
||||
options.python_version,
|
||||
options.platform,
|
||||
options.abi,
|
||||
options.implementation,
|
||||
])
|
||||
binary_only = FormatControl(set(), {':all:'})
|
||||
no_sdist_dependencies = (
|
||||
options.format_control != binary_only and
|
||||
not options.ignore_dependencies
|
||||
)
|
||||
if dist_restriction_set and no_sdist_dependencies:
|
||||
raise CommandError(
|
||||
"When restricting platform and interpreter constraints using "
|
||||
"--python-version, --platform, --abi, or --implementation, "
|
||||
"either --no-deps must be set, or --only-binary=:all: must be "
|
||||
"set and --no-binary must not be set (or must be set to "
|
||||
":none:)."
|
||||
)
|
||||
|
||||
options.src_dir = os.path.abspath(options.src_dir)
|
||||
options.download_dir = normalize_path(options.download_dir)
|
||||
|
||||
ensure_dir(options.download_dir)
|
||||
|
||||
with self._build_session(options) as session:
|
||||
finder = self._build_package_finder(
|
||||
options=options,
|
||||
session=session,
|
||||
platform=options.platform,
|
||||
python_versions=python_versions,
|
||||
abi=options.abi,
|
||||
implementation=options.implementation,
|
||||
)
|
||||
build_delete = (not (options.no_clean or options.build_dir))
|
||||
if options.cache_dir and not check_path_owner(options.cache_dir):
|
||||
logger.warning(
|
||||
"The directory '%s' or its parent directory is not owned "
|
||||
"by the current user and caching wheels has been "
|
||||
"disabled. check the permissions and owner of that "
|
||||
"directory. If executing pip with sudo, you may want "
|
||||
"sudo's -H flag.",
|
||||
options.cache_dir,
|
||||
)
|
||||
options.cache_dir = None
|
||||
|
||||
with TempDirectory(
|
||||
options.build_dir, delete=build_delete, kind="download"
|
||||
) as directory:
|
||||
|
||||
requirement_set = RequirementSet(
|
||||
require_hashes=options.require_hashes,
|
||||
)
|
||||
self.populate_requirement_set(
|
||||
requirement_set,
|
||||
args,
|
||||
options,
|
||||
finder,
|
||||
session,
|
||||
self.name,
|
||||
None
|
||||
)
|
||||
|
||||
preparer = RequirementPreparer(
|
||||
build_dir=directory.path,
|
||||
src_dir=options.src_dir,
|
||||
download_dir=options.download_dir,
|
||||
wheel_download_dir=None,
|
||||
progress_bar=options.progress_bar,
|
||||
build_isolation=options.build_isolation,
|
||||
)
|
||||
|
||||
resolver = Resolver(
|
||||
preparer=preparer,
|
||||
finder=finder,
|
||||
session=session,
|
||||
wheel_cache=None,
|
||||
use_user_site=False,
|
||||
upgrade_strategy="to-satisfy-only",
|
||||
force_reinstall=False,
|
||||
ignore_dependencies=options.ignore_dependencies,
|
||||
ignore_requires_python=False,
|
||||
ignore_installed=True,
|
||||
isolated=options.isolated_mode,
|
||||
)
|
||||
resolver.resolve(requirement_set)
|
||||
|
||||
downloaded = ' '.join([
|
||||
req.name for req in requirement_set.successfully_downloaded
|
||||
])
|
||||
if downloaded:
|
||||
logger.info('Successfully downloaded %s', downloaded)
|
||||
|
||||
# Clean up
|
||||
if not options.no_clean:
|
||||
requirement_set.cleanup_files()
|
||||
|
||||
return requirement_set
|
||||
@@ -1,96 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import sys
|
||||
|
||||
from pip._internal import index
|
||||
from pip._internal.basecommand import Command
|
||||
from pip._internal.cache import WheelCache
|
||||
from pip._internal.compat import stdlib_pkgs
|
||||
from pip._internal.operations.freeze import freeze
|
||||
|
||||
DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel'}
|
||||
|
||||
|
||||
class FreezeCommand(Command):
|
||||
"""
|
||||
Output installed packages in requirements format.
|
||||
|
||||
packages are listed in a case-insensitive sorted order.
|
||||
"""
|
||||
name = 'freeze'
|
||||
usage = """
|
||||
%prog [options]"""
|
||||
summary = 'Output installed packages in requirements format.'
|
||||
log_streams = ("ext://sys.stderr", "ext://sys.stderr")
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(FreezeCommand, self).__init__(*args, **kw)
|
||||
|
||||
self.cmd_opts.add_option(
|
||||
'-r', '--requirement',
|
||||
dest='requirements',
|
||||
action='append',
|
||||
default=[],
|
||||
metavar='file',
|
||||
help="Use the order in the given requirements file and its "
|
||||
"comments when generating output. This option can be "
|
||||
"used multiple times.")
|
||||
self.cmd_opts.add_option(
|
||||
'-f', '--find-links',
|
||||
dest='find_links',
|
||||
action='append',
|
||||
default=[],
|
||||
metavar='URL',
|
||||
help='URL for finding packages, which will be added to the '
|
||||
'output.')
|
||||
self.cmd_opts.add_option(
|
||||
'-l', '--local',
|
||||
dest='local',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='If in a virtualenv that has global access, do not output '
|
||||
'globally-installed packages.')
|
||||
self.cmd_opts.add_option(
|
||||
'--user',
|
||||
dest='user',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Only output packages installed in user-site.')
|
||||
self.cmd_opts.add_option(
|
||||
'--all',
|
||||
dest='freeze_all',
|
||||
action='store_true',
|
||||
help='Do not skip these packages in the output:'
|
||||
' %s' % ', '.join(DEV_PKGS))
|
||||
self.cmd_opts.add_option(
|
||||
'--exclude-editable',
|
||||
dest='exclude_editable',
|
||||
action='store_true',
|
||||
help='Exclude editable package from output.')
|
||||
|
||||
self.parser.insert_option_group(0, self.cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
format_control = index.FormatControl(set(), set())
|
||||
wheel_cache = WheelCache(options.cache_dir, format_control)
|
||||
skip = set(stdlib_pkgs)
|
||||
if not options.freeze_all:
|
||||
skip.update(DEV_PKGS)
|
||||
|
||||
freeze_kwargs = dict(
|
||||
requirement=options.requirements,
|
||||
find_links=options.find_links,
|
||||
local_only=options.local,
|
||||
user_only=options.user,
|
||||
skip_regex=options.skip_requirements_regex,
|
||||
isolated=options.isolated_mode,
|
||||
wheel_cache=wheel_cache,
|
||||
skip=skip,
|
||||
exclude_editable=options.exclude_editable,
|
||||
)
|
||||
|
||||
try:
|
||||
for line in freeze(**freeze_kwargs):
|
||||
sys.stdout.write(line + '\n')
|
||||
finally:
|
||||
wheel_cache.cleanup()
|
||||
@@ -1,57 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from pip._internal.basecommand import Command
|
||||
from pip._internal.status_codes import ERROR
|
||||
from pip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES
|
||||
from pip._internal.utils.misc import read_chunks
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashCommand(Command):
|
||||
"""
|
||||
Compute a hash of a local package archive.
|
||||
|
||||
These can be used with --hash in a requirements file to do repeatable
|
||||
installs.
|
||||
|
||||
"""
|
||||
name = 'hash'
|
||||
usage = '%prog [options] <file> ...'
|
||||
summary = 'Compute hashes of package archives.'
|
||||
ignore_require_venv = True
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(HashCommand, self).__init__(*args, **kw)
|
||||
self.cmd_opts.add_option(
|
||||
'-a', '--algorithm',
|
||||
dest='algorithm',
|
||||
choices=STRONG_HASHES,
|
||||
action='store',
|
||||
default=FAVORITE_HASH,
|
||||
help='The hash algorithm to use: one of %s' %
|
||||
', '.join(STRONG_HASHES))
|
||||
self.parser.insert_option_group(0, self.cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
if not args:
|
||||
self.parser.print_usage(sys.stderr)
|
||||
return ERROR
|
||||
|
||||
algorithm = options.algorithm
|
||||
for path in args:
|
||||
logger.info('%s:\n--hash=%s:%s',
|
||||
path, algorithm, _hash_of_file(path, algorithm))
|
||||
|
||||
|
||||
def _hash_of_file(path, algorithm):
|
||||
"""Return the hash digest of a file."""
|
||||
with open(path, 'rb') as archive:
|
||||
hash = hashlib.new(algorithm)
|
||||
for chunk in read_chunks(archive):
|
||||
hash.update(chunk)
|
||||
return hash.hexdigest()
|
||||
@@ -1,36 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
from pip._internal.basecommand import SUCCESS, Command
|
||||
from pip._internal.exceptions import CommandError
|
||||
|
||||
|
||||
class HelpCommand(Command):
|
||||
"""Show help for commands"""
|
||||
name = 'help'
|
||||
usage = """
|
||||
%prog <command>"""
|
||||
summary = 'Show help for commands.'
|
||||
ignore_require_venv = True
|
||||
|
||||
def run(self, options, args):
|
||||
from pip._internal.commands import commands_dict, get_similar_commands
|
||||
|
||||
try:
|
||||
# 'pip help' with no args is handled by pip.__init__.parseopt()
|
||||
cmd_name = args[0] # the command we need help for
|
||||
except IndexError:
|
||||
return SUCCESS
|
||||
|
||||
if cmd_name not in commands_dict:
|
||||
guess = get_similar_commands(cmd_name)
|
||||
|
||||
msg = ['unknown command "%s"' % cmd_name]
|
||||
if guess:
|
||||
msg.append('maybe you meant "%s"' % guess)
|
||||
|
||||
raise CommandError(' - '.join(msg))
|
||||
|
||||
command = commands_dict[cmd_name]()
|
||||
command.parser.print_help()
|
||||
|
||||
return SUCCESS
|
||||
@@ -1,502 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import operator
|
||||
import os
|
||||
import shutil
|
||||
from optparse import SUPPRESS_HELP
|
||||
|
||||
from pip._internal import cmdoptions
|
||||
from pip._internal.basecommand import RequirementCommand
|
||||
from pip._internal.cache import WheelCache
|
||||
from pip._internal.exceptions import (
|
||||
CommandError, InstallationError, PreviousBuildDirError,
|
||||
)
|
||||
from pip._internal.locations import distutils_scheme, virtualenv_no_global
|
||||
from pip._internal.operations.check import check_install_conflicts
|
||||
from pip._internal.operations.prepare import RequirementPreparer
|
||||
from pip._internal.req import RequirementSet, install_given_reqs
|
||||
from pip._internal.resolve import Resolver
|
||||
from pip._internal.status_codes import ERROR
|
||||
from pip._internal.utils.filesystem import check_path_owner
|
||||
from pip._internal.utils.misc import ensure_dir, get_installed_version
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
from pip._internal.wheel import WheelBuilder
|
||||
|
||||
try:
|
||||
import wheel
|
||||
except ImportError:
|
||||
wheel = None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InstallCommand(RequirementCommand):
|
||||
"""
|
||||
Install packages from:
|
||||
|
||||
- PyPI (and other indexes) using requirement specifiers.
|
||||
- VCS project urls.
|
||||
- Local project directories.
|
||||
- Local or remote source archives.
|
||||
|
||||
pip also supports installing from "requirements files", which provide
|
||||
an easy way to specify a whole environment to be installed.
|
||||
"""
|
||||
name = 'install'
|
||||
|
||||
usage = """
|
||||
%prog [options] <requirement specifier> [package-index-options] ...
|
||||
%prog [options] -r <requirements file> [package-index-options] ...
|
||||
%prog [options] [-e] <vcs project url> ...
|
||||
%prog [options] [-e] <local project path> ...
|
||||
%prog [options] <archive url/path> ..."""
|
||||
|
||||
summary = 'Install packages.'
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(InstallCommand, self).__init__(*args, **kw)
|
||||
|
||||
cmd_opts = self.cmd_opts
|
||||
|
||||
cmd_opts.add_option(cmdoptions.requirements())
|
||||
cmd_opts.add_option(cmdoptions.constraints())
|
||||
cmd_opts.add_option(cmdoptions.no_deps())
|
||||
cmd_opts.add_option(cmdoptions.pre())
|
||||
|
||||
cmd_opts.add_option(cmdoptions.editable())
|
||||
cmd_opts.add_option(
|
||||
'-t', '--target',
|
||||
dest='target_dir',
|
||||
metavar='dir',
|
||||
default=None,
|
||||
help='Install packages into <dir>. '
|
||||
'By default this will not replace existing files/folders in '
|
||||
'<dir>. Use --upgrade to replace existing packages in <dir> '
|
||||
'with new versions.'
|
||||
)
|
||||
cmd_opts.add_option(
|
||||
'--user',
|
||||
dest='use_user_site',
|
||||
action='store_true',
|
||||
help="Install to the Python user install directory for your "
|
||||
"platform. Typically ~/.local/, or %APPDATA%\\Python on "
|
||||
"Windows. (See the Python documentation for site.USER_BASE "
|
||||
"for full details.)")
|
||||
cmd_opts.add_option(
|
||||
'--no-user',
|
||||
dest='use_user_site',
|
||||
action='store_false',
|
||||
help=SUPPRESS_HELP)
|
||||
cmd_opts.add_option(
|
||||
'--root',
|
||||
dest='root_path',
|
||||
metavar='dir',
|
||||
default=None,
|
||||
help="Install everything relative to this alternate root "
|
||||
"directory.")
|
||||
cmd_opts.add_option(
|
||||
'--prefix',
|
||||
dest='prefix_path',
|
||||
metavar='dir',
|
||||
default=None,
|
||||
help="Installation prefix where lib, bin and other top-level "
|
||||
"folders are placed")
|
||||
|
||||
cmd_opts.add_option(cmdoptions.build_dir())
|
||||
|
||||
cmd_opts.add_option(cmdoptions.src())
|
||||
|
||||
cmd_opts.add_option(
|
||||
'-U', '--upgrade',
|
||||
dest='upgrade',
|
||||
action='store_true',
|
||||
help='Upgrade all specified packages to the newest available '
|
||||
'version. The handling of dependencies depends on the '
|
||||
'upgrade-strategy used.'
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--upgrade-strategy',
|
||||
dest='upgrade_strategy',
|
||||
default='only-if-needed',
|
||||
choices=['only-if-needed', 'eager'],
|
||||
help='Determines how dependency upgrading should be handled '
|
||||
'[default: %default]. '
|
||||
'"eager" - dependencies are upgraded regardless of '
|
||||
'whether the currently installed version satisfies the '
|
||||
'requirements of the upgraded package(s). '
|
||||
'"only-if-needed" - are upgraded only when they do not '
|
||||
'satisfy the requirements of the upgraded package(s).'
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--force-reinstall',
|
||||
dest='force_reinstall',
|
||||
action='store_true',
|
||||
help='Reinstall all packages even if they are already '
|
||||
'up-to-date.')
|
||||
|
||||
cmd_opts.add_option(
|
||||
'-I', '--ignore-installed',
|
||||
dest='ignore_installed',
|
||||
action='store_true',
|
||||
help='Ignore the installed packages (reinstalling instead).')
|
||||
|
||||
cmd_opts.add_option(cmdoptions.ignore_requires_python())
|
||||
cmd_opts.add_option(cmdoptions.no_build_isolation())
|
||||
|
||||
cmd_opts.add_option(cmdoptions.install_options())
|
||||
cmd_opts.add_option(cmdoptions.global_options())
|
||||
|
||||
cmd_opts.add_option(
|
||||
"--compile",
|
||||
action="store_true",
|
||||
dest="compile",
|
||||
default=True,
|
||||
help="Compile Python source files to bytecode",
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
"--no-compile",
|
||||
action="store_false",
|
||||
dest="compile",
|
||||
help="Do not compile Python source files to bytecode",
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
"--no-warn-script-location",
|
||||
action="store_false",
|
||||
dest="warn_script_location",
|
||||
default=True,
|
||||
help="Do not warn when installing scripts outside PATH",
|
||||
)
|
||||
cmd_opts.add_option(
|
||||
"--no-warn-conflicts",
|
||||
action="store_false",
|
||||
dest="warn_about_conflicts",
|
||||
default=True,
|
||||
help="Do not warn about broken dependencies",
|
||||
)
|
||||
|
||||
cmd_opts.add_option(cmdoptions.no_binary())
|
||||
cmd_opts.add_option(cmdoptions.only_binary())
|
||||
cmd_opts.add_option(cmdoptions.no_clean())
|
||||
cmd_opts.add_option(cmdoptions.require_hashes())
|
||||
cmd_opts.add_option(cmdoptions.progress_bar())
|
||||
|
||||
index_opts = cmdoptions.make_option_group(
|
||||
cmdoptions.index_group,
|
||||
self.parser,
|
||||
)
|
||||
|
||||
self.parser.insert_option_group(0, index_opts)
|
||||
self.parser.insert_option_group(0, cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
cmdoptions.check_install_build_global(options)
|
||||
|
||||
upgrade_strategy = "to-satisfy-only"
|
||||
if options.upgrade:
|
||||
upgrade_strategy = options.upgrade_strategy
|
||||
|
||||
if options.build_dir:
|
||||
options.build_dir = os.path.abspath(options.build_dir)
|
||||
|
||||
options.src_dir = os.path.abspath(options.src_dir)
|
||||
install_options = options.install_options or []
|
||||
if options.use_user_site:
|
||||
if options.prefix_path:
|
||||
raise CommandError(
|
||||
"Can not combine '--user' and '--prefix' as they imply "
|
||||
"different installation locations"
|
||||
)
|
||||
if virtualenv_no_global():
|
||||
raise InstallationError(
|
||||
"Can not perform a '--user' install. User site-packages "
|
||||
"are not visible in this virtualenv."
|
||||
)
|
||||
install_options.append('--user')
|
||||
install_options.append('--prefix=')
|
||||
|
||||
target_temp_dir = TempDirectory(kind="target")
|
||||
if options.target_dir:
|
||||
options.ignore_installed = True
|
||||
options.target_dir = os.path.abspath(options.target_dir)
|
||||
if (os.path.exists(options.target_dir) and not
|
||||
os.path.isdir(options.target_dir)):
|
||||
raise CommandError(
|
||||
"Target path exists but is not a directory, will not "
|
||||
"continue."
|
||||
)
|
||||
|
||||
# Create a target directory for using with the target option
|
||||
target_temp_dir.create()
|
||||
install_options.append('--home=' + target_temp_dir.path)
|
||||
|
||||
global_options = options.global_options or []
|
||||
|
||||
with self._build_session(options) as session:
|
||||
finder = self._build_package_finder(options, session)
|
||||
build_delete = (not (options.no_clean or options.build_dir))
|
||||
wheel_cache = WheelCache(options.cache_dir, options.format_control)
|
||||
|
||||
if options.cache_dir and not check_path_owner(options.cache_dir):
|
||||
logger.warning(
|
||||
"The directory '%s' or its parent directory is not owned "
|
||||
"by the current user and caching wheels has been "
|
||||
"disabled. check the permissions and owner of that "
|
||||
"directory. If executing pip with sudo, you may want "
|
||||
"sudo's -H flag.",
|
||||
options.cache_dir,
|
||||
)
|
||||
options.cache_dir = None
|
||||
|
||||
with TempDirectory(
|
||||
options.build_dir, delete=build_delete, kind="install"
|
||||
) as directory:
|
||||
requirement_set = RequirementSet(
|
||||
require_hashes=options.require_hashes,
|
||||
)
|
||||
|
||||
try:
|
||||
self.populate_requirement_set(
|
||||
requirement_set, args, options, finder, session,
|
||||
self.name, wheel_cache
|
||||
)
|
||||
preparer = RequirementPreparer(
|
||||
build_dir=directory.path,
|
||||
src_dir=options.src_dir,
|
||||
download_dir=None,
|
||||
wheel_download_dir=None,
|
||||
progress_bar=options.progress_bar,
|
||||
build_isolation=options.build_isolation,
|
||||
)
|
||||
|
||||
resolver = Resolver(
|
||||
preparer=preparer,
|
||||
finder=finder,
|
||||
session=session,
|
||||
wheel_cache=wheel_cache,
|
||||
use_user_site=options.use_user_site,
|
||||
upgrade_strategy=upgrade_strategy,
|
||||
force_reinstall=options.force_reinstall,
|
||||
ignore_dependencies=options.ignore_dependencies,
|
||||
ignore_requires_python=options.ignore_requires_python,
|
||||
ignore_installed=options.ignore_installed,
|
||||
isolated=options.isolated_mode,
|
||||
)
|
||||
resolver.resolve(requirement_set)
|
||||
|
||||
# If caching is disabled or wheel is not installed don't
|
||||
# try to build wheels.
|
||||
if wheel and options.cache_dir:
|
||||
# build wheels before install.
|
||||
wb = WheelBuilder(
|
||||
finder, preparer, wheel_cache,
|
||||
build_options=[], global_options=[],
|
||||
)
|
||||
# Ignore the result: a failed wheel will be
|
||||
# installed from the sdist/vcs whatever.
|
||||
wb.build(
|
||||
requirement_set.requirements.values(),
|
||||
session=session, autobuilding=True
|
||||
)
|
||||
|
||||
to_install = resolver.get_installation_order(
|
||||
requirement_set
|
||||
)
|
||||
|
||||
# Consistency Checking of the package set we're installing.
|
||||
should_warn_about_conflicts = (
|
||||
not options.ignore_dependencies and
|
||||
options.warn_about_conflicts
|
||||
)
|
||||
if should_warn_about_conflicts:
|
||||
self._warn_about_conflicts(to_install)
|
||||
|
||||
# Don't warn about script install locations if
|
||||
# --target has been specified
|
||||
warn_script_location = options.warn_script_location
|
||||
if options.target_dir:
|
||||
warn_script_location = False
|
||||
|
||||
installed = install_given_reqs(
|
||||
to_install,
|
||||
install_options,
|
||||
global_options,
|
||||
root=options.root_path,
|
||||
home=target_temp_dir.path,
|
||||
prefix=options.prefix_path,
|
||||
pycompile=options.compile,
|
||||
warn_script_location=warn_script_location,
|
||||
use_user_site=options.use_user_site,
|
||||
)
|
||||
|
||||
possible_lib_locations = get_lib_location_guesses(
|
||||
user=options.use_user_site,
|
||||
home=target_temp_dir.path,
|
||||
root=options.root_path,
|
||||
prefix=options.prefix_path,
|
||||
isolated=options.isolated_mode,
|
||||
)
|
||||
reqs = sorted(installed, key=operator.attrgetter('name'))
|
||||
items = []
|
||||
for req in reqs:
|
||||
item = req.name
|
||||
try:
|
||||
installed_version = get_installed_version(
|
||||
req.name, possible_lib_locations
|
||||
)
|
||||
if installed_version:
|
||||
item += '-' + installed_version
|
||||
except Exception:
|
||||
pass
|
||||
items.append(item)
|
||||
installed = ' '.join(items)
|
||||
if installed:
|
||||
logger.info('Successfully installed %s', installed)
|
||||
except EnvironmentError as error:
|
||||
show_traceback = (self.verbosity >= 1)
|
||||
|
||||
message = create_env_error_message(
|
||||
error, show_traceback, options.use_user_site,
|
||||
)
|
||||
logger.error(message, exc_info=show_traceback)
|
||||
|
||||
return ERROR
|
||||
except PreviousBuildDirError:
|
||||
options.no_clean = True
|
||||
raise
|
||||
finally:
|
||||
# Clean up
|
||||
if not options.no_clean:
|
||||
requirement_set.cleanup_files()
|
||||
wheel_cache.cleanup()
|
||||
|
||||
if options.target_dir:
|
||||
self._handle_target_dir(
|
||||
options.target_dir, target_temp_dir, options.upgrade
|
||||
)
|
||||
return requirement_set
|
||||
|
||||
def _handle_target_dir(self, target_dir, target_temp_dir, upgrade):
|
||||
ensure_dir(target_dir)
|
||||
|
||||
# Checking both purelib and platlib directories for installed
|
||||
# packages to be moved to target directory
|
||||
lib_dir_list = []
|
||||
|
||||
with target_temp_dir:
|
||||
# Checking both purelib and platlib directories for installed
|
||||
# packages to be moved to target directory
|
||||
scheme = distutils_scheme('', home=target_temp_dir.path)
|
||||
purelib_dir = scheme['purelib']
|
||||
platlib_dir = scheme['platlib']
|
||||
data_dir = scheme['data']
|
||||
|
||||
if os.path.exists(purelib_dir):
|
||||
lib_dir_list.append(purelib_dir)
|
||||
if os.path.exists(platlib_dir) and platlib_dir != purelib_dir:
|
||||
lib_dir_list.append(platlib_dir)
|
||||
if os.path.exists(data_dir):
|
||||
lib_dir_list.append(data_dir)
|
||||
|
||||
for lib_dir in lib_dir_list:
|
||||
for item in os.listdir(lib_dir):
|
||||
if lib_dir == data_dir:
|
||||
ddir = os.path.join(data_dir, item)
|
||||
if any(s.startswith(ddir) for s in lib_dir_list[:-1]):
|
||||
continue
|
||||
target_item_dir = os.path.join(target_dir, item)
|
||||
if os.path.exists(target_item_dir):
|
||||
if not upgrade:
|
||||
logger.warning(
|
||||
'Target directory %s already exists. Specify '
|
||||
'--upgrade to force replacement.',
|
||||
target_item_dir
|
||||
)
|
||||
continue
|
||||
if os.path.islink(target_item_dir):
|
||||
logger.warning(
|
||||
'Target directory %s already exists and is '
|
||||
'a link. Pip will not automatically replace '
|
||||
'links, please remove if replacement is '
|
||||
'desired.',
|
||||
target_item_dir
|
||||
)
|
||||
continue
|
||||
if os.path.isdir(target_item_dir):
|
||||
shutil.rmtree(target_item_dir)
|
||||
else:
|
||||
os.remove(target_item_dir)
|
||||
|
||||
shutil.move(
|
||||
os.path.join(lib_dir, item),
|
||||
target_item_dir
|
||||
)
|
||||
|
||||
def _warn_about_conflicts(self, to_install):
|
||||
package_set, _dep_info = check_install_conflicts(to_install)
|
||||
missing, conflicting = _dep_info
|
||||
|
||||
# NOTE: There is some duplication here from pip check
|
||||
for project_name in missing:
|
||||
version = package_set[project_name][0]
|
||||
for dependency in missing[project_name]:
|
||||
logger.critical(
|
||||
"%s %s requires %s, which is not installed.",
|
||||
project_name, version, dependency[1],
|
||||
)
|
||||
|
||||
for project_name in conflicting:
|
||||
version = package_set[project_name][0]
|
||||
for dep_name, dep_version, req in conflicting[project_name]:
|
||||
logger.critical(
|
||||
"%s %s has requirement %s, but you'll have %s %s which is "
|
||||
"incompatible.",
|
||||
project_name, version, req, dep_name, dep_version,
|
||||
)
|
||||
|
||||
|
||||
def get_lib_location_guesses(*args, **kwargs):
|
||||
scheme = distutils_scheme('', *args, **kwargs)
|
||||
return [scheme['purelib'], scheme['platlib']]
|
||||
|
||||
|
||||
def create_env_error_message(error, show_traceback, using_user_site):
|
||||
"""Format an error message for an EnvironmentError
|
||||
|
||||
It may occur anytime during the execution of the install command.
|
||||
"""
|
||||
parts = []
|
||||
|
||||
# Mention the error if we are not going to show a traceback
|
||||
parts.append("Could not install packages due to an EnvironmentError")
|
||||
if not show_traceback:
|
||||
parts.append(": ")
|
||||
parts.append(str(error))
|
||||
else:
|
||||
parts.append(".")
|
||||
|
||||
# Spilt the error indication from a helper message (if any)
|
||||
parts[-1] += "\n"
|
||||
|
||||
# Suggest useful actions to the user:
|
||||
# (1) using user site-packages or (2) verifying the permissions
|
||||
if error.errno == errno.EACCES:
|
||||
user_option_part = "Consider using the `--user` option"
|
||||
permissions_part = "Check the permissions"
|
||||
|
||||
if not using_user_site:
|
||||
parts.extend([
|
||||
user_option_part, " or ",
|
||||
permissions_part.lower(),
|
||||
])
|
||||
else:
|
||||
parts.append(permissions_part)
|
||||
parts.append(".\n")
|
||||
|
||||
return "".join(parts).strip() + "\n"
|
||||
@@ -1,343 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import json
|
||||
import logging
|
||||
import warnings
|
||||
|
||||
from pip._vendor import six
|
||||
from pip._vendor.six.moves import zip_longest
|
||||
|
||||
from pip._internal.basecommand import Command
|
||||
from pip._internal.cmdoptions import index_group, make_option_group
|
||||
from pip._internal.exceptions import CommandError
|
||||
from pip._internal.index import PackageFinder
|
||||
from pip._internal.utils.deprecation import RemovedInPip11Warning
|
||||
from pip._internal.utils.misc import (
|
||||
dist_is_editable, get_installed_distributions,
|
||||
)
|
||||
from pip._internal.utils.packaging import get_installer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ListCommand(Command):
|
||||
"""
|
||||
List installed packages, including editables.
|
||||
|
||||
Packages are listed in a case-insensitive sorted order.
|
||||
"""
|
||||
name = 'list'
|
||||
usage = """
|
||||
%prog [options]"""
|
||||
summary = 'List installed packages.'
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(ListCommand, self).__init__(*args, **kw)
|
||||
|
||||
cmd_opts = self.cmd_opts
|
||||
|
||||
cmd_opts.add_option(
|
||||
'-o', '--outdated',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='List outdated packages')
|
||||
cmd_opts.add_option(
|
||||
'-u', '--uptodate',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='List uptodate packages')
|
||||
cmd_opts.add_option(
|
||||
'-e', '--editable',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='List editable projects.')
|
||||
cmd_opts.add_option(
|
||||
'-l', '--local',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help=('If in a virtualenv that has global access, do not list '
|
||||
'globally-installed packages.'),
|
||||
)
|
||||
self.cmd_opts.add_option(
|
||||
'--user',
|
||||
dest='user',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Only output packages installed in user-site.')
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--pre',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help=("Include pre-release and development versions. By default, "
|
||||
"pip only finds stable versions."),
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--format',
|
||||
action='store',
|
||||
dest='list_format',
|
||||
default="columns",
|
||||
choices=('legacy', 'columns', 'freeze', 'json'),
|
||||
help="Select the output format among: columns (default), freeze, "
|
||||
"json, or legacy.",
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--not-required',
|
||||
action='store_true',
|
||||
dest='not_required',
|
||||
help="List packages that are not dependencies of "
|
||||
"installed packages.",
|
||||
)
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--exclude-editable',
|
||||
action='store_false',
|
||||
dest='include_editable',
|
||||
help='Exclude editable package from output.',
|
||||
)
|
||||
cmd_opts.add_option(
|
||||
'--include-editable',
|
||||
action='store_true',
|
||||
dest='include_editable',
|
||||
help='Include editable package from output.',
|
||||
default=True,
|
||||
)
|
||||
index_opts = make_option_group(index_group, self.parser)
|
||||
|
||||
self.parser.insert_option_group(0, index_opts)
|
||||
self.parser.insert_option_group(0, cmd_opts)
|
||||
|
||||
def _build_package_finder(self, options, index_urls, session):
|
||||
"""
|
||||
Create a package finder appropriate to this list command.
|
||||
"""
|
||||
return PackageFinder(
|
||||
find_links=options.find_links,
|
||||
index_urls=index_urls,
|
||||
allow_all_prereleases=options.pre,
|
||||
trusted_hosts=options.trusted_hosts,
|
||||
process_dependency_links=options.process_dependency_links,
|
||||
session=session,
|
||||
)
|
||||
|
||||
def run(self, options, args):
|
||||
if options.list_format == "legacy":
|
||||
warnings.warn(
|
||||
"The legacy format has been deprecated and will be removed "
|
||||
"in the future.",
|
||||
RemovedInPip11Warning,
|
||||
)
|
||||
|
||||
if options.outdated and options.uptodate:
|
||||
raise CommandError(
|
||||
"Options --outdated and --uptodate cannot be combined.")
|
||||
|
||||
packages = get_installed_distributions(
|
||||
local_only=options.local,
|
||||
user_only=options.user,
|
||||
editables_only=options.editable,
|
||||
include_editables=options.include_editable,
|
||||
)
|
||||
|
||||
if options.outdated:
|
||||
packages = self.get_outdated(packages, options)
|
||||
elif options.uptodate:
|
||||
packages = self.get_uptodate(packages, options)
|
||||
|
||||
if options.not_required:
|
||||
packages = self.get_not_required(packages, options)
|
||||
|
||||
self.output_package_listing(packages, options)
|
||||
|
||||
def get_outdated(self, packages, options):
|
||||
return [
|
||||
dist for dist in self.iter_packages_latest_infos(packages, options)
|
||||
if dist.latest_version > dist.parsed_version
|
||||
]
|
||||
|
||||
def get_uptodate(self, packages, options):
|
||||
return [
|
||||
dist for dist in self.iter_packages_latest_infos(packages, options)
|
||||
if dist.latest_version == dist.parsed_version
|
||||
]
|
||||
|
||||
def get_not_required(self, packages, options):
|
||||
dep_keys = set()
|
||||
for dist in packages:
|
||||
dep_keys.update(requirement.key for requirement in dist.requires())
|
||||
return {pkg for pkg in packages if pkg.key not in dep_keys}
|
||||
|
||||
def iter_packages_latest_infos(self, packages, options):
|
||||
index_urls = [options.index_url] + options.extra_index_urls
|
||||
if options.no_index:
|
||||
logger.debug('Ignoring indexes: %s', ','.join(index_urls))
|
||||
index_urls = []
|
||||
|
||||
dependency_links = []
|
||||
for dist in packages:
|
||||
if dist.has_metadata('dependency_links.txt'):
|
||||
dependency_links.extend(
|
||||
dist.get_metadata_lines('dependency_links.txt'),
|
||||
)
|
||||
|
||||
with self._build_session(options) as session:
|
||||
finder = self._build_package_finder(options, index_urls, session)
|
||||
finder.add_dependency_links(dependency_links)
|
||||
|
||||
for dist in packages:
|
||||
typ = 'unknown'
|
||||
all_candidates = finder.find_all_candidates(dist.key)
|
||||
if not options.pre:
|
||||
# Remove prereleases
|
||||
all_candidates = [candidate for candidate in all_candidates
|
||||
if not candidate.version.is_prerelease]
|
||||
|
||||
if not all_candidates:
|
||||
continue
|
||||
best_candidate = max(all_candidates,
|
||||
key=finder._candidate_sort_key)
|
||||
remote_version = best_candidate.version
|
||||
if best_candidate.location.is_wheel:
|
||||
typ = 'wheel'
|
||||
else:
|
||||
typ = 'sdist'
|
||||
# This is dirty but makes the rest of the code much cleaner
|
||||
dist.latest_version = remote_version
|
||||
dist.latest_filetype = typ
|
||||
yield dist
|
||||
|
||||
def output_legacy(self, dist, options):
|
||||
if options.verbose >= 1:
|
||||
return '%s (%s, %s, %s)' % (
|
||||
dist.project_name,
|
||||
dist.version,
|
||||
dist.location,
|
||||
get_installer(dist),
|
||||
)
|
||||
elif dist_is_editable(dist):
|
||||
return '%s (%s, %s)' % (
|
||||
dist.project_name,
|
||||
dist.version,
|
||||
dist.location,
|
||||
)
|
||||
else:
|
||||
return '%s (%s)' % (dist.project_name, dist.version)
|
||||
|
||||
def output_legacy_latest(self, dist, options):
|
||||
return '%s - Latest: %s [%s]' % (
|
||||
self.output_legacy(dist, options),
|
||||
dist.latest_version,
|
||||
dist.latest_filetype,
|
||||
)
|
||||
|
||||
def output_package_listing(self, packages, options):
|
||||
packages = sorted(
|
||||
packages,
|
||||
key=lambda dist: dist.project_name.lower(),
|
||||
)
|
||||
if options.list_format == 'columns' and packages:
|
||||
data, header = format_for_columns(packages, options)
|
||||
self.output_package_listing_columns(data, header)
|
||||
elif options.list_format == 'freeze':
|
||||
for dist in packages:
|
||||
if options.verbose >= 1:
|
||||
logger.info("%s==%s (%s)", dist.project_name,
|
||||
dist.version, dist.location)
|
||||
else:
|
||||
logger.info("%s==%s", dist.project_name, dist.version)
|
||||
elif options.list_format == 'json':
|
||||
logger.info(format_for_json(packages, options))
|
||||
elif options.list_format == "legacy":
|
||||
for dist in packages:
|
||||
if options.outdated:
|
||||
logger.info(self.output_legacy_latest(dist, options))
|
||||
else:
|
||||
logger.info(self.output_legacy(dist, options))
|
||||
|
||||
def output_package_listing_columns(self, data, header):
|
||||
# insert the header first: we need to know the size of column names
|
||||
if len(data) > 0:
|
||||
data.insert(0, header)
|
||||
|
||||
pkg_strings, sizes = tabulate(data)
|
||||
|
||||
# Create and add a separator.
|
||||
if len(data) > 0:
|
||||
pkg_strings.insert(1, " ".join(map(lambda x: '-' * x, sizes)))
|
||||
|
||||
for val in pkg_strings:
|
||||
logger.info(val)
|
||||
|
||||
|
||||
def tabulate(vals):
|
||||
# From pfmoore on GitHub:
|
||||
# https://github.com/pypa/pip/issues/3651#issuecomment-216932564
|
||||
assert len(vals) > 0
|
||||
|
||||
sizes = [0] * max(len(x) for x in vals)
|
||||
for row in vals:
|
||||
sizes = [max(s, len(str(c))) for s, c in zip_longest(sizes, row)]
|
||||
|
||||
result = []
|
||||
for row in vals:
|
||||
display = " ".join([str(c).ljust(s) if c is not None else ''
|
||||
for s, c in zip_longest(sizes, row)])
|
||||
result.append(display)
|
||||
|
||||
return result, sizes
|
||||
|
||||
|
||||
def format_for_columns(pkgs, options):
|
||||
"""
|
||||
Convert the package data into something usable
|
||||
by output_package_listing_columns.
|
||||
"""
|
||||
running_outdated = options.outdated
|
||||
# Adjust the header for the `pip list --outdated` case.
|
||||
if running_outdated:
|
||||
header = ["Package", "Version", "Latest", "Type"]
|
||||
else:
|
||||
header = ["Package", "Version"]
|
||||
|
||||
data = []
|
||||
if options.verbose >= 1 or any(dist_is_editable(x) for x in pkgs):
|
||||
header.append("Location")
|
||||
if options.verbose >= 1:
|
||||
header.append("Installer")
|
||||
|
||||
for proj in pkgs:
|
||||
# if we're working on the 'outdated' list, separate out the
|
||||
# latest_version and type
|
||||
row = [proj.project_name, proj.version]
|
||||
|
||||
if running_outdated:
|
||||
row.append(proj.latest_version)
|
||||
row.append(proj.latest_filetype)
|
||||
|
||||
if options.verbose >= 1 or dist_is_editable(proj):
|
||||
row.append(proj.location)
|
||||
if options.verbose >= 1:
|
||||
row.append(get_installer(proj))
|
||||
|
||||
data.append(row)
|
||||
|
||||
return data, header
|
||||
|
||||
|
||||
def format_for_json(packages, options):
|
||||
data = []
|
||||
for dist in packages:
|
||||
info = {
|
||||
'name': dist.project_name,
|
||||
'version': six.text_type(dist.version),
|
||||
}
|
||||
if options.verbose >= 1:
|
||||
info['location'] = dist.location
|
||||
info['installer'] = get_installer(dist)
|
||||
if options.outdated:
|
||||
info['latest_version'] = six.text_type(dist.latest_version)
|
||||
info['latest_filetype'] = dist.latest_filetype
|
||||
data.append(info)
|
||||
return json.dumps(data)
|
||||
@@ -1,135 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import textwrap
|
||||
from collections import OrderedDict
|
||||
|
||||
from pip._vendor import pkg_resources
|
||||
from pip._vendor.packaging.version import parse as parse_version
|
||||
# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
|
||||
# why we ignore the type on this import
|
||||
from pip._vendor.six.moves import xmlrpc_client # type: ignore
|
||||
|
||||
from pip._internal.basecommand import SUCCESS, Command
|
||||
from pip._internal.compat import get_terminal_size
|
||||
from pip._internal.download import PipXmlrpcTransport
|
||||
from pip._internal.exceptions import CommandError
|
||||
from pip._internal.models import PyPI
|
||||
from pip._internal.status_codes import NO_MATCHES_FOUND
|
||||
from pip._internal.utils.logging import indent_log
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SearchCommand(Command):
|
||||
"""Search for PyPI packages whose name or summary contains <query>."""
|
||||
name = 'search'
|
||||
usage = """
|
||||
%prog [options] <query>"""
|
||||
summary = 'Search PyPI for packages.'
|
||||
ignore_require_venv = True
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(SearchCommand, self).__init__(*args, **kw)
|
||||
self.cmd_opts.add_option(
|
||||
'-i', '--index',
|
||||
dest='index',
|
||||
metavar='URL',
|
||||
default=PyPI.pypi_url,
|
||||
help='Base URL of Python Package Index (default %default)')
|
||||
|
||||
self.parser.insert_option_group(0, self.cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
if not args:
|
||||
raise CommandError('Missing required argument (search query).')
|
||||
query = args
|
||||
pypi_hits = self.search(query, options)
|
||||
hits = transform_hits(pypi_hits)
|
||||
|
||||
terminal_width = None
|
||||
if sys.stdout.isatty():
|
||||
terminal_width = get_terminal_size()[0]
|
||||
|
||||
print_results(hits, terminal_width=terminal_width)
|
||||
if pypi_hits:
|
||||
return SUCCESS
|
||||
return NO_MATCHES_FOUND
|
||||
|
||||
def search(self, query, options):
|
||||
index_url = options.index
|
||||
with self._build_session(options) as session:
|
||||
transport = PipXmlrpcTransport(index_url, session)
|
||||
pypi = xmlrpc_client.ServerProxy(index_url, transport)
|
||||
hits = pypi.search({'name': query, 'summary': query}, 'or')
|
||||
return hits
|
||||
|
||||
|
||||
def transform_hits(hits):
|
||||
"""
|
||||
The list from pypi is really a list of versions. We want a list of
|
||||
packages with the list of versions stored inline. This converts the
|
||||
list from pypi into one we can use.
|
||||
"""
|
||||
packages = OrderedDict()
|
||||
for hit in hits:
|
||||
name = hit['name']
|
||||
summary = hit['summary']
|
||||
version = hit['version']
|
||||
|
||||
if name not in packages.keys():
|
||||
packages[name] = {
|
||||
'name': name,
|
||||
'summary': summary,
|
||||
'versions': [version],
|
||||
}
|
||||
else:
|
||||
packages[name]['versions'].append(version)
|
||||
|
||||
# if this is the highest version, replace summary and score
|
||||
if version == highest_version(packages[name]['versions']):
|
||||
packages[name]['summary'] = summary
|
||||
|
||||
return list(packages.values())
|
||||
|
||||
|
||||
def print_results(hits, name_column_width=None, terminal_width=None):
|
||||
if not hits:
|
||||
return
|
||||
if name_column_width is None:
|
||||
name_column_width = max([
|
||||
len(hit['name']) + len(highest_version(hit.get('versions', ['-'])))
|
||||
for hit in hits
|
||||
]) + 4
|
||||
|
||||
installed_packages = [p.project_name for p in pkg_resources.working_set]
|
||||
for hit in hits:
|
||||
name = hit['name']
|
||||
summary = hit['summary'] or ''
|
||||
latest = highest_version(hit.get('versions', ['-']))
|
||||
if terminal_width is not None:
|
||||
target_width = terminal_width - name_column_width - 5
|
||||
if target_width > 10:
|
||||
# wrap and indent summary to fit terminal
|
||||
summary = textwrap.wrap(summary, target_width)
|
||||
summary = ('\n' + ' ' * (name_column_width + 3)).join(summary)
|
||||
|
||||
line = '%-*s - %s' % (name_column_width,
|
||||
'%s (%s)' % (name, latest), summary)
|
||||
try:
|
||||
logger.info(line)
|
||||
if name in installed_packages:
|
||||
dist = pkg_resources.get_distribution(name)
|
||||
with indent_log():
|
||||
if dist.version == latest:
|
||||
logger.info('INSTALLED: %s (latest)', dist.version)
|
||||
else:
|
||||
logger.info('INSTALLED: %s', dist.version)
|
||||
logger.info('LATEST: %s', latest)
|
||||
except UnicodeEncodeError:
|
||||
pass
|
||||
|
||||
|
||||
def highest_version(versions):
|
||||
return max(versions, key=parse_version)
|
||||
@@ -1,164 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
import os
|
||||
from email.parser import FeedParser # type: ignore
|
||||
|
||||
from pip._vendor import pkg_resources
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
|
||||
from pip._internal.basecommand import Command
|
||||
from pip._internal.status_codes import ERROR, SUCCESS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ShowCommand(Command):
|
||||
"""Show information about one or more installed packages."""
|
||||
name = 'show'
|
||||
usage = """
|
||||
%prog [options] <package> ..."""
|
||||
summary = 'Show information about installed packages.'
|
||||
ignore_require_venv = True
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(ShowCommand, self).__init__(*args, **kw)
|
||||
self.cmd_opts.add_option(
|
||||
'-f', '--files',
|
||||
dest='files',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Show the full list of installed files for each package.')
|
||||
|
||||
self.parser.insert_option_group(0, self.cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
if not args:
|
||||
logger.warning('ERROR: Please provide a package name or names.')
|
||||
return ERROR
|
||||
query = args
|
||||
|
||||
results = search_packages_info(query)
|
||||
if not print_results(
|
||||
results, list_files=options.files, verbose=options.verbose):
|
||||
return ERROR
|
||||
return SUCCESS
|
||||
|
||||
|
||||
def search_packages_info(query):
|
||||
"""
|
||||
Gather details from installed distributions. Print distribution name,
|
||||
version, location, and installed files. Installed files requires a
|
||||
pip generated 'installed-files.txt' in the distributions '.egg-info'
|
||||
directory.
|
||||
"""
|
||||
installed = {}
|
||||
for p in pkg_resources.working_set:
|
||||
installed[canonicalize_name(p.project_name)] = p
|
||||
|
||||
query_names = [canonicalize_name(name) for name in query]
|
||||
|
||||
for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
|
||||
package = {
|
||||
'name': dist.project_name,
|
||||
'version': dist.version,
|
||||
'location': dist.location,
|
||||
'requires': [dep.project_name for dep in dist.requires()],
|
||||
}
|
||||
file_list = None
|
||||
metadata = None
|
||||
if isinstance(dist, pkg_resources.DistInfoDistribution):
|
||||
# RECORDs should be part of .dist-info metadatas
|
||||
if dist.has_metadata('RECORD'):
|
||||
lines = dist.get_metadata_lines('RECORD')
|
||||
paths = [l.split(',')[0] for l in lines]
|
||||
paths = [os.path.join(dist.location, p) for p in paths]
|
||||
file_list = [os.path.relpath(p, dist.location) for p in paths]
|
||||
|
||||
if dist.has_metadata('METADATA'):
|
||||
metadata = dist.get_metadata('METADATA')
|
||||
else:
|
||||
# Otherwise use pip's log for .egg-info's
|
||||
if dist.has_metadata('installed-files.txt'):
|
||||
paths = dist.get_metadata_lines('installed-files.txt')
|
||||
paths = [os.path.join(dist.egg_info, p) for p in paths]
|
||||
file_list = [os.path.relpath(p, dist.location) for p in paths]
|
||||
|
||||
if dist.has_metadata('PKG-INFO'):
|
||||
metadata = dist.get_metadata('PKG-INFO')
|
||||
|
||||
if dist.has_metadata('entry_points.txt'):
|
||||
entry_points = dist.get_metadata_lines('entry_points.txt')
|
||||
package['entry_points'] = entry_points
|
||||
|
||||
if dist.has_metadata('INSTALLER'):
|
||||
for line in dist.get_metadata_lines('INSTALLER'):
|
||||
if line.strip():
|
||||
package['installer'] = line.strip()
|
||||
break
|
||||
|
||||
# @todo: Should pkg_resources.Distribution have a
|
||||
# `get_pkg_info` method?
|
||||
feed_parser = FeedParser()
|
||||
feed_parser.feed(metadata)
|
||||
pkg_info_dict = feed_parser.close()
|
||||
for key in ('metadata-version', 'summary',
|
||||
'home-page', 'author', 'author-email', 'license'):
|
||||
package[key] = pkg_info_dict.get(key)
|
||||
|
||||
# It looks like FeedParser cannot deal with repeated headers
|
||||
classifiers = []
|
||||
for line in metadata.splitlines():
|
||||
if line.startswith('Classifier: '):
|
||||
classifiers.append(line[len('Classifier: '):])
|
||||
package['classifiers'] = classifiers
|
||||
|
||||
if file_list:
|
||||
package['files'] = sorted(file_list)
|
||||
yield package
|
||||
|
||||
|
||||
def print_results(distributions, list_files=False, verbose=False):
|
||||
"""
|
||||
Print the informations from installed distributions found.
|
||||
"""
|
||||
results_printed = False
|
||||
for i, dist in enumerate(distributions):
|
||||
results_printed = True
|
||||
if i > 0:
|
||||
logger.info("---")
|
||||
|
||||
name = dist.get('name', '')
|
||||
required_by = [
|
||||
pkg.project_name for pkg in pkg_resources.working_set
|
||||
if name in [required.name for required in pkg.requires()]
|
||||
]
|
||||
|
||||
logger.info("Name: %s", name)
|
||||
logger.info("Version: %s", dist.get('version', ''))
|
||||
logger.info("Summary: %s", dist.get('summary', ''))
|
||||
logger.info("Home-page: %s", dist.get('home-page', ''))
|
||||
logger.info("Author: %s", dist.get('author', ''))
|
||||
logger.info("Author-email: %s", dist.get('author-email', ''))
|
||||
logger.info("License: %s", dist.get('license', ''))
|
||||
logger.info("Location: %s", dist.get('location', ''))
|
||||
logger.info("Requires: %s", ', '.join(dist.get('requires', [])))
|
||||
logger.info("Required-by: %s", ', '.join(required_by))
|
||||
|
||||
if verbose:
|
||||
logger.info("Metadata-Version: %s",
|
||||
dist.get('metadata-version', ''))
|
||||
logger.info("Installer: %s", dist.get('installer', ''))
|
||||
logger.info("Classifiers:")
|
||||
for classifier in dist.get('classifiers', []):
|
||||
logger.info(" %s", classifier)
|
||||
logger.info("Entry-points:")
|
||||
for entry in dist.get('entry_points', []):
|
||||
logger.info(" %s", entry.strip())
|
||||
if list_files:
|
||||
logger.info("Files:")
|
||||
for line in dist.get('files', []):
|
||||
logger.info(" %s", line.strip())
|
||||
if "files" not in dist:
|
||||
logger.info("Cannot locate installed-files.txt")
|
||||
return results_printed
|
||||
@@ -1,71 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
|
||||
from pip._internal.basecommand import Command
|
||||
from pip._internal.exceptions import InstallationError
|
||||
from pip._internal.req import InstallRequirement, parse_requirements
|
||||
|
||||
|
||||
class UninstallCommand(Command):
|
||||
"""
|
||||
Uninstall packages.
|
||||
|
||||
pip is able to uninstall most installed packages. Known exceptions are:
|
||||
|
||||
- Pure distutils packages installed with ``python setup.py install``, which
|
||||
leave behind no metadata to determine what files were installed.
|
||||
- Script wrappers installed by ``python setup.py develop``.
|
||||
"""
|
||||
name = 'uninstall'
|
||||
usage = """
|
||||
%prog [options] <package> ...
|
||||
%prog [options] -r <requirements file> ..."""
|
||||
summary = 'Uninstall packages.'
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(UninstallCommand, self).__init__(*args, **kw)
|
||||
self.cmd_opts.add_option(
|
||||
'-r', '--requirement',
|
||||
dest='requirements',
|
||||
action='append',
|
||||
default=[],
|
||||
metavar='file',
|
||||
help='Uninstall all the packages listed in the given requirements '
|
||||
'file. This option can be used multiple times.',
|
||||
)
|
||||
self.cmd_opts.add_option(
|
||||
'-y', '--yes',
|
||||
dest='yes',
|
||||
action='store_true',
|
||||
help="Don't ask for confirmation of uninstall deletions.")
|
||||
|
||||
self.parser.insert_option_group(0, self.cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
with self._build_session(options) as session:
|
||||
reqs_to_uninstall = {}
|
||||
for name in args:
|
||||
req = InstallRequirement.from_line(
|
||||
name, isolated=options.isolated_mode,
|
||||
)
|
||||
if req.name:
|
||||
reqs_to_uninstall[canonicalize_name(req.name)] = req
|
||||
for filename in options.requirements:
|
||||
for req in parse_requirements(
|
||||
filename,
|
||||
options=options,
|
||||
session=session):
|
||||
if req.name:
|
||||
reqs_to_uninstall[canonicalize_name(req.name)] = req
|
||||
if not reqs_to_uninstall:
|
||||
raise InstallationError(
|
||||
'You must give at least one requirement to %(name)s (see '
|
||||
'"pip help %(name)s")' % dict(name=self.name)
|
||||
)
|
||||
for req in reqs_to_uninstall.values():
|
||||
uninstall_pathset = req.uninstall(
|
||||
auto_confirm=options.yes, verbose=self.verbosity > 0,
|
||||
)
|
||||
if uninstall_pathset:
|
||||
uninstall_pathset.commit()
|
||||
@@ -1,179 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from pip._internal import cmdoptions
|
||||
from pip._internal.basecommand import RequirementCommand
|
||||
from pip._internal.cache import WheelCache
|
||||
from pip._internal.exceptions import CommandError, PreviousBuildDirError
|
||||
from pip._internal.operations.prepare import RequirementPreparer
|
||||
from pip._internal.req import RequirementSet
|
||||
from pip._internal.resolve import Resolver
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
from pip._internal.wheel import WheelBuilder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WheelCommand(RequirementCommand):
|
||||
"""
|
||||
Build Wheel archives for your requirements and dependencies.
|
||||
|
||||
Wheel is a built-package format, and offers the advantage of not
|
||||
recompiling your software during every install. For more details, see the
|
||||
wheel docs: https://wheel.readthedocs.io/en/latest/
|
||||
|
||||
Requirements: setuptools>=0.8, and wheel.
|
||||
|
||||
'pip wheel' uses the bdist_wheel setuptools extension from the wheel
|
||||
package to build individual wheels.
|
||||
|
||||
"""
|
||||
|
||||
name = 'wheel'
|
||||
usage = """
|
||||
%prog [options] <requirement specifier> ...
|
||||
%prog [options] -r <requirements file> ...
|
||||
%prog [options] [-e] <vcs project url> ...
|
||||
%prog [options] [-e] <local project path> ...
|
||||
%prog [options] <archive url/path> ..."""
|
||||
|
||||
summary = 'Build wheels from your requirements.'
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super(WheelCommand, self).__init__(*args, **kw)
|
||||
|
||||
cmd_opts = self.cmd_opts
|
||||
|
||||
cmd_opts.add_option(
|
||||
'-w', '--wheel-dir',
|
||||
dest='wheel_dir',
|
||||
metavar='dir',
|
||||
default=os.curdir,
|
||||
help=("Build wheels into <dir>, where the default is the "
|
||||
"current working directory."),
|
||||
)
|
||||
cmd_opts.add_option(cmdoptions.no_binary())
|
||||
cmd_opts.add_option(cmdoptions.only_binary())
|
||||
cmd_opts.add_option(
|
||||
'--build-option',
|
||||
dest='build_options',
|
||||
metavar='options',
|
||||
action='append',
|
||||
help="Extra arguments to be supplied to 'setup.py bdist_wheel'.",
|
||||
)
|
||||
cmd_opts.add_option(cmdoptions.no_build_isolation())
|
||||
cmd_opts.add_option(cmdoptions.constraints())
|
||||
cmd_opts.add_option(cmdoptions.editable())
|
||||
cmd_opts.add_option(cmdoptions.requirements())
|
||||
cmd_opts.add_option(cmdoptions.src())
|
||||
cmd_opts.add_option(cmdoptions.ignore_requires_python())
|
||||
cmd_opts.add_option(cmdoptions.no_deps())
|
||||
cmd_opts.add_option(cmdoptions.build_dir())
|
||||
cmd_opts.add_option(cmdoptions.progress_bar())
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--global-option',
|
||||
dest='global_options',
|
||||
action='append',
|
||||
metavar='options',
|
||||
help="Extra global options to be supplied to the setup.py "
|
||||
"call before the 'bdist_wheel' command.")
|
||||
|
||||
cmd_opts.add_option(
|
||||
'--pre',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help=("Include pre-release and development versions. By default, "
|
||||
"pip only finds stable versions."),
|
||||
)
|
||||
|
||||
cmd_opts.add_option(cmdoptions.no_clean())
|
||||
cmd_opts.add_option(cmdoptions.require_hashes())
|
||||
|
||||
index_opts = cmdoptions.make_option_group(
|
||||
cmdoptions.index_group,
|
||||
self.parser,
|
||||
)
|
||||
|
||||
self.parser.insert_option_group(0, index_opts)
|
||||
self.parser.insert_option_group(0, cmd_opts)
|
||||
|
||||
def run(self, options, args):
|
||||
cmdoptions.check_install_build_global(options)
|
||||
|
||||
index_urls = [options.index_url] + options.extra_index_urls
|
||||
if options.no_index:
|
||||
logger.debug('Ignoring indexes: %s', ','.join(index_urls))
|
||||
index_urls = []
|
||||
|
||||
if options.build_dir:
|
||||
options.build_dir = os.path.abspath(options.build_dir)
|
||||
|
||||
options.src_dir = os.path.abspath(options.src_dir)
|
||||
|
||||
with self._build_session(options) as session:
|
||||
finder = self._build_package_finder(options, session)
|
||||
build_delete = (not (options.no_clean or options.build_dir))
|
||||
wheel_cache = WheelCache(options.cache_dir, options.format_control)
|
||||
|
||||
with TempDirectory(
|
||||
options.build_dir, delete=build_delete, kind="wheel"
|
||||
) as directory:
|
||||
requirement_set = RequirementSet(
|
||||
require_hashes=options.require_hashes,
|
||||
)
|
||||
|
||||
try:
|
||||
self.populate_requirement_set(
|
||||
requirement_set, args, options, finder, session,
|
||||
self.name, wheel_cache
|
||||
)
|
||||
|
||||
preparer = RequirementPreparer(
|
||||
build_dir=directory.path,
|
||||
src_dir=options.src_dir,
|
||||
download_dir=None,
|
||||
wheel_download_dir=options.wheel_dir,
|
||||
progress_bar=options.progress_bar,
|
||||
build_isolation=options.build_isolation,
|
||||
)
|
||||
|
||||
resolver = Resolver(
|
||||
preparer=preparer,
|
||||
finder=finder,
|
||||
session=session,
|
||||
wheel_cache=wheel_cache,
|
||||
use_user_site=False,
|
||||
upgrade_strategy="to-satisfy-only",
|
||||
force_reinstall=False,
|
||||
ignore_dependencies=options.ignore_dependencies,
|
||||
ignore_requires_python=options.ignore_requires_python,
|
||||
ignore_installed=True,
|
||||
isolated=options.isolated_mode,
|
||||
)
|
||||
resolver.resolve(requirement_set)
|
||||
|
||||
# build wheels
|
||||
wb = WheelBuilder(
|
||||
finder, preparer, wheel_cache,
|
||||
build_options=options.build_options or [],
|
||||
global_options=options.global_options or [],
|
||||
no_clean=options.no_clean,
|
||||
)
|
||||
wheels_built_successfully = wb.build(
|
||||
requirement_set.requirements.values(), session=session,
|
||||
)
|
||||
if not wheels_built_successfully:
|
||||
raise CommandError(
|
||||
"Failed to build one or more wheels"
|
||||
)
|
||||
except PreviousBuildDirError:
|
||||
options.no_clean = True
|
||||
raise
|
||||
finally:
|
||||
if not options.no_clean:
|
||||
requirement_set.cleanup_files()
|
||||
wheel_cache.cleanup()
|
||||
@@ -1,235 +0,0 @@
|
||||
"""Stuff that differs in different Python versions and platform
|
||||
distributions."""
|
||||
from __future__ import absolute_import, division
|
||||
|
||||
import codecs
|
||||
import locale
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
from pip._vendor.six import text_type
|
||||
|
||||
try:
|
||||
import ipaddress
|
||||
except ImportError:
|
||||
try:
|
||||
from pip._vendor import ipaddress # type: ignore
|
||||
except ImportError:
|
||||
import ipaddr as ipaddress # type: ignore
|
||||
ipaddress.ip_address = ipaddress.IPAddress
|
||||
ipaddress.ip_network = ipaddress.IPNetwork
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ipaddress", "uses_pycache", "console_to_str", "native_str",
|
||||
"get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size",
|
||||
]
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if sys.version_info >= (3, 4):
|
||||
uses_pycache = True
|
||||
from importlib.util import cache_from_source
|
||||
else:
|
||||
import imp
|
||||
|
||||
try:
|
||||
cache_from_source = imp.cache_from_source # type: ignore
|
||||
except AttributeError:
|
||||
# does not use __pycache__
|
||||
cache_from_source = None
|
||||
|
||||
uses_pycache = cache_from_source is not None
|
||||
|
||||
|
||||
if sys.version_info >= (3, 5):
|
||||
backslashreplace_decode = "backslashreplace"
|
||||
else:
|
||||
# In version 3.4 and older, backslashreplace exists
|
||||
# but does not support use for decoding.
|
||||
# We implement our own replace handler for this
|
||||
# situation, so that we can consistently use
|
||||
# backslash replacement for all versions.
|
||||
def backslashreplace_decode_fn(err):
|
||||
raw_bytes = (err.object[i] for i in range(err.start, err.end))
|
||||
if sys.version_info[0] == 2:
|
||||
# Python 2 gave us characters - convert to numeric bytes
|
||||
raw_bytes = (ord(b) for b in raw_bytes)
|
||||
return u"".join(u"\\x%x" % c for c in raw_bytes), err.end
|
||||
codecs.register_error(
|
||||
"backslashreplace_decode",
|
||||
backslashreplace_decode_fn,
|
||||
)
|
||||
backslashreplace_decode = "backslashreplace_decode"
|
||||
|
||||
|
||||
def console_to_str(data):
|
||||
"""Return a string, safe for output, of subprocess output.
|
||||
|
||||
We assume the data is in the locale preferred encoding.
|
||||
If it won't decode properly, we warn the user but decode as
|
||||
best we can.
|
||||
|
||||
We also ensure that the output can be safely written to
|
||||
standard output without encoding errors.
|
||||
"""
|
||||
|
||||
# First, get the encoding we assume. This is the preferred
|
||||
# encoding for the locale, unless that is not found, or
|
||||
# it is ASCII, in which case assume UTF-8
|
||||
encoding = locale.getpreferredencoding()
|
||||
if (not encoding) or codecs.lookup(encoding).name == "ascii":
|
||||
encoding = "utf-8"
|
||||
|
||||
# Now try to decode the data - if we fail, warn the user and
|
||||
# decode with replacement.
|
||||
try:
|
||||
s = data.decode(encoding)
|
||||
except UnicodeDecodeError:
|
||||
logger.warning(
|
||||
"Subprocess output does not appear to be encoded as %s",
|
||||
encoding,
|
||||
)
|
||||
s = data.decode(encoding, errors=backslashreplace_decode)
|
||||
|
||||
# Make sure we can print the output, by encoding it to the output
|
||||
# encoding with replacement of unencodable characters, and then
|
||||
# decoding again.
|
||||
# We use stderr's encoding because it's less likely to be
|
||||
# redirected and if we don't find an encoding we skip this
|
||||
# step (on the assumption that output is wrapped by something
|
||||
# that won't fail).
|
||||
# The double getattr is to deal with the possibility that we're
|
||||
# being called in a situation where sys.__stderr__ doesn't exist,
|
||||
# or doesn't have an encoding attribute. Neither of these cases
|
||||
# should occur in normal pip use, but there's no harm in checking
|
||||
# in case people use pip in (unsupported) unusual situations.
|
||||
output_encoding = getattr(getattr(sys, "__stderr__", None),
|
||||
"encoding", None)
|
||||
|
||||
if output_encoding:
|
||||
s = s.encode(output_encoding, errors="backslashreplace")
|
||||
s = s.decode(output_encoding)
|
||||
|
||||
return s
|
||||
|
||||
|
||||
if sys.version_info >= (3,):
|
||||
def native_str(s, replace=False):
|
||||
if isinstance(s, bytes):
|
||||
return s.decode('utf-8', 'replace' if replace else 'strict')
|
||||
return s
|
||||
|
||||
else:
|
||||
def native_str(s, replace=False):
|
||||
# Replace is ignored -- unicode to UTF-8 can't fail
|
||||
if isinstance(s, text_type):
|
||||
return s.encode('utf-8')
|
||||
return s
|
||||
|
||||
|
||||
def get_path_uid(path):
|
||||
"""
|
||||
Return path's uid.
|
||||
|
||||
Does not follow symlinks:
|
||||
https://github.com/pypa/pip/pull/935#discussion_r5307003
|
||||
|
||||
Placed this function in compat due to differences on AIX and
|
||||
Jython, that should eventually go away.
|
||||
|
||||
:raises OSError: When path is a symlink or can't be read.
|
||||
"""
|
||||
if hasattr(os, 'O_NOFOLLOW'):
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
file_uid = os.fstat(fd).st_uid
|
||||
os.close(fd)
|
||||
else: # AIX and Jython
|
||||
# WARNING: time of check vulnerability, but best we can do w/o NOFOLLOW
|
||||
if not os.path.islink(path):
|
||||
# older versions of Jython don't have `os.fstat`
|
||||
file_uid = os.stat(path).st_uid
|
||||
else:
|
||||
# raise OSError for parity with os.O_NOFOLLOW above
|
||||
raise OSError(
|
||||
"%s is a symlink; Will not return uid for symlinks" % path
|
||||
)
|
||||
return file_uid
|
||||
|
||||
|
||||
def expanduser(path):
|
||||
"""
|
||||
Expand ~ and ~user constructions.
|
||||
|
||||
Includes a workaround for http://bugs.python.org/issue14768
|
||||
"""
|
||||
expanded = os.path.expanduser(path)
|
||||
if path.startswith('~/') and expanded.startswith('//'):
|
||||
expanded = expanded[1:]
|
||||
return expanded
|
||||
|
||||
|
||||
# packages in the stdlib that may have installation metadata, but should not be
|
||||
# considered 'installed'. this theoretically could be determined based on
|
||||
# dist.location (py27:`sysconfig.get_paths()['stdlib']`,
|
||||
# py26:sysconfig.get_config_vars('LIBDEST')), but fear platform variation may
|
||||
# make this ineffective, so hard-coding
|
||||
stdlib_pkgs = {"python", "wsgiref", "argparse"}
|
||||
|
||||
|
||||
# windows detection, covers cpython and ironpython
|
||||
WINDOWS = (sys.platform.startswith("win") or
|
||||
(sys.platform == 'cli' and os.name == 'nt'))
|
||||
|
||||
|
||||
def samefile(file1, file2):
|
||||
"""Provide an alternative for os.path.samefile on Windows/Python2"""
|
||||
if hasattr(os.path, 'samefile'):
|
||||
return os.path.samefile(file1, file2)
|
||||
else:
|
||||
path1 = os.path.normcase(os.path.abspath(file1))
|
||||
path2 = os.path.normcase(os.path.abspath(file2))
|
||||
return path1 == path2
|
||||
|
||||
|
||||
if hasattr(shutil, 'get_terminal_size'):
|
||||
def get_terminal_size():
|
||||
"""
|
||||
Returns a tuple (x, y) representing the width(x) and the height(y)
|
||||
in characters of the terminal window.
|
||||
"""
|
||||
return tuple(shutil.get_terminal_size())
|
||||
else:
|
||||
def get_terminal_size():
|
||||
"""
|
||||
Returns a tuple (x, y) representing the width(x) and the height(y)
|
||||
in characters of the terminal window.
|
||||
"""
|
||||
def ioctl_GWINSZ(fd):
|
||||
try:
|
||||
import fcntl
|
||||
import termios
|
||||
import struct
|
||||
cr = struct.unpack_from(
|
||||
'hh',
|
||||
fcntl.ioctl(fd, termios.TIOCGWINSZ, '12345678')
|
||||
)
|
||||
except:
|
||||
return None
|
||||
if cr == (0, 0):
|
||||
return None
|
||||
return cr
|
||||
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
|
||||
if not cr:
|
||||
try:
|
||||
fd = os.open(os.ctermid(), os.O_RDONLY)
|
||||
cr = ioctl_GWINSZ(fd)
|
||||
os.close(fd)
|
||||
except:
|
||||
pass
|
||||
if not cr:
|
||||
cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
|
||||
return int(cr[1]), int(cr[0])
|
||||
@@ -1,378 +0,0 @@
|
||||
"""Configuration management setup
|
||||
|
||||
Some terminology:
|
||||
- name
|
||||
As written in config files.
|
||||
- value
|
||||
Value associated with a name
|
||||
- key
|
||||
Name combined with it's section (section.name)
|
||||
- variant
|
||||
A single word describing where the configuration key-value pair came from
|
||||
"""
|
||||
|
||||
import locale
|
||||
import logging
|
||||
import os
|
||||
|
||||
from pip._vendor import six
|
||||
from pip._vendor.six.moves import configparser
|
||||
|
||||
from pip._internal.exceptions import ConfigurationError
|
||||
from pip._internal.locations import (
|
||||
legacy_config_file, new_config_file, running_under_virtualenv,
|
||||
site_config_files, venv_config_file,
|
||||
)
|
||||
from pip._internal.utils.misc import ensure_dir, enum
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple
|
||||
|
||||
RawConfigParser = configparser.RawConfigParser # Shorthand
|
||||
Kind = NewType("Kind", str)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# NOTE: Maybe use the optionx attribute to normalize keynames.
|
||||
def _normalize_name(name):
|
||||
# type: (str) -> str
|
||||
"""Make a name consistent regardless of source (environment or file)
|
||||
"""
|
||||
name = name.lower().replace('_', '-')
|
||||
if name.startswith('--'):
|
||||
name = name[2:] # only prefer long opts
|
||||
return name
|
||||
|
||||
|
||||
def _disassemble_key(name):
|
||||
# type: (str) -> List[str]
|
||||
return name.split(".", 1)
|
||||
|
||||
|
||||
# The kinds of configurations there are.
|
||||
kinds = enum(
|
||||
USER="user", # User Specific
|
||||
GLOBAL="global", # System Wide
|
||||
VENV="venv", # Virtual Environment Specific
|
||||
ENV="env", # from PIP_CONFIG_FILE
|
||||
ENV_VAR="env-var", # from Environment Variables
|
||||
)
|
||||
|
||||
|
||||
class Configuration(object):
|
||||
"""Handles management of configuration.
|
||||
|
||||
Provides an interface to accessing and managing configuration files.
|
||||
|
||||
This class converts provides an API that takes "section.key-name" style
|
||||
keys and stores the value associated with it as "key-name" under the
|
||||
section "section".
|
||||
|
||||
This allows for a clean interface wherein the both the section and the
|
||||
key-name are preserved in an easy to manage form in the configuration files
|
||||
and the data stored is also nice.
|
||||
"""
|
||||
|
||||
def __init__(self, isolated, load_only=None):
|
||||
# type: (bool, Kind) -> None
|
||||
super(Configuration, self).__init__()
|
||||
|
||||
_valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.VENV, None]
|
||||
if load_only not in _valid_load_only:
|
||||
raise ConfigurationError(
|
||||
"Got invalid value for load_only - should be one of {}".format(
|
||||
", ".join(map(repr, _valid_load_only[:-1]))
|
||||
)
|
||||
)
|
||||
self.isolated = isolated # type: bool
|
||||
self.load_only = load_only # type: Optional[Kind]
|
||||
|
||||
# The order here determines the override order.
|
||||
self._override_order = [
|
||||
kinds.GLOBAL, kinds.USER, kinds.VENV, kinds.ENV, kinds.ENV_VAR
|
||||
]
|
||||
|
||||
self._ignore_env_names = ["version", "help"]
|
||||
|
||||
# Because we keep track of where we got the data from
|
||||
self._parsers = {
|
||||
variant: [] for variant in self._override_order
|
||||
} # type: Dict[Kind, List[Tuple[str, RawConfigParser]]]
|
||||
self._config = {
|
||||
variant: {} for variant in self._override_order
|
||||
} # type: Dict[Kind, Dict[str, Any]]
|
||||
self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]]
|
||||
|
||||
def load(self):
|
||||
# type: () -> None
|
||||
"""Loads configuration from configuration files and environment
|
||||
"""
|
||||
self._load_config_files()
|
||||
if not self.isolated:
|
||||
self._load_environment_vars()
|
||||
|
||||
def get_file_to_edit(self):
|
||||
# type: () -> Optional[str]
|
||||
"""Returns the file with highest priority in configuration
|
||||
"""
|
||||
assert self.load_only is not None, \
|
||||
"Need to be specified a file to be editing"
|
||||
|
||||
try:
|
||||
return self._get_parser_to_modify()[0]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def items(self):
|
||||
# type: () -> Iterable[Tuple[str, Any]]
|
||||
"""Returns key-value pairs like dict.items() representing the loaded
|
||||
configuration
|
||||
"""
|
||||
return self._dictionary.items()
|
||||
|
||||
def get_value(self, key):
|
||||
# type: (str) -> Any
|
||||
"""Get a value from the configuration.
|
||||
"""
|
||||
try:
|
||||
return self._dictionary[key]
|
||||
except KeyError:
|
||||
raise ConfigurationError("No such key - {}".format(key))
|
||||
|
||||
def set_value(self, key, value):
|
||||
# type: (str, Any) -> None
|
||||
"""Modify a value in the configuration.
|
||||
"""
|
||||
self._ensure_have_load_only()
|
||||
|
||||
fname, parser = self._get_parser_to_modify()
|
||||
|
||||
if parser is not None:
|
||||
section, name = _disassemble_key(key)
|
||||
|
||||
# Modify the parser and the configuration
|
||||
if not parser.has_section(section):
|
||||
parser.add_section(section)
|
||||
parser.set(section, name, value)
|
||||
|
||||
self._config[self.load_only][key] = value
|
||||
self._mark_as_modified(fname, parser)
|
||||
|
||||
def unset_value(self, key):
|
||||
# type: (str) -> None
|
||||
"""Unset a value in the configuration.
|
||||
"""
|
||||
self._ensure_have_load_only()
|
||||
|
||||
if key not in self._config[self.load_only]:
|
||||
raise ConfigurationError("No such key - {}".format(key))
|
||||
|
||||
fname, parser = self._get_parser_to_modify()
|
||||
|
||||
if parser is not None:
|
||||
section, name = _disassemble_key(key)
|
||||
|
||||
# Remove the key in the parser
|
||||
modified_something = False
|
||||
if parser.has_section(section):
|
||||
# Returns whether the option was removed or not
|
||||
modified_something = parser.remove_option(section, name)
|
||||
|
||||
if modified_something:
|
||||
# name removed from parser, section may now be empty
|
||||
section_iter = iter(parser.items(section))
|
||||
try:
|
||||
val = six.next(section_iter)
|
||||
except StopIteration:
|
||||
val = None
|
||||
|
||||
if val is None:
|
||||
parser.remove_section(section)
|
||||
|
||||
self._mark_as_modified(fname, parser)
|
||||
else:
|
||||
raise ConfigurationError(
|
||||
"Fatal Internal error [id=1]. Please report as a bug."
|
||||
)
|
||||
|
||||
del self._config[self.load_only][key]
|
||||
|
||||
def save(self):
|
||||
# type: () -> None
|
||||
"""Save the currentin-memory state.
|
||||
"""
|
||||
self._ensure_have_load_only()
|
||||
|
||||
for fname, parser in self._modified_parsers:
|
||||
logger.info("Writing to %s", fname)
|
||||
|
||||
# Ensure directory exists.
|
||||
ensure_dir(os.path.dirname(fname))
|
||||
|
||||
with open(fname, "w") as f:
|
||||
parser.write(f) # type: ignore
|
||||
|
||||
#
|
||||
# Private routines
|
||||
#
|
||||
|
||||
def _ensure_have_load_only(self):
|
||||
# type: () -> None
|
||||
if self.load_only is None:
|
||||
raise ConfigurationError("Needed a specific file to be modifying.")
|
||||
logger.debug("Will be working with %s variant only", self.load_only)
|
||||
|
||||
@property
|
||||
def _dictionary(self):
|
||||
# type: () -> Dict[str, Any]
|
||||
"""A dictionary representing the loaded configuration.
|
||||
"""
|
||||
# NOTE: Dictionaries are not populated if not loaded. So, conditionals
|
||||
# are not needed here.
|
||||
retval = {}
|
||||
|
||||
for variant in self._override_order:
|
||||
retval.update(self._config[variant])
|
||||
|
||||
return retval
|
||||
|
||||
def _load_config_files(self):
|
||||
# type: () -> None
|
||||
"""Loads configuration from configuration files
|
||||
"""
|
||||
config_files = dict(self._iter_config_files())
|
||||
if config_files[kinds.ENV][0:1] == [os.devnull]:
|
||||
logger.debug(
|
||||
"Skipping loading configuration files due to "
|
||||
"environment's PIP_CONFIG_FILE being os.devnull"
|
||||
)
|
||||
return
|
||||
|
||||
for variant, files in config_files.items():
|
||||
for fname in files:
|
||||
# If there's specific variant set in `load_only`, load only
|
||||
# that variant, not the others.
|
||||
if self.load_only is not None and variant != self.load_only:
|
||||
logger.debug(
|
||||
"Skipping file '%s' (variant: %s)", fname, variant
|
||||
)
|
||||
continue
|
||||
|
||||
parser = self._load_file(variant, fname)
|
||||
|
||||
# Keeping track of the parsers used
|
||||
self._parsers[variant].append((fname, parser))
|
||||
|
||||
def _load_file(self, variant, fname):
|
||||
# type: (Kind, str) -> RawConfigParser
|
||||
logger.debug("For variant '%s', will try loading '%s'", variant, fname)
|
||||
parser = self._construct_parser(fname)
|
||||
|
||||
for section in parser.sections():
|
||||
items = parser.items(section)
|
||||
self._config[variant].update(self._normalized_keys(section, items))
|
||||
|
||||
return parser
|
||||
|
||||
def _construct_parser(self, fname):
|
||||
# type: (str) -> RawConfigParser
|
||||
parser = configparser.RawConfigParser()
|
||||
# If there is no such file, don't bother reading it but create the
|
||||
# parser anyway, to hold the data.
|
||||
# Doing this is useful when modifying and saving files, where we don't
|
||||
# need to construct a parser.
|
||||
if os.path.exists(fname):
|
||||
try:
|
||||
parser.read(fname)
|
||||
except UnicodeDecodeError:
|
||||
raise ConfigurationError((
|
||||
"ERROR: "
|
||||
"Configuration file contains invalid %s characters.\n"
|
||||
"Please fix your configuration, located at %s\n"
|
||||
) % (locale.getpreferredencoding(False), fname))
|
||||
return parser
|
||||
|
||||
def _load_environment_vars(self):
|
||||
# type: () -> None
|
||||
"""Loads configuration from environment variables
|
||||
"""
|
||||
self._config[kinds.ENV_VAR].update(
|
||||
self._normalized_keys(":env:", self._get_environ_vars())
|
||||
)
|
||||
|
||||
def _normalized_keys(self, section, items):
|
||||
# type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any]
|
||||
"""Normalizes items to construct a dictionary with normalized keys.
|
||||
|
||||
This routine is where the names become keys and are made the same
|
||||
regardless of source - configuration files or environment.
|
||||
"""
|
||||
normalized = {}
|
||||
for name, val in items:
|
||||
key = section + "." + _normalize_name(name)
|
||||
normalized[key] = val
|
||||
return normalized
|
||||
|
||||
def _get_environ_vars(self):
|
||||
# type: () -> Iterable[Tuple[str, str]]
|
||||
"""Returns a generator with all environmental vars with prefix PIP_"""
|
||||
for key, val in os.environ.items():
|
||||
should_be_yielded = (
|
||||
key.startswith("PIP_") and
|
||||
key[4:].lower() not in self._ignore_env_names
|
||||
)
|
||||
if should_be_yielded:
|
||||
yield key[4:].lower(), val
|
||||
|
||||
# XXX: This is patched in the tests.
|
||||
def _iter_config_files(self):
|
||||
# type: () -> Iterable[Tuple[Kind, List[str]]]
|
||||
"""Yields variant and configuration files associated with it.
|
||||
|
||||
This should be treated like items of a dictionary.
|
||||
"""
|
||||
# SMELL: Move the conditions out of this function
|
||||
|
||||
# environment variables have the lowest priority
|
||||
config_file = os.environ.get('PIP_CONFIG_FILE', None)
|
||||
if config_file is not None:
|
||||
yield kinds.ENV, [config_file]
|
||||
else:
|
||||
yield kinds.ENV, []
|
||||
|
||||
# at the base we have any global configuration
|
||||
yield kinds.GLOBAL, list(site_config_files)
|
||||
|
||||
# per-user configuration next
|
||||
should_load_user_config = not self.isolated and not (
|
||||
config_file and os.path.exists(config_file)
|
||||
)
|
||||
if should_load_user_config:
|
||||
# The legacy config file is overridden by the new config file
|
||||
yield kinds.USER, [legacy_config_file, new_config_file]
|
||||
|
||||
# finally virtualenv configuration first trumping others
|
||||
if running_under_virtualenv():
|
||||
yield kinds.VENV, [venv_config_file]
|
||||
|
||||
def _get_parser_to_modify(self):
|
||||
# type: () -> Tuple[str, RawConfigParser]
|
||||
# Determine which parser to modify
|
||||
parsers = self._parsers[self.load_only]
|
||||
if not parsers:
|
||||
# This should not happen if everything works correctly.
|
||||
raise ConfigurationError(
|
||||
"Fatal Internal error [id=2]. Please report as a bug."
|
||||
)
|
||||
|
||||
# Use the highest priority parser.
|
||||
return parsers[-1]
|
||||
|
||||
# XXX: This is patched in the tests.
|
||||
def _mark_as_modified(self, fname, parser):
|
||||
# type: (str, RawConfigParser) -> None
|
||||
file_parser_tuple = (fname, parser)
|
||||
if file_parser_tuple not in self._modified_parsers:
|
||||
self._modified_parsers.append(file_parser_tuple)
|
||||
@@ -1,922 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import cgi
|
||||
import email.utils
|
||||
import getpass
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
from pip._vendor import requests, six, urllib3
|
||||
from pip._vendor.cachecontrol import CacheControlAdapter
|
||||
from pip._vendor.cachecontrol.caches import FileCache
|
||||
from pip._vendor.lockfile import LockError
|
||||
from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
|
||||
from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
|
||||
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
|
||||
from pip._vendor.requests.structures import CaseInsensitiveDict
|
||||
from pip._vendor.requests.utils import get_netrc_auth
|
||||
# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
|
||||
# why we ignore the type on this import
|
||||
from pip._vendor.six.moves import xmlrpc_client # type: ignore
|
||||
from pip._vendor.six.moves.urllib import parse as urllib_parse
|
||||
from pip._vendor.six.moves.urllib import request as urllib_request
|
||||
from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote
|
||||
from pip._vendor.urllib3.util import IS_PYOPENSSL
|
||||
|
||||
import pip
|
||||
from pip._internal.compat import WINDOWS
|
||||
from pip._internal.exceptions import HashMismatch, InstallationError
|
||||
from pip._internal.locations import write_delete_marker_file
|
||||
from pip._internal.models import PyPI
|
||||
from pip._internal.utils.encoding import auto_decode
|
||||
from pip._internal.utils.filesystem import check_path_owner
|
||||
from pip._internal.utils.glibc import libc_ver
|
||||
from pip._internal.utils.logging import indent_log
|
||||
from pip._internal.utils.misc import (
|
||||
ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume,
|
||||
display_path, format_size, get_installed_version, rmtree, splitext,
|
||||
unpack_file,
|
||||
)
|
||||
from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
from pip._internal.utils.ui import DownloadProgressProvider
|
||||
from pip._internal.vcs import vcs
|
||||
|
||||
try:
|
||||
import ssl # noqa
|
||||
except ImportError:
|
||||
ssl = None
|
||||
|
||||
HAS_TLS = (ssl is not None) or IS_PYOPENSSL
|
||||
|
||||
__all__ = ['get_file_content',
|
||||
'is_url', 'url_to_path', 'path_to_url',
|
||||
'is_archive_file', 'unpack_vcs_link',
|
||||
'unpack_file_url', 'is_vcs_url', 'is_file_url',
|
||||
'unpack_http_url', 'unpack_url']
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def user_agent():
|
||||
"""
|
||||
Return a string representing the user agent.
|
||||
"""
|
||||
data = {
|
||||
"installer": {"name": "pip", "version": pip.__version__},
|
||||
"python": platform.python_version(),
|
||||
"implementation": {
|
||||
"name": platform.python_implementation(),
|
||||
},
|
||||
}
|
||||
|
||||
if data["implementation"]["name"] == 'CPython':
|
||||
data["implementation"]["version"] = platform.python_version()
|
||||
elif data["implementation"]["name"] == 'PyPy':
|
||||
if sys.pypy_version_info.releaselevel == 'final':
|
||||
pypy_version_info = sys.pypy_version_info[:3]
|
||||
else:
|
||||
pypy_version_info = sys.pypy_version_info
|
||||
data["implementation"]["version"] = ".".join(
|
||||
[str(x) for x in pypy_version_info]
|
||||
)
|
||||
elif data["implementation"]["name"] == 'Jython':
|
||||
# Complete Guess
|
||||
data["implementation"]["version"] = platform.python_version()
|
||||
elif data["implementation"]["name"] == 'IronPython':
|
||||
# Complete Guess
|
||||
data["implementation"]["version"] = platform.python_version()
|
||||
|
||||
if sys.platform.startswith("linux"):
|
||||
from pip._vendor import distro
|
||||
distro_infos = dict(filter(
|
||||
lambda x: x[1],
|
||||
zip(["name", "version", "id"], distro.linux_distribution()),
|
||||
))
|
||||
libc = dict(filter(
|
||||
lambda x: x[1],
|
||||
zip(["lib", "version"], libc_ver()),
|
||||
))
|
||||
if libc:
|
||||
distro_infos["libc"] = libc
|
||||
if distro_infos:
|
||||
data["distro"] = distro_infos
|
||||
|
||||
if sys.platform.startswith("darwin") and platform.mac_ver()[0]:
|
||||
data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]}
|
||||
|
||||
if platform.system():
|
||||
data.setdefault("system", {})["name"] = platform.system()
|
||||
|
||||
if platform.release():
|
||||
data.setdefault("system", {})["release"] = platform.release()
|
||||
|
||||
if platform.machine():
|
||||
data["cpu"] = platform.machine()
|
||||
|
||||
if HAS_TLS:
|
||||
data["openssl_version"] = ssl.OPENSSL_VERSION
|
||||
|
||||
setuptools_version = get_installed_version("setuptools")
|
||||
if setuptools_version is not None:
|
||||
data["setuptools_version"] = setuptools_version
|
||||
|
||||
return "{data[installer][name]}/{data[installer][version]} {json}".format(
|
||||
data=data,
|
||||
json=json.dumps(data, separators=(",", ":"), sort_keys=True),
|
||||
)
|
||||
|
||||
|
||||
class MultiDomainBasicAuth(AuthBase):
|
||||
|
||||
def __init__(self, prompting=True):
|
||||
self.prompting = prompting
|
||||
self.passwords = {}
|
||||
|
||||
def __call__(self, req):
|
||||
parsed = urllib_parse.urlparse(req.url)
|
||||
|
||||
# Get the netloc without any embedded credentials
|
||||
netloc = parsed.netloc.rsplit("@", 1)[-1]
|
||||
|
||||
# Set the url of the request to the url without any credentials
|
||||
req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:])
|
||||
|
||||
# Use any stored credentials that we have for this netloc
|
||||
username, password = self.passwords.get(netloc, (None, None))
|
||||
|
||||
# Extract credentials embedded in the url if we have none stored
|
||||
if username is None:
|
||||
username, password = self.parse_credentials(parsed.netloc)
|
||||
|
||||
# Get creds from netrc if we still don't have them
|
||||
if username is None and password is None:
|
||||
netrc_auth = get_netrc_auth(req.url)
|
||||
username, password = netrc_auth if netrc_auth else (None, None)
|
||||
|
||||
if username or password:
|
||||
# Store the username and password
|
||||
self.passwords[netloc] = (username, password)
|
||||
|
||||
# Send the basic auth with this request
|
||||
req = HTTPBasicAuth(username or "", password or "")(req)
|
||||
|
||||
# Attach a hook to handle 401 responses
|
||||
req.register_hook("response", self.handle_401)
|
||||
|
||||
return req
|
||||
|
||||
def handle_401(self, resp, **kwargs):
|
||||
# We only care about 401 responses, anything else we want to just
|
||||
# pass through the actual response
|
||||
if resp.status_code != 401:
|
||||
return resp
|
||||
|
||||
# We are not able to prompt the user so simply return the response
|
||||
if not self.prompting:
|
||||
return resp
|
||||
|
||||
parsed = urllib_parse.urlparse(resp.url)
|
||||
|
||||
# Prompt the user for a new username and password
|
||||
username = six.moves.input("User for %s: " % parsed.netloc)
|
||||
password = getpass.getpass("Password: ")
|
||||
|
||||
# Store the new username and password to use for future requests
|
||||
if username or password:
|
||||
self.passwords[parsed.netloc] = (username, password)
|
||||
|
||||
# Consume content and release the original connection to allow our new
|
||||
# request to reuse the same one.
|
||||
resp.content
|
||||
resp.raw.release_conn()
|
||||
|
||||
# Add our new username and password to the request
|
||||
req = HTTPBasicAuth(username or "", password or "")(resp.request)
|
||||
|
||||
# Send our new request
|
||||
new_resp = resp.connection.send(req, **kwargs)
|
||||
new_resp.history.append(resp)
|
||||
|
||||
return new_resp
|
||||
|
||||
def parse_credentials(self, netloc):
|
||||
if "@" in netloc:
|
||||
userinfo = netloc.rsplit("@", 1)[0]
|
||||
if ":" in userinfo:
|
||||
user, pwd = userinfo.split(":", 1)
|
||||
return (urllib_unquote(user), urllib_unquote(pwd))
|
||||
return urllib_unquote(userinfo), None
|
||||
return None, None
|
||||
|
||||
|
||||
class LocalFSAdapter(BaseAdapter):
|
||||
|
||||
def send(self, request, stream=None, timeout=None, verify=None, cert=None,
|
||||
proxies=None):
|
||||
pathname = url_to_path(request.url)
|
||||
|
||||
resp = Response()
|
||||
resp.status_code = 200
|
||||
resp.url = request.url
|
||||
|
||||
try:
|
||||
stats = os.stat(pathname)
|
||||
except OSError as exc:
|
||||
resp.status_code = 404
|
||||
resp.raw = exc
|
||||
else:
|
||||
modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
|
||||
content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
|
||||
resp.headers = CaseInsensitiveDict({
|
||||
"Content-Type": content_type,
|
||||
"Content-Length": stats.st_size,
|
||||
"Last-Modified": modified,
|
||||
})
|
||||
|
||||
resp.raw = open(pathname, "rb")
|
||||
resp.close = resp.raw.close
|
||||
|
||||
return resp
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class SafeFileCache(FileCache):
|
||||
"""
|
||||
A file based cache which is safe to use even when the target directory may
|
||||
not be accessible or writable.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(SafeFileCache, self).__init__(*args, **kwargs)
|
||||
|
||||
# Check to ensure that the directory containing our cache directory
|
||||
# is owned by the user current executing pip. If it does not exist
|
||||
# we will check the parent directory until we find one that does exist.
|
||||
# If it is not owned by the user executing pip then we will disable
|
||||
# the cache and log a warning.
|
||||
if not check_path_owner(self.directory):
|
||||
logger.warning(
|
||||
"The directory '%s' or its parent directory is not owned by "
|
||||
"the current user and the cache has been disabled. Please "
|
||||
"check the permissions and owner of that directory. If "
|
||||
"executing pip with sudo, you may want sudo's -H flag.",
|
||||
self.directory,
|
||||
)
|
||||
|
||||
# Set our directory to None to disable the Cache
|
||||
self.directory = None
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
# If we don't have a directory, then the cache should be a no-op.
|
||||
if self.directory is None:
|
||||
return
|
||||
|
||||
try:
|
||||
return super(SafeFileCache, self).get(*args, **kwargs)
|
||||
except (LockError, OSError, IOError):
|
||||
# We intentionally silence this error, if we can't access the cache
|
||||
# then we can just skip caching and process the request as if
|
||||
# caching wasn't enabled.
|
||||
pass
|
||||
|
||||
def set(self, *args, **kwargs):
|
||||
# If we don't have a directory, then the cache should be a no-op.
|
||||
if self.directory is None:
|
||||
return
|
||||
|
||||
try:
|
||||
return super(SafeFileCache, self).set(*args, **kwargs)
|
||||
except (LockError, OSError, IOError):
|
||||
# We intentionally silence this error, if we can't access the cache
|
||||
# then we can just skip caching and process the request as if
|
||||
# caching wasn't enabled.
|
||||
pass
|
||||
|
||||
def delete(self, *args, **kwargs):
|
||||
# If we don't have a directory, then the cache should be a no-op.
|
||||
if self.directory is None:
|
||||
return
|
||||
|
||||
try:
|
||||
return super(SafeFileCache, self).delete(*args, **kwargs)
|
||||
except (LockError, OSError, IOError):
|
||||
# We intentionally silence this error, if we can't access the cache
|
||||
# then we can just skip caching and process the request as if
|
||||
# caching wasn't enabled.
|
||||
pass
|
||||
|
||||
|
||||
class InsecureHTTPAdapter(HTTPAdapter):
|
||||
|
||||
def cert_verify(self, conn, url, verify, cert):
|
||||
conn.cert_reqs = 'CERT_NONE'
|
||||
conn.ca_certs = None
|
||||
|
||||
|
||||
class PipSession(requests.Session):
|
||||
|
||||
timeout = None
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
retries = kwargs.pop("retries", 0)
|
||||
cache = kwargs.pop("cache", None)
|
||||
insecure_hosts = kwargs.pop("insecure_hosts", [])
|
||||
|
||||
super(PipSession, self).__init__(*args, **kwargs)
|
||||
|
||||
# Attach our User Agent to the request
|
||||
self.headers["User-Agent"] = user_agent()
|
||||
|
||||
# Attach our Authentication handler to the session
|
||||
self.auth = MultiDomainBasicAuth()
|
||||
|
||||
# Create our urllib3.Retry instance which will allow us to customize
|
||||
# how we handle retries.
|
||||
retries = urllib3.Retry(
|
||||
# Set the total number of retries that a particular request can
|
||||
# have.
|
||||
total=retries,
|
||||
|
||||
# A 503 error from PyPI typically means that the Fastly -> Origin
|
||||
# connection got interrupted in some way. A 503 error in general
|
||||
# is typically considered a transient error so we'll go ahead and
|
||||
# retry it.
|
||||
# A 500 may indicate transient error in Amazon S3
|
||||
# A 520 or 527 - may indicate transient error in CloudFlare
|
||||
status_forcelist=[500, 503, 520, 527],
|
||||
|
||||
# Add a small amount of back off between failed requests in
|
||||
# order to prevent hammering the service.
|
||||
backoff_factor=0.25,
|
||||
)
|
||||
|
||||
# We want to _only_ cache responses on securely fetched origins. We do
|
||||
# this because we can't validate the response of an insecurely fetched
|
||||
# origin, and we don't want someone to be able to poison the cache and
|
||||
# require manual eviction from the cache to fix it.
|
||||
if cache:
|
||||
secure_adapter = CacheControlAdapter(
|
||||
cache=SafeFileCache(cache, use_dir_lock=True),
|
||||
max_retries=retries,
|
||||
)
|
||||
else:
|
||||
secure_adapter = HTTPAdapter(max_retries=retries)
|
||||
|
||||
# Our Insecure HTTPAdapter disables HTTPS validation. It does not
|
||||
# support caching (see above) so we'll use it for all http:// URLs as
|
||||
# well as any https:// host that we've marked as ignoring TLS errors
|
||||
# for.
|
||||
insecure_adapter = InsecureHTTPAdapter(max_retries=retries)
|
||||
|
||||
self.mount("https://", secure_adapter)
|
||||
self.mount("http://", insecure_adapter)
|
||||
|
||||
# Enable file:// urls
|
||||
self.mount("file://", LocalFSAdapter())
|
||||
|
||||
# We want to use a non-validating adapter for any requests which are
|
||||
# deemed insecure.
|
||||
for host in insecure_hosts:
|
||||
self.mount("https://{}/".format(host), insecure_adapter)
|
||||
|
||||
def request(self, method, url, *args, **kwargs):
|
||||
# Allow setting a default timeout on a session
|
||||
kwargs.setdefault("timeout", self.timeout)
|
||||
|
||||
# Dispatch the actual request
|
||||
return super(PipSession, self).request(method, url, *args, **kwargs)
|
||||
|
||||
|
||||
def get_file_content(url, comes_from=None, session=None):
|
||||
"""Gets the content of a file; it may be a filename, file: URL, or
|
||||
http: URL. Returns (location, content). Content is unicode.
|
||||
|
||||
:param url: File path or url.
|
||||
:param comes_from: Origin description of requirements.
|
||||
:param session: Instance of pip.download.PipSession.
|
||||
"""
|
||||
if session is None:
|
||||
raise TypeError(
|
||||
"get_file_content() missing 1 required keyword argument: 'session'"
|
||||
)
|
||||
|
||||
match = _scheme_re.search(url)
|
||||
if match:
|
||||
scheme = match.group(1).lower()
|
||||
if (scheme == 'file' and comes_from and
|
||||
comes_from.startswith('http')):
|
||||
raise InstallationError(
|
||||
'Requirements file %s references URL %s, which is local'
|
||||
% (comes_from, url))
|
||||
if scheme == 'file':
|
||||
path = url.split(':', 1)[1]
|
||||
path = path.replace('\\', '/')
|
||||
match = _url_slash_drive_re.match(path)
|
||||
if match:
|
||||
path = match.group(1) + ':' + path.split('|', 1)[1]
|
||||
path = urllib_parse.unquote(path)
|
||||
if path.startswith('/'):
|
||||
path = '/' + path.lstrip('/')
|
||||
url = path
|
||||
else:
|
||||
# FIXME: catch some errors
|
||||
resp = session.get(url)
|
||||
resp.raise_for_status()
|
||||
return resp.url, resp.text
|
||||
try:
|
||||
with open(url, 'rb') as f:
|
||||
content = auto_decode(f.read())
|
||||
except IOError as exc:
|
||||
raise InstallationError(
|
||||
'Could not open requirements file: %s' % str(exc)
|
||||
)
|
||||
return url, content
|
||||
|
||||
|
||||
_scheme_re = re.compile(r'^(http|https|file):', re.I)
|
||||
_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)
|
||||
|
||||
|
||||
def is_url(name):
|
||||
"""Returns true if the name looks like a URL"""
|
||||
if ':' not in name:
|
||||
return False
|
||||
scheme = name.split(':', 1)[0].lower()
|
||||
return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes
|
||||
|
||||
|
||||
def url_to_path(url):
|
||||
"""
|
||||
Convert a file: URL to a path.
|
||||
"""
|
||||
assert url.startswith('file:'), (
|
||||
"You can only turn file: urls into filenames (not %r)" % url)
|
||||
|
||||
_, netloc, path, _, _ = urllib_parse.urlsplit(url)
|
||||
|
||||
# if we have a UNC path, prepend UNC share notation
|
||||
if netloc:
|
||||
netloc = '\\\\' + netloc
|
||||
|
||||
path = urllib_request.url2pathname(netloc + path)
|
||||
return path
|
||||
|
||||
|
||||
def path_to_url(path):
|
||||
"""
|
||||
Convert a path to a file: URL. The path will be made absolute and have
|
||||
quoted path parts.
|
||||
"""
|
||||
path = os.path.normpath(os.path.abspath(path))
|
||||
url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path))
|
||||
return url
|
||||
|
||||
|
||||
def is_archive_file(name):
|
||||
"""Return True if `name` is a considered as an archive file."""
|
||||
ext = splitext(name)[1].lower()
|
||||
if ext in ARCHIVE_EXTENSIONS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def unpack_vcs_link(link, location):
|
||||
vcs_backend = _get_used_vcs_backend(link)
|
||||
vcs_backend.unpack(location)
|
||||
|
||||
|
||||
def _get_used_vcs_backend(link):
|
||||
for backend in vcs.backends:
|
||||
if link.scheme in backend.schemes:
|
||||
vcs_backend = backend(link.url)
|
||||
return vcs_backend
|
||||
|
||||
|
||||
def is_vcs_url(link):
|
||||
return bool(_get_used_vcs_backend(link))
|
||||
|
||||
|
||||
def is_file_url(link):
|
||||
return link.url.lower().startswith('file:')
|
||||
|
||||
|
||||
def is_dir_url(link):
|
||||
"""Return whether a file:// Link points to a directory.
|
||||
|
||||
``link`` must not have any other scheme but file://. Call is_file_url()
|
||||
first.
|
||||
|
||||
"""
|
||||
link_path = url_to_path(link.url_without_fragment)
|
||||
return os.path.isdir(link_path)
|
||||
|
||||
|
||||
def _progress_indicator(iterable, *args, **kwargs):
|
||||
return iterable
|
||||
|
||||
|
||||
def _download_url(resp, link, content_file, hashes, progress_bar):
|
||||
try:
|
||||
total_length = int(resp.headers['content-length'])
|
||||
except (ValueError, KeyError, TypeError):
|
||||
total_length = 0
|
||||
|
||||
cached_resp = getattr(resp, "from_cache", False)
|
||||
if logger.getEffectiveLevel() > logging.INFO:
|
||||
show_progress = False
|
||||
elif cached_resp:
|
||||
show_progress = False
|
||||
elif total_length > (40 * 1000):
|
||||
show_progress = True
|
||||
elif not total_length:
|
||||
show_progress = True
|
||||
else:
|
||||
show_progress = False
|
||||
|
||||
show_url = link.show_url
|
||||
|
||||
def resp_read(chunk_size):
|
||||
try:
|
||||
# Special case for urllib3.
|
||||
for chunk in resp.raw.stream(
|
||||
chunk_size,
|
||||
# We use decode_content=False here because we don't
|
||||
# want urllib3 to mess with the raw bytes we get
|
||||
# from the server. If we decompress inside of
|
||||
# urllib3 then we cannot verify the checksum
|
||||
# because the checksum will be of the compressed
|
||||
# file. This breakage will only occur if the
|
||||
# server adds a Content-Encoding header, which
|
||||
# depends on how the server was configured:
|
||||
# - Some servers will notice that the file isn't a
|
||||
# compressible file and will leave the file alone
|
||||
# and with an empty Content-Encoding
|
||||
# - Some servers will notice that the file is
|
||||
# already compressed and will leave the file
|
||||
# alone and will add a Content-Encoding: gzip
|
||||
# header
|
||||
# - Some servers won't notice anything at all and
|
||||
# will take a file that's already been compressed
|
||||
# and compress it again and set the
|
||||
# Content-Encoding: gzip header
|
||||
#
|
||||
# By setting this not to decode automatically we
|
||||
# hope to eliminate problems with the second case.
|
||||
decode_content=False):
|
||||
yield chunk
|
||||
except AttributeError:
|
||||
# Standard file-like object.
|
||||
while True:
|
||||
chunk = resp.raw.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
def written_chunks(chunks):
|
||||
for chunk in chunks:
|
||||
content_file.write(chunk)
|
||||
yield chunk
|
||||
|
||||
progress_indicator = _progress_indicator
|
||||
|
||||
if link.netloc == PyPI.netloc:
|
||||
url = show_url
|
||||
else:
|
||||
url = link.url_without_fragment
|
||||
|
||||
if show_progress: # We don't show progress on cached responses
|
||||
progress_indicator = DownloadProgressProvider(progress_bar,
|
||||
max=total_length)
|
||||
if total_length:
|
||||
logger.info("Downloading %s (%s)", url, format_size(total_length))
|
||||
else:
|
||||
logger.info("Downloading %s", url)
|
||||
elif cached_resp:
|
||||
logger.info("Using cached %s", url)
|
||||
else:
|
||||
logger.info("Downloading %s", url)
|
||||
|
||||
logger.debug('Downloading from URL %s', link)
|
||||
|
||||
downloaded_chunks = written_chunks(
|
||||
progress_indicator(
|
||||
resp_read(CONTENT_CHUNK_SIZE),
|
||||
CONTENT_CHUNK_SIZE
|
||||
)
|
||||
)
|
||||
if hashes:
|
||||
hashes.check_against_chunks(downloaded_chunks)
|
||||
else:
|
||||
consume(downloaded_chunks)
|
||||
|
||||
|
||||
def _copy_file(filename, location, link):
|
||||
copy = True
|
||||
download_location = os.path.join(location, link.filename)
|
||||
if os.path.exists(download_location):
|
||||
response = ask_path_exists(
|
||||
'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' %
|
||||
display_path(download_location), ('i', 'w', 'b', 'a'))
|
||||
if response == 'i':
|
||||
copy = False
|
||||
elif response == 'w':
|
||||
logger.warning('Deleting %s', display_path(download_location))
|
||||
os.remove(download_location)
|
||||
elif response == 'b':
|
||||
dest_file = backup_dir(download_location)
|
||||
logger.warning(
|
||||
'Backing up %s to %s',
|
||||
display_path(download_location),
|
||||
display_path(dest_file),
|
||||
)
|
||||
shutil.move(download_location, dest_file)
|
||||
elif response == 'a':
|
||||
sys.exit(-1)
|
||||
if copy:
|
||||
shutil.copy(filename, download_location)
|
||||
logger.info('Saved %s', display_path(download_location))
|
||||
|
||||
|
||||
def unpack_http_url(link, location, download_dir=None,
|
||||
session=None, hashes=None, progress_bar="on"):
|
||||
if session is None:
|
||||
raise TypeError(
|
||||
"unpack_http_url() missing 1 required keyword argument: 'session'"
|
||||
)
|
||||
|
||||
with TempDirectory(kind="unpack") as temp_dir:
|
||||
# If a download dir is specified, is the file already downloaded there?
|
||||
already_downloaded_path = None
|
||||
if download_dir:
|
||||
already_downloaded_path = _check_download_dir(link,
|
||||
download_dir,
|
||||
hashes)
|
||||
|
||||
if already_downloaded_path:
|
||||
from_path = already_downloaded_path
|
||||
content_type = mimetypes.guess_type(from_path)[0]
|
||||
else:
|
||||
# let's download to a tmp dir
|
||||
from_path, content_type = _download_http_url(link,
|
||||
session,
|
||||
temp_dir.path,
|
||||
hashes,
|
||||
progress_bar)
|
||||
|
||||
# unpack the archive to the build dir location. even when only
|
||||
# downloading archives, they have to be unpacked to parse dependencies
|
||||
unpack_file(from_path, location, content_type, link)
|
||||
|
||||
# a download dir is specified; let's copy the archive there
|
||||
if download_dir and not already_downloaded_path:
|
||||
_copy_file(from_path, download_dir, link)
|
||||
|
||||
if not already_downloaded_path:
|
||||
os.unlink(from_path)
|
||||
|
||||
|
||||
def unpack_file_url(link, location, download_dir=None, hashes=None):
|
||||
"""Unpack link into location.
|
||||
|
||||
If download_dir is provided and link points to a file, make a copy
|
||||
of the link file inside download_dir.
|
||||
"""
|
||||
link_path = url_to_path(link.url_without_fragment)
|
||||
|
||||
# If it's a url to a local directory
|
||||
if is_dir_url(link):
|
||||
if os.path.isdir(location):
|
||||
rmtree(location)
|
||||
shutil.copytree(link_path, location, symlinks=True)
|
||||
if download_dir:
|
||||
logger.info('Link is a directory, ignoring download_dir')
|
||||
return
|
||||
|
||||
# If --require-hashes is off, `hashes` is either empty, the
|
||||
# link's embedded hash, or MissingHashes; it is required to
|
||||
# match. If --require-hashes is on, we are satisfied by any
|
||||
# hash in `hashes` matching: a URL-based or an option-based
|
||||
# one; no internet-sourced hash will be in `hashes`.
|
||||
if hashes:
|
||||
hashes.check_against_path(link_path)
|
||||
|
||||
# If a download dir is specified, is the file already there and valid?
|
||||
already_downloaded_path = None
|
||||
if download_dir:
|
||||
already_downloaded_path = _check_download_dir(link,
|
||||
download_dir,
|
||||
hashes)
|
||||
|
||||
if already_downloaded_path:
|
||||
from_path = already_downloaded_path
|
||||
else:
|
||||
from_path = link_path
|
||||
|
||||
content_type = mimetypes.guess_type(from_path)[0]
|
||||
|
||||
# unpack the archive to the build dir location. even when only downloading
|
||||
# archives, they have to be unpacked to parse dependencies
|
||||
unpack_file(from_path, location, content_type, link)
|
||||
|
||||
# a download dir is specified and not already downloaded
|
||||
if download_dir and not already_downloaded_path:
|
||||
_copy_file(from_path, download_dir, link)
|
||||
|
||||
|
||||
def _copy_dist_from_dir(link_path, location):
|
||||
"""Copy distribution files in `link_path` to `location`.
|
||||
|
||||
Invoked when user requests to install a local directory. E.g.:
|
||||
|
||||
pip install .
|
||||
pip install ~/dev/git-repos/python-prompt-toolkit
|
||||
|
||||
"""
|
||||
|
||||
# Note: This is currently VERY SLOW if you have a lot of data in the
|
||||
# directory, because it copies everything with `shutil.copytree`.
|
||||
# What it should really do is build an sdist and install that.
|
||||
# See https://github.com/pypa/pip/issues/2195
|
||||
|
||||
if os.path.isdir(location):
|
||||
rmtree(location)
|
||||
|
||||
# build an sdist
|
||||
setup_py = 'setup.py'
|
||||
sdist_args = [sys.executable]
|
||||
sdist_args.append('-c')
|
||||
sdist_args.append(SETUPTOOLS_SHIM % setup_py)
|
||||
sdist_args.append('sdist')
|
||||
sdist_args += ['--dist-dir', location]
|
||||
logger.info('Running setup.py sdist for %s', link_path)
|
||||
|
||||
with indent_log():
|
||||
call_subprocess(sdist_args, cwd=link_path, show_stdout=False)
|
||||
|
||||
# unpack sdist into `location`
|
||||
sdist = os.path.join(location, os.listdir(location)[0])
|
||||
logger.info('Unpacking sdist %s into %s', sdist, location)
|
||||
unpack_file(sdist, location, content_type=None, link=None)
|
||||
|
||||
|
||||
class PipXmlrpcTransport(xmlrpc_client.Transport):
|
||||
"""Provide a `xmlrpclib.Transport` implementation via a `PipSession`
|
||||
object.
|
||||
"""
|
||||
|
||||
def __init__(self, index_url, session, use_datetime=False):
|
||||
xmlrpc_client.Transport.__init__(self, use_datetime)
|
||||
index_parts = urllib_parse.urlparse(index_url)
|
||||
self._scheme = index_parts.scheme
|
||||
self._session = session
|
||||
|
||||
def request(self, host, handler, request_body, verbose=False):
|
||||
parts = (self._scheme, host, handler, None, None, None)
|
||||
url = urllib_parse.urlunparse(parts)
|
||||
try:
|
||||
headers = {'Content-Type': 'text/xml'}
|
||||
response = self._session.post(url, data=request_body,
|
||||
headers=headers, stream=True)
|
||||
response.raise_for_status()
|
||||
self.verbose = verbose
|
||||
return self.parse_response(response.raw)
|
||||
except requests.HTTPError as exc:
|
||||
logger.critical(
|
||||
"HTTP error %s while getting %s",
|
||||
exc.response.status_code, url,
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def unpack_url(link, location, download_dir=None,
|
||||
only_download=False, session=None, hashes=None,
|
||||
progress_bar="on"):
|
||||
"""Unpack link.
|
||||
If link is a VCS link:
|
||||
if only_download, export into download_dir and ignore location
|
||||
else unpack into location
|
||||
for other types of link:
|
||||
- unpack into location
|
||||
- if download_dir, copy the file into download_dir
|
||||
- if only_download, mark location for deletion
|
||||
|
||||
:param hashes: A Hashes object, one of whose embedded hashes must match,
|
||||
or HashMismatch will be raised. If the Hashes is empty, no matches are
|
||||
required, and unhashable types of requirements (like VCS ones, which
|
||||
would ordinarily raise HashUnsupported) are allowed.
|
||||
"""
|
||||
# non-editable vcs urls
|
||||
if is_vcs_url(link):
|
||||
unpack_vcs_link(link, location)
|
||||
|
||||
# file urls
|
||||
elif is_file_url(link):
|
||||
unpack_file_url(link, location, download_dir, hashes=hashes)
|
||||
|
||||
# http urls
|
||||
else:
|
||||
if session is None:
|
||||
session = PipSession()
|
||||
|
||||
unpack_http_url(
|
||||
link,
|
||||
location,
|
||||
download_dir,
|
||||
session,
|
||||
hashes=hashes,
|
||||
progress_bar=progress_bar
|
||||
)
|
||||
if only_download:
|
||||
write_delete_marker_file(location)
|
||||
|
||||
|
||||
def _download_http_url(link, session, temp_dir, hashes, progress_bar):
|
||||
"""Download link url into temp_dir using provided session"""
|
||||
target_url = link.url.split('#', 1)[0]
|
||||
try:
|
||||
resp = session.get(
|
||||
target_url,
|
||||
# We use Accept-Encoding: identity here because requests
|
||||
# defaults to accepting compressed responses. This breaks in
|
||||
# a variety of ways depending on how the server is configured.
|
||||
# - Some servers will notice that the file isn't a compressible
|
||||
# file and will leave the file alone and with an empty
|
||||
# Content-Encoding
|
||||
# - Some servers will notice that the file is already
|
||||
# compressed and will leave the file alone and will add a
|
||||
# Content-Encoding: gzip header
|
||||
# - Some servers won't notice anything at all and will take
|
||||
# a file that's already been compressed and compress it again
|
||||
# and set the Content-Encoding: gzip header
|
||||
# By setting this to request only the identity encoding We're
|
||||
# hoping to eliminate the third case. Hopefully there does not
|
||||
# exist a server which when given a file will notice it is
|
||||
# already compressed and that you're not asking for a
|
||||
# compressed file and will then decompress it before sending
|
||||
# because if that's the case I don't think it'll ever be
|
||||
# possible to make this work.
|
||||
headers={"Accept-Encoding": "identity"},
|
||||
stream=True,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except requests.HTTPError as exc:
|
||||
logger.critical(
|
||||
"HTTP error %s while getting %s", exc.response.status_code, link,
|
||||
)
|
||||
raise
|
||||
|
||||
content_type = resp.headers.get('content-type', '')
|
||||
filename = link.filename # fallback
|
||||
# Have a look at the Content-Disposition header for a better guess
|
||||
content_disposition = resp.headers.get('content-disposition')
|
||||
if content_disposition:
|
||||
type, params = cgi.parse_header(content_disposition)
|
||||
# We use ``or`` here because we don't want to use an "empty" value
|
||||
# from the filename param.
|
||||
filename = params.get('filename') or filename
|
||||
ext = splitext(filename)[1]
|
||||
if not ext:
|
||||
ext = mimetypes.guess_extension(content_type)
|
||||
if ext:
|
||||
filename += ext
|
||||
if not ext and link.url != resp.url:
|
||||
ext = os.path.splitext(resp.url)[1]
|
||||
if ext:
|
||||
filename += ext
|
||||
file_path = os.path.join(temp_dir, filename)
|
||||
with open(file_path, 'wb') as content_file:
|
||||
_download_url(resp, link, content_file, hashes, progress_bar)
|
||||
return file_path, content_type
|
||||
|
||||
|
||||
def _check_download_dir(link, download_dir, hashes):
|
||||
""" Check download_dir for previously downloaded file with correct hash
|
||||
If a correct file is found return its path else None
|
||||
"""
|
||||
download_path = os.path.join(download_dir, link.filename)
|
||||
if os.path.exists(download_path):
|
||||
# If already downloaded, does its hash match?
|
||||
logger.info('File was already downloaded %s', download_path)
|
||||
if hashes:
|
||||
try:
|
||||
hashes.check_against_path(download_path)
|
||||
except HashMismatch:
|
||||
logger.warning(
|
||||
'Previously-downloaded file %s has bad hash. '
|
||||
'Re-downloading.',
|
||||
download_path
|
||||
)
|
||||
os.unlink(download_path)
|
||||
return None
|
||||
return download_path
|
||||
return None
|
||||
@@ -1,249 +0,0 @@
|
||||
"""Exceptions used throughout package"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
from itertools import chain, groupby, repeat
|
||||
|
||||
from pip._vendor.six import iteritems
|
||||
|
||||
|
||||
class PipError(Exception):
|
||||
"""Base pip exception"""
|
||||
|
||||
|
||||
class ConfigurationError(PipError):
|
||||
"""General exception in configuration"""
|
||||
|
||||
|
||||
class InstallationError(PipError):
|
||||
"""General exception during installation"""
|
||||
|
||||
|
||||
class UninstallationError(PipError):
|
||||
"""General exception during uninstallation"""
|
||||
|
||||
|
||||
class DistributionNotFound(InstallationError):
|
||||
"""Raised when a distribution cannot be found to satisfy a requirement"""
|
||||
|
||||
|
||||
class RequirementsFileParseError(InstallationError):
|
||||
"""Raised when a general error occurs parsing a requirements file line."""
|
||||
|
||||
|
||||
class BestVersionAlreadyInstalled(PipError):
|
||||
"""Raised when the most up-to-date version of a package is already
|
||||
installed."""
|
||||
|
||||
|
||||
class BadCommand(PipError):
|
||||
"""Raised when virtualenv or a command is not found"""
|
||||
|
||||
|
||||
class CommandError(PipError):
|
||||
"""Raised when there is an error in command-line arguments"""
|
||||
|
||||
|
||||
class PreviousBuildDirError(PipError):
|
||||
"""Raised when there's a previous conflicting build directory"""
|
||||
|
||||
|
||||
class InvalidWheelFilename(InstallationError):
|
||||
"""Invalid wheel filename."""
|
||||
|
||||
|
||||
class UnsupportedWheel(InstallationError):
|
||||
"""Unsupported wheel."""
|
||||
|
||||
|
||||
class HashErrors(InstallationError):
|
||||
"""Multiple HashError instances rolled into one for reporting"""
|
||||
|
||||
def __init__(self):
|
||||
self.errors = []
|
||||
|
||||
def append(self, error):
|
||||
self.errors.append(error)
|
||||
|
||||
def __str__(self):
|
||||
lines = []
|
||||
self.errors.sort(key=lambda e: e.order)
|
||||
for cls, errors_of_cls in groupby(self.errors, lambda e: e.__class__):
|
||||
lines.append(cls.head)
|
||||
lines.extend(e.body() for e in errors_of_cls)
|
||||
if lines:
|
||||
return '\n'.join(lines)
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.errors)
|
||||
|
||||
def __bool__(self):
|
||||
return self.__nonzero__()
|
||||
|
||||
|
||||
class HashError(InstallationError):
|
||||
"""
|
||||
A failure to verify a package against known-good hashes
|
||||
|
||||
:cvar order: An int sorting hash exception classes by difficulty of
|
||||
recovery (lower being harder), so the user doesn't bother fretting
|
||||
about unpinned packages when he has deeper issues, like VCS
|
||||
dependencies, to deal with. Also keeps error reports in a
|
||||
deterministic order.
|
||||
:cvar head: A section heading for display above potentially many
|
||||
exceptions of this kind
|
||||
:ivar req: The InstallRequirement that triggered this error. This is
|
||||
pasted on after the exception is instantiated, because it's not
|
||||
typically available earlier.
|
||||
|
||||
"""
|
||||
req = None
|
||||
head = ''
|
||||
|
||||
def body(self):
|
||||
"""Return a summary of me for display under the heading.
|
||||
|
||||
This default implementation simply prints a description of the
|
||||
triggering requirement.
|
||||
|
||||
:param req: The InstallRequirement that provoked this error, with
|
||||
populate_link() having already been called
|
||||
|
||||
"""
|
||||
return ' %s' % self._requirement_name()
|
||||
|
||||
def __str__(self):
|
||||
return '%s\n%s' % (self.head, self.body())
|
||||
|
||||
def _requirement_name(self):
|
||||
"""Return a description of the requirement that triggered me.
|
||||
|
||||
This default implementation returns long description of the req, with
|
||||
line numbers
|
||||
|
||||
"""
|
||||
return str(self.req) if self.req else 'unknown package'
|
||||
|
||||
|
||||
class VcsHashUnsupported(HashError):
|
||||
"""A hash was provided for a version-control-system-based requirement, but
|
||||
we don't have a method for hashing those."""
|
||||
|
||||
order = 0
|
||||
head = ("Can't verify hashes for these requirements because we don't "
|
||||
"have a way to hash version control repositories:")
|
||||
|
||||
|
||||
class DirectoryUrlHashUnsupported(HashError):
|
||||
"""A hash was provided for a version-control-system-based requirement, but
|
||||
we don't have a method for hashing those."""
|
||||
|
||||
order = 1
|
||||
head = ("Can't verify hashes for these file:// requirements because they "
|
||||
"point to directories:")
|
||||
|
||||
|
||||
class HashMissing(HashError):
|
||||
"""A hash was needed for a requirement but is absent."""
|
||||
|
||||
order = 2
|
||||
head = ('Hashes are required in --require-hashes mode, but they are '
|
||||
'missing from some requirements. Here is a list of those '
|
||||
'requirements along with the hashes their downloaded archives '
|
||||
'actually had. Add lines like these to your requirements files to '
|
||||
'prevent tampering. (If you did not enable --require-hashes '
|
||||
'manually, note that it turns on automatically when any package '
|
||||
'has a hash.)')
|
||||
|
||||
def __init__(self, gotten_hash):
|
||||
"""
|
||||
:param gotten_hash: The hash of the (possibly malicious) archive we
|
||||
just downloaded
|
||||
"""
|
||||
self.gotten_hash = gotten_hash
|
||||
|
||||
def body(self):
|
||||
# Dodge circular import.
|
||||
from pip._internal.utils.hashes import FAVORITE_HASH
|
||||
|
||||
package = None
|
||||
if self.req:
|
||||
# In the case of URL-based requirements, display the original URL
|
||||
# seen in the requirements file rather than the package name,
|
||||
# so the output can be directly copied into the requirements file.
|
||||
package = (self.req.original_link if self.req.original_link
|
||||
# In case someone feeds something downright stupid
|
||||
# to InstallRequirement's constructor.
|
||||
else getattr(self.req, 'req', None))
|
||||
return ' %s --hash=%s:%s' % (package or 'unknown package',
|
||||
FAVORITE_HASH,
|
||||
self.gotten_hash)
|
||||
|
||||
|
||||
class HashUnpinned(HashError):
|
||||
"""A requirement had a hash specified but was not pinned to a specific
|
||||
version."""
|
||||
|
||||
order = 3
|
||||
head = ('In --require-hashes mode, all requirements must have their '
|
||||
'versions pinned with ==. These do not:')
|
||||
|
||||
|
||||
class HashMismatch(HashError):
|
||||
"""
|
||||
Distribution file hash values don't match.
|
||||
|
||||
:ivar package_name: The name of the package that triggered the hash
|
||||
mismatch. Feel free to write to this after the exception is raise to
|
||||
improve its error message.
|
||||
|
||||
"""
|
||||
order = 4
|
||||
head = ('THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS '
|
||||
'FILE. If you have updated the package versions, please update '
|
||||
'the hashes. Otherwise, examine the package contents carefully; '
|
||||
'someone may have tampered with them.')
|
||||
|
||||
def __init__(self, allowed, gots):
|
||||
"""
|
||||
:param allowed: A dict of algorithm names pointing to lists of allowed
|
||||
hex digests
|
||||
:param gots: A dict of algorithm names pointing to hashes we
|
||||
actually got from the files under suspicion
|
||||
"""
|
||||
self.allowed = allowed
|
||||
self.gots = gots
|
||||
|
||||
def body(self):
|
||||
return ' %s:\n%s' % (self._requirement_name(),
|
||||
self._hash_comparison())
|
||||
|
||||
def _hash_comparison(self):
|
||||
"""
|
||||
Return a comparison of actual and expected hash values.
|
||||
|
||||
Example::
|
||||
|
||||
Expected sha256 abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde
|
||||
or 123451234512345123451234512345123451234512345
|
||||
Got bcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdef
|
||||
|
||||
"""
|
||||
def hash_then_or(hash_name):
|
||||
# For now, all the decent hashes have 6-char names, so we can get
|
||||
# away with hard-coding space literals.
|
||||
return chain([hash_name], repeat(' or'))
|
||||
|
||||
lines = []
|
||||
for hash_name, expecteds in iteritems(self.allowed):
|
||||
prefix = hash_then_or(hash_name)
|
||||
lines.extend((' Expected %s %s' % (next(prefix), e))
|
||||
for e in expecteds)
|
||||
lines.append(' Got %s\n' %
|
||||
self.gots[hash_name].hexdigest())
|
||||
prefix = ' or'
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
class UnsupportedPythonVersion(InstallationError):
|
||||
"""Unsupported python version according to Requires-Python package
|
||||
metadata."""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,194 +0,0 @@
|
||||
"""Locations where we look for configs, install stuff, etc"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import os.path
|
||||
import platform
|
||||
import site
|
||||
import sys
|
||||
import sysconfig
|
||||
from distutils import sysconfig as distutils_sysconfig
|
||||
from distutils.command.install import SCHEME_KEYS, install # type: ignore
|
||||
|
||||
from pip._internal.compat import WINDOWS, expanduser
|
||||
from pip._internal.utils import appdirs
|
||||
|
||||
# Application Directories
|
||||
USER_CACHE_DIR = appdirs.user_cache_dir("pip")
|
||||
|
||||
|
||||
DELETE_MARKER_MESSAGE = '''\
|
||||
This file is placed here by pip to indicate the source was put
|
||||
here by pip.
|
||||
|
||||
Once this package is successfully installed this source code will be
|
||||
deleted (unless you remove this file).
|
||||
'''
|
||||
PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt'
|
||||
|
||||
|
||||
def write_delete_marker_file(directory):
|
||||
"""
|
||||
Write the pip delete marker file into this directory.
|
||||
"""
|
||||
filepath = os.path.join(directory, PIP_DELETE_MARKER_FILENAME)
|
||||
with open(filepath, 'w') as marker_fp:
|
||||
marker_fp.write(DELETE_MARKER_MESSAGE)
|
||||
|
||||
|
||||
def running_under_virtualenv():
|
||||
"""
|
||||
Return True if we're running inside a virtualenv, False otherwise.
|
||||
|
||||
"""
|
||||
if hasattr(sys, 'real_prefix'):
|
||||
return True
|
||||
elif sys.prefix != getattr(sys, "base_prefix", sys.prefix):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def virtualenv_no_global():
|
||||
"""
|
||||
Return True if in a venv and no system site packages.
|
||||
"""
|
||||
# this mirrors the logic in virtualenv.py for locating the
|
||||
# no-global-site-packages.txt file
|
||||
site_mod_dir = os.path.dirname(os.path.abspath(site.__file__))
|
||||
no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt')
|
||||
if running_under_virtualenv() and os.path.isfile(no_global_file):
|
||||
return True
|
||||
|
||||
|
||||
if running_under_virtualenv():
|
||||
src_prefix = os.path.join(sys.prefix, 'src')
|
||||
else:
|
||||
# FIXME: keep src in cwd for now (it is not a temporary folder)
|
||||
try:
|
||||
src_prefix = os.path.join(os.getcwd(), 'src')
|
||||
except OSError:
|
||||
# In case the current working directory has been renamed or deleted
|
||||
sys.exit(
|
||||
"The folder you are executing pip from can no longer be found."
|
||||
)
|
||||
|
||||
# under macOS + virtualenv sys.prefix is not properly resolved
|
||||
# it is something like /path/to/python/bin/..
|
||||
# Note: using realpath due to tmp dirs on OSX being symlinks
|
||||
src_prefix = os.path.abspath(src_prefix)
|
||||
|
||||
# FIXME doesn't account for venv linked to global site-packages
|
||||
|
||||
site_packages = sysconfig.get_path("purelib")
|
||||
# This is because of a bug in PyPy's sysconfig module, see
|
||||
# https://bitbucket.org/pypy/pypy/issues/2506/sysconfig-returns-incorrect-paths
|
||||
# for more information.
|
||||
if platform.python_implementation().lower() == "pypy":
|
||||
site_packages = distutils_sysconfig.get_python_lib()
|
||||
try:
|
||||
# Use getusersitepackages if this is present, as it ensures that the
|
||||
# value is initialised properly.
|
||||
user_site = site.getusersitepackages()
|
||||
except AttributeError:
|
||||
user_site = site.USER_SITE
|
||||
user_dir = expanduser('~')
|
||||
if WINDOWS:
|
||||
bin_py = os.path.join(sys.prefix, 'Scripts')
|
||||
bin_user = os.path.join(user_site, 'Scripts')
|
||||
# buildout uses 'bin' on Windows too?
|
||||
if not os.path.exists(bin_py):
|
||||
bin_py = os.path.join(sys.prefix, 'bin')
|
||||
bin_user = os.path.join(user_site, 'bin')
|
||||
|
||||
config_basename = 'pip.ini'
|
||||
|
||||
legacy_storage_dir = os.path.join(user_dir, 'pip')
|
||||
legacy_config_file = os.path.join(
|
||||
legacy_storage_dir,
|
||||
config_basename,
|
||||
)
|
||||
else:
|
||||
bin_py = os.path.join(sys.prefix, 'bin')
|
||||
bin_user = os.path.join(user_site, 'bin')
|
||||
|
||||
config_basename = 'pip.conf'
|
||||
|
||||
legacy_storage_dir = os.path.join(user_dir, '.pip')
|
||||
legacy_config_file = os.path.join(
|
||||
legacy_storage_dir,
|
||||
config_basename,
|
||||
)
|
||||
# Forcing to use /usr/local/bin for standard macOS framework installs
|
||||
# Also log to ~/Library/Logs/ for use with the Console.app log viewer
|
||||
if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/':
|
||||
bin_py = '/usr/local/bin'
|
||||
|
||||
site_config_files = [
|
||||
os.path.join(path, config_basename)
|
||||
for path in appdirs.site_config_dirs('pip')
|
||||
]
|
||||
|
||||
venv_config_file = os.path.join(sys.prefix, config_basename)
|
||||
new_config_file = os.path.join(appdirs.user_config_dir("pip"), config_basename)
|
||||
|
||||
|
||||
def distutils_scheme(dist_name, user=False, home=None, root=None,
|
||||
isolated=False, prefix=None):
|
||||
"""
|
||||
Return a distutils install scheme
|
||||
"""
|
||||
from distutils.dist import Distribution
|
||||
|
||||
scheme = {}
|
||||
|
||||
if isolated:
|
||||
extra_dist_args = {"script_args": ["--no-user-cfg"]}
|
||||
else:
|
||||
extra_dist_args = {}
|
||||
dist_args = {'name': dist_name}
|
||||
dist_args.update(extra_dist_args)
|
||||
|
||||
d = Distribution(dist_args)
|
||||
d.parse_config_files()
|
||||
i = d.get_command_obj('install', create=True)
|
||||
# NOTE: setting user or home has the side-effect of creating the home dir
|
||||
# or user base for installations during finalize_options()
|
||||
# ideally, we'd prefer a scheme class that has no side-effects.
|
||||
assert not (user and prefix), "user={} prefix={}".format(user, prefix)
|
||||
i.user = user or i.user
|
||||
if user:
|
||||
i.prefix = ""
|
||||
i.prefix = prefix or i.prefix
|
||||
i.home = home or i.home
|
||||
i.root = root or i.root
|
||||
i.finalize_options()
|
||||
for key in SCHEME_KEYS:
|
||||
scheme[key] = getattr(i, 'install_' + key)
|
||||
|
||||
# install_lib specified in setup.cfg should install *everything*
|
||||
# into there (i.e. it takes precedence over both purelib and
|
||||
# platlib). Note, i.install_lib is *always* set after
|
||||
# finalize_options(); we only want to override here if the user
|
||||
# has explicitly requested it hence going back to the config
|
||||
if 'install_lib' in d.get_option_dict('install'):
|
||||
scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib))
|
||||
|
||||
if running_under_virtualenv():
|
||||
scheme['headers'] = os.path.join(
|
||||
sys.prefix,
|
||||
'include',
|
||||
'site',
|
||||
'python' + sys.version[:3],
|
||||
dist_name,
|
||||
)
|
||||
|
||||
if root is not None:
|
||||
path_no_drive = os.path.splitdrive(
|
||||
os.path.abspath(scheme["headers"]))[1]
|
||||
scheme["headers"] = os.path.join(
|
||||
root,
|
||||
path_no_drive[1:],
|
||||
)
|
||||
|
||||
return scheme
|
||||
@@ -1,4 +0,0 @@
|
||||
from pip._internal.models.index import Index, PyPI
|
||||
|
||||
|
||||
__all__ = ["Index", "PyPI"]
|
||||
@@ -1,15 +0,0 @@
|
||||
from pip._vendor.six.moves.urllib import parse as urllib_parse
|
||||
|
||||
|
||||
class Index(object):
|
||||
def __init__(self, url):
|
||||
self.url = url
|
||||
self.netloc = urllib_parse.urlsplit(url).netloc
|
||||
self.simple_url = self.url_to_path('simple')
|
||||
self.pypi_url = self.url_to_path('pypi')
|
||||
|
||||
def url_to_path(self, path):
|
||||
return urllib_parse.urljoin(self.url, path)
|
||||
|
||||
|
||||
PyPI = Index('https://pypi.org/')
|
||||
@@ -1,106 +0,0 @@
|
||||
"""Validation of dependencies of packages
|
||||
"""
|
||||
|
||||
from collections import namedtuple
|
||||
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
|
||||
from pip._internal.operations.prepare import make_abstract_dist
|
||||
|
||||
from pip._internal.utils.misc import get_installed_distributions
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from pip._internal.req.req_install import InstallRequirement
|
||||
from typing import Any, Dict, Iterator, Set, Tuple, List
|
||||
|
||||
# Shorthands
|
||||
PackageSet = Dict[str, 'PackageDetails']
|
||||
Missing = Tuple[str, Any]
|
||||
Conflicting = Tuple[str, str, Any]
|
||||
|
||||
MissingDict = Dict[str, List[Missing]]
|
||||
ConflictingDict = Dict[str, List[Conflicting]]
|
||||
CheckResult = Tuple[MissingDict, ConflictingDict]
|
||||
|
||||
PackageDetails = namedtuple('PackageDetails', ['version', 'requires'])
|
||||
|
||||
|
||||
def create_package_set_from_installed(**kwargs):
|
||||
# type: (**Any) -> PackageSet
|
||||
"""Converts a list of distributions into a PackageSet.
|
||||
"""
|
||||
# Default to using all packages installed on the system
|
||||
if kwargs == {}:
|
||||
kwargs = {"local_only": False, "skip": ()}
|
||||
retval = {}
|
||||
for dist in get_installed_distributions(**kwargs):
|
||||
name = canonicalize_name(dist.project_name)
|
||||
retval[name] = PackageDetails(dist.version, dist.requires())
|
||||
return retval
|
||||
|
||||
|
||||
def check_package_set(package_set):
|
||||
# type: (PackageSet) -> CheckResult
|
||||
"""Check if a package set is consistent
|
||||
"""
|
||||
missing = dict()
|
||||
conflicting = dict()
|
||||
|
||||
for package_name in package_set:
|
||||
# Info about dependencies of package_name
|
||||
missing_deps = set() # type: Set[Missing]
|
||||
conflicting_deps = set() # type: Set[Conflicting]
|
||||
|
||||
for req in package_set[package_name].requires:
|
||||
name = canonicalize_name(req.project_name) # type: str
|
||||
|
||||
# Check if it's missing
|
||||
if name not in package_set:
|
||||
missed = True
|
||||
if req.marker is not None:
|
||||
missed = req.marker.evaluate()
|
||||
if missed:
|
||||
missing_deps.add((name, req))
|
||||
continue
|
||||
|
||||
# Check if there's a conflict
|
||||
version = package_set[name].version # type: str
|
||||
if not req.specifier.contains(version, prereleases=True):
|
||||
conflicting_deps.add((name, version, req))
|
||||
|
||||
def str_key(x):
|
||||
return str(x)
|
||||
|
||||
if missing_deps:
|
||||
missing[package_name] = sorted(missing_deps, key=str_key)
|
||||
if conflicting_deps:
|
||||
conflicting[package_name] = sorted(conflicting_deps, key=str_key)
|
||||
|
||||
return missing, conflicting
|
||||
|
||||
|
||||
def check_install_conflicts(to_install):
|
||||
# type: (List[InstallRequirement]) -> Tuple[PackageSet, CheckResult]
|
||||
"""For checking if the dependency graph would be consistent after \
|
||||
installing given requirements
|
||||
"""
|
||||
# Start from the current state
|
||||
state = create_package_set_from_installed()
|
||||
_simulate_installation_of(to_install, state)
|
||||
return state, check_package_set(state)
|
||||
|
||||
|
||||
# NOTE from @pradyunsg
|
||||
# This required a minor update in dependency link handling logic over at
|
||||
# operations.prepare.IsSDist.dist() to get it working
|
||||
def _simulate_installation_of(to_install, state):
|
||||
# type: (List[InstallRequirement], PackageSet) -> None
|
||||
"""Computes the version of packages after installing to_install.
|
||||
"""
|
||||
|
||||
# Modify it as installing requirement_set would (assuming no errors)
|
||||
for inst_req in to_install:
|
||||
dist = make_abstract_dist(inst_req).dist(finder=None)
|
||||
name = canonicalize_name(dist.key)
|
||||
state[name] = PackageDetails(dist.version, dist.requires())
|
||||
@@ -1,252 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import collections
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import warnings
|
||||
|
||||
from pip._vendor import pkg_resources, six
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
from pip._vendor.pkg_resources import RequirementParseError
|
||||
|
||||
from pip._internal.exceptions import InstallationError
|
||||
from pip._internal.req import InstallRequirement
|
||||
from pip._internal.req.req_file import COMMENT_RE
|
||||
from pip._internal.utils.deprecation import RemovedInPip11Warning
|
||||
from pip._internal.utils.misc import (
|
||||
dist_is_editable, get_installed_distributions,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def freeze(
|
||||
requirement=None,
|
||||
find_links=None, local_only=None, user_only=None, skip_regex=None,
|
||||
isolated=False,
|
||||
wheel_cache=None,
|
||||
exclude_editable=False,
|
||||
skip=()):
|
||||
find_links = find_links or []
|
||||
skip_match = None
|
||||
|
||||
if skip_regex:
|
||||
skip_match = re.compile(skip_regex).search
|
||||
|
||||
dependency_links = []
|
||||
|
||||
for dist in pkg_resources.working_set:
|
||||
if dist.has_metadata('dependency_links.txt'):
|
||||
dependency_links.extend(
|
||||
dist.get_metadata_lines('dependency_links.txt')
|
||||
)
|
||||
for link in find_links:
|
||||
if '#egg=' in link:
|
||||
dependency_links.append(link)
|
||||
for link in find_links:
|
||||
yield '-f %s' % link
|
||||
installations = {}
|
||||
for dist in get_installed_distributions(local_only=local_only,
|
||||
skip=(),
|
||||
user_only=user_only):
|
||||
try:
|
||||
req = FrozenRequirement.from_dist(
|
||||
dist,
|
||||
dependency_links
|
||||
)
|
||||
except RequirementParseError:
|
||||
logger.warning(
|
||||
"Could not parse requirement: %s",
|
||||
dist.project_name
|
||||
)
|
||||
continue
|
||||
if exclude_editable and req.editable:
|
||||
continue
|
||||
installations[req.name] = req
|
||||
|
||||
if requirement:
|
||||
# the options that don't get turned into an InstallRequirement
|
||||
# should only be emitted once, even if the same option is in multiple
|
||||
# requirements files, so we need to keep track of what has been emitted
|
||||
# so that we don't emit it again if it's seen again
|
||||
emitted_options = set()
|
||||
# keep track of which files a requirement is in so that we can
|
||||
# give an accurate warning if a requirement appears multiple times.
|
||||
req_files = collections.defaultdict(list)
|
||||
for req_file_path in requirement:
|
||||
with open(req_file_path) as req_file:
|
||||
for line in req_file:
|
||||
if (not line.strip() or
|
||||
line.strip().startswith('#') or
|
||||
(skip_match and skip_match(line)) or
|
||||
line.startswith((
|
||||
'-r', '--requirement',
|
||||
'-Z', '--always-unzip',
|
||||
'-f', '--find-links',
|
||||
'-i', '--index-url',
|
||||
'--pre',
|
||||
'--trusted-host',
|
||||
'--process-dependency-links',
|
||||
'--extra-index-url'))):
|
||||
line = line.rstrip()
|
||||
if line not in emitted_options:
|
||||
emitted_options.add(line)
|
||||
yield line
|
||||
continue
|
||||
|
||||
if line.startswith('-e') or line.startswith('--editable'):
|
||||
if line.startswith('-e'):
|
||||
line = line[2:].strip()
|
||||
else:
|
||||
line = line[len('--editable'):].strip().lstrip('=')
|
||||
line_req = InstallRequirement.from_editable(
|
||||
line,
|
||||
isolated=isolated,
|
||||
wheel_cache=wheel_cache,
|
||||
)
|
||||
else:
|
||||
line_req = InstallRequirement.from_line(
|
||||
COMMENT_RE.sub('', line).strip(),
|
||||
isolated=isolated,
|
||||
wheel_cache=wheel_cache,
|
||||
)
|
||||
|
||||
if not line_req.name:
|
||||
logger.info(
|
||||
"Skipping line in requirement file [%s] because "
|
||||
"it's not clear what it would install: %s",
|
||||
req_file_path, line.strip(),
|
||||
)
|
||||
logger.info(
|
||||
" (add #egg=PackageName to the URL to avoid"
|
||||
" this warning)"
|
||||
)
|
||||
elif line_req.name not in installations:
|
||||
# either it's not installed, or it is installed
|
||||
# but has been processed already
|
||||
if not req_files[line_req.name]:
|
||||
logger.warning(
|
||||
"Requirement file [%s] contains %s, but that "
|
||||
"package is not installed",
|
||||
req_file_path,
|
||||
COMMENT_RE.sub('', line).strip(),
|
||||
)
|
||||
else:
|
||||
req_files[line_req.name].append(req_file_path)
|
||||
else:
|
||||
yield str(installations[line_req.name]).rstrip()
|
||||
del installations[line_req.name]
|
||||
req_files[line_req.name].append(req_file_path)
|
||||
|
||||
# Warn about requirements that were included multiple times (in a
|
||||
# single requirements file or in different requirements files).
|
||||
for name, files in six.iteritems(req_files):
|
||||
if len(files) > 1:
|
||||
logger.warning("Requirement %s included multiple times [%s]",
|
||||
name, ', '.join(sorted(set(files))))
|
||||
|
||||
yield(
|
||||
'## The following requirements were added by '
|
||||
'pip freeze:'
|
||||
)
|
||||
for installation in sorted(
|
||||
installations.values(), key=lambda x: x.name.lower()):
|
||||
if canonicalize_name(installation.name) not in skip:
|
||||
yield str(installation).rstrip()
|
||||
|
||||
|
||||
class FrozenRequirement(object):
|
||||
def __init__(self, name, req, editable, comments=()):
|
||||
self.name = name
|
||||
self.req = req
|
||||
self.editable = editable
|
||||
self.comments = comments
|
||||
|
||||
_rev_re = re.compile(r'-r(\d+)$')
|
||||
_date_re = re.compile(r'-(20\d\d\d\d\d\d)$')
|
||||
|
||||
@classmethod
|
||||
def from_dist(cls, dist, dependency_links):
|
||||
location = os.path.normcase(os.path.abspath(dist.location))
|
||||
comments = []
|
||||
from pip._internal.vcs import vcs, get_src_requirement
|
||||
if dist_is_editable(dist) and vcs.get_backend_name(location):
|
||||
editable = True
|
||||
try:
|
||||
req = get_src_requirement(dist, location)
|
||||
except InstallationError as exc:
|
||||
logger.warning(
|
||||
"Error when trying to get requirement for VCS system %s, "
|
||||
"falling back to uneditable format", exc
|
||||
)
|
||||
req = None
|
||||
if req is None:
|
||||
logger.warning(
|
||||
'Could not determine repository location of %s', location
|
||||
)
|
||||
comments.append(
|
||||
'## !! Could not determine repository location'
|
||||
)
|
||||
req = dist.as_requirement()
|
||||
editable = False
|
||||
else:
|
||||
editable = False
|
||||
req = dist.as_requirement()
|
||||
specs = req.specs
|
||||
assert len(specs) == 1 and specs[0][0] in ["==", "==="], \
|
||||
'Expected 1 spec with == or ===; specs = %r; dist = %r' % \
|
||||
(specs, dist)
|
||||
version = specs[0][1]
|
||||
ver_match = cls._rev_re.search(version)
|
||||
date_match = cls._date_re.search(version)
|
||||
if ver_match or date_match:
|
||||
svn_backend = vcs.get_backend('svn')
|
||||
if svn_backend:
|
||||
svn_location = svn_backend().get_location(
|
||||
dist,
|
||||
dependency_links,
|
||||
)
|
||||
if not svn_location:
|
||||
logger.warning(
|
||||
'Warning: cannot find svn location for %s', req,
|
||||
)
|
||||
comments.append(
|
||||
'## FIXME: could not find svn URL in dependency_links '
|
||||
'for this package:'
|
||||
)
|
||||
else:
|
||||
warnings.warn(
|
||||
"SVN editable detection based on dependency links "
|
||||
"will be dropped in the future.",
|
||||
RemovedInPip11Warning,
|
||||
)
|
||||
comments.append(
|
||||
'# Installing as editable to satisfy requirement %s:' %
|
||||
req
|
||||
)
|
||||
if ver_match:
|
||||
rev = ver_match.group(1)
|
||||
else:
|
||||
rev = '{%s}' % date_match.group(1)
|
||||
editable = True
|
||||
req = '%s@%s#egg=%s' % (
|
||||
svn_location,
|
||||
rev,
|
||||
cls.egg_name(dist)
|
||||
)
|
||||
return cls(dist.project_name, req, editable, comments)
|
||||
|
||||
@staticmethod
|
||||
def egg_name(dist):
|
||||
name = dist.egg_name()
|
||||
match = re.search(r'-py\d\.\d$', name)
|
||||
if match:
|
||||
name = name[:match.start()]
|
||||
return name
|
||||
|
||||
def __str__(self):
|
||||
req = self.req
|
||||
if self.editable:
|
||||
req = '-e %s' % req
|
||||
return '\n'.join(list(self.comments) + [str(req)]) + '\n'
|
||||
@@ -1,380 +0,0 @@
|
||||
"""Prepares a distribution for installation
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from copy import copy
|
||||
|
||||
from pip._vendor import pkg_resources, requests
|
||||
|
||||
from pip._internal.build_env import NoOpBuildEnvironment
|
||||
from pip._internal.compat import expanduser
|
||||
from pip._internal.download import (
|
||||
is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path,
|
||||
)
|
||||
from pip._internal.exceptions import (
|
||||
DirectoryUrlHashUnsupported, HashUnpinned, InstallationError,
|
||||
PreviousBuildDirError, VcsHashUnsupported,
|
||||
)
|
||||
from pip._internal.index import FormatControl
|
||||
from pip._internal.req.req_install import InstallRequirement
|
||||
from pip._internal.utils.hashes import MissingHashes
|
||||
from pip._internal.utils.logging import indent_log
|
||||
from pip._internal.utils.misc import (
|
||||
call_subprocess, display_path, normalize_path,
|
||||
)
|
||||
from pip._internal.utils.ui import open_spinner
|
||||
from pip._internal.vcs import vcs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_abstract_dist(req):
|
||||
"""Factory to make an abstract dist object.
|
||||
|
||||
Preconditions: Either an editable req with a source_dir, or satisfied_by or
|
||||
a wheel link, or a non-editable req with a source_dir.
|
||||
|
||||
:return: A concrete DistAbstraction.
|
||||
"""
|
||||
if req.editable:
|
||||
return IsSDist(req)
|
||||
elif req.link and req.link.is_wheel:
|
||||
return IsWheel(req)
|
||||
else:
|
||||
return IsSDist(req)
|
||||
|
||||
|
||||
def _install_build_reqs(finder, prefix, build_requirements):
|
||||
# NOTE: What follows is not a very good thing.
|
||||
# Eventually, this should move into the BuildEnvironment class and
|
||||
# that should handle all the isolation and sub-process invocation.
|
||||
finder = copy(finder)
|
||||
finder.format_control = FormatControl(set(), set([":all:"]))
|
||||
urls = [
|
||||
finder.find_requirement(
|
||||
InstallRequirement.from_line(r), upgrade=False).url
|
||||
for r in build_requirements
|
||||
]
|
||||
args = [
|
||||
sys.executable, '-m', 'pip', 'install', '--ignore-installed',
|
||||
'--no-user', '--prefix', prefix,
|
||||
] + list(urls)
|
||||
|
||||
with open_spinner("Installing build dependencies") as spinner:
|
||||
call_subprocess(args, show_stdout=False, spinner=spinner)
|
||||
|
||||
|
||||
class DistAbstraction(object):
|
||||
"""Abstracts out the wheel vs non-wheel Resolver.resolve() logic.
|
||||
|
||||
The requirements for anything installable are as follows:
|
||||
- we must be able to determine the requirement name
|
||||
(or we can't correctly handle the non-upgrade case).
|
||||
- we must be able to generate a list of run-time dependencies
|
||||
without installing any additional packages (or we would
|
||||
have to either burn time by doing temporary isolated installs
|
||||
or alternatively violate pips 'don't start installing unless
|
||||
all requirements are available' rule - neither of which are
|
||||
desirable).
|
||||
- for packages with setup requirements, we must also be able
|
||||
to determine their requirements without installing additional
|
||||
packages (for the same reason as run-time dependencies)
|
||||
- we must be able to create a Distribution object exposing the
|
||||
above metadata.
|
||||
"""
|
||||
|
||||
def __init__(self, req):
|
||||
self.req = req
|
||||
|
||||
def dist(self, finder):
|
||||
"""Return a setuptools Dist object."""
|
||||
raise NotImplementedError(self.dist)
|
||||
|
||||
def prep_for_dist(self, finder):
|
||||
"""Ensure that we can get a Dist for this requirement."""
|
||||
raise NotImplementedError(self.dist)
|
||||
|
||||
|
||||
class IsWheel(DistAbstraction):
|
||||
|
||||
def dist(self, finder):
|
||||
return list(pkg_resources.find_distributions(
|
||||
self.req.source_dir))[0]
|
||||
|
||||
def prep_for_dist(self, finder, build_isolation):
|
||||
# FIXME:https://github.com/pypa/pip/issues/1112
|
||||
pass
|
||||
|
||||
|
||||
class IsSDist(DistAbstraction):
|
||||
|
||||
def dist(self, finder):
|
||||
dist = self.req.get_dist()
|
||||
# FIXME: shouldn't be globally added.
|
||||
if finder and dist.has_metadata('dependency_links.txt'):
|
||||
finder.add_dependency_links(
|
||||
dist.get_metadata_lines('dependency_links.txt')
|
||||
)
|
||||
return dist
|
||||
|
||||
def prep_for_dist(self, finder, build_isolation):
|
||||
# Before calling "setup.py egg_info", we need to set-up the build
|
||||
# environment.
|
||||
build_requirements, isolate = self.req.get_pep_518_info()
|
||||
should_isolate = build_isolation and isolate
|
||||
|
||||
minimum_requirements = ('setuptools', 'wheel')
|
||||
missing_requirements = set(minimum_requirements) - set(
|
||||
pkg_resources.Requirement(r).key
|
||||
for r in build_requirements
|
||||
)
|
||||
if missing_requirements:
|
||||
def format_reqs(rs):
|
||||
return ' and '.join(map(repr, sorted(rs)))
|
||||
logger.warning(
|
||||
"Missing build time requirements in pyproject.toml for %s: "
|
||||
"%s.", self.req, format_reqs(missing_requirements)
|
||||
)
|
||||
logger.warning(
|
||||
"This version of pip does not implement PEP 517 so it cannot "
|
||||
"build a wheel without %s.", format_reqs(minimum_requirements)
|
||||
)
|
||||
|
||||
if should_isolate:
|
||||
with self.req.build_env:
|
||||
pass
|
||||
_install_build_reqs(finder, self.req.build_env.path,
|
||||
build_requirements)
|
||||
else:
|
||||
self.req.build_env = NoOpBuildEnvironment(no_clean=False)
|
||||
|
||||
self.req.run_egg_info()
|
||||
self.req.assert_source_matches_version()
|
||||
|
||||
|
||||
class Installed(DistAbstraction):
|
||||
|
||||
def dist(self, finder):
|
||||
return self.req.satisfied_by
|
||||
|
||||
def prep_for_dist(self, finder):
|
||||
pass
|
||||
|
||||
|
||||
class RequirementPreparer(object):
|
||||
"""Prepares a Requirement
|
||||
"""
|
||||
|
||||
def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir,
|
||||
progress_bar, build_isolation):
|
||||
super(RequirementPreparer, self).__init__()
|
||||
|
||||
self.src_dir = src_dir
|
||||
self.build_dir = build_dir
|
||||
|
||||
# Where still packed archives should be written to. If None, they are
|
||||
# not saved, and are deleted immediately after unpacking.
|
||||
self.download_dir = download_dir
|
||||
|
||||
# Where still-packed .whl files should be written to. If None, they are
|
||||
# written to the download_dir parameter. Separate to download_dir to
|
||||
# permit only keeping wheel archives for pip wheel.
|
||||
if wheel_download_dir:
|
||||
wheel_download_dir = normalize_path(wheel_download_dir)
|
||||
self.wheel_download_dir = wheel_download_dir
|
||||
|
||||
# NOTE
|
||||
# download_dir and wheel_download_dir overlap semantically and may
|
||||
# be combined if we're willing to have non-wheel archives present in
|
||||
# the wheelhouse output by 'pip wheel'.
|
||||
|
||||
self.progress_bar = progress_bar
|
||||
|
||||
# Is build isolation allowed?
|
||||
self.build_isolation = build_isolation
|
||||
|
||||
@property
|
||||
def _download_should_save(self):
|
||||
# TODO: Modify to reduce indentation needed
|
||||
if self.download_dir:
|
||||
self.download_dir = expanduser(self.download_dir)
|
||||
if os.path.exists(self.download_dir):
|
||||
return True
|
||||
else:
|
||||
logger.critical('Could not find download directory')
|
||||
raise InstallationError(
|
||||
"Could not find or access download directory '%s'"
|
||||
% display_path(self.download_dir))
|
||||
return False
|
||||
|
||||
def prepare_linked_requirement(self, req, session, finder,
|
||||
upgrade_allowed, require_hashes):
|
||||
"""Prepare a requirement that would be obtained from req.link
|
||||
"""
|
||||
# TODO: Breakup into smaller functions
|
||||
if req.link and req.link.scheme == 'file':
|
||||
path = url_to_path(req.link.url)
|
||||
logger.info('Processing %s', display_path(path))
|
||||
else:
|
||||
logger.info('Collecting %s', req)
|
||||
|
||||
with indent_log():
|
||||
# @@ if filesystem packages are not marked
|
||||
# editable in a req, a non deterministic error
|
||||
# occurs when the script attempts to unpack the
|
||||
# build directory
|
||||
req.ensure_has_source_dir(self.build_dir)
|
||||
# If a checkout exists, it's unwise to keep going. version
|
||||
# inconsistencies are logged later, but do not fail the
|
||||
# installation.
|
||||
# FIXME: this won't upgrade when there's an existing
|
||||
# package unpacked in `req.source_dir`
|
||||
# package unpacked in `req.source_dir`
|
||||
if os.path.exists(os.path.join(req.source_dir, 'setup.py')):
|
||||
raise PreviousBuildDirError(
|
||||
"pip can't proceed with requirements '%s' due to a"
|
||||
" pre-existing build directory (%s). This is "
|
||||
"likely due to a previous installation that failed"
|
||||
". pip is being responsible and not assuming it "
|
||||
"can delete this. Please delete it and try again."
|
||||
% (req, req.source_dir)
|
||||
)
|
||||
req.populate_link(finder, upgrade_allowed, require_hashes)
|
||||
|
||||
# We can't hit this spot and have populate_link return None.
|
||||
# req.satisfied_by is None here (because we're
|
||||
# guarded) and upgrade has no impact except when satisfied_by
|
||||
# is not None.
|
||||
# Then inside find_requirement existing_applicable -> False
|
||||
# If no new versions are found, DistributionNotFound is raised,
|
||||
# otherwise a result is guaranteed.
|
||||
assert req.link
|
||||
link = req.link
|
||||
|
||||
# Now that we have the real link, we can tell what kind of
|
||||
# requirements we have and raise some more informative errors
|
||||
# than otherwise. (For example, we can raise VcsHashUnsupported
|
||||
# for a VCS URL rather than HashMissing.)
|
||||
if require_hashes:
|
||||
# We could check these first 2 conditions inside
|
||||
# unpack_url and save repetition of conditions, but then
|
||||
# we would report less-useful error messages for
|
||||
# unhashable requirements, complaining that there's no
|
||||
# hash provided.
|
||||
if is_vcs_url(link):
|
||||
raise VcsHashUnsupported()
|
||||
elif is_file_url(link) and is_dir_url(link):
|
||||
raise DirectoryUrlHashUnsupported()
|
||||
if not req.original_link and not req.is_pinned:
|
||||
# Unpinned packages are asking for trouble when a new
|
||||
# version is uploaded. This isn't a security check, but
|
||||
# it saves users a surprising hash mismatch in the
|
||||
# future.
|
||||
#
|
||||
# file:/// URLs aren't pinnable, so don't complain
|
||||
# about them not being pinned.
|
||||
raise HashUnpinned()
|
||||
|
||||
hashes = req.hashes(trust_internet=not require_hashes)
|
||||
if require_hashes and not hashes:
|
||||
# Known-good hashes are missing for this requirement, so
|
||||
# shim it with a facade object that will provoke hash
|
||||
# computation and then raise a HashMissing exception
|
||||
# showing the user what the hash should be.
|
||||
hashes = MissingHashes()
|
||||
|
||||
try:
|
||||
download_dir = self.download_dir
|
||||
# We always delete unpacked sdists after pip ran.
|
||||
autodelete_unpacked = True
|
||||
if req.link.is_wheel and self.wheel_download_dir:
|
||||
# when doing 'pip wheel` we download wheels to a
|
||||
# dedicated dir.
|
||||
download_dir = self.wheel_download_dir
|
||||
if req.link.is_wheel:
|
||||
if download_dir:
|
||||
# When downloading, we only unpack wheels to get
|
||||
# metadata.
|
||||
autodelete_unpacked = True
|
||||
else:
|
||||
# When installing a wheel, we use the unpacked
|
||||
# wheel.
|
||||
autodelete_unpacked = False
|
||||
unpack_url(
|
||||
req.link, req.source_dir,
|
||||
download_dir, autodelete_unpacked,
|
||||
session=session, hashes=hashes,
|
||||
progress_bar=self.progress_bar
|
||||
)
|
||||
except requests.HTTPError as exc:
|
||||
logger.critical(
|
||||
'Could not install requirement %s because of error %s',
|
||||
req,
|
||||
exc,
|
||||
)
|
||||
raise InstallationError(
|
||||
'Could not install requirement %s because of HTTP '
|
||||
'error %s for URL %s' %
|
||||
(req, exc, req.link)
|
||||
)
|
||||
abstract_dist = make_abstract_dist(req)
|
||||
abstract_dist.prep_for_dist(finder, self.build_isolation)
|
||||
if self._download_should_save:
|
||||
# Make a .zip of the source_dir we already created.
|
||||
if req.link.scheme in vcs.all_schemes:
|
||||
req.archive(self.download_dir)
|
||||
return abstract_dist
|
||||
|
||||
def prepare_editable_requirement(self, req, require_hashes, use_user_site,
|
||||
finder):
|
||||
"""Prepare an editable requirement
|
||||
"""
|
||||
assert req.editable, "cannot prepare a non-editable req as editable"
|
||||
|
||||
logger.info('Obtaining %s', req)
|
||||
|
||||
with indent_log():
|
||||
if require_hashes:
|
||||
raise InstallationError(
|
||||
'The editable requirement %s cannot be installed when '
|
||||
'requiring hashes, because there is no single file to '
|
||||
'hash.' % req
|
||||
)
|
||||
req.ensure_has_source_dir(self.src_dir)
|
||||
req.update_editable(not self._download_should_save)
|
||||
|
||||
abstract_dist = make_abstract_dist(req)
|
||||
abstract_dist.prep_for_dist(finder, self.build_isolation)
|
||||
|
||||
if self._download_should_save:
|
||||
req.archive(self.download_dir)
|
||||
req.check_if_exists(use_user_site)
|
||||
|
||||
return abstract_dist
|
||||
|
||||
def prepare_installed_requirement(self, req, require_hashes, skip_reason):
|
||||
"""Prepare an already-installed requirement
|
||||
"""
|
||||
assert req.satisfied_by, "req should have been satisfied but isn't"
|
||||
assert skip_reason is not None, (
|
||||
"did not get skip reason skipped but req.satisfied_by "
|
||||
"is set to %r" % (req.satisfied_by,)
|
||||
)
|
||||
logger.info(
|
||||
'Requirement %s: %s (%s)',
|
||||
skip_reason, req, req.satisfied_by.version
|
||||
)
|
||||
with indent_log():
|
||||
if require_hashes:
|
||||
logger.debug(
|
||||
'Since it is already installed, we are trusting this '
|
||||
'package without checking its hash. To ensure a '
|
||||
'completely repeatable environment, install into an '
|
||||
'empty virtualenv.'
|
||||
)
|
||||
abstract_dist = Installed(req)
|
||||
|
||||
return abstract_dist
|
||||
@@ -1,317 +0,0 @@
|
||||
"""Generate and work with PEP 425 Compatibility Tags."""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import distutils.util
|
||||
import logging
|
||||
import platform
|
||||
import re
|
||||
import sys
|
||||
import sysconfig
|
||||
import warnings
|
||||
from collections import OrderedDict
|
||||
|
||||
import pip._internal.utils.glibc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)')
|
||||
|
||||
|
||||
def get_config_var(var):
|
||||
try:
|
||||
return sysconfig.get_config_var(var)
|
||||
except IOError as e: # Issue #1074
|
||||
warnings.warn("{}".format(e), RuntimeWarning)
|
||||
return None
|
||||
|
||||
|
||||
def get_abbr_impl():
|
||||
"""Return abbreviated implementation name."""
|
||||
if hasattr(sys, 'pypy_version_info'):
|
||||
pyimpl = 'pp'
|
||||
elif sys.platform.startswith('java'):
|
||||
pyimpl = 'jy'
|
||||
elif sys.platform == 'cli':
|
||||
pyimpl = 'ip'
|
||||
else:
|
||||
pyimpl = 'cp'
|
||||
return pyimpl
|
||||
|
||||
|
||||
def get_impl_ver():
|
||||
"""Return implementation version."""
|
||||
impl_ver = get_config_var("py_version_nodot")
|
||||
if not impl_ver or get_abbr_impl() == 'pp':
|
||||
impl_ver = ''.join(map(str, get_impl_version_info()))
|
||||
return impl_ver
|
||||
|
||||
|
||||
def get_impl_version_info():
|
||||
"""Return sys.version_info-like tuple for use in decrementing the minor
|
||||
version."""
|
||||
if get_abbr_impl() == 'pp':
|
||||
# as per https://github.com/pypa/pip/issues/2882
|
||||
return (sys.version_info[0], sys.pypy_version_info.major,
|
||||
sys.pypy_version_info.minor)
|
||||
else:
|
||||
return sys.version_info[0], sys.version_info[1]
|
||||
|
||||
|
||||
def get_impl_tag():
|
||||
"""
|
||||
Returns the Tag for this specific implementation.
|
||||
"""
|
||||
return "{}{}".format(get_abbr_impl(), get_impl_ver())
|
||||
|
||||
|
||||
def get_flag(var, fallback, expected=True, warn=True):
|
||||
"""Use a fallback method for determining SOABI flags if the needed config
|
||||
var is unset or unavailable."""
|
||||
val = get_config_var(var)
|
||||
if val is None:
|
||||
if warn:
|
||||
logger.debug("Config variable '%s' is unset, Python ABI tag may "
|
||||
"be incorrect", var)
|
||||
return fallback()
|
||||
return val == expected
|
||||
|
||||
|
||||
def get_abi_tag():
|
||||
"""Return the ABI tag based on SOABI (if available) or emulate SOABI
|
||||
(CPython 2, PyPy)."""
|
||||
soabi = get_config_var('SOABI')
|
||||
impl = get_abbr_impl()
|
||||
if not soabi and impl in {'cp', 'pp'} and hasattr(sys, 'maxunicode'):
|
||||
d = ''
|
||||
m = ''
|
||||
u = ''
|
||||
if get_flag('Py_DEBUG',
|
||||
lambda: hasattr(sys, 'gettotalrefcount'),
|
||||
warn=(impl == 'cp')):
|
||||
d = 'd'
|
||||
if get_flag('WITH_PYMALLOC',
|
||||
lambda: impl == 'cp',
|
||||
warn=(impl == 'cp')):
|
||||
m = 'm'
|
||||
if get_flag('Py_UNICODE_SIZE',
|
||||
lambda: sys.maxunicode == 0x10ffff,
|
||||
expected=4,
|
||||
warn=(impl == 'cp' and
|
||||
sys.version_info < (3, 3))) \
|
||||
and sys.version_info < (3, 3):
|
||||
u = 'u'
|
||||
abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u)
|
||||
elif soabi and soabi.startswith('cpython-'):
|
||||
abi = 'cp' + soabi.split('-')[1]
|
||||
elif soabi:
|
||||
abi = soabi.replace('.', '_').replace('-', '_')
|
||||
else:
|
||||
abi = None
|
||||
return abi
|
||||
|
||||
|
||||
def _is_running_32bit():
|
||||
return sys.maxsize == 2147483647
|
||||
|
||||
|
||||
def get_platform():
|
||||
"""Return our platform name 'win32', 'linux_x86_64'"""
|
||||
if sys.platform == 'darwin':
|
||||
# distutils.util.get_platform() returns the release based on the value
|
||||
# of MACOSX_DEPLOYMENT_TARGET on which Python was built, which may
|
||||
# be significantly older than the user's current machine.
|
||||
release, _, machine = platform.mac_ver()
|
||||
split_ver = release.split('.')
|
||||
|
||||
if machine == "x86_64" and _is_running_32bit():
|
||||
machine = "i386"
|
||||
elif machine == "ppc64" and _is_running_32bit():
|
||||
machine = "ppc"
|
||||
|
||||
return 'macosx_{}_{}_{}'.format(split_ver[0], split_ver[1], machine)
|
||||
|
||||
# XXX remove distutils dependency
|
||||
result = distutils.util.get_platform().replace('.', '_').replace('-', '_')
|
||||
if result == "linux_x86_64" and _is_running_32bit():
|
||||
# 32 bit Python program (running on a 64 bit Linux): pip should only
|
||||
# install and run 32 bit compiled extensions in that case.
|
||||
result = "linux_i686"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def is_manylinux1_compatible():
|
||||
# Only Linux, and only x86-64 / i686
|
||||
if get_platform() not in {"linux_x86_64", "linux_i686"}:
|
||||
return False
|
||||
|
||||
# Check for presence of _manylinux module
|
||||
try:
|
||||
import _manylinux
|
||||
return bool(_manylinux.manylinux1_compatible)
|
||||
except (ImportError, AttributeError):
|
||||
# Fall through to heuristic check below
|
||||
pass
|
||||
|
||||
# Check glibc version. CentOS 5 uses glibc 2.5.
|
||||
return pip._internal.utils.glibc.have_compatible_glibc(2, 5)
|
||||
|
||||
|
||||
def get_darwin_arches(major, minor, machine):
|
||||
"""Return a list of supported arches (including group arches) for
|
||||
the given major, minor and machine architecture of an macOS machine.
|
||||
"""
|
||||
arches = []
|
||||
|
||||
def _supports_arch(major, minor, arch):
|
||||
# Looking at the application support for macOS versions in the chart
|
||||
# provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears
|
||||
# our timeline looks roughly like:
|
||||
#
|
||||
# 10.0 - Introduces ppc support.
|
||||
# 10.4 - Introduces ppc64, i386, and x86_64 support, however the ppc64
|
||||
# and x86_64 support is CLI only, and cannot be used for GUI
|
||||
# applications.
|
||||
# 10.5 - Extends ppc64 and x86_64 support to cover GUI applications.
|
||||
# 10.6 - Drops support for ppc64
|
||||
# 10.7 - Drops support for ppc
|
||||
#
|
||||
# Given that we do not know if we're installing a CLI or a GUI
|
||||
# application, we must be conservative and assume it might be a GUI
|
||||
# application and behave as if ppc64 and x86_64 support did not occur
|
||||
# until 10.5.
|
||||
#
|
||||
# Note: The above information is taken from the "Application support"
|
||||
# column in the chart not the "Processor support" since I believe
|
||||
# that we care about what instruction sets an application can use
|
||||
# not which processors the OS supports.
|
||||
if arch == 'ppc':
|
||||
return (major, minor) <= (10, 5)
|
||||
if arch == 'ppc64':
|
||||
return (major, minor) == (10, 5)
|
||||
if arch == 'i386':
|
||||
return (major, minor) >= (10, 4)
|
||||
if arch == 'x86_64':
|
||||
return (major, minor) >= (10, 5)
|
||||
if arch in groups:
|
||||
for garch in groups[arch]:
|
||||
if _supports_arch(major, minor, garch):
|
||||
return True
|
||||
return False
|
||||
|
||||
groups = OrderedDict([
|
||||
("fat", ("i386", "ppc")),
|
||||
("intel", ("x86_64", "i386")),
|
||||
("fat64", ("x86_64", "ppc64")),
|
||||
("fat32", ("x86_64", "i386", "ppc")),
|
||||
])
|
||||
|
||||
if _supports_arch(major, minor, machine):
|
||||
arches.append(machine)
|
||||
|
||||
for garch in groups:
|
||||
if machine in groups[garch] and _supports_arch(major, minor, garch):
|
||||
arches.append(garch)
|
||||
|
||||
arches.append('universal')
|
||||
|
||||
return arches
|
||||
|
||||
|
||||
def get_supported(versions=None, noarch=False, platform=None,
|
||||
impl=None, abi=None):
|
||||
"""Return a list of supported tags for each version specified in
|
||||
`versions`.
|
||||
|
||||
:param versions: a list of string versions, of the form ["33", "32"],
|
||||
or None. The first version will be assumed to support our ABI.
|
||||
:param platform: specify the exact platform you want valid
|
||||
tags for, or None. If None, use the local system platform.
|
||||
:param impl: specify the exact implementation you want valid
|
||||
tags for, or None. If None, use the local interpreter impl.
|
||||
:param abi: specify the exact abi you want valid
|
||||
tags for, or None. If None, use the local interpreter abi.
|
||||
"""
|
||||
supported = []
|
||||
|
||||
# Versions must be given with respect to the preference
|
||||
if versions is None:
|
||||
versions = []
|
||||
version_info = get_impl_version_info()
|
||||
major = version_info[:-1]
|
||||
# Support all previous minor Python versions.
|
||||
for minor in range(version_info[-1], -1, -1):
|
||||
versions.append(''.join(map(str, major + (minor,))))
|
||||
|
||||
impl = impl or get_abbr_impl()
|
||||
|
||||
abis = []
|
||||
|
||||
abi = abi or get_abi_tag()
|
||||
if abi:
|
||||
abis[0:0] = [abi]
|
||||
|
||||
abi3s = set()
|
||||
import imp
|
||||
for suffix in imp.get_suffixes():
|
||||
if suffix[0].startswith('.abi'):
|
||||
abi3s.add(suffix[0].split('.', 2)[1])
|
||||
|
||||
abis.extend(sorted(list(abi3s)))
|
||||
|
||||
abis.append('none')
|
||||
|
||||
if not noarch:
|
||||
arch = platform or get_platform()
|
||||
if arch.startswith('macosx'):
|
||||
# support macosx-10.6-intel on macosx-10.9-x86_64
|
||||
match = _osx_arch_pat.match(arch)
|
||||
if match:
|
||||
name, major, minor, actual_arch = match.groups()
|
||||
tpl = '{}_{}_%i_%s'.format(name, major)
|
||||
arches = []
|
||||
for m in reversed(range(int(minor) + 1)):
|
||||
for a in get_darwin_arches(int(major), m, actual_arch):
|
||||
arches.append(tpl % (m, a))
|
||||
else:
|
||||
# arch pattern didn't match (?!)
|
||||
arches = [arch]
|
||||
elif platform is None and is_manylinux1_compatible():
|
||||
arches = [arch.replace('linux', 'manylinux1'), arch]
|
||||
else:
|
||||
arches = [arch]
|
||||
|
||||
# Current version, current API (built specifically for our Python):
|
||||
for abi in abis:
|
||||
for arch in arches:
|
||||
supported.append(('%s%s' % (impl, versions[0]), abi, arch))
|
||||
|
||||
# abi3 modules compatible with older version of Python
|
||||
for version in versions[1:]:
|
||||
# abi3 was introduced in Python 3.2
|
||||
if version in {'31', '30'}:
|
||||
break
|
||||
for abi in abi3s: # empty set if not Python 3
|
||||
for arch in arches:
|
||||
supported.append(("%s%s" % (impl, version), abi, arch))
|
||||
|
||||
# Has binaries, does not use the Python API:
|
||||
for arch in arches:
|
||||
supported.append(('py%s' % (versions[0][0]), 'none', arch))
|
||||
|
||||
# No abi / arch, but requires our implementation:
|
||||
supported.append(('%s%s' % (impl, versions[0]), 'none', 'any'))
|
||||
# Tagged specifically as being cross-version compatible
|
||||
# (with just the major version specified)
|
||||
supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any'))
|
||||
|
||||
# No abi / arch, generic Python
|
||||
for i, version in enumerate(versions):
|
||||
supported.append(('py%s' % (version,), 'none', 'any'))
|
||||
if i == 0:
|
||||
supported.append(('py%s' % (version[0]), 'none', 'any'))
|
||||
|
||||
return supported
|
||||
|
||||
|
||||
implementation_tag = get_impl_tag()
|
||||
@@ -1,69 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
|
||||
from .req_install import InstallRequirement
|
||||
from .req_set import RequirementSet
|
||||
from .req_file import parse_requirements
|
||||
from pip._internal.utils.logging import indent_log
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RequirementSet", "InstallRequirement",
|
||||
"parse_requirements", "install_given_reqs",
|
||||
]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def install_given_reqs(to_install, install_options, global_options=(),
|
||||
*args, **kwargs):
|
||||
"""
|
||||
Install everything in the given list.
|
||||
|
||||
(to be called after having downloaded and unpacked the packages)
|
||||
"""
|
||||
|
||||
if to_install:
|
||||
logger.info(
|
||||
'Installing collected packages: %s',
|
||||
', '.join([req.name for req in to_install]),
|
||||
)
|
||||
|
||||
with indent_log():
|
||||
for requirement in to_install:
|
||||
if requirement.conflicts_with:
|
||||
logger.info(
|
||||
'Found existing installation: %s',
|
||||
requirement.conflicts_with,
|
||||
)
|
||||
with indent_log():
|
||||
uninstalled_pathset = requirement.uninstall(
|
||||
auto_confirm=True
|
||||
)
|
||||
try:
|
||||
requirement.install(
|
||||
install_options,
|
||||
global_options,
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
except:
|
||||
should_rollback = (
|
||||
requirement.conflicts_with and
|
||||
not requirement.install_succeeded
|
||||
)
|
||||
# if install did not succeed, rollback previous uninstall
|
||||
if should_rollback:
|
||||
uninstalled_pathset.rollback()
|
||||
raise
|
||||
else:
|
||||
should_commit = (
|
||||
requirement.conflicts_with and
|
||||
requirement.install_succeeded
|
||||
)
|
||||
if should_commit:
|
||||
uninstalled_pathset.commit()
|
||||
requirement.remove_temporary_source()
|
||||
|
||||
return to_install
|
||||
@@ -1,338 +0,0 @@
|
||||
"""
|
||||
Requirements file parsing
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import optparse
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import sys
|
||||
|
||||
from pip._vendor.six.moves import filterfalse
|
||||
from pip._vendor.six.moves.urllib import parse as urllib_parse
|
||||
|
||||
from pip._internal import cmdoptions
|
||||
from pip._internal.download import get_file_content
|
||||
from pip._internal.exceptions import RequirementsFileParseError
|
||||
from pip._internal.req.req_install import InstallRequirement
|
||||
|
||||
__all__ = ['parse_requirements']
|
||||
|
||||
SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
|
||||
COMMENT_RE = re.compile(r'(^|\s)+#.*$')
|
||||
|
||||
# Matches environment variable-style values in '${MY_VARIABLE_1}' with the
|
||||
# variable name consisting of only uppercase letters, digits or the '_'
|
||||
# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
|
||||
# 2013 Edition.
|
||||
ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})')
|
||||
|
||||
SUPPORTED_OPTIONS = [
|
||||
cmdoptions.constraints,
|
||||
cmdoptions.editable,
|
||||
cmdoptions.requirements,
|
||||
cmdoptions.no_index,
|
||||
cmdoptions.index_url,
|
||||
cmdoptions.find_links,
|
||||
cmdoptions.extra_index_url,
|
||||
cmdoptions.always_unzip,
|
||||
cmdoptions.no_binary,
|
||||
cmdoptions.only_binary,
|
||||
cmdoptions.pre,
|
||||
cmdoptions.process_dependency_links,
|
||||
cmdoptions.trusted_host,
|
||||
cmdoptions.require_hashes,
|
||||
]
|
||||
|
||||
# options to be passed to requirements
|
||||
SUPPORTED_OPTIONS_REQ = [
|
||||
cmdoptions.install_options,
|
||||
cmdoptions.global_options,
|
||||
cmdoptions.hash,
|
||||
]
|
||||
|
||||
# the 'dest' string values
|
||||
SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ]
|
||||
|
||||
|
||||
def parse_requirements(filename, finder=None, comes_from=None, options=None,
|
||||
session=None, constraint=False, wheel_cache=None):
|
||||
"""Parse a requirements file and yield InstallRequirement instances.
|
||||
|
||||
:param filename: Path or url of requirements file.
|
||||
:param finder: Instance of pip.index.PackageFinder.
|
||||
:param comes_from: Origin description of requirements.
|
||||
:param options: cli options.
|
||||
:param session: Instance of pip.download.PipSession.
|
||||
:param constraint: If true, parsing a constraint file rather than
|
||||
requirements file.
|
||||
:param wheel_cache: Instance of pip.wheel.WheelCache
|
||||
"""
|
||||
if session is None:
|
||||
raise TypeError(
|
||||
"parse_requirements() missing 1 required keyword argument: "
|
||||
"'session'"
|
||||
)
|
||||
|
||||
_, content = get_file_content(
|
||||
filename, comes_from=comes_from, session=session
|
||||
)
|
||||
|
||||
lines_enum = preprocess(content, options)
|
||||
|
||||
for line_number, line in lines_enum:
|
||||
req_iter = process_line(line, filename, line_number, finder,
|
||||
comes_from, options, session, wheel_cache,
|
||||
constraint=constraint)
|
||||
for req in req_iter:
|
||||
yield req
|
||||
|
||||
|
||||
def preprocess(content, options):
|
||||
"""Split, filter, and join lines, and return a line iterator
|
||||
|
||||
:param content: the content of the requirements file
|
||||
:param options: cli options
|
||||
"""
|
||||
lines_enum = enumerate(content.splitlines(), start=1)
|
||||
lines_enum = join_lines(lines_enum)
|
||||
lines_enum = ignore_comments(lines_enum)
|
||||
lines_enum = skip_regex(lines_enum, options)
|
||||
lines_enum = expand_env_variables(lines_enum)
|
||||
return lines_enum
|
||||
|
||||
|
||||
def process_line(line, filename, line_number, finder=None, comes_from=None,
|
||||
options=None, session=None, wheel_cache=None,
|
||||
constraint=False):
|
||||
"""Process a single requirements line; This can result in creating/yielding
|
||||
requirements, or updating the finder.
|
||||
|
||||
For lines that contain requirements, the only options that have an effect
|
||||
are from SUPPORTED_OPTIONS_REQ, and they are scoped to the
|
||||
requirement. Other options from SUPPORTED_OPTIONS may be present, but are
|
||||
ignored.
|
||||
|
||||
For lines that do not contain requirements, the only options that have an
|
||||
effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may
|
||||
be present, but are ignored. These lines may contain multiple options
|
||||
(although our docs imply only one is supported), and all our parsed and
|
||||
affect the finder.
|
||||
|
||||
:param constraint: If True, parsing a constraints file.
|
||||
:param options: OptionParser options that we may update
|
||||
"""
|
||||
parser = build_parser(line)
|
||||
defaults = parser.get_default_values()
|
||||
defaults.index_url = None
|
||||
if finder:
|
||||
# `finder.format_control` will be updated during parsing
|
||||
defaults.format_control = finder.format_control
|
||||
args_str, options_str = break_args_options(line)
|
||||
if sys.version_info < (2, 7, 3):
|
||||
# Prior to 2.7.3, shlex cannot deal with unicode entries
|
||||
options_str = options_str.encode('utf8')
|
||||
opts, _ = parser.parse_args(shlex.split(options_str), defaults)
|
||||
|
||||
# preserve for the nested code path
|
||||
line_comes_from = '%s %s (line %s)' % (
|
||||
'-c' if constraint else '-r', filename, line_number,
|
||||
)
|
||||
|
||||
# yield a line requirement
|
||||
if args_str:
|
||||
isolated = options.isolated_mode if options else False
|
||||
if options:
|
||||
cmdoptions.check_install_build_global(options, opts)
|
||||
# get the options that apply to requirements
|
||||
req_options = {}
|
||||
for dest in SUPPORTED_OPTIONS_REQ_DEST:
|
||||
if dest in opts.__dict__ and opts.__dict__[dest]:
|
||||
req_options[dest] = opts.__dict__[dest]
|
||||
yield InstallRequirement.from_line(
|
||||
args_str, line_comes_from, constraint=constraint,
|
||||
isolated=isolated, options=req_options, wheel_cache=wheel_cache
|
||||
)
|
||||
|
||||
# yield an editable requirement
|
||||
elif opts.editables:
|
||||
isolated = options.isolated_mode if options else False
|
||||
yield InstallRequirement.from_editable(
|
||||
opts.editables[0], comes_from=line_comes_from,
|
||||
constraint=constraint, isolated=isolated, wheel_cache=wheel_cache
|
||||
)
|
||||
|
||||
# parse a nested requirements file
|
||||
elif opts.requirements or opts.constraints:
|
||||
if opts.requirements:
|
||||
req_path = opts.requirements[0]
|
||||
nested_constraint = False
|
||||
else:
|
||||
req_path = opts.constraints[0]
|
||||
nested_constraint = True
|
||||
# original file is over http
|
||||
if SCHEME_RE.search(filename):
|
||||
# do a url join so relative paths work
|
||||
req_path = urllib_parse.urljoin(filename, req_path)
|
||||
# original file and nested file are paths
|
||||
elif not SCHEME_RE.search(req_path):
|
||||
# do a join so relative paths work
|
||||
req_path = os.path.join(os.path.dirname(filename), req_path)
|
||||
# TODO: Why not use `comes_from='-r {} (line {})'` here as well?
|
||||
parser = parse_requirements(
|
||||
req_path, finder, comes_from, options, session,
|
||||
constraint=nested_constraint, wheel_cache=wheel_cache
|
||||
)
|
||||
for req in parser:
|
||||
yield req
|
||||
|
||||
# percolate hash-checking option upward
|
||||
elif opts.require_hashes:
|
||||
options.require_hashes = opts.require_hashes
|
||||
|
||||
# set finder options
|
||||
elif finder:
|
||||
if opts.index_url:
|
||||
finder.index_urls = [opts.index_url]
|
||||
if opts.no_index is True:
|
||||
finder.index_urls = []
|
||||
if opts.extra_index_urls:
|
||||
finder.index_urls.extend(opts.extra_index_urls)
|
||||
if opts.find_links:
|
||||
# FIXME: it would be nice to keep track of the source
|
||||
# of the find_links: support a find-links local path
|
||||
# relative to a requirements file.
|
||||
value = opts.find_links[0]
|
||||
req_dir = os.path.dirname(os.path.abspath(filename))
|
||||
relative_to_reqs_file = os.path.join(req_dir, value)
|
||||
if os.path.exists(relative_to_reqs_file):
|
||||
value = relative_to_reqs_file
|
||||
finder.find_links.append(value)
|
||||
if opts.pre:
|
||||
finder.allow_all_prereleases = True
|
||||
if opts.process_dependency_links:
|
||||
finder.process_dependency_links = True
|
||||
if opts.trusted_hosts:
|
||||
finder.secure_origins.extend(
|
||||
("*", host, "*") for host in opts.trusted_hosts)
|
||||
|
||||
|
||||
def break_args_options(line):
|
||||
"""Break up the line into an args and options string. We only want to shlex
|
||||
(and then optparse) the options, not the args. args can contain markers
|
||||
which are corrupted by shlex.
|
||||
"""
|
||||
tokens = line.split(' ')
|
||||
args = []
|
||||
options = tokens[:]
|
||||
for token in tokens:
|
||||
if token.startswith('-') or token.startswith('--'):
|
||||
break
|
||||
else:
|
||||
args.append(token)
|
||||
options.pop(0)
|
||||
return ' '.join(args), ' '.join(options)
|
||||
|
||||
|
||||
def build_parser(line):
|
||||
"""
|
||||
Return a parser for parsing requirement lines
|
||||
"""
|
||||
parser = optparse.OptionParser(add_help_option=False)
|
||||
|
||||
option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ
|
||||
for option_factory in option_factories:
|
||||
option = option_factory()
|
||||
parser.add_option(option)
|
||||
|
||||
# By default optparse sys.exits on parsing errors. We want to wrap
|
||||
# that in our own exception.
|
||||
def parser_exit(self, msg):
|
||||
# add offending line
|
||||
msg = 'Invalid requirement: %s\n%s' % (line, msg)
|
||||
raise RequirementsFileParseError(msg)
|
||||
parser.exit = parser_exit
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def join_lines(lines_enum):
|
||||
"""Joins a line ending in '\' with the previous line (except when following
|
||||
comments). The joined line takes on the index of the first line.
|
||||
"""
|
||||
primary_line_number = None
|
||||
new_line = []
|
||||
for line_number, line in lines_enum:
|
||||
if not line.endswith('\\') or COMMENT_RE.match(line):
|
||||
if COMMENT_RE.match(line):
|
||||
# this ensures comments are always matched later
|
||||
line = ' ' + line
|
||||
if new_line:
|
||||
new_line.append(line)
|
||||
yield primary_line_number, ''.join(new_line)
|
||||
new_line = []
|
||||
else:
|
||||
yield line_number, line
|
||||
else:
|
||||
if not new_line:
|
||||
primary_line_number = line_number
|
||||
new_line.append(line.strip('\\'))
|
||||
|
||||
# last line contains \
|
||||
if new_line:
|
||||
yield primary_line_number, ''.join(new_line)
|
||||
|
||||
# TODO: handle space after '\'.
|
||||
|
||||
|
||||
def ignore_comments(lines_enum):
|
||||
"""
|
||||
Strips comments and filter empty lines.
|
||||
"""
|
||||
for line_number, line in lines_enum:
|
||||
line = COMMENT_RE.sub('', line)
|
||||
line = line.strip()
|
||||
if line:
|
||||
yield line_number, line
|
||||
|
||||
|
||||
def skip_regex(lines_enum, options):
|
||||
"""
|
||||
Skip lines that match '--skip-requirements-regex' pattern
|
||||
|
||||
Note: the regex pattern is only built once
|
||||
"""
|
||||
skip_regex = options.skip_requirements_regex if options else None
|
||||
if skip_regex:
|
||||
pattern = re.compile(skip_regex)
|
||||
lines_enum = filterfalse(lambda e: pattern.search(e[1]), lines_enum)
|
||||
return lines_enum
|
||||
|
||||
|
||||
def expand_env_variables(lines_enum):
|
||||
"""Replace all environment variables that can be retrieved via `os.getenv`.
|
||||
|
||||
The only allowed format for environment variables defined in the
|
||||
requirement file is `${MY_VARIABLE_1}` to ensure two things:
|
||||
|
||||
1. Strings that contain a `$` aren't accidentally (partially) expanded.
|
||||
2. Ensure consistency across platforms for requirement files.
|
||||
|
||||
These points are the result of a discusssion on the `github pull
|
||||
request #3514 <https://github.com/pypa/pip/pull/3514>`_.
|
||||
|
||||
Valid characters in variable names follow the `POSIX standard
|
||||
<http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
|
||||
to uppercase letter, digits and the `_` (underscore).
|
||||
"""
|
||||
for line_number, line in lines_enum:
|
||||
for env_var, var_name in ENV_VAR_RE.findall(line):
|
||||
value = os.getenv(var_name)
|
||||
if not value:
|
||||
continue
|
||||
|
||||
line = line.replace(env_var, value)
|
||||
|
||||
yield line_number, line
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,164 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
from collections import OrderedDict
|
||||
|
||||
from pip._internal.exceptions import InstallationError
|
||||
from pip._internal.utils.logging import indent_log
|
||||
from pip._internal.wheel import Wheel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RequirementSet(object):
|
||||
|
||||
def __init__(self, require_hashes=False):
|
||||
"""Create a RequirementSet.
|
||||
|
||||
:param wheel_cache: The pip wheel cache, for passing to
|
||||
InstallRequirement.
|
||||
"""
|
||||
|
||||
self.requirements = OrderedDict()
|
||||
self.require_hashes = require_hashes
|
||||
|
||||
# Mapping of alias: real_name
|
||||
self.requirement_aliases = {}
|
||||
self.unnamed_requirements = []
|
||||
self.successfully_downloaded = []
|
||||
self.reqs_to_cleanup = []
|
||||
|
||||
def __str__(self):
|
||||
reqs = [req for req in self.requirements.values()
|
||||
if not req.comes_from]
|
||||
reqs.sort(key=lambda req: req.name.lower())
|
||||
return ' '.join([str(req.req) for req in reqs])
|
||||
|
||||
def __repr__(self):
|
||||
reqs = [req for req in self.requirements.values()]
|
||||
reqs.sort(key=lambda req: req.name.lower())
|
||||
reqs_str = ', '.join([str(req.req) for req in reqs])
|
||||
return ('<%s object; %d requirement(s): %s>'
|
||||
% (self.__class__.__name__, len(reqs), reqs_str))
|
||||
|
||||
def add_requirement(self, install_req, parent_req_name=None,
|
||||
extras_requested=None):
|
||||
"""Add install_req as a requirement to install.
|
||||
|
||||
:param parent_req_name: The name of the requirement that needed this
|
||||
added. The name is used because when multiple unnamed requirements
|
||||
resolve to the same name, we could otherwise end up with dependency
|
||||
links that point outside the Requirements set. parent_req must
|
||||
already be added. Note that None implies that this is a user
|
||||
supplied requirement, vs an inferred one.
|
||||
:param extras_requested: an iterable of extras used to evaluate the
|
||||
environment markers.
|
||||
:return: Additional requirements to scan. That is either [] if
|
||||
the requirement is not applicable, or [install_req] if the
|
||||
requirement is applicable and has just been added.
|
||||
"""
|
||||
name = install_req.name
|
||||
if not install_req.match_markers(extras_requested):
|
||||
logger.info("Ignoring %s: markers '%s' don't match your "
|
||||
"environment", install_req.name,
|
||||
install_req.markers)
|
||||
return [], None
|
||||
|
||||
# This check has to come after we filter requirements with the
|
||||
# environment markers.
|
||||
if install_req.link and install_req.link.is_wheel:
|
||||
wheel = Wheel(install_req.link.filename)
|
||||
if not wheel.supported():
|
||||
raise InstallationError(
|
||||
"%s is not a supported wheel on this platform." %
|
||||
wheel.filename
|
||||
)
|
||||
|
||||
# This next bit is really a sanity check.
|
||||
assert install_req.is_direct == (parent_req_name is None), (
|
||||
"a direct req shouldn't have a parent and also, "
|
||||
"a non direct req should have a parent"
|
||||
)
|
||||
|
||||
if not name:
|
||||
# url or path requirement w/o an egg fragment
|
||||
self.unnamed_requirements.append(install_req)
|
||||
return [install_req], None
|
||||
else:
|
||||
try:
|
||||
existing_req = self.get_requirement(name)
|
||||
except KeyError:
|
||||
existing_req = None
|
||||
if (parent_req_name is None and existing_req and not
|
||||
existing_req.constraint and
|
||||
existing_req.extras == install_req.extras and not
|
||||
existing_req.req.specifier == install_req.req.specifier):
|
||||
raise InstallationError(
|
||||
'Double requirement given: %s (already in %s, name=%r)'
|
||||
% (install_req, existing_req, name))
|
||||
if not existing_req:
|
||||
# Add requirement
|
||||
self.requirements[name] = install_req
|
||||
# FIXME: what about other normalizations? E.g., _ vs. -?
|
||||
if name.lower() != name:
|
||||
self.requirement_aliases[name.lower()] = name
|
||||
result = [install_req]
|
||||
else:
|
||||
# Assume there's no need to scan, and that we've already
|
||||
# encountered this for scanning.
|
||||
result = []
|
||||
if not install_req.constraint and existing_req.constraint:
|
||||
if (install_req.link and not (existing_req.link and
|
||||
install_req.link.path == existing_req.link.path)):
|
||||
self.reqs_to_cleanup.append(install_req)
|
||||
raise InstallationError(
|
||||
"Could not satisfy constraints for '%s': "
|
||||
"installation from path or url cannot be "
|
||||
"constrained to a version" % name,
|
||||
)
|
||||
# If we're now installing a constraint, mark the existing
|
||||
# object for real installation.
|
||||
existing_req.constraint = False
|
||||
existing_req.extras = tuple(
|
||||
sorted(set(existing_req.extras).union(
|
||||
set(install_req.extras))))
|
||||
logger.debug("Setting %s extras to: %s",
|
||||
existing_req, existing_req.extras)
|
||||
# And now we need to scan this.
|
||||
result = [existing_req]
|
||||
# Canonicalise to the already-added object for the backref
|
||||
# check below.
|
||||
install_req = existing_req
|
||||
|
||||
# We return install_req here to allow for the caller to add it to
|
||||
# the dependency information for the parent package.
|
||||
return result, install_req
|
||||
|
||||
def has_requirement(self, project_name):
|
||||
name = project_name.lower()
|
||||
if (name in self.requirements and
|
||||
not self.requirements[name].constraint or
|
||||
name in self.requirement_aliases and
|
||||
not self.requirements[self.requirement_aliases[name]].constraint):
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def has_requirements(self):
|
||||
return list(req for req in self.requirements.values() if not
|
||||
req.constraint) or self.unnamed_requirements
|
||||
|
||||
def get_requirement(self, project_name):
|
||||
for name in project_name, project_name.lower():
|
||||
if name in self.requirements:
|
||||
return self.requirements[name]
|
||||
if name in self.requirement_aliases:
|
||||
return self.requirements[self.requirement_aliases[name]]
|
||||
raise KeyError("No project with the name %r" % project_name)
|
||||
|
||||
def cleanup_files(self):
|
||||
"""Clean up files, remove builds."""
|
||||
logger.debug('Cleaning up...')
|
||||
with indent_log():
|
||||
for req in self.reqs_to_cleanup:
|
||||
req.remove_temporary_source()
|
||||
@@ -1,455 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import csv
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import sysconfig
|
||||
|
||||
from pip._vendor import pkg_resources
|
||||
|
||||
from pip._internal.compat import WINDOWS, cache_from_source, uses_pycache
|
||||
from pip._internal.exceptions import UninstallationError
|
||||
from pip._internal.locations import bin_py, bin_user
|
||||
from pip._internal.utils.logging import indent_log
|
||||
from pip._internal.utils.misc import (
|
||||
FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local,
|
||||
normalize_path, renames,
|
||||
)
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _script_names(dist, script_name, is_gui):
|
||||
"""Create the fully qualified name of the files created by
|
||||
{console,gui}_scripts for the given ``dist``.
|
||||
Returns the list of file names
|
||||
"""
|
||||
if dist_in_usersite(dist):
|
||||
bin_dir = bin_user
|
||||
else:
|
||||
bin_dir = bin_py
|
||||
exe_name = os.path.join(bin_dir, script_name)
|
||||
paths_to_remove = [exe_name]
|
||||
if WINDOWS:
|
||||
paths_to_remove.append(exe_name + '.exe')
|
||||
paths_to_remove.append(exe_name + '.exe.manifest')
|
||||
if is_gui:
|
||||
paths_to_remove.append(exe_name + '-script.pyw')
|
||||
else:
|
||||
paths_to_remove.append(exe_name + '-script.py')
|
||||
return paths_to_remove
|
||||
|
||||
|
||||
def _unique(fn):
|
||||
@functools.wraps(fn)
|
||||
def unique(*args, **kw):
|
||||
seen = set()
|
||||
for item in fn(*args, **kw):
|
||||
if item not in seen:
|
||||
seen.add(item)
|
||||
yield item
|
||||
return unique
|
||||
|
||||
|
||||
@_unique
|
||||
def uninstallation_paths(dist):
|
||||
"""
|
||||
Yield all the uninstallation paths for dist based on RECORD-without-.pyc
|
||||
|
||||
Yield paths to all the files in RECORD. For each .py file in RECORD, add
|
||||
the .pyc in the same directory.
|
||||
|
||||
UninstallPathSet.add() takes care of the __pycache__ .pyc.
|
||||
"""
|
||||
r = csv.reader(FakeFile(dist.get_metadata_lines('RECORD')))
|
||||
for row in r:
|
||||
path = os.path.join(dist.location, row[0])
|
||||
yield path
|
||||
if path.endswith('.py'):
|
||||
dn, fn = os.path.split(path)
|
||||
base = fn[:-3]
|
||||
path = os.path.join(dn, base + '.pyc')
|
||||
yield path
|
||||
|
||||
|
||||
def compact(paths):
|
||||
"""Compact a path set to contain the minimal number of paths
|
||||
necessary to contain all paths in the set. If /a/path/ and
|
||||
/a/path/to/a/file.txt are both in the set, leave only the
|
||||
shorter path."""
|
||||
|
||||
sep = os.path.sep
|
||||
short_paths = set()
|
||||
for path in sorted(paths, key=len):
|
||||
should_add = any(
|
||||
path.startswith(shortpath.rstrip("*")) and
|
||||
path[len(shortpath.rstrip("*").rstrip(sep))] == sep
|
||||
for shortpath in short_paths
|
||||
)
|
||||
if not should_add:
|
||||
short_paths.add(path)
|
||||
return short_paths
|
||||
|
||||
|
||||
def compress_for_output_listing(paths):
|
||||
"""Returns a tuple of 2 sets of which paths to display to user
|
||||
|
||||
The first set contains paths that would be deleted. Files of a package
|
||||
are not added and the top-level directory of the package has a '*' added
|
||||
at the end - to signify that all it's contents are removed.
|
||||
|
||||
The second set contains files that would have been skipped in the above
|
||||
folders.
|
||||
"""
|
||||
|
||||
will_remove = list(paths)
|
||||
will_skip = set()
|
||||
|
||||
# Determine folders and files
|
||||
folders = set()
|
||||
files = set()
|
||||
for path in will_remove:
|
||||
if path.endswith(".pyc"):
|
||||
continue
|
||||
if path.endswith("__init__.py") or ".dist-info" in path:
|
||||
folders.add(os.path.dirname(path))
|
||||
files.add(path)
|
||||
|
||||
folders = compact(folders)
|
||||
|
||||
# This walks the tree using os.walk to not miss extra folders
|
||||
# that might get added.
|
||||
for folder in folders:
|
||||
for dirpath, _, dirfiles in os.walk(folder):
|
||||
for fname in dirfiles:
|
||||
if fname.endswith(".pyc"):
|
||||
continue
|
||||
|
||||
file_ = os.path.normcase(os.path.join(dirpath, fname))
|
||||
if os.path.isfile(file_) and file_ not in files:
|
||||
# We are skipping this file. Add it to the set.
|
||||
will_skip.add(file_)
|
||||
|
||||
will_remove = files | {
|
||||
os.path.join(folder, "*") for folder in folders
|
||||
}
|
||||
|
||||
return will_remove, will_skip
|
||||
|
||||
|
||||
class UninstallPathSet(object):
|
||||
"""A set of file paths to be removed in the uninstallation of a
|
||||
requirement."""
|
||||
def __init__(self, dist):
|
||||
self.paths = set()
|
||||
self._refuse = set()
|
||||
self.pth = {}
|
||||
self.dist = dist
|
||||
self.save_dir = TempDirectory(kind="uninstall")
|
||||
self._moved_paths = []
|
||||
|
||||
def _permitted(self, path):
|
||||
"""
|
||||
Return True if the given path is one we are permitted to
|
||||
remove/modify, False otherwise.
|
||||
|
||||
"""
|
||||
return is_local(path)
|
||||
|
||||
def add(self, path):
|
||||
head, tail = os.path.split(path)
|
||||
|
||||
# we normalize the head to resolve parent directory symlinks, but not
|
||||
# the tail, since we only want to uninstall symlinks, not their targets
|
||||
path = os.path.join(normalize_path(head), os.path.normcase(tail))
|
||||
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
if self._permitted(path):
|
||||
self.paths.add(path)
|
||||
else:
|
||||
self._refuse.add(path)
|
||||
|
||||
# __pycache__ files can show up after 'installed-files.txt' is created,
|
||||
# due to imports
|
||||
if os.path.splitext(path)[1] == '.py' and uses_pycache:
|
||||
self.add(cache_from_source(path))
|
||||
|
||||
def add_pth(self, pth_file, entry):
|
||||
pth_file = normalize_path(pth_file)
|
||||
if self._permitted(pth_file):
|
||||
if pth_file not in self.pth:
|
||||
self.pth[pth_file] = UninstallPthEntries(pth_file)
|
||||
self.pth[pth_file].add(entry)
|
||||
else:
|
||||
self._refuse.add(pth_file)
|
||||
|
||||
def _stash(self, path):
|
||||
return os.path.join(
|
||||
self.save_dir.path, os.path.splitdrive(path)[1].lstrip(os.path.sep)
|
||||
)
|
||||
|
||||
def remove(self, auto_confirm=False, verbose=False):
|
||||
"""Remove paths in ``self.paths`` with confirmation (unless
|
||||
``auto_confirm`` is True)."""
|
||||
|
||||
if not self.paths:
|
||||
logger.info(
|
||||
"Can't uninstall '%s'. No files were found to uninstall.",
|
||||
self.dist.project_name,
|
||||
)
|
||||
return
|
||||
|
||||
dist_name_version = (
|
||||
self.dist.project_name + "-" + self.dist.version
|
||||
)
|
||||
logger.info('Uninstalling %s:', dist_name_version)
|
||||
|
||||
with indent_log():
|
||||
if auto_confirm or self._allowed_to_proceed(verbose):
|
||||
self.save_dir.create()
|
||||
|
||||
for path in sorted(compact(self.paths)):
|
||||
new_path = self._stash(path)
|
||||
logger.debug('Removing file or directory %s', path)
|
||||
self._moved_paths.append(path)
|
||||
renames(path, new_path)
|
||||
for pth in self.pth.values():
|
||||
pth.remove()
|
||||
|
||||
logger.info('Successfully uninstalled %s', dist_name_version)
|
||||
|
||||
def _allowed_to_proceed(self, verbose):
|
||||
"""Display which files would be deleted and prompt for confirmation
|
||||
"""
|
||||
|
||||
def _display(msg, paths):
|
||||
if not paths:
|
||||
return
|
||||
|
||||
logger.info(msg)
|
||||
with indent_log():
|
||||
for path in sorted(compact(paths)):
|
||||
logger.info(path)
|
||||
|
||||
if not verbose:
|
||||
will_remove, will_skip = compress_for_output_listing(self.paths)
|
||||
else:
|
||||
# In verbose mode, display all the files that are going to be
|
||||
# deleted.
|
||||
will_remove = list(self.paths)
|
||||
will_skip = set()
|
||||
|
||||
_display('Would remove:', will_remove)
|
||||
_display('Would not remove (might be manually added):', will_skip)
|
||||
_display('Would not remove (outside of prefix):', self._refuse)
|
||||
|
||||
return ask('Proceed (y/n)? ', ('y', 'n')) == 'y'
|
||||
|
||||
def rollback(self):
|
||||
"""Rollback the changes previously made by remove()."""
|
||||
if self.save_dir.path is None:
|
||||
logger.error(
|
||||
"Can't roll back %s; was not uninstalled",
|
||||
self.dist.project_name,
|
||||
)
|
||||
return False
|
||||
logger.info('Rolling back uninstall of %s', self.dist.project_name)
|
||||
for path in self._moved_paths:
|
||||
tmp_path = self._stash(path)
|
||||
logger.debug('Replacing %s', path)
|
||||
renames(tmp_path, path)
|
||||
for pth in self.pth.values():
|
||||
pth.rollback()
|
||||
|
||||
def commit(self):
|
||||
"""Remove temporary save dir: rollback will no longer be possible."""
|
||||
self.save_dir.cleanup()
|
||||
self._moved_paths = []
|
||||
|
||||
@classmethod
|
||||
def from_dist(cls, dist):
|
||||
dist_path = normalize_path(dist.location)
|
||||
if not dist_is_local(dist):
|
||||
logger.info(
|
||||
"Not uninstalling %s at %s, outside environment %s",
|
||||
dist.key,
|
||||
dist_path,
|
||||
sys.prefix,
|
||||
)
|
||||
return cls(dist)
|
||||
|
||||
if dist_path in {p for p in {sysconfig.get_path("stdlib"),
|
||||
sysconfig.get_path("platstdlib")}
|
||||
if p}:
|
||||
logger.info(
|
||||
"Not uninstalling %s at %s, as it is in the standard library.",
|
||||
dist.key,
|
||||
dist_path,
|
||||
)
|
||||
return cls(dist)
|
||||
|
||||
paths_to_remove = cls(dist)
|
||||
develop_egg_link = egg_link_path(dist)
|
||||
develop_egg_link_egg_info = '{}.egg-info'.format(
|
||||
pkg_resources.to_filename(dist.project_name))
|
||||
egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info)
|
||||
# Special case for distutils installed package
|
||||
distutils_egg_info = getattr(dist._provider, 'path', None)
|
||||
|
||||
# Uninstall cases order do matter as in the case of 2 installs of the
|
||||
# same package, pip needs to uninstall the currently detected version
|
||||
if (egg_info_exists and dist.egg_info.endswith('.egg-info') and
|
||||
not dist.egg_info.endswith(develop_egg_link_egg_info)):
|
||||
# if dist.egg_info.endswith(develop_egg_link_egg_info), we
|
||||
# are in fact in the develop_egg_link case
|
||||
paths_to_remove.add(dist.egg_info)
|
||||
if dist.has_metadata('installed-files.txt'):
|
||||
for installed_file in dist.get_metadata(
|
||||
'installed-files.txt').splitlines():
|
||||
path = os.path.normpath(
|
||||
os.path.join(dist.egg_info, installed_file)
|
||||
)
|
||||
paths_to_remove.add(path)
|
||||
# FIXME: need a test for this elif block
|
||||
# occurs with --single-version-externally-managed/--record outside
|
||||
# of pip
|
||||
elif dist.has_metadata('top_level.txt'):
|
||||
if dist.has_metadata('namespace_packages.txt'):
|
||||
namespaces = dist.get_metadata('namespace_packages.txt')
|
||||
else:
|
||||
namespaces = []
|
||||
for top_level_pkg in [
|
||||
p for p
|
||||
in dist.get_metadata('top_level.txt').splitlines()
|
||||
if p and p not in namespaces]:
|
||||
path = os.path.join(dist.location, top_level_pkg)
|
||||
paths_to_remove.add(path)
|
||||
paths_to_remove.add(path + '.py')
|
||||
paths_to_remove.add(path + '.pyc')
|
||||
paths_to_remove.add(path + '.pyo')
|
||||
|
||||
elif distutils_egg_info:
|
||||
raise UninstallationError(
|
||||
"Cannot uninstall {!r}. It is a distutils installed project "
|
||||
"and thus we cannot accurately determine which files belong "
|
||||
"to it which would lead to only a partial uninstall.".format(
|
||||
dist.project_name,
|
||||
)
|
||||
)
|
||||
|
||||
elif dist.location.endswith('.egg'):
|
||||
# package installed by easy_install
|
||||
# We cannot match on dist.egg_name because it can slightly vary
|
||||
# i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg
|
||||
paths_to_remove.add(dist.location)
|
||||
easy_install_egg = os.path.split(dist.location)[1]
|
||||
easy_install_pth = os.path.join(os.path.dirname(dist.location),
|
||||
'easy-install.pth')
|
||||
paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg)
|
||||
|
||||
elif egg_info_exists and dist.egg_info.endswith('.dist-info'):
|
||||
for path in uninstallation_paths(dist):
|
||||
paths_to_remove.add(path)
|
||||
|
||||
elif develop_egg_link:
|
||||
# develop egg
|
||||
with open(develop_egg_link, 'r') as fh:
|
||||
link_pointer = os.path.normcase(fh.readline().strip())
|
||||
assert (link_pointer == dist.location), (
|
||||
'Egg-link %s does not match installed location of %s '
|
||||
'(at %s)' % (link_pointer, dist.project_name, dist.location)
|
||||
)
|
||||
paths_to_remove.add(develop_egg_link)
|
||||
easy_install_pth = os.path.join(os.path.dirname(develop_egg_link),
|
||||
'easy-install.pth')
|
||||
paths_to_remove.add_pth(easy_install_pth, dist.location)
|
||||
|
||||
else:
|
||||
logger.debug(
|
||||
'Not sure how to uninstall: %s - Check: %s',
|
||||
dist, dist.location,
|
||||
)
|
||||
|
||||
# find distutils scripts= scripts
|
||||
if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'):
|
||||
for script in dist.metadata_listdir('scripts'):
|
||||
if dist_in_usersite(dist):
|
||||
bin_dir = bin_user
|
||||
else:
|
||||
bin_dir = bin_py
|
||||
paths_to_remove.add(os.path.join(bin_dir, script))
|
||||
if WINDOWS:
|
||||
paths_to_remove.add(os.path.join(bin_dir, script) + '.bat')
|
||||
|
||||
# find console_scripts
|
||||
_scripts_to_remove = []
|
||||
console_scripts = dist.get_entry_map(group='console_scripts')
|
||||
for name in console_scripts.keys():
|
||||
_scripts_to_remove.extend(_script_names(dist, name, False))
|
||||
# find gui_scripts
|
||||
gui_scripts = dist.get_entry_map(group='gui_scripts')
|
||||
for name in gui_scripts.keys():
|
||||
_scripts_to_remove.extend(_script_names(dist, name, True))
|
||||
|
||||
for s in _scripts_to_remove:
|
||||
paths_to_remove.add(s)
|
||||
|
||||
return paths_to_remove
|
||||
|
||||
|
||||
class UninstallPthEntries(object):
|
||||
def __init__(self, pth_file):
|
||||
if not os.path.isfile(pth_file):
|
||||
raise UninstallationError(
|
||||
"Cannot remove entries from nonexistent file %s" % pth_file
|
||||
)
|
||||
self.file = pth_file
|
||||
self.entries = set()
|
||||
self._saved_lines = None
|
||||
|
||||
def add(self, entry):
|
||||
entry = os.path.normcase(entry)
|
||||
# On Windows, os.path.normcase converts the entry to use
|
||||
# backslashes. This is correct for entries that describe absolute
|
||||
# paths outside of site-packages, but all the others use forward
|
||||
# slashes.
|
||||
if WINDOWS and not os.path.splitdrive(entry)[0]:
|
||||
entry = entry.replace('\\', '/')
|
||||
self.entries.add(entry)
|
||||
|
||||
def remove(self):
|
||||
logger.debug('Removing pth entries from %s:', self.file)
|
||||
with open(self.file, 'rb') as fh:
|
||||
# windows uses '\r\n' with py3k, but uses '\n' with py2.x
|
||||
lines = fh.readlines()
|
||||
self._saved_lines = lines
|
||||
if any(b'\r\n' in line for line in lines):
|
||||
endline = '\r\n'
|
||||
else:
|
||||
endline = '\n'
|
||||
# handle missing trailing newline
|
||||
if lines and not lines[-1].endswith(endline.encode("utf-8")):
|
||||
lines[-1] = lines[-1] + endline.encode("utf-8")
|
||||
for entry in self.entries:
|
||||
try:
|
||||
logger.debug('Removing entry: %s', entry)
|
||||
lines.remove((entry + endline).encode("utf-8"))
|
||||
except ValueError:
|
||||
pass
|
||||
with open(self.file, 'wb') as fh:
|
||||
fh.writelines(lines)
|
||||
|
||||
def rollback(self):
|
||||
if self._saved_lines is None:
|
||||
logger.error(
|
||||
'Cannot roll back changes to %s, none were made', self.file
|
||||
)
|
||||
return False
|
||||
logger.debug('Rolling %s back to previous state', self.file)
|
||||
with open(self.file, 'wb') as fh:
|
||||
fh.writelines(self._saved_lines)
|
||||
return True
|
||||
@@ -1,354 +0,0 @@
|
||||
"""Dependency Resolution
|
||||
|
||||
The dependency resolution in pip is performed as follows:
|
||||
|
||||
for top-level requirements:
|
||||
a. only one spec allowed per project, regardless of conflicts or not.
|
||||
otherwise a "double requirement" exception is raised
|
||||
b. they override sub-dependency requirements.
|
||||
for sub-dependencies
|
||||
a. "first found, wins" (where the order is breadth first)
|
||||
"""
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from itertools import chain
|
||||
|
||||
from pip._internal.exceptions import (
|
||||
BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors,
|
||||
UnsupportedPythonVersion,
|
||||
)
|
||||
|
||||
from pip._internal.req.req_install import InstallRequirement
|
||||
from pip._internal.utils.logging import indent_log
|
||||
from pip._internal.utils.misc import dist_in_usersite, ensure_dir
|
||||
from pip._internal.utils.packaging import check_dist_requires_python
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Resolver(object):
|
||||
"""Resolves which packages need to be installed/uninstalled to perform \
|
||||
the requested operation without breaking the requirements of any package.
|
||||
"""
|
||||
|
||||
_allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"}
|
||||
|
||||
def __init__(self, preparer, session, finder, wheel_cache, use_user_site,
|
||||
ignore_dependencies, ignore_installed, ignore_requires_python,
|
||||
force_reinstall, isolated, upgrade_strategy):
|
||||
super(Resolver, self).__init__()
|
||||
assert upgrade_strategy in self._allowed_strategies
|
||||
|
||||
self.preparer = preparer
|
||||
self.finder = finder
|
||||
self.session = session
|
||||
|
||||
# NOTE: This would eventually be replaced with a cache that can give
|
||||
# information about both sdist and wheels transparently.
|
||||
self.wheel_cache = wheel_cache
|
||||
|
||||
self.require_hashes = None # This is set in resolve
|
||||
|
||||
self.upgrade_strategy = upgrade_strategy
|
||||
self.force_reinstall = force_reinstall
|
||||
self.isolated = isolated
|
||||
self.ignore_dependencies = ignore_dependencies
|
||||
self.ignore_installed = ignore_installed
|
||||
self.ignore_requires_python = ignore_requires_python
|
||||
self.use_user_site = use_user_site
|
||||
|
||||
self._discovered_dependencies = defaultdict(list)
|
||||
|
||||
def resolve(self, requirement_set):
|
||||
"""Resolve what operations need to be done
|
||||
|
||||
As a side-effect of this method, the packages (and their dependencies)
|
||||
are downloaded, unpacked and prepared for installation. This
|
||||
preparation is done by ``pip.operations.prepare``.
|
||||
|
||||
Once PyPI has static dependency metadata available, it would be
|
||||
possible to move the preparation to become a step separated from
|
||||
dependency resolution.
|
||||
"""
|
||||
# make the wheelhouse
|
||||
if self.preparer.wheel_download_dir:
|
||||
ensure_dir(self.preparer.wheel_download_dir)
|
||||
|
||||
# If any top-level requirement has a hash specified, enter
|
||||
# hash-checking mode, which requires hashes from all.
|
||||
root_reqs = (
|
||||
requirement_set.unnamed_requirements +
|
||||
list(requirement_set.requirements.values())
|
||||
)
|
||||
self.require_hashes = (
|
||||
requirement_set.require_hashes or
|
||||
any(req.has_hash_options for req in root_reqs)
|
||||
)
|
||||
|
||||
# Display where finder is looking for packages
|
||||
locations = self.finder.get_formatted_locations()
|
||||
if locations:
|
||||
logger.info(locations)
|
||||
|
||||
# Actually prepare the files, and collect any exceptions. Most hash
|
||||
# exceptions cannot be checked ahead of time, because
|
||||
# req.populate_link() needs to be called before we can make decisions
|
||||
# based on link type.
|
||||
discovered_reqs = []
|
||||
hash_errors = HashErrors()
|
||||
for req in chain(root_reqs, discovered_reqs):
|
||||
try:
|
||||
discovered_reqs.extend(
|
||||
self._resolve_one(requirement_set, req)
|
||||
)
|
||||
except HashError as exc:
|
||||
exc.req = req
|
||||
hash_errors.append(exc)
|
||||
|
||||
if hash_errors:
|
||||
raise hash_errors
|
||||
|
||||
def _is_upgrade_allowed(self, req):
|
||||
if self.upgrade_strategy == "to-satisfy-only":
|
||||
return False
|
||||
elif self.upgrade_strategy == "eager":
|
||||
return True
|
||||
else:
|
||||
assert self.upgrade_strategy == "only-if-needed"
|
||||
return req.is_direct
|
||||
|
||||
def _set_req_to_reinstall(self, req):
|
||||
"""
|
||||
Set a requirement to be installed.
|
||||
"""
|
||||
# Don't uninstall the conflict if doing a user install and the
|
||||
# conflict is not a user install.
|
||||
if not self.use_user_site or dist_in_usersite(req.satisfied_by):
|
||||
req.conflicts_with = req.satisfied_by
|
||||
req.satisfied_by = None
|
||||
|
||||
# XXX: Stop passing requirement_set for options
|
||||
def _check_skip_installed(self, req_to_install):
|
||||
"""Check if req_to_install should be skipped.
|
||||
|
||||
This will check if the req is installed, and whether we should upgrade
|
||||
or reinstall it, taking into account all the relevant user options.
|
||||
|
||||
After calling this req_to_install will only have satisfied_by set to
|
||||
None if the req_to_install is to be upgraded/reinstalled etc. Any
|
||||
other value will be a dist recording the current thing installed that
|
||||
satisfies the requirement.
|
||||
|
||||
Note that for vcs urls and the like we can't assess skipping in this
|
||||
routine - we simply identify that we need to pull the thing down,
|
||||
then later on it is pulled down and introspected to assess upgrade/
|
||||
reinstalls etc.
|
||||
|
||||
:return: A text reason for why it was skipped, or None.
|
||||
"""
|
||||
if self.ignore_installed:
|
||||
return None
|
||||
|
||||
req_to_install.check_if_exists(self.use_user_site)
|
||||
if not req_to_install.satisfied_by:
|
||||
return None
|
||||
|
||||
if self.force_reinstall:
|
||||
self._set_req_to_reinstall(req_to_install)
|
||||
return None
|
||||
|
||||
if not self._is_upgrade_allowed(req_to_install):
|
||||
if self.upgrade_strategy == "only-if-needed":
|
||||
return 'not upgraded as not directly required'
|
||||
return 'already satisfied'
|
||||
|
||||
# Check for the possibility of an upgrade. For link-based
|
||||
# requirements we have to pull the tree down and inspect to assess
|
||||
# the version #, so it's handled way down.
|
||||
if not req_to_install.link:
|
||||
try:
|
||||
self.finder.find_requirement(req_to_install, upgrade=True)
|
||||
except BestVersionAlreadyInstalled:
|
||||
# Then the best version is installed.
|
||||
return 'already up-to-date'
|
||||
except DistributionNotFound:
|
||||
# No distribution found, so we squash the error. It will
|
||||
# be raised later when we re-try later to do the install.
|
||||
# Why don't we just raise here?
|
||||
pass
|
||||
|
||||
self._set_req_to_reinstall(req_to_install)
|
||||
return None
|
||||
|
||||
def _get_abstract_dist_for(self, req):
|
||||
"""Takes a InstallRequirement and returns a single AbstractDist \
|
||||
representing a prepared variant of the same.
|
||||
"""
|
||||
assert self.require_hashes is not None, (
|
||||
"require_hashes should have been set in Resolver.resolve()"
|
||||
)
|
||||
|
||||
if req.editable:
|
||||
return self.preparer.prepare_editable_requirement(
|
||||
req, self.require_hashes, self.use_user_site, self.finder,
|
||||
)
|
||||
|
||||
# satisfied_by is only evaluated by calling _check_skip_installed,
|
||||
# so it must be None here.
|
||||
assert req.satisfied_by is None
|
||||
skip_reason = self._check_skip_installed(req)
|
||||
|
||||
if req.satisfied_by:
|
||||
return self.preparer.prepare_installed_requirement(
|
||||
req, self.require_hashes, skip_reason
|
||||
)
|
||||
|
||||
upgrade_allowed = self._is_upgrade_allowed(req)
|
||||
abstract_dist = self.preparer.prepare_linked_requirement(
|
||||
req, self.session, self.finder, upgrade_allowed,
|
||||
self.require_hashes
|
||||
)
|
||||
|
||||
# NOTE
|
||||
# The following portion is for determining if a certain package is
|
||||
# going to be re-installed/upgraded or not and reporting to the user.
|
||||
# This should probably get cleaned up in a future refactor.
|
||||
|
||||
# req.req is only avail after unpack for URL
|
||||
# pkgs repeat check_if_exists to uninstall-on-upgrade
|
||||
# (#14)
|
||||
if not self.ignore_installed:
|
||||
req.check_if_exists(self.use_user_site)
|
||||
|
||||
if req.satisfied_by:
|
||||
should_modify = (
|
||||
self.upgrade_strategy != "to-satisfy-only" or
|
||||
self.force_reinstall or
|
||||
self.ignore_installed or
|
||||
req.link.scheme == 'file'
|
||||
)
|
||||
if should_modify:
|
||||
self._set_req_to_reinstall(req)
|
||||
else:
|
||||
logger.info(
|
||||
'Requirement already satisfied (use --upgrade to upgrade):'
|
||||
' %s', req,
|
||||
)
|
||||
|
||||
return abstract_dist
|
||||
|
||||
def _resolve_one(self, requirement_set, req_to_install):
|
||||
"""Prepare a single requirements file.
|
||||
|
||||
:return: A list of additional InstallRequirements to also install.
|
||||
"""
|
||||
# Tell user what we are doing for this requirement:
|
||||
# obtain (editable), skipping, processing (local url), collecting
|
||||
# (remote url or package name)
|
||||
if req_to_install.constraint or req_to_install.prepared:
|
||||
return []
|
||||
|
||||
req_to_install.prepared = True
|
||||
|
||||
# register tmp src for cleanup in case something goes wrong
|
||||
requirement_set.reqs_to_cleanup.append(req_to_install)
|
||||
|
||||
abstract_dist = self._get_abstract_dist_for(req_to_install)
|
||||
|
||||
# Parse and return dependencies
|
||||
dist = abstract_dist.dist(self.finder)
|
||||
try:
|
||||
check_dist_requires_python(dist)
|
||||
except UnsupportedPythonVersion as err:
|
||||
if self.ignore_requires_python:
|
||||
logger.warning(err.args[0])
|
||||
else:
|
||||
raise
|
||||
|
||||
more_reqs = []
|
||||
|
||||
def add_req(subreq, extras_requested):
|
||||
sub_install_req = InstallRequirement.from_req(
|
||||
str(subreq),
|
||||
req_to_install,
|
||||
isolated=self.isolated,
|
||||
wheel_cache=self.wheel_cache,
|
||||
)
|
||||
parent_req_name = req_to_install.name
|
||||
to_scan_again, add_to_parent = requirement_set.add_requirement(
|
||||
sub_install_req,
|
||||
parent_req_name=parent_req_name,
|
||||
extras_requested=extras_requested,
|
||||
)
|
||||
if parent_req_name and add_to_parent:
|
||||
self._discovered_dependencies[parent_req_name].append(
|
||||
add_to_parent
|
||||
)
|
||||
more_reqs.extend(to_scan_again)
|
||||
|
||||
with indent_log():
|
||||
# We add req_to_install before its dependencies, so that we
|
||||
# can refer to it when adding dependencies.
|
||||
if not requirement_set.has_requirement(req_to_install.name):
|
||||
# 'unnamed' requirements will get added here
|
||||
req_to_install.is_direct = True
|
||||
requirement_set.add_requirement(
|
||||
req_to_install, parent_req_name=None,
|
||||
)
|
||||
|
||||
if not self.ignore_dependencies:
|
||||
if req_to_install.extras:
|
||||
logger.debug(
|
||||
"Installing extra requirements: %r",
|
||||
','.join(req_to_install.extras),
|
||||
)
|
||||
missing_requested = sorted(
|
||||
set(req_to_install.extras) - set(dist.extras)
|
||||
)
|
||||
for missing in missing_requested:
|
||||
logger.warning(
|
||||
'%s does not provide the extra \'%s\'',
|
||||
dist, missing
|
||||
)
|
||||
|
||||
available_requested = sorted(
|
||||
set(dist.extras) & set(req_to_install.extras)
|
||||
)
|
||||
for subreq in dist.requires(available_requested):
|
||||
add_req(subreq, extras_requested=available_requested)
|
||||
|
||||
if not req_to_install.editable and not req_to_install.satisfied_by:
|
||||
# XXX: --no-install leads this to report 'Successfully
|
||||
# downloaded' for only non-editable reqs, even though we took
|
||||
# action on them.
|
||||
requirement_set.successfully_downloaded.append(req_to_install)
|
||||
|
||||
return more_reqs
|
||||
|
||||
def get_installation_order(self, req_set):
|
||||
"""Create the installation order.
|
||||
|
||||
The installation order is topological - requirements are installed
|
||||
before the requiring thing. We break cycles at an arbitrary point,
|
||||
and make no other guarantees.
|
||||
"""
|
||||
# The current implementation, which we may change at any point
|
||||
# installs the user specified things in the order given, except when
|
||||
# dependencies must come earlier to achieve topological order.
|
||||
order = []
|
||||
ordered_reqs = set()
|
||||
|
||||
def schedule(req):
|
||||
if req.satisfied_by or req in ordered_reqs:
|
||||
return
|
||||
if req.constraint:
|
||||
return
|
||||
ordered_reqs.add(req)
|
||||
for dep in self._discovered_dependencies[req.name]:
|
||||
schedule(dep)
|
||||
order.append(req)
|
||||
|
||||
for install_req in req_set.requirements.values():
|
||||
schedule(install_req)
|
||||
return order
|
||||
@@ -1,8 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
SUCCESS = 0
|
||||
ERROR = 1
|
||||
UNKNOWN_ERROR = 2
|
||||
VIRTUALENV_NOT_FOUND = 3
|
||||
PREVIOUS_BUILD_DIR_ERROR = 4
|
||||
NO_MATCHES_FOUND = 23
|
||||
@@ -1,258 +0,0 @@
|
||||
"""
|
||||
This code was taken from https://github.com/ActiveState/appdirs and modified
|
||||
to suit our purposes.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from pip._vendor.six import PY2, text_type
|
||||
|
||||
from pip._internal.compat import WINDOWS, expanduser
|
||||
|
||||
|
||||
def user_cache_dir(appname):
|
||||
r"""
|
||||
Return full path to the user-specific cache dir for this application.
|
||||
|
||||
"appname" is the name of application.
|
||||
|
||||
Typical user cache directories are:
|
||||
macOS: ~/Library/Caches/<AppName>
|
||||
Unix: ~/.cache/<AppName> (XDG default)
|
||||
Windows: C:\Users\<username>\AppData\Local\<AppName>\Cache
|
||||
|
||||
On Windows the only suggestion in the MSDN docs is that local settings go
|
||||
in the `CSIDL_LOCAL_APPDATA` directory. This is identical to the
|
||||
non-roaming app data dir (the default returned by `user_data_dir`). Apps
|
||||
typically put cache data somewhere *under* the given dir here. Some
|
||||
examples:
|
||||
...\Mozilla\Firefox\Profiles\<ProfileName>\Cache
|
||||
...\Acme\SuperApp\Cache\1.0
|
||||
|
||||
OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value.
|
||||
"""
|
||||
if WINDOWS:
|
||||
# Get the base path
|
||||
path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
|
||||
|
||||
# When using Python 2, return paths as bytes on Windows like we do on
|
||||
# other operating systems. See helper function docs for more details.
|
||||
if PY2 and isinstance(path, text_type):
|
||||
path = _win_path_to_bytes(path)
|
||||
|
||||
# Add our app name and Cache directory to it
|
||||
path = os.path.join(path, appname, "Cache")
|
||||
elif sys.platform == "darwin":
|
||||
# Get the base path
|
||||
path = expanduser("~/Library/Caches")
|
||||
|
||||
# Add our app name to it
|
||||
path = os.path.join(path, appname)
|
||||
else:
|
||||
# Get the base path
|
||||
path = os.getenv("XDG_CACHE_HOME", expanduser("~/.cache"))
|
||||
|
||||
# Add our app name to it
|
||||
path = os.path.join(path, appname)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def user_data_dir(appname, roaming=False):
|
||||
r"""
|
||||
Return full path to the user-specific data dir for this application.
|
||||
|
||||
"appname" is the name of application.
|
||||
If None, just the system directory is returned.
|
||||
"roaming" (boolean, default False) can be set True to use the Windows
|
||||
roaming appdata directory. That means that for users on a Windows
|
||||
network setup for roaming profiles, this user data will be
|
||||
sync'd on login. See
|
||||
<http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
|
||||
for a discussion of issues.
|
||||
|
||||
Typical user data directories are:
|
||||
macOS: ~/Library/Application Support/<AppName>
|
||||
if it exists, else ~/.config/<AppName>
|
||||
Unix: ~/.local/share/<AppName> # or in
|
||||
$XDG_DATA_HOME, if defined
|
||||
Win XP (not roaming): C:\Documents and Settings\<username>\ ...
|
||||
...Application Data\<AppName>
|
||||
Win XP (roaming): C:\Documents and Settings\<username>\Local ...
|
||||
...Settings\Application Data\<AppName>
|
||||
Win 7 (not roaming): C:\\Users\<username>\AppData\Local\<AppName>
|
||||
Win 7 (roaming): C:\\Users\<username>\AppData\Roaming\<AppName>
|
||||
|
||||
For Unix, we follow the XDG spec and support $XDG_DATA_HOME.
|
||||
That means, by default "~/.local/share/<AppName>".
|
||||
"""
|
||||
if WINDOWS:
|
||||
const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA"
|
||||
path = os.path.join(os.path.normpath(_get_win_folder(const)), appname)
|
||||
elif sys.platform == "darwin":
|
||||
path = os.path.join(
|
||||
expanduser('~/Library/Application Support/'),
|
||||
appname,
|
||||
) if os.path.isdir(os.path.join(
|
||||
expanduser('~/Library/Application Support/'),
|
||||
appname,
|
||||
)
|
||||
) else os.path.join(
|
||||
expanduser('~/.config/'),
|
||||
appname,
|
||||
)
|
||||
else:
|
||||
path = os.path.join(
|
||||
os.getenv('XDG_DATA_HOME', expanduser("~/.local/share")),
|
||||
appname,
|
||||
)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def user_config_dir(appname, roaming=True):
|
||||
"""Return full path to the user-specific config dir for this application.
|
||||
|
||||
"appname" is the name of application.
|
||||
If None, just the system directory is returned.
|
||||
"roaming" (boolean, default True) can be set False to not use the
|
||||
Windows roaming appdata directory. That means that for users on a
|
||||
Windows network setup for roaming profiles, this user data will be
|
||||
sync'd on login. See
|
||||
<http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
|
||||
for a discussion of issues.
|
||||
|
||||
Typical user data directories are:
|
||||
macOS: same as user_data_dir
|
||||
Unix: ~/.config/<AppName>
|
||||
Win *: same as user_data_dir
|
||||
|
||||
For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME.
|
||||
That means, by default "~/.config/<AppName>".
|
||||
"""
|
||||
if WINDOWS:
|
||||
path = user_data_dir(appname, roaming=roaming)
|
||||
elif sys.platform == "darwin":
|
||||
path = user_data_dir(appname)
|
||||
else:
|
||||
path = os.getenv('XDG_CONFIG_HOME', expanduser("~/.config"))
|
||||
path = os.path.join(path, appname)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
# for the discussion regarding site_config_dirs locations
|
||||
# see <https://github.com/pypa/pip/issues/1733>
|
||||
def site_config_dirs(appname):
|
||||
r"""Return a list of potential user-shared config dirs for this application.
|
||||
|
||||
"appname" is the name of application.
|
||||
|
||||
Typical user config directories are:
|
||||
macOS: /Library/Application Support/<AppName>/
|
||||
Unix: /etc or $XDG_CONFIG_DIRS[i]/<AppName>/ for each value in
|
||||
$XDG_CONFIG_DIRS
|
||||
Win XP: C:\Documents and Settings\All Users\Application ...
|
||||
...Data\<AppName>\
|
||||
Vista: (Fail! "C:\ProgramData" is a hidden *system* directory
|
||||
on Vista.)
|
||||
Win 7: Hidden, but writeable on Win 7:
|
||||
C:\ProgramData\<AppName>\
|
||||
"""
|
||||
if WINDOWS:
|
||||
path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA"))
|
||||
pathlist = [os.path.join(path, appname)]
|
||||
elif sys.platform == 'darwin':
|
||||
pathlist = [os.path.join('/Library/Application Support', appname)]
|
||||
else:
|
||||
# try looking in $XDG_CONFIG_DIRS
|
||||
xdg_config_dirs = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg')
|
||||
if xdg_config_dirs:
|
||||
pathlist = [
|
||||
os.path.join(expanduser(x), appname)
|
||||
for x in xdg_config_dirs.split(os.pathsep)
|
||||
]
|
||||
else:
|
||||
pathlist = []
|
||||
|
||||
# always look in /etc directly as well
|
||||
pathlist.append('/etc')
|
||||
|
||||
return pathlist
|
||||
|
||||
|
||||
# -- Windows support functions --
|
||||
|
||||
def _get_win_folder_from_registry(csidl_name):
|
||||
"""
|
||||
This is a fallback technique at best. I'm not sure if using the
|
||||
registry for this guarantees us the correct answer for all CSIDL_*
|
||||
names.
|
||||
"""
|
||||
import _winreg
|
||||
|
||||
shell_folder_name = {
|
||||
"CSIDL_APPDATA": "AppData",
|
||||
"CSIDL_COMMON_APPDATA": "Common AppData",
|
||||
"CSIDL_LOCAL_APPDATA": "Local AppData",
|
||||
}[csidl_name]
|
||||
|
||||
key = _winreg.OpenKey(
|
||||
_winreg.HKEY_CURRENT_USER,
|
||||
r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
|
||||
)
|
||||
directory, _type = _winreg.QueryValueEx(key, shell_folder_name)
|
||||
return directory
|
||||
|
||||
|
||||
def _get_win_folder_with_ctypes(csidl_name):
|
||||
csidl_const = {
|
||||
"CSIDL_APPDATA": 26,
|
||||
"CSIDL_COMMON_APPDATA": 35,
|
||||
"CSIDL_LOCAL_APPDATA": 28,
|
||||
}[csidl_name]
|
||||
|
||||
buf = ctypes.create_unicode_buffer(1024)
|
||||
ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
|
||||
|
||||
# Downgrade to short path name if have highbit chars. See
|
||||
# <http://bugs.activestate.com/show_bug.cgi?id=85099>.
|
||||
has_high_char = False
|
||||
for c in buf:
|
||||
if ord(c) > 255:
|
||||
has_high_char = True
|
||||
break
|
||||
if has_high_char:
|
||||
buf2 = ctypes.create_unicode_buffer(1024)
|
||||
if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024):
|
||||
buf = buf2
|
||||
|
||||
return buf.value
|
||||
|
||||
|
||||
if WINDOWS:
|
||||
try:
|
||||
import ctypes
|
||||
_get_win_folder = _get_win_folder_with_ctypes
|
||||
except ImportError:
|
||||
_get_win_folder = _get_win_folder_from_registry
|
||||
|
||||
|
||||
def _win_path_to_bytes(path):
|
||||
"""Encode Windows paths to bytes. Only used on Python 2.
|
||||
|
||||
Motivation is to be consistent with other operating systems where paths
|
||||
are also returned as bytes. This avoids problems mixing bytes and Unicode
|
||||
elsewhere in the codebase. For more details and discussion see
|
||||
<https://github.com/pypa/pip/issues/3463>.
|
||||
|
||||
If encoding using ASCII and MBCS fails, return the original Unicode path.
|
||||
"""
|
||||
for encoding in ('ASCII', 'MBCS'):
|
||||
try:
|
||||
return path.encode(encoding)
|
||||
except (UnicodeEncodeError, LookupError):
|
||||
pass
|
||||
return path
|
||||
@@ -1,77 +0,0 @@
|
||||
"""
|
||||
A module that implements tooling to enable easy warnings about deprecations.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
import warnings
|
||||
|
||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
|
||||
|
||||
if MYPY_CHECK_RUNNING:
|
||||
from typing import Any
|
||||
|
||||
|
||||
class PipDeprecationWarning(Warning):
|
||||
pass
|
||||
|
||||
|
||||
class Pending(object):
|
||||
pass
|
||||
|
||||
|
||||
class RemovedInPip11Warning(PipDeprecationWarning):
|
||||
pass
|
||||
|
||||
|
||||
class RemovedInPip12Warning(PipDeprecationWarning, Pending):
|
||||
pass
|
||||
|
||||
|
||||
# Warnings <-> Logging Integration
|
||||
|
||||
|
||||
_warnings_showwarning = None # type: Any
|
||||
|
||||
|
||||
def _showwarning(message, category, filename, lineno, file=None, line=None):
|
||||
if file is not None:
|
||||
if _warnings_showwarning is not None:
|
||||
_warnings_showwarning(
|
||||
message, category, filename, lineno, file, line,
|
||||
)
|
||||
else:
|
||||
if issubclass(category, PipDeprecationWarning):
|
||||
# We use a specially named logger which will handle all of the
|
||||
# deprecation messages for pip.
|
||||
logger = logging.getLogger("pip._internal.deprecations")
|
||||
|
||||
# This is purposely using the % formatter here instead of letting
|
||||
# the logging module handle the interpolation. This is because we
|
||||
# want it to appear as if someone typed this entire message out.
|
||||
log_message = "DEPRECATION: %s" % message
|
||||
|
||||
# PipDeprecationWarnings that are Pending still have at least 2
|
||||
# versions to go until they are removed so they can just be
|
||||
# warnings. Otherwise, they will be removed in the very next
|
||||
# version of pip. We want these to be more obvious so we use the
|
||||
# ERROR logging level.
|
||||
if issubclass(category, Pending):
|
||||
logger.warning(log_message)
|
||||
else:
|
||||
logger.error(log_message)
|
||||
else:
|
||||
_warnings_showwarning(
|
||||
message, category, filename, lineno, file, line,
|
||||
)
|
||||
|
||||
|
||||
def install_warning_logger():
|
||||
# Enable our Deprecation Warnings
|
||||
warnings.simplefilter("default", PipDeprecationWarning, append=True)
|
||||
|
||||
global _warnings_showwarning
|
||||
|
||||
if _warnings_showwarning is None:
|
||||
_warnings_showwarning = warnings.showwarning
|
||||
warnings.showwarning = _showwarning
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user