initial

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="JavaScriptSettings">
+    <option name="languageLevel" value="ES6" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (thue)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/thue.iml" filepath="$PROJECT_DIR$/.idea/thue.iml" />
+    </modules>
+  </component>
+</project>

.idea/thue.iml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>

requirements.txt

@@ -0,0 +1 @@
+lark-parser

rethue.py

@@ -0,0 +1,49 @@
+from lark import Lark
+
+grammar = r'''
+suite: prod*
+
+prod: _named_prod
+
+_named_prod: _direct_prod | alias
+_direct_prod: _compound_prod | _rule_prod | _literal_prod | _abstract_prod
+_abstract_prod: output
+_compound_prod: cont | sing
+_rule_prod: full | part
+_literal_prod: lit | ref
+
+cont: "{" suite "}"             // continual production
+sing: "[" suite "]"             // singular production
+
+alias: ref "=" prod             // production alias
+full: _direct_prod "=>" prod    // full application
+part: ctx "::=" prod            // partial application
+
+output: "~" prod
+
+lit: STRING
+ref: CNAME
+ctx: REGEX
+
+CNAME: /[a-z_][a-z0-9_]*/i
+STRING: /(r)?(['"])(.*?)(?<!\\)\2/
+REGEX: /(r)?([\/])(.*?)(?<!\\)\2(i)?/
+
+%import common.WS
+%ignore WS
+'''
+
+
+def main():
+    lrk = Lark(grammar, start='suite')
+
+    with open('sample.ret') as f:
+        src = f.read()
+
+    tree = lrk.parse(src)
+
+    print(tree.pretty())
+
+
+if __name__ == '__main__':
+    main()

sample.ret

@@ -0,0 +1,22 @@
+UNARY = ';1;\0' => {
+    ~ 'Conversion step: \0'
+
+    /(.*);(.*);(.*)0/ ::= '\1;\2\2;\3'
+    /(.*);(.*);(.*)1/ ::= '\1\2;\2\2;\3'
+
+    /(.*);(.*);$/ ::= '\1'
+}
+
+
+FIZZ     = UNARY => /(.{3})*/ ::= ~ 'fizz'
+BUZZ     = UNARY => /(.{5})*/ ::= ~ 'buzz'
+FIZZBUZZ = UNARY => /(.{15})*/ ::= ~ 'fizzbuzz'
+
+FIZZ_BUZZ = [
+    FIZZBUZZ
+    FIZZ
+    BUZZ
+    ~ '\0'
+]
+
+':::' => FIZZ_BUZZ

venv/bin/activate

@@ -0,0 +1,76 @@
+# This file must be used with "source bin/activate" *from bash*
+# you cannot run it directly
+
+deactivate () {
+    # reset old environment variables
+    if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
+        PATH="${_OLD_VIRTUAL_PATH:-}"
+        export PATH
+        unset _OLD_VIRTUAL_PATH
+    fi
+    if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
+        PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
+        export PYTHONHOME
+        unset _OLD_VIRTUAL_PYTHONHOME
+    fi
+
+    # This should detect bash and zsh, which have a hash command that must
+    # be called to get it to forget past commands.  Without forgetting
+    # past commands the $PATH changes we made may not be respected
+    if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
+        hash -r
+    fi
+
+    if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
+        PS1="${_OLD_VIRTUAL_PS1:-}"
+        export PS1
+        unset _OLD_VIRTUAL_PS1
+    fi
+
+    unset VIRTUAL_ENV
+    if [ ! "$1" = "nondestructive" ] ; then
+    # Self destruct!
+        unset -f deactivate
+    fi
+}
+
+# unset irrelevant variables
+deactivate nondestructive
+
+VIRTUAL_ENV="/home/allem/PycharmProjects/thue/venv"
+export VIRTUAL_ENV
+
+_OLD_VIRTUAL_PATH="$PATH"
+PATH="$VIRTUAL_ENV/bin:$PATH"
+export PATH
+
+# unset PYTHONHOME if set
+# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
+# could use `if (set -u; : $PYTHONHOME) ;` in bash
+if [ -n "${PYTHONHOME:-}" ] ; then
+    _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
+    unset PYTHONHOME
+fi
+
+if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
+    _OLD_VIRTUAL_PS1="${PS1:-}"
+    if [ "x(venv) " != x ] ; then
+	PS1="(venv) ${PS1:-}"
+    else
+    if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
+        # special case for Aspen magic directories
+        # see http://www.zetadev.com/software/aspen/
+        PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
+    else
+        PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
+    fi
+    fi
+    export PS1
+fi
+
+# This should detect bash and zsh, which have a hash command that must
+# be called to get it to forget past commands.  Without forgetting
+# past commands the $PATH changes we made may not be respected
+if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
+    hash -r
+fi

venv/bin/activate.csh

@@ -0,0 +1,37 @@
+# This file must be used with "source bin/activate.csh" *from csh*.
+# You cannot run it directly.
+# Created by Davide Di Blasi <davidedb@gmail.com>.
+# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com>
+
+alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate'
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+setenv VIRTUAL_ENV "/home/allem/PycharmProjects/thue/venv"
+
+set _OLD_VIRTUAL_PATH="$PATH"
+setenv PATH "$VIRTUAL_ENV/bin:$PATH"
+
+
+set _OLD_VIRTUAL_PROMPT="$prompt"
+
+if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
+    if ("venv" != "") then
+        set env_name = "venv"
+    else
+        if (`basename "VIRTUAL_ENV"` == "__") then
+            # special case for Aspen magic directories
+            # see http://www.zetadev.com/software/aspen/
+            set env_name = `basename \`dirname "$VIRTUAL_ENV"\``
+        else
+            set env_name = `basename "$VIRTUAL_ENV"`
+        endif
+    endif
+    set prompt = "[$env_name] $prompt"
+    unset env_name
+endif
+
+alias pydoc python -m pydoc
+
+rehash

venv/bin/activate.fish

@@ -0,0 +1,75 @@
+# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org)
+# you cannot run it directly
+
+function deactivate  -d "Exit virtualenv and return to normal shell environment"
+    # reset old environment variables
+    if test -n "$_OLD_VIRTUAL_PATH"
+        set -gx PATH $_OLD_VIRTUAL_PATH
+        set -e _OLD_VIRTUAL_PATH
+    end
+    if test -n "$_OLD_VIRTUAL_PYTHONHOME"
+        set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
+        set -e _OLD_VIRTUAL_PYTHONHOME
+    end
+
+    if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
+        functions -e fish_prompt
+        set -e _OLD_FISH_PROMPT_OVERRIDE
+        functions -c _old_fish_prompt fish_prompt
+        functions -e _old_fish_prompt
+    end
+
+    set -e VIRTUAL_ENV
+    if test "$argv[1]" != "nondestructive"
+        # Self destruct!
+        functions -e deactivate
+    end
+end
+
+# unset irrelevant variables
+deactivate nondestructive
+
+set -gx VIRTUAL_ENV "/home/allem/PycharmProjects/thue/venv"
+
+set -gx _OLD_VIRTUAL_PATH $PATH
+set -gx PATH "$VIRTUAL_ENV/bin" $PATH
+
+# unset PYTHONHOME if set
+if set -q PYTHONHOME
+    set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
+    set -e PYTHONHOME
+end
+
+if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
+    # fish uses a function instead of an env var to generate the prompt.
+
+    # save the current fish_prompt function as the function _old_fish_prompt
+    functions -c fish_prompt _old_fish_prompt
+
+    # with the original prompt function renamed, we can override with our own.
+    function fish_prompt
+        # Save the return status of the last command
+        set -l old_status $status
+
+        # Prompt override?
+        if test -n "(venv) "            
+            printf "%s%s" "(venv) " (set_color normal)
+        else
+            # ...Otherwise, prepend env
+            set -l _checkbase (basename "$VIRTUAL_ENV")
+            if test $_checkbase = "__"
+                # special case for Aspen magic directories
+                # see http://www.zetadev.com/software/aspen/
+                printf "%s[%s]%s " (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal)
+            else
+                printf "%s(%s)%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal)
+            end
+        end
+
+        # Restore the return status of the previous command.
+        echo "exit $old_status" | .
+        _old_fish_prompt
+    end
+
+    set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
+end

venv/bin/easy_install

@@ -0,0 +1,12 @@
+#!/home/allem/PycharmProjects/thue/venv/bin/python
+# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==39.1.0','console_scripts','easy_install'
+__requires__ = 'setuptools==39.1.0'
+import re
+import sys
+from pkg_resources import load_entry_point
+
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+    sys.exit(
+        load_entry_point('setuptools==39.1.0', 'console_scripts', 'easy_install')()
+    )

venv/bin/easy_install-3.6

@@ -0,0 +1,12 @@
+#!/home/allem/PycharmProjects/thue/venv/bin/python
+# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==39.1.0','console_scripts','easy_install-3.6'
+__requires__ = 'setuptools==39.1.0'
+import re
+import sys
+from pkg_resources import load_entry_point
+
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+    sys.exit(
+        load_entry_point('setuptools==39.1.0', 'console_scripts', 'easy_install-3.6')()
+    )

venv/bin/pip

@@ -0,0 +1,12 @@
+#!/home/allem/PycharmProjects/thue/venv/bin/python
+# EASY-INSTALL-ENTRY-SCRIPT: 'pip==10.0.1','console_scripts','pip'
+__requires__ = 'pip==10.0.1'
+import re
+import sys
+from pkg_resources import load_entry_point
+
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+    sys.exit(
+        load_entry_point('pip==10.0.1', 'console_scripts', 'pip')()
+    )

venv/bin/pip3

@@ -0,0 +1,12 @@
+#!/home/allem/PycharmProjects/thue/venv/bin/python
+# EASY-INSTALL-ENTRY-SCRIPT: 'pip==10.0.1','console_scripts','pip3'
+__requires__ = 'pip==10.0.1'
+import re
+import sys
+from pkg_resources import load_entry_point
+
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+    sys.exit(
+        load_entry_point('pip==10.0.1', 'console_scripts', 'pip3')()
+    )

venv/bin/pip3.6

@@ -0,0 +1,12 @@
+#!/home/allem/PycharmProjects/thue/venv/bin/python
+# EASY-INSTALL-ENTRY-SCRIPT: 'pip==10.0.1','console_scripts','pip3.6'
+__requires__ = 'pip==10.0.1'
+import re
+import sys
+from pkg_resources import load_entry_point
+
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+    sys.exit(
+        load_entry_point('pip==10.0.1', 'console_scripts', 'pip3.6')()
+    )

venv/lib/python3.6/site-packages/easy-install.pth

@@ -0,0 +1,2 @@
+./setuptools-39.1.0-py3.6.egg
+./pip-10.0.1-py3.6.egg

venv/lib/python3.6/site-packages/lark/__init__.py

@@ -0,0 +1,8 @@
+from .tree import Tree
+from .visitors import Transformer, Visitor, v_args, Discard
+from .visitors import InlineTransformer, inline_args   # XXX Deprecated
+from .exceptions import ParseError, LexError, GrammarError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters
+from .lexer import Token
+from .lark import Lark
+
+__version__ = "0.6.5"

venv/lib/python3.6/site-packages/lark/common.py

@@ -0,0 +1,16 @@
+
+
+class LexerConf:
+    def __init__(self, tokens, ignore=(), postlex=None, callbacks=None):
+        self.tokens = tokens
+        self.ignore = ignore
+        self.postlex = postlex
+        self.callbacks = callbacks or {}
+
+class ParserConf:
+    def __init__(self, rules, callback, start):
+        self.rules = rules
+        self.callback = callback
+        self.start = start
+
+

venv/lib/python3.6/site-packages/lark/exceptions.py

@@ -0,0 +1,89 @@
+from .utils import STRING_TYPE
+
+###{standalone
+class LarkError(Exception):
+    pass
+
+class GrammarError(LarkError):
+    pass
+
+class ParseError(LarkError):
+    pass
+
+class LexError(LarkError):
+    pass
+
+class UnexpectedInput(LarkError):
+    pos_in_stream = None
+
+    def get_context(self, text, span=40):
+        pos = self.pos_in_stream
+        start = max(pos - span, 0)
+        end = pos + span
+        before = text[start:pos].rsplit('\n', 1)[-1]
+        after = text[pos:end].split('\n', 1)[0]
+        return before + after + '\n' + ' ' * len(before) + '^\n'
+
+    def match_examples(self, parse_fn, examples):
+        """ Given a parser instance and a dictionary mapping some label with
+            some malformed syntax examples, it'll return the label for the
+            example that bests matches the current error.
+        """
+        assert self.state is not None, "Not supported for this exception"
+
+        candidate = None
+        for label, example in examples.items():
+            assert not isinstance(example, STRING_TYPE)
+
+            for malformed in example:
+                try:
+                    parse_fn(malformed)
+                except UnexpectedInput as ut:
+                    if ut.state == self.state:
+                        try:
+                            if ut.token == self.token:  # Try exact match first
+                                return label
+                        except AttributeError:
+                            pass
+                        if not candidate:
+                            candidate = label
+
+        return candidate
+
+
+class UnexpectedCharacters(LexError, UnexpectedInput):
+    def __init__(self, seq, lex_pos, line, column, allowed=None, considered_tokens=None, state=None):
+        message = "No terminal defined for '%s' at line %d col %d" % (seq[lex_pos], line, column)
+
+        self.line = line
+        self.column = column
+        self.allowed = allowed
+        self.considered_tokens = considered_tokens
+        self.pos_in_stream = lex_pos
+        self.state = state
+
+        message += '\n\n' + self.get_context(seq)
+        if allowed:
+            message += '\nExpecting: %s\n' % allowed
+
+        super(UnexpectedCharacters, self).__init__(message)
+
+
+
+class UnexpectedToken(ParseError, UnexpectedInput):
+    def __init__(self, token, expected, considered_rules=None, state=None):
+        self.token = token
+        self.expected = expected     # XXX str shouldn't necessary
+        self.line = getattr(token, 'line', '?')
+        self.column = getattr(token, 'column', '?')
+        self.considered_rules = considered_rules
+        self.state = state
+        self.pos_in_stream = getattr(token, 'pos_in_stream', None)
+
+        message = ("Unexpected token %r at line %s, column %s.\n"
+                   "Expected one of: \n\t* %s\n"
+                   % (token, self.line, self.column, '\n\t* '.join(self.expected)))
+
+        super(UnexpectedToken, self).__init__(message)
+
+###}

venv/lib/python3.6/site-packages/lark/grammar.py

@@ -0,0 +1,66 @@
+class Symbol(object):
+    is_term = NotImplemented
+
+    def __init__(self, name):
+        self.name = name
+
+    def __eq__(self, other):
+        assert isinstance(other, Symbol), other
+        return self.is_term == other.is_term and self.name == other.name
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __hash__(self):
+        return hash(self.name)
+
+    def __repr__(self):
+        return '%s(%r)' % (type(self).__name__, self.name)
+
+    fullrepr = property(__repr__)
+
+class Terminal(Symbol):
+    is_term = True
+
+    def __init__(self, name, filter_out=False):
+        self.name = name
+        self.filter_out = filter_out
+
+    @property
+    def fullrepr(self):
+        return '%s(%r, %r)' % (type(self).__name__, self.name, self.filter_out)
+
+
+class NonTerminal(Symbol):
+    is_term = False
+
+class Rule(object):
+    """
+        origin : a symbol
+        expansion : a list of symbols
+    """
+    def __init__(self, origin, expansion, alias=None, options=None):
+        self.origin = origin
+        self.expansion = expansion
+        self.alias = alias
+        self.options = options
+
+    def __str__(self):
+        return '<%s : %s>' % (self.origin, ' '.join(map(str,self.expansion)))
+
+    def __repr__(self):
+        return 'Rule(%r, %r, %r, %r)' % (self.origin, self.expansion, self.alias, self.options)
+
+
+class RuleOptions:
+    def __init__(self, keep_all_tokens=False, expand1=False, priority=None):
+        self.keep_all_tokens = keep_all_tokens
+        self.expand1 = expand1
+        self.priority = priority
+
+    def __repr__(self):
+        return 'RuleOptions(%r, %r, %r)' % (
+            self.keep_all_tokens,
+            self.expand1,
+            self.priority,
+        )

venv/lib/python3.6/site-packages/lark/grammars/common.lark

@@ -0,0 +1,49 @@
+//
+// Numbers
+//
+
+DIGIT: "0".."9"
+HEXDIGIT: "a".."f"|"A".."F"|DIGIT
+
+INT: DIGIT+
+SIGNED_INT: ["+"|"-"] INT
+DECIMAL: INT "." INT? | "." INT
+
+// float = /-?\d+(\.\d+)?([eE][+-]?\d+)?/
+_EXP: ("e"|"E") SIGNED_INT
+FLOAT: INT _EXP | DECIMAL _EXP?
+SIGNED_FLOAT: ["+"|"-"] FLOAT
+
+NUMBER: FLOAT | INT
+SIGNED_NUMBER: ["+"|"-"] NUMBER
+
+//
+// Strings
+//
+//STRING: /"(\\\"|\\\\|[^"\n])*?"i?/
+STRING_INNER: ("\\\""|/[^"]/)
+ESCAPED_STRING: "\"" STRING_INNER* "\""
+
+
+//
+// Names (Variables)
+//
+LCASE_LETTER: "a".."z"
+UCASE_LETTER: "A".."Z"
+
+LETTER: UCASE_LETTER | LCASE_LETTER
+WORD: LETTER+
+
+CNAME: ("_"|LETTER) ("_"|LETTER|DIGIT)*
+
+
+//
+// Whitespace
+//
+WS_INLINE: (" "|/\t/)+
+WS: /[ \t\f\r\n]/+
+
+CR : /\r/
+LF : /\n/
+NEWLINE: (CR? LF)+
+

venv/lib/python3.6/site-packages/lark/indenter.py

@@ -0,0 +1,55 @@
+"Provides Indentation services for languages with indentation similar to Python"
+
+from .lexer import Token
+
+###{standalone
+class Indenter:
+    def __init__(self):
+        self.paren_level = 0
+        self.indent_level = [0]
+
+    def handle_NL(self, token):
+        if self.paren_level > 0:
+            return
+
+        yield token
+
+        indent_str = token.rsplit('\n', 1)[1] # Tabs and spaces
+        indent = indent_str.count(' ') + indent_str.count('\t') * self.tab_len
+
+        if indent > self.indent_level[-1]:
+            self.indent_level.append(indent)
+            yield Token.new_borrow_pos(self.INDENT_type, indent_str, token)
+        else:
+            while indent < self.indent_level[-1]:
+                self.indent_level.pop()
+                yield Token.new_borrow_pos(self.DEDENT_type, indent_str, token)
+
+            assert indent == self.indent_level[-1], '%s != %s' % (indent, self.indent_level[-1])
+
+    def process(self, stream):
+        for token in stream:
+            if token.type == self.NL_type:
+                for t in self.handle_NL(token):
+                    yield t
+            else:
+                yield token
+
+            if token.type in self.OPEN_PAREN_types:
+                self.paren_level += 1
+            elif token.type in self.CLOSE_PAREN_types:
+                self.paren_level -= 1
+                assert self.paren_level >= 0
+
+        while len(self.indent_level) > 1:
+            self.indent_level.pop()
+            yield Token(self.DEDENT_type, '')
+
+        assert self.indent_level == [0], self.indent_level
+
+    # XXX Hack for ContextualLexer. Maybe there's a more elegant solution?
+    @property
+    def always_accept(self):
+        return (self.NL_type,)
+
+###}

venv/lib/python3.6/site-packages/lark/lark.py

@@ -0,0 +1,235 @@
+from __future__ import absolute_import
+
+import os
+import time
+from collections import defaultdict
+from io import open
+
+from .utils import STRING_TYPE
+from .load_grammar import load_grammar
+from .tree import Tree
+from .common import LexerConf, ParserConf
+
+from .lexer import Lexer, TraditionalLexer
+from .parse_tree_builder import ParseTreeBuilder
+from .parser_frontends import get_frontend
+
+
+class LarkOptions(object):
+    """Specifies the options for Lark
+
+    """
+    OPTIONS_DOC = """
+        parser - Decides which parser engine to use, "earley" or "lalr". (Default: "earley")
+                 Note: "lalr" requires a lexer
+
+        lexer - Decides whether or not to use a lexer stage
+            "standard": Use a standard lexer
+            "contextual": Stronger lexer (only works with parser="lalr")
+            "dynamic": Flexible and powerful (only with parser="earley")
+            "dynamic_complete": Same as dynamic, but tries *every* variation
+                                of tokenizing possible. (only with parser="earley")
+            "auto" (default): Choose for me based on grammar and parser
+
+        ambiguity - Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
+            "resolve": The parser will automatically choose the simplest derivation
+                       (it chooses consistently: greedy for tokens, non-greedy for rules)
+            "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
+
+        transformer - Applies the transformer to every parse tree
+        debug - Affects verbosity (default: False)
+        keep_all_tokens - Don't automagically remove "punctuation" tokens (default: False)
+        cache_grammar - Cache the Lark grammar (Default: False)
+        postlex - Lexer post-processing (Default: None) Only works with the standard and contextual lexers.
+        start - The start symbol (Default: start)
+        profile - Measure run-time usage in Lark. Read results from the profiler proprety (Default: False)
+        propagate_positions - Propagates [line, column, end_line, end_column] attributes into all tree branches.
+        lexer_callbacks - Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
+    """
+    __doc__ += OPTIONS_DOC
+    def __init__(self, options_dict):
+        o = dict(options_dict)
+
+        self.debug = bool(o.pop('debug', False))
+        self.keep_all_tokens = bool(o.pop('keep_all_tokens', False))
+        self.tree_class = o.pop('tree_class', Tree)
+        self.cache_grammar = o.pop('cache_grammar', False)
+        self.postlex = o.pop('postlex', None)
+        self.parser = o.pop('parser', 'earley')
+        self.lexer = o.pop('lexer', 'auto')
+        self.transformer = o.pop('transformer', None)
+        self.start = o.pop('start', 'start')
+        self.profile = o.pop('profile', False)
+        self.ambiguity = o.pop('ambiguity', 'auto')
+        self.propagate_positions = o.pop('propagate_positions', False)
+        self.earley__predict_all = o.pop('earley__predict_all', False)
+        self.lexer_callbacks = o.pop('lexer_callbacks', {})
+
+        assert self.parser in ('earley', 'lalr', 'cyk', None)
+
+        if self.parser == 'earley' and self.transformer:
+            raise ValueError('Cannot specify an embedded transformer when using the Earley algorithm.'
+                             'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. lalr)')
+
+        if o:
+            raise ValueError("Unknown options: %s" % o.keys())
+
+
+class Profiler:
+    def __init__(self):
+        self.total_time = defaultdict(float)
+        self.cur_section = '__init__'
+        self.last_enter_time = time.time()
+
+    def enter_section(self, name):
+        cur_time = time.time()
+        self.total_time[self.cur_section] += cur_time - self.last_enter_time
+        self.last_enter_time = cur_time
+        self.cur_section = name
+
+    def make_wrapper(self, name, f):
+        def wrapper(*args, **kwargs):
+            last_section = self.cur_section
+            self.enter_section(name)
+            try:
+                return f(*args, **kwargs)
+            finally:
+                self.enter_section(last_section)
+
+        return wrapper
+
+
+class Lark:
+    def __init__(self, grammar, **options):
+        """
+            grammar : a string or file-object containing the grammar spec (using Lark's ebnf syntax)
+            options : a dictionary controlling various aspects of Lark.
+        """
+        self.options = LarkOptions(options)
+
+        # Some, but not all file-like objects have a 'name' attribute
+        try:
+            self.source = grammar.name
+        except AttributeError:
+            self.source = '<string>'
+            cache_file = "larkcache_%s" % str(hash(grammar)%(2**32))
+        else:
+            cache_file = "larkcache_%s" % os.path.basename(self.source)
+
+        # Drain file-like objects to get their contents
+        try:
+            read = grammar.read
+        except AttributeError:
+            pass
+        else:
+            grammar = read()
+
+        assert isinstance(grammar, STRING_TYPE)
+
+        if self.options.cache_grammar:
+            raise NotImplementedError("Not available yet")
+
+        assert not self.options.profile, "Feature temporarily disabled"
+        self.profiler = Profiler() if self.options.profile else None
+
+        if self.options.lexer == 'auto':
+            if self.options.parser == 'lalr':
+                self.options.lexer = 'contextual'
+            elif self.options.parser == 'earley':
+                self.options.lexer = 'dynamic'
+            elif self.options.parser == 'cyk':
+                self.options.lexer = 'standard'
+            else:
+                assert False, self.options.parser
+        lexer = self.options.lexer
+        assert lexer in ('standard', 'contextual', 'dynamic', 'dynamic_complete') or issubclass(lexer, Lexer)
+
+        if self.options.ambiguity == 'auto':
+            if self.options.parser == 'earley':
+                self.options.ambiguity = 'resolve'
+        else:
+            disambig_parsers = ['earley', 'cyk']
+            assert self.options.parser in disambig_parsers, (
+                'Only %s supports disambiguation right now') % ', '.join(disambig_parsers)
+        assert self.options.ambiguity in ('resolve', 'explicit', 'auto', 'resolve__antiscore_sum')
+
+        # Parse the grammar file and compose the grammars (TODO)
+        self.grammar = load_grammar(grammar, self.source)
+
+        # Compile the EBNF grammar into BNF
+        self.terminals, self.rules, self.ignore_tokens = self.grammar.compile()
+
+        self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, self.options.lexer_callbacks)
+
+        if self.options.parser:
+            self.parser = self._build_parser()
+        elif lexer:
+            self.lexer = self._build_lexer()
+
+        if self.profiler: self.profiler.enter_section('outside_lark')
+
+    __init__.__doc__ += "\nOPTIONS:" + LarkOptions.OPTIONS_DOC
+
+    def _build_lexer(self):
+        return TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks)
+
+    def _build_parser(self):
+        self.parser_class = get_frontend(self.options.parser, self.options.lexer)
+
+        self._parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class, self.options.propagate_positions, self.options.keep_all_tokens, self.options.parser!='lalr')
+        callback = self._parse_tree_builder.create_callback(self.options.transformer)
+        if self.profiler:
+            for f in dir(callback):
+                if not (f.startswith('__') and f.endswith('__')):
+                    setattr(callback, f, self.profiler.make_wrapper('transformer', getattr(callback, f)))
+
+        parser_conf = ParserConf(self.rules, callback, self.options.start)
+
+        return self.parser_class(self.lexer_conf, parser_conf, options=self.options)
+
+    @classmethod
+    def open(cls, grammar_filename, rel_to=None, **options):
+        """Create an instance of Lark with the grammar given by its filename
+
+        If rel_to is provided, the function will find the grammar filename in relation to it.
+
+        Example:
+
+            >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr")
+            Lark(...)
+
+        """
+        if rel_to:
+            basepath = os.path.dirname(rel_to)
+            grammar_filename = os.path.join(basepath, grammar_filename)
+        with open(grammar_filename, encoding='utf8') as f:
+            return cls(f, **options)
+
+    def __repr__(self):
+        return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source, self.options.parser, self.options.lexer)
+
+
+    def lex(self, text):
+        "Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard'"
+        if not hasattr(self, 'lexer'):
+            self.lexer = self._build_lexer()
+        stream = self.lexer.lex(text)
+        if self.options.postlex:
+            return self.options.postlex.process(stream)
+        return stream
+
+    def parse(self, text):
+        "Parse the given text, according to the options provided. Returns a tree, unless specified otherwise."
+        return self.parser.parse(text)
+
+        # if self.profiler:
+        #     self.profiler.enter_section('lex')
+        #     l = list(self.lex(text))
+        #     self.profiler.enter_section('parse')
+        #     try:
+        #         return self.parser.parse(l)
+        #     finally:
+        #         self.profiler.enter_section('outside_lark')
+        # else:
+        #     l = list(self.lex(text))
+        #     return self.parser.parse(l)

venv/lib/python3.6/site-packages/lark/lexer.py

@@ -0,0 +1,324 @@
+## Lexer Implementation
+
+import re
+
+from .utils import Str, classify, get_regexp_width, Py36
+from .exceptions import UnexpectedCharacters, LexError
+
+class Pattern(object):
+    def __init__(self, value, flags=()):
+        self.value = value
+        self.flags = frozenset(flags)
+
+    def __repr__(self):
+        return repr(self.to_regexp())
+
+    # Pattern Hashing assumes all subclasses have a different priority!
+    def __hash__(self):
+        return hash((type(self), self.value, self.flags))
+    def __eq__(self, other):
+        return type(self) == type(other) and self.value == other.value and self.flags == other.flags
+
+    def to_regexp(self):
+        raise NotImplementedError()
+
+    if Py36:
+        # Python 3.6 changed syntax for flags in regular expression
+        def _get_flags(self, value):
+            for f in self.flags:
+                value = ('(?%s:%s)' % (f, value))
+            return value
+
+    else:
+        def _get_flags(self, value):
+            for f in self.flags:
+                value = ('(?%s)' % f) + value
+            return value
+
+class PatternStr(Pattern):
+    def to_regexp(self):
+        return self._get_flags(re.escape(self.value))
+
+    @property
+    def min_width(self):
+        return len(self.value)
+    max_width = min_width
+
+class PatternRE(Pattern):
+    def to_regexp(self):
+        return self._get_flags(self.value)
+
+    @property
+    def min_width(self):
+        return get_regexp_width(self.to_regexp())[0]
+    @property
+    def max_width(self):
+        return get_regexp_width(self.to_regexp())[1]
+
+class TerminalDef(object):
+    def __init__(self, name, pattern, priority=1):
+        assert isinstance(pattern, Pattern), pattern
+        self.name = name
+        self.pattern = pattern
+        self.priority = priority
+
+    def __repr__(self):
+        return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern)
+
+
+
+###{standalone
+class Token(Str):
+    __slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column')
+
+    def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None):
+        self = super(Token, cls).__new__(cls, value)
+        self.type = type_
+        self.pos_in_stream = pos_in_stream
+        self.value = value
+        self.line = line
+        self.column = column
+        self.end_line = None
+        self.end_column = None
+        return self
+
+    @classmethod
+    def new_borrow_pos(cls, type_, value, borrow_t):
+        return cls(type_, value, borrow_t.pos_in_stream, line=borrow_t.line, column=borrow_t.column)
+
+    def __reduce__(self):
+        return (self.__class__, (self.type, self.value, self.pos_in_stream, self.line, self.column, ))
+
+    def __repr__(self):
+        return 'Token(%s, %r)' % (self.type, self.value)
+
+    def __deepcopy__(self, memo):
+        return Token(self.type, self.value, self.pos_in_stream, self.line, self.column)
+
+    def __eq__(self, other):
+        if isinstance(other, Token) and self.type != other.type:
+            return False
+
+        return Str.__eq__(self, other)
+
+    __hash__ = Str.__hash__
+
+
+class LineCounter:
+    def __init__(self):
+        self.newline_char = '\n'
+        self.char_pos = 0
+        self.line = 1
+        self.column = 1
+        self.line_start_pos = 0
+
+    def feed(self, token, test_newline=True):
+        """Consume a token and calculate the new line & column.
+
+        As an optional optimization, set test_newline=False is token doesn't contain a newline.
+        """
+        if test_newline:
+            newlines = token.count(self.newline_char)
+            if newlines:
+                self.line += newlines
+                self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1
+
+        self.char_pos += len(token)
+        self.column = self.char_pos - self.line_start_pos + 1
+
+class _Lex:
+    "Built to serve both Lexer and ContextualLexer"
+    def __init__(self, lexer, state=None):
+        self.lexer = lexer
+        self.state = state
+
+    def lex(self, stream, newline_types, ignore_types):
+        newline_types = frozenset(newline_types)
+        ignore_types = frozenset(ignore_types)
+        line_ctr = LineCounter()
+
+        while line_ctr.char_pos < len(stream):
+            lexer = self.lexer
+            for mre, type_from_index in lexer.mres:
+                m = mre.match(stream, line_ctr.char_pos)
+                if not m:
+                    continue
+
+                t = None
+                value = m.group(0)
+                type_ = type_from_index[m.lastindex]
+                if type_ not in ignore_types:
+                    t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
+                    if t.type in lexer.callback:
+                        t = lexer.callback[t.type](t)
+                    yield t
+                else:
+                    if type_ in lexer.callback:
+                        t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
+                        lexer.callback[type_](t)
+
+                line_ctr.feed(value, type_ in newline_types)
+                if t:
+                    t.end_line = line_ctr.line
+                    t.end_column = line_ctr.column
+
+                break
+            else:
+                raise UnexpectedCharacters(stream, line_ctr.char_pos, line_ctr.line, line_ctr.column, state=self.state)
+
+
+class UnlessCallback:
+    def __init__(self, mres):
+        self.mres = mres
+
+    def __call__(self, t):
+        for mre, type_from_index in self.mres:
+            m = mre.match(t.value)
+            if m:
+                t.type = type_from_index[m.lastindex]
+                break
+        return t
+
+###}
+
+
+
+def _create_unless(terminals):
+    tokens_by_type = classify(terminals, lambda t: type(t.pattern))
+    assert len(tokens_by_type) <= 2, tokens_by_type.keys()
+    embedded_strs = set()
+    callback = {}
+    for retok in tokens_by_type.get(PatternRE, []):
+        unless = [] # {}
+        for strtok in tokens_by_type.get(PatternStr, []):
+            if strtok.priority > retok.priority:
+                continue
+            s = strtok.pattern.value
+            m = re.match(retok.pattern.to_regexp(), s)
+            if m and m.group(0) == s:
+                unless.append(strtok)
+                if strtok.pattern.flags <= retok.pattern.flags:
+                    embedded_strs.add(strtok)
+        if unless:
+            callback[retok.name] = UnlessCallback(build_mres(unless, match_whole=True))
+
+    terminals = [t for t in terminals if t not in embedded_strs]
+    return terminals, callback
+
+
+def _build_mres(terminals, max_size, match_whole):
+    # Python sets an unreasonable group limit (currently 100) in its re module
+    # Worse, the only way to know we reached it is by catching an AssertionError!
+    # This function recursively tries less and less groups until it's successful.
+    postfix = '$' if match_whole else ''
+    mres = []
+    while terminals:
+        try:
+            mre = re.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in terminals[:max_size]))
+        except AssertionError:  # Yes, this is what Python provides us.. :/
+            return _build_mres(terminals, max_size//2, match_whole)
+
+        # terms_from_name = {t.name: t for t in terminals[:max_size]}
+        mres.append((mre, {i:n for n,i in mre.groupindex.items()} ))
+        terminals = terminals[max_size:]
+    return mres
+
+def build_mres(terminals, match_whole=False):
+    return _build_mres(terminals, len(terminals), match_whole)
+
+def _regexp_has_newline(r):
+    """Expressions that may indicate newlines in a regexp:
+        - newlines (\n)
+        - escaped newline (\\n)
+        - anything but ([^...])
+        - any-char (.) when the flag (?s) exists
+    """
+    return '\n' in r or '\\n' in r or '[^' in r or ('(?s' in r and '.' in r)
+
+class Lexer:
+    """Lexer interface
+
+    Method Signatures:
+        lex(self, stream) -> Iterator[Token]
+
+        set_parser_state(self, state)   # Optional
+    """
+    set_parser_state = NotImplemented
+    lex = NotImplemented
+
+class TraditionalLexer(Lexer):
+    def __init__(self, terminals, ignore=(), user_callbacks={}):
+        assert all(isinstance(t, TerminalDef) for t in terminals), terminals
+
+        terminals = list(terminals)
+
+        # Sanitization
+        for t in terminals:
+            try:
+                re.compile(t.pattern.to_regexp())
+            except:
+                raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern))
+
+            if t.pattern.min_width == 0:
+                raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern))
+
+        assert set(ignore) <= {t.name for t in terminals}
+
+        # Init
+        self.newline_types = [t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp())]
+        self.ignore_types = list(ignore)
+
+        terminals.sort(key=lambda x:(-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name))
+
+        terminals, self.callback = _create_unless(terminals)
+        assert all(self.callback.values())
+
+        for type_, f in user_callbacks.items():
+            assert type_ not in self.callback
+            self.callback[type_] = f
+
+        self.terminals = terminals
+
+        self.mres = build_mres(terminals)
+
+
+    def lex(self, stream):
+        return _Lex(self).lex(stream, self.newline_types, self.ignore_types)
+
+
+class ContextualLexer(Lexer):
+    def __init__(self, terminals, states, ignore=(), always_accept=(), user_callbacks={}):
+        tokens_by_name = {}
+        for t in terminals:
+            assert t.name not in tokens_by_name, t
+            tokens_by_name[t.name] = t
+
+        lexer_by_tokens = {}
+        self.lexers = {}
+        for state, accepts in states.items():
+            key = frozenset(accepts)
+            try:
+                lexer = lexer_by_tokens[key]
+            except KeyError:
+                accepts = set(accepts) | set(ignore) | set(always_accept)
+                state_tokens = [tokens_by_name[n] for n in accepts if n and n in tokens_by_name]
+                lexer = TraditionalLexer(state_tokens, ignore=ignore, user_callbacks=user_callbacks)
+                lexer_by_tokens[key] = lexer
+
+            self.lexers[state] = lexer
+
+        self.root_lexer = TraditionalLexer(terminals, ignore=ignore, user_callbacks=user_callbacks)
+
+        self.set_parser_state(None) # Needs to be set on the outside
+
+    def set_parser_state(self, state):
+        self.parser_state = state
+
+    def lex(self, stream):
+        l = _Lex(self.lexers[self.parser_state], self.parser_state)
+        for x in l.lex(stream, self.root_lexer.newline_types, self.root_lexer.ignore_types):
+            yield x
+            l.lexer = self.lexers[self.parser_state]
+            l.state = self.parser_state
+
+

venv/lib/python3.6/site-packages/lark/load_grammar.py

@@ -0,0 +1,779 @@
+"Parses and creates Grammar objects"
+
+import os.path
+import sys
+from ast import literal_eval
+from copy import deepcopy
+
+from .utils import bfs
+from .lexer import Token, TerminalDef, PatternStr, PatternRE
+
+from .parse_tree_builder import ParseTreeBuilder
+from .parser_frontends import LALR_TraditionalLexer
+from .common import LexerConf, ParserConf
+from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
+from .utils import classify, suppress
+from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken
+
+from .tree import Tree, SlottedTree as ST
+from .visitors import Transformer, Visitor, v_args, Transformer_InPlace
+inline_args = v_args(inline=True)
+
+__path__ = os.path.dirname(__file__)
+IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
+
+EXT = '.lark'
+
+_RE_FLAGS = 'imslux'
+
+_TERMINAL_NAMES = {
+    '.' : 'DOT',
+    ',' : 'COMMA',
+    ':' : 'COLON',
+    ';' : 'SEMICOLON',
+    '+' : 'PLUS',
+    '-' : 'MINUS',
+    '*' : 'STAR',
+    '/' : 'SLASH',
+    '\\' : 'BACKSLASH',
+    '|' : 'VBAR',
+    '?' : 'QMARK',
+    '!' : 'BANG',
+    '@' : 'AT',
+    '#' : 'HASH',
+    '$' : 'DOLLAR',
+    '%' : 'PERCENT',
+    '^' : 'CIRCUMFLEX',
+    '&' : 'AMPERSAND',
+    '_' : 'UNDERSCORE',
+    '<' : 'LESSTHAN',
+    '>' : 'MORETHAN',
+    '=' : 'EQUAL',
+    '"' : 'DBLQUOTE',
+    '\'' : 'QUOTE',
+    '`' : 'BACKQUOTE',
+    '~' : 'TILDE',
+    '(' : 'LPAR',
+    ')' : 'RPAR',
+    '{' : 'LBRACE',
+    '}' : 'RBRACE',
+    '[' : 'LSQB',
+    ']' : 'RSQB',
+    '\n' : 'NEWLINE',
+    '\r\n' : 'CRLF',
+    '\t' : 'TAB',
+    ' ' : 'SPACE',
+}
+
+# Grammar Parser
+TERMINALS = {
+    '_LPAR': r'\(',
+    '_RPAR': r'\)',
+    '_LBRA': r'\[',
+    '_RBRA': r'\]',
+    'OP': '[+*][?]?|[?](?![a-z])',
+    '_COLON': ':',
+    '_COMMA': ',',
+    '_OR': r'\|',
+    '_DOT': r'\.',
+    'TILDE': '~',
+    'RULE': '!?[_?]?[a-z][_a-z0-9]*',
+    'TERMINAL': '_?[A-Z][_A-Z0-9]*',
+    'STRING': r'"(\\"|\\\\|[^"\n])*?"i?',
+    'REGEXP': r'/(?!/)(\\/|\\\\|[^/\n])*?/[%s]*' % _RE_FLAGS,
+    '_NL': r'(\r?\n)+\s*',
+    'WS': r'[ \t]+',
+    'COMMENT': r'//[^\n]*',
+    '_TO': '->',
+    '_IGNORE': r'%ignore',
+    '_DECLARE': r'%declare',
+    '_IMPORT': r'%import',
+    'NUMBER': r'\d+',
+}
+
+RULES = {
+    'start': ['_list'],
+    '_list':  ['_item', '_list _item'],
+    '_item':  ['rule', 'term', 'statement', '_NL'],
+
+    'rule': ['RULE _COLON expansions _NL',
+             'RULE _DOT NUMBER _COLON expansions _NL'],
+    'expansions': ['alias',
+                   'expansions _OR alias',
+                   'expansions _NL _OR alias'],
+
+    '?alias':     ['expansion _TO RULE', 'expansion'],
+    'expansion': ['_expansion'],
+
+    '_expansion': ['', '_expansion expr'],
+
+    '?expr': ['atom',
+              'atom OP',
+              'atom TILDE NUMBER',
+              'atom TILDE NUMBER _DOT _DOT NUMBER',
+              ],
+
+    '?atom': ['_LPAR expansions _RPAR',
+              'maybe',
+              'value'],
+
+    'value': ['terminal',
+              'nonterminal',
+              'literal',
+              'range'],
+
+    'terminal': ['TERMINAL'],
+    'nonterminal': ['RULE'],
+
+    '?name': ['RULE', 'TERMINAL'],
+
+    'maybe': ['_LBRA expansions _RBRA'],
+    'range': ['STRING _DOT _DOT STRING'],
+
+    'term': ['TERMINAL _COLON expansions _NL',
+              'TERMINAL _DOT NUMBER _COLON expansions _NL'],
+    'statement': ['ignore', 'import', 'declare'],
+    'ignore': ['_IGNORE expansions _NL'],
+    'declare': ['_DECLARE _declare_args _NL'],
+    'import': ['_IMPORT _import_path _NL',
+               '_IMPORT _import_path _LPAR name_list _RPAR _NL',
+               '_IMPORT _import_path _TO TERMINAL _NL'],
+
+    '_import_path': ['import_lib', 'import_rel'],
+    'import_lib': ['_import_args'],
+    'import_rel': ['_DOT _import_args'],
+    '_import_args': ['name', '_import_args _DOT name'],
+
+    'name_list': ['_name_list'],
+    '_name_list': ['name', '_name_list _COMMA name'],
+
+    '_declare_args': ['name', '_declare_args name'],
+    'literal': ['REGEXP', 'STRING'],
+}
+
+
+@inline_args
+class EBNF_to_BNF(Transformer_InPlace):
+    def __init__(self):
+        self.new_rules = []
+        self.rules_by_expr = {}
+        self.prefix = 'anon'
+        self.i = 0
+        self.rule_options = None
+
+    def _add_recurse_rule(self, type_, expr):
+        if expr in self.rules_by_expr:
+            return self.rules_by_expr[expr]
+
+        new_name = '__%s_%s_%d' % (self.prefix, type_, self.i)
+        self.i += 1
+        t = NonTerminal(new_name)
+        tree = ST('expansions', [ST('expansion', [expr]), ST('expansion', [t, expr])])
+        self.new_rules.append((new_name, tree, self.rule_options))
+        self.rules_by_expr[expr] = t
+        return t
+
+    def expr(self, rule, op, *args):
+        if op.value == '?':
+            return ST('expansions', [rule, ST('expansion', [])])
+        elif op.value == '+':
+            # a : b c+ d
+            #   -->
+            # a : b _c d
+            # _c : _c c | c;
+            return self._add_recurse_rule('plus', rule)
+        elif op.value == '*':
+            # a : b c* d
+            #   -->
+            # a : b _c? d
+            # _c : _c c | c;
+            new_name = self._add_recurse_rule('star', rule)
+            return ST('expansions', [new_name, ST('expansion', [])])
+        elif op.value == '~':
+            if len(args) == 1:
+                mn = mx = int(args[0])
+            else:
+                mn, mx = map(int, args)
+                if mx < mn:
+                    raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (rule, mn, mx))
+            return ST('expansions', [ST('expansion', [rule] * n) for n in range(mn, mx+1)])
+        assert False, op
+
+
+class SimplifyRule_Visitor(Visitor):
+
+    @staticmethod
+    def _flatten(tree):
+        while True:
+            to_expand = [i for i, child in enumerate(tree.children)
+                         if isinstance(child, Tree) and child.data == tree.data]
+            if not to_expand:
+                break
+            tree.expand_kids_by_index(*to_expand)
+
+    def expansion(self, tree):
+        # rules_list unpacking
+        # a : b (c|d) e
+        #  -->
+        # a : b c e | b d e
+        #
+        # In AST terms:
+        # expansion(b, expansions(c, d), e)
+        #   -->
+        # expansions( expansion(b, c, e), expansion(b, d, e) )
+
+        self._flatten(tree)
+
+        for i, child in enumerate(tree.children):
+            if isinstance(child, Tree) and child.data == 'expansions':
+                tree.data = 'expansions'
+                tree.children = [self.visit(ST('expansion', [option if i==j else other
+                                                            for j, other in enumerate(tree.children)]))
+                                    for option in set(child.children)]
+                self._flatten(tree)
+                break
+
+    def alias(self, tree):
+        rule, alias_name = tree.children
+        if rule.data == 'expansions':
+            aliases = []
+            for child in tree.children[0].children:
+                aliases.append(ST('alias', [child, alias_name]))
+            tree.data = 'expansions'
+            tree.children = aliases
+
+    def expansions(self, tree):
+        self._flatten(tree)
+        tree.children = list(set(tree.children))
+
+
+class RuleTreeToText(Transformer):
+    def expansions(self, x):
+        return x
+    def expansion(self, symbols):
+        return symbols, None
+    def alias(self, x):
+        (expansion, _alias), alias = x
+        assert _alias is None, (alias, expansion, '-', _alias)  # Double alias not allowed
+        return expansion, alias.value
+
+
+@inline_args
+class CanonizeTree(Transformer_InPlace):
+    def maybe(self, expr):
+        return ST('expr', [expr, Token('OP', '?', -1)])
+
+    def tokenmods(self, *args):
+        if len(args) == 1:
+            return list(args)
+        tokenmods, value = args
+        return tokenmods + [value]
+
+class PrepareAnonTerminals(Transformer_InPlace):
+    "Create a unique list of anonymous terminals. Attempt to give meaningful names to them when we add them"
+
+    def __init__(self, terminals):
+        self.terminals = terminals
+        self.term_set = {td.name for td in self.terminals}
+        self.term_reverse = {td.pattern: td for td in terminals}
+        self.i = 0
+
+
+    @inline_args
+    def pattern(self, p):
+        value = p.value
+        if p in self.term_reverse and p.flags != self.term_reverse[p].pattern.flags:
+            raise GrammarError(u'Conflicting flags for the same terminal: %s' % p)
+
+        term_name = None
+
+        if isinstance(p, PatternStr):
+            try:
+                # If already defined, use the user-defined terminal name
+                term_name = self.term_reverse[p].name
+            except KeyError:
+                # Try to assign an indicative anon-terminal name
+                try:
+                    term_name = _TERMINAL_NAMES[value]
+                except KeyError:
+                    if value.isalnum() and value[0].isalpha() and value.upper() not in self.term_set:
+                        with suppress(UnicodeEncodeError):
+                            value.upper().encode('ascii') # Make sure we don't have unicode in our terminal names
+                            term_name = value.upper()
+
+                if term_name in self.term_set:
+                    term_name = None
+
+        elif isinstance(p, PatternRE):
+            if p in self.term_reverse: # Kind of a wierd placement.name
+                term_name = self.term_reverse[p].name
+        else:
+            assert False, p
+
+        if term_name is None:
+            term_name = '__ANON_%d' % self.i
+            self.i += 1
+
+        if term_name not in self.term_set:
+            assert p not in self.term_reverse
+            self.term_set.add(term_name)
+            termdef = TerminalDef(term_name, p)
+            self.term_reverse[p] = termdef
+            self.terminals.append(termdef)
+
+        return Terminal(term_name, filter_out=isinstance(p, PatternStr))
+
+
+def _rfind(s, choices):
+    return max(s.rfind(c) for c in choices)
+
+
+
+def _fix_escaping(s):
+    w = ''
+    i = iter(s)
+    for n in i:
+        w += n
+        if n == '\\':
+            n2 = next(i)
+            if n2 == '\\':
+                w += '\\\\'
+            elif n2 not in 'unftr':
+                w += '\\'
+            w += n2
+    w = w.replace('\\"', '"').replace("'", "\\'")
+
+    to_eval = "u'''%s'''" % w
+    try:
+        s = literal_eval(to_eval)
+    except SyntaxError as e:
+        raise ValueError(s, e)
+
+    return s
+
+
+def _literal_to_pattern(literal):
+    v = literal.value
+    flag_start = _rfind(v, '/"')+1
+    assert flag_start > 0
+    flags = v[flag_start:]
+    assert all(f in _RE_FLAGS for f in flags), flags
+
+    v = v[:flag_start]
+    assert v[0] == v[-1] and v[0] in '"/'
+    x = v[1:-1]
+
+    s = _fix_escaping(x)
+
+    if literal.type == 'STRING':
+        s = s.replace('\\\\', '\\')
+
+    return { 'STRING': PatternStr,
+             'REGEXP': PatternRE }[literal.type](s, flags)
+
+
+@inline_args
+class PrepareLiterals(Transformer_InPlace):
+    def literal(self, literal):
+        return ST('pattern', [_literal_to_pattern(literal)])
+
+    def range(self, start, end):
+        assert start.type == end.type == 'STRING'
+        start = start.value[1:-1]
+        end = end.value[1:-1]
+        assert len(start) == len(end) == 1, (start, end, len(start), len(end))
+        regexp = '[%s-%s]' % (start, end)
+        return ST('pattern', [PatternRE(regexp)])
+
+
+class TerminalTreeToPattern(Transformer):
+    def pattern(self, ps):
+        p ,= ps
+        return p
+
+    def expansion(self, items):
+        assert items
+        if len(items) == 1:
+            return items[0]
+        if len({i.flags for i in items}) > 1:
+            raise GrammarError("Lark doesn't support joining terminals with conflicting flags!")
+        return PatternRE(''.join(i.to_regexp() for i in items), items[0].flags if items else ())
+
+    def expansions(self, exps):
+        if len(exps) == 1:
+            return exps[0]
+        if len({i.flags for i in exps}) > 1:
+            raise GrammarError("Lark doesn't support joining terminals with conflicting flags!")
+        return PatternRE('(?:%s)' % ('|'.join(i.to_regexp() for i in exps)), exps[0].flags)
+
+    def expr(self, args):
+        inner, op = args[:2]
+        if op == '~':
+            if len(args) == 3:
+                op = "{%d}" % int(args[2])
+            else:
+                mn, mx = map(int, args[2:])
+                if mx < mn:
+                    raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (inner, mn, mx))
+                op = "{%d,%d}" % (mn, mx)
+        else:
+            assert len(args) == 2
+        return PatternRE('(?:%s)%s' % (inner.to_regexp(), op), inner.flags)
+
+    def alias(self, t):
+        raise GrammarError("Aliasing not allowed in terminals (You used -> in the wrong place)")
+
+    def value(self, v):
+        return v[0]
+
+class PrepareSymbols(Transformer_InPlace):
+    def value(self, v):
+        v ,= v
+        if isinstance(v, Tree):
+            return v
+        elif v.type == 'RULE':
+            return NonTerminal(v.value)
+        elif v.type == 'TERMINAL':
+            return Terminal(v.value, filter_out=v.startswith('_'))
+        assert False
+
+def _choice_of_rules(rules):
+    return ST('expansions', [ST('expansion', [Token('RULE', name)]) for name in rules])
+
+class Grammar:
+    def __init__(self, rule_defs, term_defs, ignore):
+        self.term_defs = term_defs
+        self.rule_defs = rule_defs
+        self.ignore = ignore
+
+    def compile(self):
+        # We change the trees in-place (to support huge grammars)
+        # So deepcopy allows calling compile more than once.
+        term_defs = deepcopy(list(self.term_defs))
+        rule_defs = deepcopy(self.rule_defs)
+
+        # ===================
+        #  Compile Terminals
+        # ===================
+
+        # Convert terminal-trees to strings/regexps
+        transformer = PrepareLiterals() * TerminalTreeToPattern()
+        for name, (term_tree, priority) in term_defs:
+            if term_tree is None:  # Terminal added through %declare
+                continue
+            expansions = list(term_tree.find_data('expansion'))
+            if len(expansions) == 1 and not expansions[0].children:
+                raise GrammarError("Terminals cannot be empty (%s)" % name)
+
+        terminals = [TerminalDef(name, transformer.transform(term_tree), priority)
+                  for name, (term_tree, priority) in term_defs if term_tree]
+
+        # =================
+        #  Compile Rules
+        # =================
+
+        # 1. Pre-process terminals
+        transformer = PrepareLiterals() * PrepareSymbols() * PrepareAnonTerminals(terminals)   # Adds to terminals
+
+        # 2. Convert EBNF to BNF (and apply step 1)
+        ebnf_to_bnf = EBNF_to_BNF()
+        rules = []
+        for name, rule_tree, options in rule_defs:
+            ebnf_to_bnf.rule_options = RuleOptions(keep_all_tokens=True) if options and options.keep_all_tokens else None
+            tree = transformer.transform(rule_tree)
+            rules.append((name, ebnf_to_bnf.transform(tree), options))
+        rules += ebnf_to_bnf.new_rules
+
+        assert len(rules) == len({name for name, _t, _o in rules}), "Whoops, name collision"
+
+        # 3. Compile tree to Rule objects
+        rule_tree_to_text = RuleTreeToText()
+
+        simplify_rule = SimplifyRule_Visitor()
+        compiled_rules = []
+        for name, tree, options in rules:
+            simplify_rule.visit(tree)
+            expansions = rule_tree_to_text.transform(tree)
+
+            for expansion, alias in expansions:
+                if alias and name.startswith('_'):
+                    raise GrammarError("Rule %s is marked for expansion (it starts with an underscore) and isn't allowed to have aliases (alias=%s)" % (name, alias))
+
+                assert all(isinstance(x, Symbol) for x in expansion), expansion
+
+                rule = Rule(NonTerminal(name), expansion, alias, options)
+                compiled_rules.append(rule)
+
+        return terminals, compiled_rules, self.ignore
+
+
+
+_imported_grammars = {}
+def import_grammar(grammar_path, base_paths=[]):
+    if grammar_path not in _imported_grammars:
+        import_paths = base_paths + IMPORT_PATHS
+        for import_path in import_paths:
+            with suppress(IOError):
+                with open(os.path.join(import_path, grammar_path)) as f:
+                    text = f.read()
+                grammar = load_grammar(text, grammar_path)
+                _imported_grammars[grammar_path] = grammar
+                break
+        else:
+            open(grammar_path)
+            assert False
+
+    return _imported_grammars[grammar_path]
+
+def import_from_grammar_into_namespace(grammar, namespace, aliases):
+    imported_terms = dict(grammar.term_defs)
+    imported_rules = {n:(n,t,o) for n,t,o in grammar.rule_defs}
+    
+    term_defs = []
+    rule_defs = []
+
+    def rule_dependencies(symbol):
+        if symbol.type != 'RULE':
+            return []
+        _, tree, _ = imported_rules[symbol]
+        return tree.scan_values(lambda x: x.type in ('RULE', 'TERMINAL'))
+
+    def get_namespace_name(name):
+        try:
+            return aliases[name].value
+        except KeyError:
+            return '%s.%s' % (namespace, name)
+
+    to_import = list(bfs(aliases, rule_dependencies))
+    for symbol in to_import:
+        if symbol.type == 'TERMINAL':
+            term_defs.append([get_namespace_name(symbol), imported_terms[symbol]])
+        else:
+            assert symbol.type == 'RULE'
+            rule = imported_rules[symbol]
+            for t in rule[1].iter_subtrees():
+                for i, c in enumerate(t.children):
+                    if isinstance(c, Token) and c.type in ('RULE', 'TERMINAL'):
+                        t.children[i] = Token(c.type, get_namespace_name(c))
+            rule_defs.append((get_namespace_name(symbol), rule[1], rule[2]))
+
+    return term_defs, rule_defs
+
+
+
+def resolve_term_references(term_defs):
+    # TODO Cycles detection
+    # TODO Solve with transitive closure (maybe)
+
+    token_dict = {k:t for k, (t,_p) in term_defs}
+    assert len(token_dict) == len(term_defs), "Same name defined twice?"
+
+    while True:
+        changed = False
+        for name, (token_tree, _p) in term_defs:
+            if token_tree is None:  # Terminal added through %declare
+                continue
+            for exp in token_tree.find_data('value'):
+                item ,= exp.children
+                if isinstance(item, Token):
+                    if item.type == 'RULE':
+                        raise GrammarError("Rules aren't allowed inside terminals (%s in %s)" % (item, name))
+                    if item.type == 'TERMINAL':
+                        exp.children[0] = token_dict[item]
+                        changed = True
+        if not changed:
+            break
+
+def options_from_rule(name, *x):
+    if len(x) > 1:
+        priority, expansions = x
+        priority = int(priority)
+    else:
+        expansions ,= x
+        priority = None
+
+    keep_all_tokens = name.startswith('!')
+    name = name.lstrip('!')
+    expand1 = name.startswith('?')
+    name = name.lstrip('?')
+
+    return name, expansions, RuleOptions(keep_all_tokens, expand1, priority=priority)
+
+
+def symbols_from_strcase(expansion):
+    return [Terminal(x, filter_out=x.startswith('_')) if x.isupper() else NonTerminal(x) for x in expansion]
+
+@inline_args
+class PrepareGrammar(Transformer_InPlace):
+    def terminal(self, name):
+        return name
+    def nonterminal(self, name):
+        return name
+
+
+class GrammarLoader:
+    def __init__(self):
+        terminals = [TerminalDef(name, PatternRE(value)) for name, value in TERMINALS.items()]
+
+        rules = [options_from_rule(name, x) for name, x in  RULES.items()]
+        rules = [Rule(NonTerminal(r), symbols_from_strcase(x.split()), None, o) for r, xs, o in rules for x in xs]
+        callback = ParseTreeBuilder(rules, ST).create_callback()
+        lexer_conf = LexerConf(terminals, ['WS', 'COMMENT'])
+
+        parser_conf = ParserConf(rules, callback, 'start')
+        self.parser = LALR_TraditionalLexer(lexer_conf, parser_conf)
+
+        self.canonize_tree = CanonizeTree()
+
+    def load_grammar(self, grammar_text, grammar_name='<?>'):
+        "Parse grammar_text, verify, and create Grammar object. Display nice messages on error."
+
+        try:
+            tree = self.canonize_tree.transform( self.parser.parse(grammar_text+'\n') )
+        except UnexpectedCharacters as e:
+            context = e.get_context(grammar_text)
+            raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
+                               (e.line, e.column, grammar_name, context))
+        except UnexpectedToken as e:
+            context = e.get_context(grammar_text)
+            error = e.match_examples(self.parser.parse, {
+                'Unclosed parenthesis': ['a: (\n'],
+                'Umatched closing parenthesis': ['a: )\n', 'a: [)\n', 'a: (]\n'],
+                'Expecting rule or terminal definition (missing colon)': ['a\n', 'a->\n', 'A->\n', 'a A\n'],
+                'Alias expects lowercase name': ['a: -> "a"\n'],
+                'Unexpected colon': ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n'],
+                'Misplaced operator': ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n'],
+                'Expecting option ("|") or a new rule or terminal definition': ['a:a\n()\n'],
+                '%import expects a name': ['%import "a"\n'],
+                '%ignore expects a value': ['%ignore %import\n'],
+            })
+            if error:
+                raise GrammarError("%s at line %s column %s\n\n%s" % (error, e.line, e.column, context))
+            elif 'STRING' in e.expected:
+                raise GrammarError("Expecting a value at line %s column %s\n\n%s" % (e.line, e.column, context))
+            raise
+
+        tree = PrepareGrammar().transform(tree)
+
+        # Extract grammar items
+        defs = classify(tree.children, lambda c: c.data, lambda c: c.children)
+        term_defs = defs.pop('term', [])
+        rule_defs = defs.pop('rule', [])
+        statements = defs.pop('statement', [])
+        assert not defs
+
+        term_defs = [td if len(td)==3 else (td[0], 1, td[1]) for td in term_defs]
+        term_defs = [(name.value, (t, int(p))) for name, p, t in term_defs]
+        rule_defs = [options_from_rule(*x) for x in rule_defs]
+
+        # Execute statements
+        ignore = []
+        for (stmt,) in statements:
+            if stmt.data == 'ignore':
+                t ,= stmt.children
+                ignore.append(t)
+            elif stmt.data == 'import':
+                if len(stmt.children) > 1:
+                    path_node, arg1 = stmt.children
+                else:
+                    path_node ,= stmt.children
+                    arg1 = None
+
+                if isinstance(arg1, Tree):  # Multi import
+                    dotted_path = path_node.children
+                    names = arg1.children
+                    aliases = names  # Can't have aliased multi import, so all aliases will be the same as names
+                else:  # Single import
+                    dotted_path = path_node.children[:-1]
+                    names = [path_node.children[-1]]  # Get name from dotted path
+                    aliases = [arg1] if arg1 else names  # Aliases if exist
+
+                grammar_path = os.path.join(*dotted_path) + EXT
+
+                if path_node.data == 'import_lib':  # Import from library
+                    g = import_grammar(grammar_path)
+                else:  # Relative import
+                    if grammar_name == '<string>':  # Import relative to script file path if grammar is coded in script
+                        base_file = os.path.abspath(sys.modules['__main__'].__file__)
+                    else:
+                        base_file = grammar_name  # Import relative to grammar file path if external grammar file
+                    base_path = os.path.split(base_file)[0]
+                    g = import_grammar(grammar_path, base_paths=[base_path])
+
+                aliases_dict = dict(zip(names, aliases))
+                new_td, new_rd = import_from_grammar_into_namespace(g, '.'.join(dotted_path), aliases_dict)
+
+                term_defs += new_td
+                rule_defs += new_rd
+
+            elif stmt.data == 'declare':
+                for t in stmt.children:
+                    term_defs.append([t.value, (None, None)])
+            else:
+                assert False, stmt
+
+
+        # Verify correctness 1
+        for name, _ in term_defs:
+            if name.startswith('__'):
+                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
+
+        # Handle ignore tokens
+        # XXX A slightly hacky solution. Recognition of %ignore TERMINAL as separate comes from the lexer's
+        #     inability to handle duplicate terminals (two names, one value)
+        ignore_names = []
+        for t in ignore:
+            if t.data=='expansions' and len(t.children) == 1:
+                t2 ,= t.children
+                if t2.data=='expansion' and len(t2.children) == 1:
+                    item ,= t2.children
+                    if item.data == 'value':
+                        item ,= item.children
+                        if isinstance(item, Token) and item.type == 'TERMINAL':
+                            ignore_names.append(item.value)
+                            continue
+
+            name = '__IGNORE_%d'% len(ignore_names)
+            ignore_names.append(name)
+            term_defs.append((name, (t, 0)))
+
+        # Verify correctness 2
+        terminal_names = set()
+        for name, _ in term_defs:
+            if name in terminal_names:
+                raise GrammarError("Terminal '%s' defined more than once" % name)
+            terminal_names.add(name)
+
+        if set(ignore_names) > terminal_names:
+            raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(ignore_names) - terminal_names))
+
+        resolve_term_references(term_defs)
+
+        rules = rule_defs
+
+        rule_names = set()
+        for name, _x, _o in rules:
+            if name.startswith('__'):
+                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
+            if name in rule_names:
+                raise GrammarError("Rule '%s' defined more than once" % name)
+            rule_names.add(name)
+
+        for name, expansions, _o in rules:
+            used_symbols = {t for x in expansions.find_data('expansion')
+                              for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
+            for sym in used_symbols:
+                if sym.type == 'TERMINAL':
+                    if sym not in terminal_names:
+                        raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, name))
+                else:
+                    if sym not in rule_names:
+                        raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, name))
+
+        # TODO don't include unused terminals, they can only cause trouble!
+
+        return Grammar(rules, term_defs, ignore_names)
+
+
+
+load_grammar = GrammarLoader().load_grammar

venv/lib/python3.6/site-packages/lark/parse_tree_builder.py

@@ -0,0 +1,174 @@
+from .exceptions import GrammarError
+from .utils import suppress
+from .lexer import Token
+from .grammar import Rule
+from .tree import Tree
+from .visitors import InlineTransformer # XXX Deprecated
+
+###{standalone
+from functools import partial, wraps
+
+
+class ExpandSingleChild:
+    def __init__(self, node_builder):
+        self.node_builder = node_builder
+
+    def __call__(self, children):
+        if len(children) == 1:
+            return children[0]
+        else:
+            return self.node_builder(children)
+
+
+class PropagatePositions:
+    def __init__(self, node_builder):
+        self.node_builder = node_builder
+
+    def __call__(self, children):
+        res = self.node_builder(children)
+
+        if isinstance(res, Tree):
+            res.meta.empty = True
+
+            for c in children:
+                if isinstance(c, Tree) and c.children and not c.meta.empty:
+                    res.meta.line = c.meta.line
+                    res.meta.column = c.meta.column
+                    res.meta.start_pos = c.meta.start_pos
+                    res.meta.empty = False
+                    break
+                elif isinstance(c, Token):
+                    res.meta.line = c.line
+                    res.meta.column = c.column
+                    res.meta.start_pos = c.pos_in_stream
+                    res.meta.empty = False
+                    break
+
+            for c in reversed(children):
+                if isinstance(c, Tree) and c.children and not c.meta.empty:
+                    res.meta.end_line = c.meta.end_line
+                    res.meta.end_column = c.meta.end_column
+                    res.meta.end_pos = c.meta.end_pos
+                    res.meta.empty = False
+                    break
+                elif isinstance(c, Token):
+                    res.meta.end_line = c.end_line
+                    res.meta.end_column = c.end_column
+                    res.meta.end_pos = c.pos_in_stream + len(c.value)
+                    res.meta.empty = False
+                    break
+
+        return res
+
+
+class ChildFilter:
+    def __init__(self, to_include, node_builder):
+        self.node_builder = node_builder
+        self.to_include = to_include
+
+    def __call__(self, children):
+        filtered = []
+        for i, to_expand in self.to_include:
+            if to_expand:
+                filtered += children[i].children
+            else:
+                filtered.append(children[i])
+
+        return self.node_builder(filtered)
+
+class ChildFilterLALR(ChildFilter):
+    "Optimized childfilter for LALR (assumes no duplication in parse tree, so it's safe to change it)"
+
+    def __call__(self, children):
+        filtered = []
+        for i, to_expand in self.to_include:
+            if to_expand:
+                if filtered:
+                    filtered += children[i].children
+                else:   # Optimize for left-recursion
+                    filtered = children[i].children
+            else:
+                filtered.append(children[i])
+
+        return self.node_builder(filtered)
+
+def _should_expand(sym):
+    return not sym.is_term and sym.name.startswith('_')
+
+def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous):
+    to_include = [(i, _should_expand(sym)) for i, sym in enumerate(expansion)
+                  if keep_all_tokens or not (sym.is_term and sym.filter_out)]
+
+    if len(to_include) < len(expansion) or any(to_expand for i, to_expand in to_include):
+        return partial(ChildFilter if ambiguous else ChildFilterLALR, to_include)
+
+
+class Callback(object):
+    pass
+
+
+def ptb_inline_args(func):
+    @wraps(func)
+    def f(children):
+        return func(*children)
+    return f
+
+
+
+class ParseTreeBuilder:
+    def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False):
+        self.tree_class = tree_class
+        self.propagate_positions = propagate_positions
+        self.always_keep_all_tokens = keep_all_tokens
+        self.ambiguous = ambiguous
+
+        self.rule_builders = list(self._init_builders(rules))
+
+        self.user_aliases = {}
+
+    def _init_builders(self, rules):
+        for rule in rules:
+            options = rule.options
+            keep_all_tokens = self.always_keep_all_tokens or (options.keep_all_tokens if options else False)
+            expand_single_child = options.expand1 if options else False
+
+            wrapper_chain = filter(None, [
+                (expand_single_child and not rule.alias) and ExpandSingleChild,
+                maybe_create_child_filter(rule.expansion, keep_all_tokens, self.ambiguous),
+                self.propagate_positions and PropagatePositions,
+            ])
+
+            yield rule, wrapper_chain
+
+
+    def create_callback(self, transformer=None):
+        callback = Callback()
+
+        i = 0
+        for rule, wrapper_chain in self.rule_builders:
+            internal_callback_name = '_cb%d_%s' % (i, rule.origin)
+            i += 1
+
+            user_callback_name = rule.alias or rule.origin.name
+            try:
+                f = getattr(transformer, user_callback_name)
+                assert not getattr(f, 'meta', False), "Meta args not supported for internal transformer"
+                # XXX InlineTransformer is deprecated!
+                if getattr(f, 'inline', False) or isinstance(transformer, InlineTransformer):
+                    f = ptb_inline_args(f)
+            except AttributeError:
+                f = partial(self.tree_class, user_callback_name)
+
+            self.user_aliases[rule] = rule.alias
+            rule.alias = internal_callback_name
+
+            for w in wrapper_chain:
+                f = w(f)
+
+            if hasattr(callback, internal_callback_name):
+                raise GrammarError("Rule '%s' already exists" % (rule,))
+            setattr(callback, internal_callback_name, f)
+
+        return callback
+
+###}

venv/lib/python3.6/site-packages/lark/parser_frontends.py

@@ -0,0 +1,192 @@
+import re
+from functools import partial
+
+from .utils import get_regexp_width
+from .parsers.grammar_analysis import GrammarAnalyzer
+from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token
+
+from .parsers import lalr_parser, earley, xearley, resolve_ambig, cyk
+from .tree import Tree
+
+class WithLexer:
+    lexer = None
+    parser = None
+    lexer_conf = None
+
+    def init_traditional_lexer(self, lexer_conf):
+        self.lexer_conf = lexer_conf
+        self.lexer = TraditionalLexer(lexer_conf.tokens, ignore=lexer_conf.ignore, user_callbacks=lexer_conf.callbacks)
+
+    def init_contextual_lexer(self, lexer_conf):
+        self.lexer_conf = lexer_conf
+        states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()}
+        always_accept = lexer_conf.postlex.always_accept if lexer_conf.postlex else ()
+        self.lexer = ContextualLexer(lexer_conf.tokens, states,
+                                     ignore=lexer_conf.ignore,
+                                     always_accept=always_accept,
+                                     user_callbacks=lexer_conf.callbacks)
+
+    def lex(self, text):
+        stream = self.lexer.lex(text)
+        if self.lexer_conf.postlex:
+            return self.lexer_conf.postlex.process(stream)
+        return stream
+
+    def parse(self, text):
+        token_stream = self.lex(text)
+        sps = self.lexer.set_parser_state
+        return self.parser.parse(token_stream, *[sps] if sps is not NotImplemented else [])
+
+class LALR_TraditionalLexer(WithLexer):
+    def __init__(self, lexer_conf, parser_conf, options=None):
+        self.parser = lalr_parser.Parser(parser_conf)
+        self.init_traditional_lexer(lexer_conf)
+
+class LALR_ContextualLexer(WithLexer):
+    def __init__(self, lexer_conf, parser_conf, options=None):
+        self.parser = lalr_parser.Parser(parser_conf)
+        self.init_contextual_lexer(lexer_conf)
+
+class LALR_CustomLexer(WithLexer):
+    def __init__(self, lexer_cls, lexer_conf, parser_conf, options=None):
+        self.parser = lalr_parser.Parser(parser_conf)
+        self.lexer_conf = lexer_conf
+        self.lexer = lexer_cls(lexer_conf)
+
+
+def get_ambiguity_resolver(options):
+    if not options or options.ambiguity == 'resolve':
+        return resolve_ambig.standard_resolve_ambig
+    elif options.ambiguity == 'resolve__antiscore_sum':
+        return resolve_ambig.antiscore_sum_resolve_ambig
+    elif options.ambiguity == 'explicit':
+        return None
+    raise ValueError(options)
+
+def tokenize_text(text):
+    line = 1
+    col_start_pos = 0
+    for i, ch in enumerate(text):
+        if '\n' in ch:
+            line += ch.count('\n')
+            col_start_pos = i + ch.rindex('\n')
+        yield Token('CHAR', ch, line=line, column=i - col_start_pos)
+
+class Earley(WithLexer):
+    def __init__(self, lexer_conf, parser_conf, options=None):
+        self.init_traditional_lexer(lexer_conf)
+
+        self.parser = earley.Parser(parser_conf, self.match,
+                                    resolve_ambiguity=get_ambiguity_resolver(options))
+
+    def match(self, term, token):
+        return term.name == token.type
+
+
+class XEarley:
+    def __init__(self, lexer_conf, parser_conf, options=None, **kw):
+        self.token_by_name = {t.name:t for t in lexer_conf.tokens}
+
+        self._prepare_match(lexer_conf)
+
+        self.parser = xearley.Parser(parser_conf,
+                                    self.match,
+                                    resolve_ambiguity=get_ambiguity_resolver(options),
+                                    ignore=lexer_conf.ignore,
+                                    predict_all=options.earley__predict_all,
+                                    **kw
+                                    )
+
+    def match(self, term, text, index=0):
+        return self.regexps[term.name].match(text, index)
+
+    def _prepare_match(self, lexer_conf):
+        self.regexps = {}
+        for t in lexer_conf.tokens:
+            regexp = t.pattern.to_regexp()
+            try:
+                width = get_regexp_width(regexp)[0]
+            except ValueError:
+                raise ValueError("Bad regexp in token %s: %s" % (t.name, regexp))
+            else:
+                if width == 0:
+                    raise ValueError("Dynamic Earley doesn't allow zero-width regexps", t)
+
+            self.regexps[t.name] = re.compile(regexp)
+
+    def parse(self, text):
+        return self.parser.parse(text)
+
+class XEarley_CompleteLex(XEarley):
+    def __init__(self, *args, **kw):
+        super(self).__init__(*args, complete_lex=True, **kw)
+
+
+
+class CYK(WithLexer):
+
+    def __init__(self, lexer_conf, parser_conf, options=None):
+        self.init_traditional_lexer(lexer_conf)
+
+        self._analysis = GrammarAnalyzer(parser_conf)
+        self._parser = cyk.Parser(parser_conf.rules, parser_conf.start)
+
+        self._postprocess = {}
+        for rule in parser_conf.rules:
+            a = rule.alias
+            self._postprocess[a] = a if callable(a) else (a and getattr(parser_conf.callback, a))
+
+    def parse(self, text):
+        tokens = list(self.lex(text))
+        parse = self._parser.parse(tokens)
+        parse = self._transform(parse)
+        return parse
+
+    def _transform(self, tree):
+        subtrees = list(tree.iter_subtrees())
+        for subtree in subtrees:
+            subtree.children = [self._apply_callback(c) if isinstance(c, Tree) else c for c in subtree.children]
+
+        return self._apply_callback(tree)
+
+    def _apply_callback(self, tree):
+        children = tree.children
+        callback = self._postprocess[tree.rule.alias]
+        assert callback, tree.rule.alias
+        r = callback(children)
+        return r
+
+
+def get_frontend(parser, lexer):
+    if parser=='lalr':
+        if lexer is None:
+            raise ValueError('The LALR parser requires use of a lexer')
+        elif lexer == 'standard':
+            return LALR_TraditionalLexer
+        elif lexer == 'contextual':
+            return LALR_ContextualLexer
+        elif issubclass(lexer, Lexer):
+            return partial(LALR_CustomLexer, lexer)
+        else:
+            raise ValueError('Unknown lexer: %s' % lexer)
+    elif parser=='earley':
+        if lexer=='standard':
+            return Earley
+        elif lexer=='dynamic':
+            return XEarley
+        elif lexer=='dynamic_complete':
+            return XEarley_CompleteLex
+        elif lexer=='contextual':
+            raise ValueError('The Earley parser does not support the contextual parser')
+        else:
+            raise ValueError('Unknown lexer: %s' % lexer)
+    elif parser == 'cyk':
+        if lexer == 'standard':
+            return CYK
+        else:
+            raise ValueError('CYK parser requires using standard parser.')
+    else:
+        raise ValueError('Unknown parser: %s' % parser)
+
+
+

venv/lib/python3.6/site-packages/lark/parsers/ablalr.py

@@ -0,0 +1,98 @@
+"""This module implements an ABLALR(1) Parser
+
+Ambiguous Backtracking Look-Ahead(1) LR(0)
+"""
+# Author: Erez Shinan (2017)
+# Email : erezshin@gmail.com
+
+from ..common import UnexpectedToken
+
+from .lalr_analysis import LALR_Analyzer, Shift
+
+class Parser:
+    def __init__(self, parser_conf):
+        assert all(r.options is None or r.options.priority is None
+                   for r in parser_conf.rules), "LALR doesn't yet support prioritization"
+        self.analysis = analysis = LALR_Analyzer(parser_conf)
+        analysis.compute_lookahead()
+        callbacks = {rule: getattr(parser_conf.callback, rule.alias or rule.origin, None)
+                          for rule in analysis.rules}
+
+        self.parser_conf = parser_conf
+        self.parser = _Parser(analysis.parse_table, callbacks)
+        self.parse = self.parser.parse
+
+###{standalone
+
+class _Parser:
+    def __init__(self, parse_table, callbacks):
+        self.states = parse_table.states
+        self.start_state = parse_table.start_state
+        self.end_state = parse_table.end_state
+        self.callbacks = callbacks
+
+    def parse(self, seq, set_state=None):
+        i = 0
+        token = None
+        stream = iter(seq)
+        states = self.states
+
+        state_stack = [self.start_state]
+        value_stack = []
+
+        if set_state: set_state(self.start_state)
+
+        def get_action(key):
+            state = state_stack[-1]
+            try:
+                return states[state][key]
+            except KeyError:
+                expected = states[state].keys()
+
+                raise UnexpectedToken(token, expected, seq, i)
+
+        def reduce(rule):
+            size = len(rule.expansion)
+            if size:
+                s = value_stack[-size:]
+                del state_stack[-size:]
+                del value_stack[-size:]
+            else:
+                s = []
+
+            value = self.callbacks[rule](s)
+
+            _action, new_state = get_action(rule.origin)
+            assert _action is Shift
+            state_stack.append(new_state)
+            value_stack.append(value)
+
+        # Main LALR-parser loop
+        try:
+            token = next(stream)
+            i += 1
+            while True:
+                action, arg = get_action(token.type)
+                assert arg != self.end_state
+
+                if action is Shift:
+                    state_stack.append(arg)
+                    value_stack.append(token)
+                    if set_state: set_state(arg)
+                    token = next(stream)
+                    i += 1
+                else:
+                    reduce(arg)
+        except StopIteration:
+            pass
+
+        while True:
+            _action, arg = get_action('$END')
+            if _action is Shift:
+                assert arg == self.end_state
+                val ,= value_stack
+                return val
+            else:
+                reduce(arg)
+
+###}

venv/lib/python3.6/site-packages/lark/parsers/cyk.py

@@ -0,0 +1,343 @@
+"""This module implements a CYK parser."""
+
+# Author: https://github.com/ehudt (2018)
+#
+# Adapted by Erez
+
+
+from collections import defaultdict
+import itertools
+
+from ..exceptions import ParseError
+from ..lexer import Token
+from ..tree import Tree
+from ..grammar import Terminal as T, NonTerminal as NT, Symbol
+
+try:
+    xrange
+except NameError:
+    xrange = range
+
+def match(t, s):
+    assert isinstance(t, T)
+    return t.name == s.type
+
+
+class Rule(object):
+    """Context-free grammar rule."""
+
+    def __init__(self, lhs, rhs, weight, alias):
+        super(Rule, self).__init__()
+        assert isinstance(lhs, NT), lhs
+        assert all(isinstance(x, NT) or isinstance(x, T) for x in rhs), rhs
+        self.lhs = lhs
+        self.rhs = rhs
+        self.weight = weight
+        self.alias = alias
+
+    def __str__(self):
+        return '%s -> %s' % (str(self.lhs), ' '.join(str(x) for x in self.rhs))
+
+    def __repr__(self):
+        return str(self)
+
+    def __hash__(self):
+        return hash((self.lhs, tuple(self.rhs)))
+
+    def __eq__(self, other):
+        return self.lhs == other.lhs and self.rhs == other.rhs
+
+    def __ne__(self, other):
+        return not (self == other)
+
+
+class Grammar(object):
+    """Context-free grammar."""
+
+    def __init__(self, rules):
+        self.rules = frozenset(rules)
+
+    def __eq__(self, other):
+        return self.rules == other.rules
+
+    def __str__(self):
+        return '\n' + '\n'.join(sorted(repr(x) for x in self.rules)) + '\n'
+
+    def __repr__(self):
+        return str(self)
+
+
+# Parse tree data structures
+class RuleNode(object):
+    """A node in the parse tree, which also contains the full rhs rule."""
+
+    def __init__(self, rule, children, weight=0):
+        self.rule = rule
+        self.children = children
+        self.weight = weight
+
+    def __repr__(self):
+        return 'RuleNode(%s, [%s])' % (repr(self.rule.lhs), ', '.join(str(x) for x in self.children))
+
+
+
+class Parser(object):
+    """Parser wrapper."""
+
+    def __init__(self, rules, start):
+        super(Parser, self).__init__()
+        self.orig_rules = {rule.alias: rule for rule in rules}
+        rules = [self._to_rule(rule) for rule in rules]
+        self.grammar = to_cnf(Grammar(rules))
+        self.start = NT(start)
+
+    def _to_rule(self, lark_rule):
+        """Converts a lark rule, (lhs, rhs, callback, options), to a Rule."""
+        assert isinstance(lark_rule.origin, NT)
+        assert all(isinstance(x, Symbol) for x in lark_rule.expansion)
+        return Rule(
+            lark_rule.origin, lark_rule.expansion,
+            weight=lark_rule.options.priority if lark_rule.options and lark_rule.options.priority else 0,
+            alias=lark_rule.alias)
+
+    def parse(self, tokenized):  # pylint: disable=invalid-name
+        """Parses input, which is a list of tokens."""
+        table, trees = _parse(tokenized, self.grammar)
+        # Check if the parse succeeded.
+        if all(r.lhs != self.start for r in table[(0, len(tokenized) - 1)]):
+            raise ParseError('Parsing failed.')
+        parse = trees[(0, len(tokenized) - 1)][self.start]
+        return self._to_tree(revert_cnf(parse))
+
+    def _to_tree(self, rule_node):
+        """Converts a RuleNode parse tree to a lark Tree."""
+        orig_rule = self.orig_rules[rule_node.rule.alias]
+        children = []
+        for child in rule_node.children:
+            if isinstance(child, RuleNode):
+                children.append(self._to_tree(child))
+            else:
+                assert isinstance(child.name, Token)
+                children.append(child.name)
+        t = Tree(orig_rule.origin, children)
+        t.rule=orig_rule
+        return t
+
+
+def print_parse(node, indent=0):
+    if isinstance(node, RuleNode):
+        print(' ' * (indent * 2) + str(node.rule.lhs))
+        for child in node.children:
+            print_parse(child, indent + 1)
+    else:
+        print(' ' * (indent * 2) + str(node.s))
+
+
+def _parse(s, g):
+    """Parses sentence 's' using CNF grammar 'g'."""
+    # The CYK table. Indexed with a 2-tuple: (start pos, end pos)
+    table = defaultdict(set)
+    # Top-level structure is similar to the CYK table. Each cell is a dict from
+    # rule name to the best (lightest) tree for that rule.
+    trees = defaultdict(dict)
+    # Populate base case with existing terminal production rules
+    for i, w in enumerate(s):
+        for terminal, rules in g.terminal_rules.items():
+            if match(terminal, w):
+                for rule in rules:
+                    table[(i, i)].add(rule)
+                    if (rule.lhs not in trees[(i, i)] or
+                        rule.weight < trees[(i, i)][rule.lhs].weight):
+                        trees[(i, i)][rule.lhs] = RuleNode(rule, [T(w)], weight=rule.weight)
+
+    # Iterate over lengths of sub-sentences
+    for l in xrange(2, len(s) + 1):
+        # Iterate over sub-sentences with the given length
+        for i in xrange(len(s) - l + 1):
+            # Choose partition of the sub-sentence in [1, l)
+            for p in xrange(i + 1, i + l):
+                span1 = (i, p - 1)
+                span2 = (p, i + l - 1)
+                for r1, r2 in itertools.product(table[span1], table[span2]):
+                    for rule in g.nonterminal_rules.get((r1.lhs, r2.lhs), []):
+                        table[(i, i + l - 1)].add(rule)
+                        r1_tree = trees[span1][r1.lhs]
+                        r2_tree = trees[span2][r2.lhs]
+                        rule_total_weight = rule.weight + r1_tree.weight + r2_tree.weight
+                        if (rule.lhs not in trees[(i, i + l - 1)]
+                            or rule_total_weight < trees[(i, i + l - 1)][rule.lhs].weight):
+                            trees[(i, i + l - 1)][rule.lhs] = RuleNode(rule, [r1_tree, r2_tree], weight=rule_total_weight)
+    return table, trees
+
+
+# This section implements context-free grammar converter to Chomsky normal form.
+# It also implements a conversion of parse trees from its CNF to the original
+# grammar.
+# Overview:
+# Applies the following operations in this order:
+# * TERM: Eliminates non-solitary terminals from all rules
+# * BIN: Eliminates rules with more than 2 symbols on their right-hand-side.
+# * UNIT: Eliminates non-terminal unit rules
+#
+# The following grammar characteristics aren't featured:
+# * Start symbol appears on RHS
+# * Empty rules (epsilon rules)
+
+
+class CnfWrapper(object):
+    """CNF wrapper for grammar.
+
+  Validates that the input grammar is CNF and provides helper data structures.
+  """
+
+    def __init__(self, grammar):
+        super(CnfWrapper, self).__init__()
+        self.grammar = grammar
+        self.rules = grammar.rules
+        self.terminal_rules = defaultdict(list)
+        self.nonterminal_rules = defaultdict(list)
+        for r in self.rules:
+            # Validate that the grammar is CNF and populate auxiliary data structures.
+            assert isinstance(r.lhs, NT), r
+            if len(r.rhs) not in [1, 2]:
+                raise ParseError("CYK doesn't support empty rules")
+            if len(r.rhs) == 1 and isinstance(r.rhs[0], T):
+                self.terminal_rules[r.rhs[0]].append(r)
+            elif len(r.rhs) == 2 and all(isinstance(x, NT) for x in r.rhs):
+                self.nonterminal_rules[tuple(r.rhs)].append(r)
+            else:
+                assert False, r
+
+    def __eq__(self, other):
+        return self.grammar == other.grammar
+
+    def __repr__(self):
+        return repr(self.grammar)
+
+
+class UnitSkipRule(Rule):
+    """A rule that records NTs that were skipped during transformation."""
+
+    def __init__(self, lhs, rhs, skipped_rules, weight, alias):
+        super(UnitSkipRule, self).__init__(lhs, rhs, weight, alias)
+        self.skipped_rules = skipped_rules
+
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and self.skipped_rules == other.skipped_rules
+
+    __hash__ = Rule.__hash__
+
+
+def build_unit_skiprule(unit_rule, target_rule):
+    skipped_rules = []
+    if isinstance(unit_rule, UnitSkipRule):
+        skipped_rules += unit_rule.skipped_rules
+    skipped_rules.append(target_rule)
+    if isinstance(target_rule, UnitSkipRule):
+        skipped_rules += target_rule.skipped_rules
+    return UnitSkipRule(unit_rule.lhs, target_rule.rhs, skipped_rules,
+                      weight=unit_rule.weight + target_rule.weight, alias=unit_rule.alias)
+
+
+def get_any_nt_unit_rule(g):
+    """Returns a non-terminal unit rule from 'g', or None if there is none."""
+    for rule in g.rules:
+        if len(rule.rhs) == 1 and isinstance(rule.rhs[0], NT):
+            return rule
+    return None
+
+
+def _remove_unit_rule(g, rule):
+    """Removes 'rule' from 'g' without changing the langugage produced by 'g'."""
+    new_rules = [x for x in g.rules if x != rule]
+    refs = [x for x in g.rules if x.lhs == rule.rhs[0]]
+    new_rules += [build_unit_skiprule(rule, ref) for ref in refs]
+    return Grammar(new_rules)
+
+
+def _split(rule):
+    """Splits a rule whose len(rhs) > 2 into shorter rules."""
+    rule_str = str(rule.lhs) + '__' + '_'.join(str(x) for x in rule.rhs)
+    rule_name = '__SP_%s' % (rule_str) + '_%d'
+    yield Rule(rule.lhs, [rule.rhs[0], NT(rule_name % 1)], weight=rule.weight, alias=rule.alias)
+    for i in xrange(1, len(rule.rhs) - 2):
+        yield Rule(NT(rule_name % i), [rule.rhs[i], NT(rule_name % (i + 1))], weight=0, alias='Split')
+    yield Rule(NT(rule_name % (len(rule.rhs) - 2)), rule.rhs[-2:], weight=0, alias='Split')
+
+
+def _term(g):
+    """Applies the TERM rule on 'g' (see top comment)."""
+    all_t = {x for rule in g.rules for x in rule.rhs if isinstance(x, T)}
+    t_rules = {t: Rule(NT('__T_%s' % str(t)), [t], weight=0, alias='Term') for t in all_t}
+    new_rules = []
+    for rule in g.rules:
+        if len(rule.rhs) > 1 and any(isinstance(x, T) for x in rule.rhs):
+            new_rhs = [t_rules[x].lhs if isinstance(x, T) else x for x in rule.rhs]
+            new_rules.append(Rule(rule.lhs, new_rhs, weight=rule.weight, alias=rule.alias))
+            new_rules.extend(v for k, v in t_rules.items() if k in rule.rhs)
+        else:
+            new_rules.append(rule)
+    return Grammar(new_rules)
+
+
+def _bin(g):
+    """Applies the BIN rule to 'g' (see top comment)."""
+    new_rules = []
+    for rule in g.rules:
+        if len(rule.rhs) > 2:
+            new_rules += _split(rule)
+        else:
+            new_rules.append(rule)
+    return Grammar(new_rules)
+
+
+def _unit(g):
+    """Applies the UNIT rule to 'g' (see top comment)."""
+    nt_unit_rule = get_any_nt_unit_rule(g)
+    while nt_unit_rule:
+        g = _remove_unit_rule(g, nt_unit_rule)
+        nt_unit_rule = get_any_nt_unit_rule(g)
+    return g
+
+
+def to_cnf(g):
+    """Creates a CNF grammar from a general context-free grammar 'g'."""
+    g = _unit(_bin(_term(g)))
+    return CnfWrapper(g)
+
+
+def unroll_unit_skiprule(lhs, orig_rhs, skipped_rules, children, weight, alias):
+    if not skipped_rules:
+        return RuleNode(Rule(lhs, orig_rhs, weight=weight, alias=alias), children, weight=weight)
+    else:
+        weight = weight - skipped_rules[0].weight
+        return RuleNode(
+            Rule(lhs, [skipped_rules[0].lhs], weight=weight, alias=alias), [
+                unroll_unit_skiprule(skipped_rules[0].lhs, orig_rhs,
+                                skipped_rules[1:], children,
+                                skipped_rules[0].weight, skipped_rules[0].alias)
+            ], weight=weight)
+
+
+def revert_cnf(node):
+    """Reverts a parse tree (RuleNode) to its original non-CNF form (Node)."""
+    if isinstance(node, T):
+        return node
+    # Reverts TERM rule.
+    if node.rule.lhs.name.startswith('__T_'):
+        return node.children[0]
+    else:
+        children = []
+        for child in map(revert_cnf, node.children):
+            # Reverts BIN rule.
+            if isinstance(child, RuleNode) and child.rule.lhs.name.startswith('__SP_'):
+                children += child.children
+            else:
+                children.append(child)
+        # Reverts UNIT rule.
+        if isinstance(node.rule, UnitSkipRule):
+            return unroll_unit_skiprule(node.rule.lhs, node.rule.rhs,
+                                    node.rule.skipped_rules, children,
+                                    node.rule.weight, node.rule.alias)
+        else:
+            return RuleNode(node.rule, children)

venv/lib/python3.6/site-packages/lark/parsers/earley.py

@@ -0,0 +1,239 @@
+"This module implements an Earley Parser"
+
+# The parser uses a parse-forest to keep track of derivations and ambiguations.
+# When the parse ends successfully, a disambiguation stage resolves all ambiguity
+# (right now ambiguity resolution is not developed beyond the needs of lark)
+# Afterwards the parse tree is reduced (transformed) according to user callbacks.
+# I use the no-recursion version of Transformer, because the tree might be
+# deeper than Python's recursion limit (a bit absurd, but that's life)
+#
+# The algorithm keeps track of each state set, using a corresponding Column instance.
+# Column keeps track of new items using NewsList instances.
+#
+# Author: Erez Shinan (2017)
+# Email : erezshin@gmail.com
+
+from ..tree import Tree
+from ..visitors import Transformer_InPlace, v_args
+from ..exceptions import ParseError, UnexpectedToken
+from .grammar_analysis import GrammarAnalyzer
+from ..grammar import NonTerminal
+
+
+class Derivation(Tree):
+    def __init__(self, rule, items=None):
+        Tree.__init__(self, 'drv', items or [])
+        self.meta.rule = rule
+        self._hash = None
+
+    def _pretty_label(self):    # Nicer pretty for debugging the parser
+        return self.meta.rule.origin.name if self.meta.rule else self.data
+
+    def __hash__(self):
+        if self._hash is None:
+            self._hash = Tree.__hash__(self)
+        return self._hash
+
+class Item(object):
+    "An Earley Item, the atom of the algorithm."
+
+    def __init__(self, rule, ptr, start, tree):
+        self.rule = rule
+        self.ptr = ptr
+        self.start = start
+        self.tree = tree if tree is not None else Derivation(self.rule)
+
+    @property
+    def expect(self):
+        return self.rule.expansion[self.ptr]
+
+    @property
+    def is_complete(self):
+        return self.ptr == len(self.rule.expansion)
+
+    def advance(self, tree):
+        assert self.tree.data == 'drv'
+        new_tree = Derivation(self.rule, self.tree.children + [tree])
+        return self.__class__(self.rule, self.ptr+1, self.start, new_tree)
+
+    def __eq__(self, other):
+        return self.start is other.start and self.ptr == other.ptr and self.rule == other.rule
+
+    def __hash__(self):
+        return hash((self.rule, self.ptr, id(self.start)))   # Always runs Derivation.__hash__
+
+    def __repr__(self):
+        before = list(map(str, self.rule.expansion[:self.ptr]))
+        after = list(map(str, self.rule.expansion[self.ptr:]))
+        return '<(%d) %s : %s * %s>' % (id(self.start), self.rule.origin, ' '.join(before), ' '.join(after))
+
+class NewsList(list):
+    "Keeps track of newly added items (append-only)"
+
+    def __init__(self, initial=None):
+        list.__init__(self, initial or [])
+        self.last_iter = 0
+
+    def get_news(self):
+        i = self.last_iter
+        self.last_iter = len(self)
+        return self[i:]
+
+
+
+class Column:
+    "An entry in the table, aka Earley Chart. Contains lists of items."
+    def __init__(self, i, FIRST, predict_all=False):
+        self.i = i
+        self.to_reduce = NewsList()
+        self.to_predict = NewsList()
+        self.to_scan = []
+        self.item_count = 0
+        self.FIRST = FIRST
+
+        self.predicted = set()
+        self.completed = {}
+        self.predict_all = predict_all
+
+    def add(self, items):
+        """Sort items into scan/predict/reduce newslists
+
+        Makes sure only unique items are added.
+        """
+        for item in items:
+
+            item_key = item, item.tree  # Elsewhere, tree is not part of the comparison
+            if item.is_complete:
+                # XXX Potential bug: What happens if there's ambiguity in an empty rule?
+                if item.rule.expansion and item_key in self.completed:
+                    old_tree = self.completed[item_key].tree
+                    if old_tree == item.tree:
+                        is_empty = not self.FIRST[item.rule.origin]
+                        if not is_empty:
+                            continue
+
+                    if old_tree.data != '_ambig':
+                        new_tree = old_tree.copy()
+                        new_tree.meta.rule = old_tree.meta.rule
+                        old_tree.set('_ambig', [new_tree])
+                        old_tree.meta.rule = None    # No longer a 'drv' node
+
+                    if item.tree.children[0] is old_tree:   # XXX a little hacky!
+                        raise ParseError("Infinite recursion in grammar! (Rule %s)" % item.rule)
+
+                    if item.tree not in old_tree.children:
+                        old_tree.children.append(item.tree)
+                    # old_tree.children.append(item.tree)
+                else:
+                    self.completed[item_key] = item
+                self.to_reduce.append(item)
+            else:
+                if item.expect.is_term:
+                    self.to_scan.append(item)
+                else:
+                    k = item_key if self.predict_all else item
+                    if k in self.predicted:
+                        continue
+                    self.predicted.add(k)
+                    self.to_predict.append(item)
+
+            self.item_count += 1    # Only count if actually added
+
+
+    def __bool__(self):
+        return bool(self.item_count)
+    __nonzero__ = __bool__  # Py2 backwards-compatibility
+
+class Parser:
+    def __init__(self, parser_conf, term_matcher, resolve_ambiguity=None):
+        analysis = GrammarAnalyzer(parser_conf)
+        self.parser_conf = parser_conf
+        self.resolve_ambiguity = resolve_ambiguity
+
+        self.FIRST = analysis.FIRST
+        self.postprocess = {}
+        self.predictions = {}
+        for rule in parser_conf.rules:
+            self.postprocess[rule] = rule.alias if callable(rule.alias) else getattr(parser_conf.callback, rule.alias)
+            self.predictions[rule.origin] = [x.rule for x in analysis.expand_rule(rule.origin)]
+
+        self.term_matcher = term_matcher
+
+
+    def parse(self, stream, start_symbol=None):
+        # Define parser functions
+        start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
+
+        _Item = Item
+        match = self.term_matcher
+
+        def predict(nonterm, column):
+            assert not nonterm.is_term, nonterm
+            return [_Item(rule, 0, column, None) for rule in self.predictions[nonterm]]
+
+        def complete(item):
+            name = item.rule.origin
+            return [i.advance(item.tree) for i in item.start.to_predict if i.expect == name]
+
+        def predict_and_complete(column):
+            while True:
+                to_predict = {x.expect for x in column.to_predict.get_news()
+                              if x.ptr}  # if not part of an already predicted batch
+                to_reduce = set(column.to_reduce.get_news())
+                if not (to_predict or to_reduce):
+                    break
+
+                for nonterm in to_predict:
+                    column.add( predict(nonterm, column) )
+
+                for item in to_reduce:
+                    new_items = list(complete(item))
+                    if item in new_items:
+                        raise ParseError('Infinite recursion detected! (rule %s)' % item.rule)
+                    column.add(new_items)
+
+        def scan(i, token, column):
+            next_set = Column(i, self.FIRST)
+            next_set.add(item.advance(token) for item in column.to_scan if match(item.expect, token))
+
+            if not next_set:
+                expect = {i.expect.name for i in column.to_scan}
+                raise UnexpectedToken(token, expect, considered_rules=set(column.to_scan))
+
+            return next_set
+
+        # Main loop starts
+        column0 = Column(0, self.FIRST)
+        column0.add(predict(start_symbol, column0))
+
+        column = column0
+        for i, token in enumerate(stream):
+            predict_and_complete(column)
+            column = scan(i, token, column)
+
+        predict_and_complete(column)
+
+        # Parse ended. Now build a parse tree
+        solutions = [n.tree for n in column.to_reduce
+                     if n.rule.origin==start_symbol and n.start is column0]
+
+        if not solutions:
+            raise ParseError('Incomplete parse: Could not find a solution to input')
+        elif len(solutions) == 1:
+            tree = solutions[0]
+        else:
+            tree = Tree('_ambig', solutions)
+
+        if self.resolve_ambiguity:
+            tree = self.resolve_ambiguity(tree)
+
+        return ApplyCallbacks(self.postprocess).transform(tree)
+
+
+class ApplyCallbacks(Transformer_InPlace):
+    def __init__(self, postprocess):
+        self.postprocess = postprocess
+
+    @v_args(meta=True)
+    def drv(self, children, meta):
+        return self.postprocess[meta.rule](children)

venv/lib/python3.6/site-packages/lark/parsers/grammar_analysis.py

@@ -0,0 +1,149 @@
+
+from ..utils import bfs, fzset, classify
+from ..exceptions import GrammarError
+from ..grammar import Rule, Terminal, NonTerminal
+
+
+class RulePtr(object):
+    __slots__ = ('rule', 'index')
+
+    def __init__(self, rule, index):
+        assert isinstance(rule, Rule)
+        assert index <= len(rule.expansion)
+        self.rule = rule
+        self.index = index
+
+    def __repr__(self):
+        before = self.rule.expansion[:self.index]
+        after = self.rule.expansion[self.index:]
+        return '<%s : %s * %s>' % (self.rule.origin, ' '.join(before), ' '.join(after))
+
+    @property
+    def next(self):
+        return self.rule.expansion[self.index]
+
+    def advance(self, sym):
+        assert self.next == sym
+        return RulePtr(self.rule, self.index+1)
+
+    @property
+    def is_satisfied(self):
+        return self.index == len(self.rule.expansion)
+
+    def __eq__(self, other):
+        return self.rule == other.rule and self.index == other.index
+    def __hash__(self):
+        return hash((self.rule, self.index))
+
+
+def update_set(set1, set2):
+    if not set2:
+        return False
+
+    copy = set(set1)
+    set1 |= set2
+    return set1 != copy
+
+def calculate_sets(rules):
+    """Calculate FOLLOW sets.
+
+    Adapted from: http://lara.epfl.ch/w/cc09:algorithm_for_first_and_follow_sets"""
+    symbols = {sym for rule in rules for sym in rule.expansion} | {rule.origin for rule in rules}
+
+    # foreach grammar rule X ::= Y(1) ... Y(k)
+    # if k=0 or {Y(1),...,Y(k)} subset of NULLABLE then
+    #   NULLABLE = NULLABLE union {X}
+    # for i = 1 to k
+    #   if i=1 or {Y(1),...,Y(i-1)} subset of NULLABLE then
+    #     FIRST(X) = FIRST(X) union FIRST(Y(i))
+    #   for j = i+1 to k
+    #     if i=k or {Y(i+1),...Y(k)} subset of NULLABLE then
+    #       FOLLOW(Y(i)) = FOLLOW(Y(i)) union FOLLOW(X)
+    #     if i+1=j or {Y(i+1),...,Y(j-1)} subset of NULLABLE then
+    #       FOLLOW(Y(i)) = FOLLOW(Y(i)) union FIRST(Y(j))
+    # until none of NULLABLE,FIRST,FOLLOW changed in last iteration
+
+    NULLABLE = set()
+    FIRST = {}
+    FOLLOW = {}
+    for sym in symbols:
+        FIRST[sym]={sym} if sym.is_term else set()
+        FOLLOW[sym]=set()
+
+    # Calculate NULLABLE and FIRST
+    changed = True
+    while changed:
+        changed = False
+
+        for rule in rules:
+            if set(rule.expansion) <= NULLABLE:
+                if update_set(NULLABLE, {rule.origin}):
+                    changed = True
+
+            for i, sym in enumerate(rule.expansion):
+                if set(rule.expansion[:i]) <= NULLABLE:
+                    if update_set(FIRST[rule.origin], FIRST[sym]):
+                        changed = True
+
+    # Calculate FOLLOW
+    changed = True
+    while changed:
+        changed = False
+
+        for rule in rules:
+            for i, sym in enumerate(rule.expansion):
+                if i==len(rule.expansion)-1 or set(rule.expansion[i+1:]) <= NULLABLE:
+                    if update_set(FOLLOW[sym], FOLLOW[rule.origin]):
+                        changed = True
+
+                for j in range(i+1, len(rule.expansion)):
+                    if set(rule.expansion[i+1:j]) <= NULLABLE:
+                        if update_set(FOLLOW[sym], FIRST[rule.expansion[j]]):
+                            changed = True
+
+    return FIRST, FOLLOW, NULLABLE
+
+
+class GrammarAnalyzer(object):
+    def __init__(self, parser_conf, debug=False):
+        self.debug = debug
+
+        rules = parser_conf.rules + [Rule(NonTerminal('$root'), [NonTerminal(parser_conf.start), Terminal('$END')])]
+        self.rules_by_origin = classify(rules, lambda r: r.origin)
+
+        assert len(rules) == len(set(rules))
+        for r in rules:
+            for sym in r.expansion:
+                if not (sym.is_term or sym in self.rules_by_origin):
+                    raise GrammarError("Using an undefined rule: %s" % sym) # TODO test validation
+
+        self.start_state = self.expand_rule(NonTerminal('$root'))
+
+        self.FIRST, self.FOLLOW, self.NULLABLE = calculate_sets(rules)
+
+    def expand_rule(self, rule):
+        "Returns all init_ptrs accessible by rule (recursive)"
+        init_ptrs = set()
+        def _expand_rule(rule):
+            assert not rule.is_term, rule
+
+            for r in self.rules_by_origin[rule]:
+                init_ptr = RulePtr(r, 0)
+                init_ptrs.add(init_ptr)
+
+                if r.expansion: # if not empty rule
+                    new_r = init_ptr.next
+                    if not new_r.is_term:
+                        yield new_r
+
+        for _ in bfs([rule], _expand_rule):
+            pass
+
+        return fzset(init_ptrs)
+
+    def _first(self, r):
+        if r.is_term:
+            return {r}
+        else:
+            return {rp.next for rp in self.expand_rule(r) if rp.next.is_term}
+

venv/lib/python3.6/site-packages/lark/parsers/lalr_analysis.py

@@ -0,0 +1,109 @@
+"""This module builds a LALR(1) transition-table for lalr_parser.py
+
+For now, shift/reduce conflicts are automatically resolved as shifts.
+"""
+
+# Author: Erez Shinan (2017)
+# Email : erezshin@gmail.com
+
+import logging
+from collections import defaultdict
+
+from ..utils import classify, classify_bool, bfs, fzset
+from ..exceptions import GrammarError
+
+from .grammar_analysis import GrammarAnalyzer, Terminal
+
+class Action:
+    def __init__(self, name):
+        self.name = name
+    def __str__(self):
+        return self.name
+    def __repr__(self):
+        return str(self)
+
+Shift = Action('Shift')
+Reduce = Action('Reduce')
+
+class ParseTable:
+    def __init__(self, states, start_state, end_state):
+        self.states = states
+        self.start_state = start_state
+        self.end_state = end_state
+
+class IntParseTable(ParseTable):
+
+    @classmethod
+    def from_ParseTable(cls, parse_table):
+        enum = list(parse_table.states)
+        state_to_idx = {s:i for i,s in enumerate(enum)}
+        int_states = {}
+
+        for s, la in parse_table.states.items():
+            la = {k:(v[0], state_to_idx[v[1]]) if v[0] is Shift else v
+                  for k,v in la.items()}
+            int_states[ state_to_idx[s] ] = la
+
+
+        start_state = state_to_idx[parse_table.start_state]
+        end_state = state_to_idx[parse_table.end_state]
+        return cls(int_states, start_state, end_state)
+
+
+
+
+class LALR_Analyzer(GrammarAnalyzer):
+
+    def compute_lookahead(self):
+        self.end_states = []
+
+        self.states = {}
+        def step(state):
+            lookahead = defaultdict(list)
+            sat, unsat = classify_bool(state, lambda rp: rp.is_satisfied)
+            for rp in sat:
+                for term in self.FOLLOW.get(rp.rule.origin, ()):
+                    lookahead[term].append((Reduce, rp.rule))
+
+            d = classify(unsat, lambda rp: rp.next)
+            for sym, rps in d.items():
+                rps = {rp.advance(sym) for rp in rps}
+
+                for rp in set(rps):
+                    if not rp.is_satisfied and not rp.next.is_term:
+                        rps |= self.expand_rule(rp.next)
+
+                new_state = fzset(rps)
+                lookahead[sym].append((Shift, new_state))
+                if sym == Terminal('$END'):
+                    self.end_states.append( new_state )
+                yield new_state
+
+            for k, v in lookahead.items():
+                if len(v) > 1:
+                    if self.debug:
+                        logging.warn("Shift/reduce conflict for %s: %s. Resolving as shift.", k, v)
+                    for x in v:
+                        # XXX resolving shift/reduce into shift, like PLY
+                        # Give a proper warning
+                        if x[0] is Shift:
+                            lookahead[k] = [x]
+
+            for k, v in lookahead.items():
+                if not len(v) == 1:
+                    raise GrammarError("Collision in %s: %s" %(k, ', '.join(['\n  * %s: %s' % x for x in v])))
+
+            self.states[state] = {k.name:v[0] for k, v in lookahead.items()}
+
+        for _ in bfs([self.start_state], step):
+            pass
+
+        self.end_state ,= self.end_states
+
+        self._parse_table = ParseTable(self.states, self.start_state, self.end_state)
+
+        if self.debug:
+            self.parse_table = self._parse_table
+        else:
+            self.parse_table = IntParseTable.from_ParseTable(self._parse_table)
+

venv/lib/python3.6/site-packages/lark/parsers/lalr_parser.py

@@ -0,0 +1,91 @@
+"""This module implements a LALR(1) Parser
+"""
+# Author: Erez Shinan (2017)
+# Email : erezshin@gmail.com
+from ..exceptions import UnexpectedToken
+from ..lexer import Token
+
+from .lalr_analysis import LALR_Analyzer, Shift
+
+class Parser:
+    def __init__(self, parser_conf):
+        assert all(r.options is None or r.options.priority is None
+                   for r in parser_conf.rules), "LALR doesn't yet support prioritization"
+        analysis = LALR_Analyzer(parser_conf)
+        analysis.compute_lookahead()
+        callbacks = {rule: getattr(parser_conf.callback, rule.alias or rule.origin, None)
+                          for rule in parser_conf.rules}
+
+        self._parse_table = analysis.parse_table
+        self.parser_conf = parser_conf
+        self.parser = _Parser(analysis.parse_table, callbacks)
+        self.parse = self.parser.parse
+
+###{standalone
+
+class _Parser:
+    def __init__(self, parse_table, callbacks):
+        self.states = parse_table.states
+        self.start_state = parse_table.start_state
+        self.end_state = parse_table.end_state
+        self.callbacks = callbacks
+
+    def parse(self, seq, set_state=None):
+        token = None
+        stream = iter(seq)
+        states = self.states
+
+        state_stack = [self.start_state]
+        value_stack = []
+
+        if set_state: set_state(self.start_state)
+
+        def get_action(token):
+            state = state_stack[-1]
+            try:
+                return states[state][token.type]
+            except KeyError:
+                expected = [s for s in states[state].keys() if s.isupper()]
+                raise UnexpectedToken(token, expected, state=state)
+
+        def reduce(rule):
+            size = len(rule.expansion)
+            if size:
+                s = value_stack[-size:]
+                del state_stack[-size:]
+                del value_stack[-size:]
+            else:
+                s = []
+
+            value = self.callbacks[rule](s)
+
+            _action, new_state = states[state_stack[-1]][rule.origin.name]
+            assert _action is Shift
+            state_stack.append(new_state)
+            value_stack.append(value)
+
+        # Main LALR-parser loop
+        for token in stream:
+            while True:
+                action, arg = get_action(token)
+                assert arg != self.end_state
+
+                if action is Shift:
+                    state_stack.append(arg)
+                    value_stack.append(token)
+                    if set_state: set_state(arg)
+                    break # next token
+                else:
+                    reduce(arg)
+
+        token = Token.new_borrow_pos('$END', '', token) if token else Token('$END', '', 0, 1, 1)
+        while True:
+            _action, arg = get_action(token)
+            if _action is Shift:
+                assert arg == self.end_state
+                val ,= value_stack
+                return val
+            else:
+                reduce(arg)
+
+###}

venv/lib/python3.6/site-packages/lark/parsers/resolve_ambig.py

@@ -0,0 +1,109 @@
+from ..utils import compare
+from functools import cmp_to_key
+
+from ..tree import Tree
+
+
+# Standard ambiguity resolver (uses comparison)
+#
+# Author: Erez Sh
+
+def _compare_rules(rule1, rule2):
+    return -compare( len(rule1.expansion), len(rule2.expansion))
+
+def _sum_priority(tree):
+    p = 0
+
+    for n in tree.iter_subtrees():
+        try:
+            p += n.meta.rule.options.priority or 0
+        except AttributeError:
+            pass
+
+    return p
+
+def _compare_priority(tree1, tree2):
+    tree1.iter_subtrees()
+
+def _compare_drv(tree1, tree2):
+    try:
+        rule1 = tree1.meta.rule
+    except AttributeError:
+        rule1 = None
+
+    try:
+        rule2 = tree2.meta.rule
+    except AttributeError:
+        rule2 = None
+
+    if None == rule1 == rule2:
+        return compare(tree1, tree2)
+    elif rule1 is None:
+        return -1
+    elif rule2 is None:
+        return 1
+
+    assert tree1.data != '_ambig'
+    assert tree2.data != '_ambig'
+
+    p1 = _sum_priority(tree1)
+    p2 = _sum_priority(tree2)
+    c = (p1 or p2) and compare(p1, p2)
+    if c:
+        return c
+
+    c = _compare_rules(tree1.meta.rule, tree2.meta.rule)
+    if c:
+        return c
+
+    # rules are "equal", so compare trees
+    if len(tree1.children) == len(tree2.children):
+        for t1, t2 in zip(tree1.children, tree2.children):
+            c = _compare_drv(t1, t2)
+            if c:
+                return c
+
+    return compare(len(tree1.children), len(tree2.children))
+
+
+def _standard_resolve_ambig(tree):
+    assert tree.data == '_ambig'
+    key_f = cmp_to_key(_compare_drv)
+    best = max(tree.children, key=key_f)
+    assert best.data == 'drv'
+    tree.set('drv', best.children)
+    tree.meta.rule = best.meta.rule   # needed for applying callbacks
+
+def standard_resolve_ambig(tree):
+    for ambig in tree.find_data('_ambig'):
+        _standard_resolve_ambig(ambig)
+
+    return tree
+
+
+
+
+# Anti-score Sum
+#
+# Author: Uriva (https://github.com/uriva)
+
+def _antiscore_sum_drv(tree):
+    if not isinstance(tree, Tree):
+        return 0
+
+    assert tree.data != '_ambig'
+
+    return _sum_priority(tree)
+
+def _antiscore_sum_resolve_ambig(tree):
+    assert tree.data == '_ambig'
+    best = min(tree.children, key=_antiscore_sum_drv)
+    assert best.data == 'drv'
+    tree.set('drv', best.children)
+    tree.meta.rule = best.meta.rule   # needed for applying callbacks
+
+def antiscore_sum_resolve_ambig(tree):
+    for ambig in tree.find_data('_ambig'):
+        _antiscore_sum_resolve_ambig(ambig)
+
+    return tree

venv/lib/python3.6/site-packages/lark/parsers/xearley.py

@@ -0,0 +1,158 @@
+"This module implements an experimental Earley Parser with a dynamic lexer"
+
+# The parser uses a parse-forest to keep track of derivations and ambiguations.
+# When the parse ends successfully, a disambiguation stage resolves all ambiguity
+# (right now ambiguity resolution is not developed beyond the needs of lark)
+# Afterwards the parse tree is reduced (transformed) according to user callbacks.
+# I use the no-recursion version of Transformer and Visitor, because the tree might be
+# deeper than Python's recursion limit (a bit absurd, but that's life)
+#
+# The algorithm keeps track of each state set, using a corresponding Column instance.
+# Column keeps track of new items using NewsList instances.
+#
+# Instead of running a lexer beforehand, or using a costy char-by-char method, this parser
+# uses regular expressions by necessity, achieving high-performance while maintaining all of
+# Earley's power in parsing any CFG.
+#
+#
+# Author: Erez Shinan (2017)
+# Email : erezshin@gmail.com
+
+from collections import defaultdict
+
+from ..exceptions import ParseError, UnexpectedCharacters
+from ..lexer import Token
+from ..tree import Tree
+from .grammar_analysis import GrammarAnalyzer
+from ..grammar import NonTerminal, Terminal
+
+from .earley import ApplyCallbacks, Item, Column
+
+
+class Parser:
+    def __init__(self,  parser_conf, term_matcher, resolve_ambiguity=None, ignore=(), predict_all=False, complete_lex=False):
+        self.analysis = GrammarAnalyzer(parser_conf)
+        self.parser_conf = parser_conf
+        self.resolve_ambiguity = resolve_ambiguity
+        self.ignore = [Terminal(t) for t in ignore]
+        self.predict_all = predict_all
+        self.complete_lex = complete_lex
+
+        self.FIRST = self.analysis.FIRST
+        self.postprocess = {}
+        self.predictions = {}
+        for rule in parser_conf.rules:
+            self.postprocess[rule] = getattr(parser_conf.callback, rule.alias)
+            self.predictions[rule.origin] = [x.rule for x in self.analysis.expand_rule(rule.origin)]
+
+        self.term_matcher = term_matcher
+
+
+    def parse(self, stream, start_symbol=None):
+        # Define parser functions
+        start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
+        delayed_matches = defaultdict(list)
+        match = self.term_matcher
+
+        text_line = 1
+        text_column = 1
+
+        def predict(nonterm, column):
+            assert not nonterm.is_term, nonterm
+            return [Item(rule, 0, column, None) for rule in self.predictions[nonterm]]
+
+        def complete(item):
+            name = item.rule.origin
+            return [i.advance(item.tree) for i in item.start.to_predict if i.expect == name]
+
+        def predict_and_complete(column):
+            while True:
+                to_predict = {x.expect for x in column.to_predict.get_news()
+                              if x.ptr}  # if not part of an already predicted batch
+                to_reduce = column.to_reduce.get_news()
+                if not (to_predict or to_reduce):
+                    break
+
+                for nonterm in to_predict:
+                    column.add( predict(nonterm, column) )
+                for item in to_reduce:
+                    new_items = list(complete(item))
+                    if item in new_items:
+                        raise ParseError('Infinite recursion detected! (rule %s)' % item.rule)
+                    column.add(new_items)
+
+        def scan(i, column):
+            to_scan = column.to_scan
+
+            for x in self.ignore:
+                m = match(x, stream, i)
+                if m:
+                    delayed_matches[m.end()] += set(to_scan)
+                    delayed_matches[m.end()] += set(column.to_reduce)
+
+                    # TODO add partial matches for ignore too?
+                    # s = m.group(0)
+                    # for j in range(1, len(s)):
+                    #     m = x.match(s[:-j])
+                    #     if m:
+                    #         delayed_matches[m.end()] += to_scan
+
+            for item in to_scan:
+                m = match(item.expect, stream, i)
+                if m:
+                    t = Token(item.expect.name, m.group(0), i, text_line, text_column)
+                    delayed_matches[m.end()].append(item.advance(t))
+
+                    if self.complete_lex:
+                        s = m.group(0)
+                        for j in range(1, len(s)):
+                            m = match(item.expect, s[:-j])
+                            if m:
+                                t = Token(item.expect.name, m.group(0), i, text_line, text_column)
+                                delayed_matches[i+m.end()].append(item.advance(t))
+
+            next_set = Column(i+1, self.FIRST, predict_all=self.predict_all)
+            next_set.add(delayed_matches[i+1])
+            del delayed_matches[i+1]    # No longer needed, so unburden memory
+
+            if not next_set and not delayed_matches:
+                raise UnexpectedCharacters(stream, i, text_line, text_column, {item.expect for item in to_scan}, set(to_scan))
+
+            return next_set
+
+        # Main loop starts
+        column0 = Column(0, self.FIRST, predict_all=self.predict_all)
+        column0.add(predict(start_symbol, column0))
+
+        column = column0
+        for i, token in enumerate(stream):
+            predict_and_complete(column)
+            column = scan(i, column)
+
+            if token == '\n':
+                text_line += 1
+                text_column = 1
+            else:
+                text_column += 1
+
+        predict_and_complete(column)
+
+        # Parse ended. Now build a parse tree
+        solutions = [n.tree for n in column.to_reduce
+                     if n.rule.origin==start_symbol and n.start is column0]
+
+        if not solutions:
+            expected_tokens = [t.expect for t in column.to_scan]
+            raise ParseError('Unexpected end of input! Expecting a terminal of: %s' % expected_tokens)
+
+        elif len(solutions) == 1:
+            tree = solutions[0]
+        else:
+            tree = Tree('_ambig', solutions)
+
+        if self.resolve_ambiguity:
+            tree = self.resolve_ambiguity(tree)
+
+        return ApplyCallbacks(self.postprocess).transform(tree)
+
+

venv/lib/python3.6/site-packages/lark/reconstruct.py

@@ -0,0 +1,130 @@
+from collections import defaultdict
+
+from .tree import Tree
+from .visitors import Transformer_InPlace
+from .common import ParserConf
+from .lexer import Token, PatternStr
+from .parsers import earley, resolve_ambig
+from .grammar import Rule, Terminal, NonTerminal
+
+
+
+def is_discarded_terminal(t):
+    return t.is_term and t.filter_out
+
+def is_iter_empty(i):
+    try:
+        _ = next(i)
+        return False
+    except StopIteration:
+        return True
+
+class WriteTokensTransformer(Transformer_InPlace):
+    def __init__(self, tokens):
+        self.tokens = tokens
+
+    def __default__(self, data, children, meta):
+        #  if not isinstance(t, MatchTree):
+            #  return t
+        if not getattr(meta, 'match_tree', False):
+            return Tree(data, children)
+
+        iter_args = iter(children)
+        to_write = []
+        for sym in meta.orig_expansion:
+            if is_discarded_terminal(sym):
+                t = self.tokens[sym.name]
+                assert isinstance(t.pattern, PatternStr)
+                to_write.append(t.pattern.value)
+            else:
+                x = next(iter_args)
+                if isinstance(x, list):
+                    to_write += x
+                else:
+                    if isinstance(x, Token):
+                        assert Terminal(x.type) == sym, x
+                    else:
+                        assert NonTerminal(x.data) == sym, (sym, x)
+                    to_write.append(x)
+
+        assert is_iter_empty(iter_args)
+        return to_write
+
+
+class MatchTree(Tree):
+    pass
+
+class MakeMatchTree:
+    def __init__(self, name, expansion):
+        self.name = name
+        self.expansion = expansion
+
+    def __call__(self, args):
+        t = MatchTree(self.name, args)
+        t.meta.match_tree = True
+        t.meta.orig_expansion = self.expansion
+        return t
+
+class Reconstructor:
+    def __init__(self, parser):
+        # XXX TODO calling compile twice returns different results!
+        tokens, rules, _grammar_extra = parser.grammar.compile()
+
+        self.write_tokens = WriteTokensTransformer({t.name:t for t in tokens})
+        self.rules = list(self._build_recons_rules(rules))
+
+    def _build_recons_rules(self, rules):
+        expand1s = {r.origin for r in rules if r.options and r.options.expand1}
+
+        aliases = defaultdict(list)
+        for r in rules:
+            if r.alias:
+                aliases[r.origin].append( r.alias )
+
+        rule_names = {r.origin for r in rules}
+        nonterminals = {sym for sym in rule_names
+                       if sym.name.startswith('_') or sym in expand1s or sym in aliases }
+
+        for r in rules:
+            recons_exp = [sym if sym in nonterminals else Terminal(sym.name)
+                          for sym in r.expansion if not is_discarded_terminal(sym)]
+
+            # Skip self-recursive constructs
+            if recons_exp == [r.origin]:
+                continue
+
+            sym = NonTerminal(r.alias) if r.alias else r.origin
+
+            yield Rule(sym, recons_exp, MakeMatchTree(sym.name, r.expansion))
+
+        for origin, rule_aliases in aliases.items():
+            for alias in rule_aliases:
+                yield Rule(origin, [Terminal(alias)], MakeMatchTree(origin.name, [NonTerminal(alias)]))
+            
+            yield Rule(origin, [Terminal(origin.name)], MakeMatchTree(origin.name, [origin]))
+        
+
+
+    def _match(self, term, token):
+        if isinstance(token, Tree):
+            return Terminal(token.data) == term
+        elif isinstance(token, Token):
+            return term == Terminal(token.type)
+        assert False
+
+    def _reconstruct(self, tree):
+        # TODO: ambiguity?
+        parser = earley.Parser(ParserConf(self.rules, None, tree.data), self._match, resolve_ambiguity=resolve_ambig.standard_resolve_ambig)
+        unreduced_tree = parser.parse(tree.children)   # find a full derivation
+        assert unreduced_tree.data == tree.data
+        res = self.write_tokens.transform(unreduced_tree)
+        for item in res:
+            if isinstance(item, Tree):
+                for x in self._reconstruct(item):
+                    yield x
+            else:
+                yield item
+
+    def reconstruct(self, tree):
+        return ''.join(self._reconstruct(tree))
+

venv/lib/python3.6/site-packages/lark/reconstruct2.py

@@ -0,0 +1,107 @@
+import re
+from collections import defaultdict
+
+from .tree import Tree
+from .common import is_terminal, ParserConf, PatternStr, Terminal
+from .lexer import Token
+from .parsers import earley
+
+
+
+def is_discarded_terminal(t):
+    return is_terminal(t) and t.startswith('_')
+
+def is_iter_empty(i):
+    try:
+        _ = next(i)
+        return False
+    except StopIteration:
+        return True
+
+class Reconstructor:
+    def __init__(self, parser):
+        # Recreate the rules to assume a standard lexer
+        _tokens, rules, _grammar_extra = parser.grammar.compile(lexer='standard', start='whatever')
+        tokens = {t.name:t for t in _tokens}
+
+        token_res = {t.name:re.compile(t.pattern.to_regexp()) for t in _tokens}
+
+        class MatchTerminal(Terminal):
+            def match(self, other):
+                if isinstance(other, Tree):
+                    return False
+                return token_res[self.data].match(other) is not None
+
+        class MatchTree(Terminal):
+            def match(self, other):
+                try:
+                    return self.data == other.data
+                except AttributeError:
+                    return False
+
+        class WriteTokens:
+            def __init__(self, name, expansion):
+                self.name = name
+                self.expansion = expansion
+
+            def f(self, args):
+                args2 = iter(args)
+                to_write = []
+                for sym in self.expansion:
+                    if is_discarded_terminal(sym):
+                        t = tokens[sym]
+                        assert isinstance(t.pattern, PatternStr)
+                        to_write.append(t.pattern.value)
+                    else:
+                        x = next(args2)
+                        if isinstance(x, list):
+                            to_write += x
+                        else:
+                            if isinstance(x, Token):
+                                assert x.type == sym, x
+                            else:
+                                assert x.data == sym, x
+                            to_write.append(x)
+
+                assert is_iter_empty(args2)
+
+                return to_write
+
+        d = defaultdict(list)
+        for name, (expansions, _o) in rules.items():
+            for expansion, alias in expansions:
+                if alias:
+                    d[alias].append(expansion)
+                    d[name].append([alias])
+                else:
+                    d[name].append(expansion)
+
+        rules = []
+        expand1s = {name for name, (_x, options) in parser.rules.items()
+                    if options and options.expand1}
+
+        for name, expansions in d.items():
+            for expansion in expansions:
+                reduced = [sym if sym.startswith('_') or sym in expand1s else
+                           MatchTerminal(sym) if is_terminal(sym) else MatchTree(sym)
+                           for sym in expansion if not is_discarded_terminal(sym)]
+
+                rules.append((name, reduced, WriteTokens(name, expansion).f, None))
+        self.rules = rules
+
+
+    def _reconstruct(self, tree):
+        # TODO: ambiguity?
+        parser = earley.Parser(self.rules, tree.data, {})
+        res = parser.parse(tree.children)
+        for item in res:
+            if isinstance(item, Tree):
+                for x in self._reconstruct(item):
+                    yield x
+            else:
+                yield item
+
+    def reconstruct(self, tree):
+        return ''.join(self._reconstruct(tree))
+
+

venv/lib/python3.6/site-packages/lark/tools/nearley.py

@@ -0,0 +1,186 @@
+"Converts between Lark and Nearley grammars. Work in progress!"
+
+import os.path
+import sys
+import codecs
+
+
+from lark import Lark, InlineTransformer
+
+nearley_grammar = r"""
+    start: (ruledef|directive)+
+
+    directive: "@" NAME (STRING|NAME)
+             | "@" JS  -> js_code
+    ruledef: NAME "->" expansions
+           | NAME REGEXP "->" expansions -> macro
+    expansions: expansion ("|" expansion)*
+
+    expansion: expr+ js
+
+    ?expr: item [":" /[+*?]/]
+
+    ?item: rule|string|regexp
+         | "(" expansions ")"
+
+    rule: NAME
+    string: STRING
+    regexp: REGEXP
+    JS: /{%.*?%}/s
+    js: JS?
+
+    NAME: /[a-zA-Z_$]\w*/
+    COMMENT: /#[^\n]*/
+    REGEXP: /\[.*?\]/
+    STRING: /".*?"/
+
+    %import common.WS
+    %ignore WS
+    %ignore COMMENT
+
+    """
+
+nearley_grammar_parser = Lark(nearley_grammar, parser='earley', lexer='standard')
+
+def _get_rulename(name):
+    name = {'_': '_ws_maybe', '__':'_ws'}.get(name, name)
+    return 'n_' + name.replace('$', '__DOLLAR__').lower()
+
+class NearleyToLark(InlineTransformer):
+    def __init__(self):
+        self._count = 0
+        self.extra_rules = {}
+        self.extra_rules_rev = {}
+        self.alias_js_code = {}
+
+    def _new_function(self, code):
+        name = 'alias_%d' % self._count
+        self._count += 1
+
+        self.alias_js_code[name] = code
+        return name
+
+    def _extra_rule(self, rule):
+        if rule in self.extra_rules_rev:
+            return self.extra_rules_rev[rule]
+
+        name = 'xrule_%d' % len(self.extra_rules)
+        assert name not in self.extra_rules
+        self.extra_rules[name] = rule
+        self.extra_rules_rev[rule] = name
+        return name
+
+    def rule(self, name):
+        return _get_rulename(name)
+
+    def ruledef(self, name, exps):
+        return '!%s: %s' % (_get_rulename(name), exps)
+
+    def expr(self, item, op):
+        rule = '(%s)%s' % (item, op)
+        return self._extra_rule(rule)
+
+    def regexp(self, r):
+        return '/%s/' % r
+
+    def string(self, s):
+        return self._extra_rule(s)
+
+    def expansion(self, *x):
+        x, js = x[:-1], x[-1]
+        if js.children:
+            js_code ,= js.children
+            js_code = js_code[2:-2]
+            alias = '-> ' + self._new_function(js_code)
+        else:
+            alias = ''
+        return ' '.join(x) + alias
+
+    def expansions(self, *x):
+        return '%s' % ('\n    |'.join(x))
+
+    def start(self, *rules):
+        return '\n'.join(filter(None, rules))
+
+def _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, includes):
+    rule_defs = []
+
+    tree = nearley_grammar_parser.parse(g)
+    for statement in tree.children:
+        if statement.data == 'directive':
+            directive, arg = statement.children
+            if directive in ('builtin', 'include'):
+                folder = builtin_path if directive == 'builtin' else folder_path
+                path = os.path.join(folder, arg[1:-1])
+                if path not in includes:
+                    includes.add(path)
+                    with codecs.open(path, encoding='utf8') as f:
+                        text = f.read()
+                    rule_defs += _nearley_to_lark(text, builtin_path, n2l, js_code, os.path.abspath(os.path.dirname(path)), includes)
+            else:
+                assert False, directive
+        elif statement.data == 'js_code':
+            code ,= statement.children
+            code = code[2:-2]
+            js_code.append(code)
+        elif statement.data == 'macro':
+            pass    # TODO Add support for macros!
+        elif statement.data == 'ruledef':
+            rule_defs.append( n2l.transform(statement) )
+        else:
+            raise Exception("Unknown statement: %s" % statement)
+
+    return rule_defs
+
+
+def create_code_for_nearley_grammar(g, start, builtin_path, folder_path):
+    import js2py
+
+    emit_code = []
+    def emit(x=None):
+        if x:
+            emit_code.append(x)
+        emit_code.append('\n')
+
+    js_code = ['function id(x) {return x[0];}']
+    n2l = NearleyToLark()
+    rule_defs = _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, set())
+    lark_g = '\n'.join(rule_defs)
+    lark_g += '\n'+'\n'.join('!%s: %s' % item for item in n2l.extra_rules.items())
+
+    emit('from lark import Lark, Transformer')
+    emit()
+    emit('grammar = ' + repr(lark_g))
+    emit()
+
+    for alias, code in n2l.alias_js_code.items():
+        js_code.append('%s = (%s);' % (alias, code))
+
+    emit(js2py.translate_js('\n'.join(js_code)))
+    emit('class TransformNearley(Transformer):')
+    for alias in n2l.alias_js_code:
+        emit("    %s = var.get('%s').to_python()" % (alias, alias))
+    emit("    __default__ = lambda self, n, c, m: c if c else None")
+
+    emit()
+    emit('parser = Lark(grammar, start="n_%s")' % start)
+    emit('def parse(text):')
+    emit('    return TransformNearley().transform(parser.parse(text))')
+
+    return ''.join(emit_code)
+
+def main(fn, start, nearley_lib):
+    with codecs.open(fn, encoding='utf8') as f:
+        grammar = f.read()
+    return create_code_for_nearley_grammar(grammar, start, os.path.join(nearley_lib, 'builtin'), os.path.abspath(os.path.dirname(fn)))
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 4:
+        print("Reads Nearley grammar (with js functions) outputs an equivalent lark parser.")
+        print("Usage: %s <nearley_grammar_path> <start_rule> <nearley_lib_path>" % sys.argv[0])
+        sys.exit(1)
+
+    fn, start, nearley_lib = sys.argv[1:]
+
+    print(main(fn, start, nearley_lib))

venv/lib/python3.6/site-packages/lark/tools/standalone.py

@@ -0,0 +1,251 @@
+###{standalone
+#
+#
+#   Lark Stand-alone Generator Tool
+# ----------------------------------
+# Generates a stand-alone LALR(1) parser with a standard lexer
+#
+# Git:    https://github.com/erezsh/lark
+# Author: Erez Shinan (erezshin@gmail.com)
+#
+#
+#    >>> LICENSE
+#
+#    This tool and its generated code use a separate license from Lark.
+#
+#    It is licensed under GPLv2 or above.
+#
+#    If you wish to purchase a commercial license for this tool and its
+#    generated code, contact me via email.
+#
+#    If GPL is incompatible with your free or open-source project,
+#    contact me and we'll work it out (for free).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    See <http://www.gnu.org/licenses/>.
+#
+#
+###}
+
+import codecs
+import sys
+import os
+from pprint import pprint
+from os import path
+from collections import defaultdict
+
+import lark
+from lark import Lark
+from lark.parsers.lalr_analysis import Shift, Reduce
+
+from ..grammar import Rule
+
+_dir = path.dirname(__file__)
+_larkdir = path.join(_dir, path.pardir)
+
+
+EXTRACT_STANDALONE_FILES = [
+    'tools/standalone.py',
+    'exceptions.py',
+    'utils.py',
+    'tree.py',
+    'visitors.py',
+    'indenter.py',
+    'lexer.py',
+    'parse_tree_builder.py',
+    'parsers/lalr_parser.py',
+]
+
+
+def extract_sections(lines):
+    section = None
+    text = []
+    sections = defaultdict(list)
+    for l in lines:
+        if l.startswith('###'):
+            if l[3] == '{':
+                section = l[4:].strip()
+            elif l[3] == '}':
+                sections[section] += text
+                section = None
+                text = []
+            else:
+                raise ValueError(l)
+        elif section:
+            text.append(l)
+
+    return {name:''.join(text) for name, text in sections.items()}
+
+def _prepare_mres(mres):
+    return [(p.pattern,{i: t for i, t in d.items()}) for p,d in mres]
+
+class TraditionalLexerAtoms:
+    def __init__(self, lexer):
+        self.mres = _prepare_mres(lexer.mres)
+        self.newline_types = lexer.newline_types
+        self.ignore_types = lexer.ignore_types
+        self.callback = {name:_prepare_mres(c.mres)
+                         for name, c in lexer.callback.items()}
+
+    def print_python(self):
+        print('import re')
+        print('class LexerRegexps: pass')
+        print('NEWLINE_TYPES = %s' % self.newline_types)
+        print('IGNORE_TYPES = %s' % self.ignore_types)
+        self._print_python('lexer')
+
+    def _print_python(self, var_name):
+        print('MRES = (')
+        pprint(self.mres)
+        print(')')
+        print('LEXER_CALLBACK = (')
+        pprint(self.callback)
+        print(')')
+        print('lexer_regexps = LexerRegexps()')
+        print('lexer_regexps.mres = [(re.compile(p), d) for p, d in MRES]')
+        print('lexer_regexps.callback = {n: UnlessCallback([(re.compile(p), d) for p, d in mres])')
+        print('                          for n, mres in LEXER_CALLBACK.items()}')
+        print('%s = (lexer_regexps)' % var_name)
+
+
+class ContextualLexerAtoms:
+    def __init__(self, lexer):
+        self.lexer_atoms = {state: TraditionalLexerAtoms(lexer) for state, lexer in lexer.lexers.items()}
+        self.root_lexer_atoms = TraditionalLexerAtoms(lexer.root_lexer)
+
+    def print_python(self):
+        print('import re')
+        print('class LexerRegexps: pass')
+        print('NEWLINE_TYPES = %s' % self.root_lexer_atoms.newline_types)
+        print('IGNORE_TYPES = %s' % self.root_lexer_atoms.ignore_types)
+
+        print('LEXERS = {}')
+        for state, lexer_atoms in self.lexer_atoms.items():
+            lexer_atoms._print_python('LEXERS[%d]' % state)
+
+        print('class ContextualLexer:')
+        print('    def __init__(self):')
+        print('        self.lexers = LEXERS')
+        print('        self.set_parser_state(None)')
+        print('    def set_parser_state(self, state):')
+        print('        self.parser_state = state')
+        print('    def lex(self, stream):')
+        print('        newline_types = NEWLINE_TYPES')
+        print('        ignore_types = IGNORE_TYPES')
+        print('        lexers = LEXERS')
+        print('        l = _Lex(lexers[self.parser_state], self.parser_state)')
+        print('        for x in l.lex(stream, newline_types, ignore_types):')
+        print('            yield x')
+        print('            l.lexer = lexers[self.parser_state]')
+        print('            l.state = self.parser_state')
+
+        print('CON_LEXER = ContextualLexer()')
+        print('def lex(stream):')
+        print('    return CON_LEXER.lex(stream)')
+
+class GetRule:
+    def __init__(self, rule_id):
+        self.rule_id = rule_id
+
+    def __repr__(self):
+        return 'RULES[%d]' % self.rule_id
+
+rule_ids = {}
+token_types = {}
+
+def _get_token_type(token_type):
+    if token_type not in token_types:
+        token_types[token_type] = len(token_types)
+    return token_types[token_type]
+
+class ParserAtoms:
+    def __init__(self, parser):
+        self.parse_table = parser._parse_table
+
+    def print_python(self):
+        print('class ParseTable: pass')
+        print('parse_table = ParseTable()')
+        print('STATES = {')
+        for state, actions in self.parse_table.states.items():
+            print('  %r: %r,' % (state, {_get_token_type(token): ((1, rule_ids[arg]) if action is Reduce else (0, arg))
+                            for token, (action, arg) in actions.items()}))
+        print('}')
+        print('TOKEN_TYPES = (')
+        pprint({v:k for k, v in token_types.items()})
+        print(')')
+        print('parse_table.states = {s: {TOKEN_TYPES[t]: (a, RULES[x] if a is Reduce else x) for t, (a, x) in acts.items()}')
+        print('                      for s, acts in STATES.items()}')
+        print('parse_table.start_state = %s' % self.parse_table.start_state)
+        print('parse_table.end_state = %s' % self.parse_table.end_state)
+        print('class Lark_StandAlone:')
+        print('  def __init__(self, transformer=None, postlex=None):')
+        print('     callback = parse_tree_builder.create_callback(transformer=transformer)')
+        print('     callbacks = {rule: getattr(callback, rule.alias or rule.origin, None) for rule in RULES.values()}')
+        print('     self.parser = _Parser(parse_table, callbacks)')
+        print('     self.postlex = postlex')
+        print('  def parse(self, stream):')
+        print('     tokens = lex(stream)')
+        print('     sps = CON_LEXER.set_parser_state')
+        print('     if self.postlex: tokens = self.postlex.process(tokens)')
+        print('     return self.parser.parse(tokens, sps)')
+
+class TreeBuilderAtoms:
+    def __init__(self, lark):
+        self.rules = lark.rules
+        self.ptb = lark._parse_tree_builder
+
+    def print_python(self):
+        # print('class InlineTransformer: pass')
+        print('RULES = {')
+        for i, r in enumerate(self.rules):
+            rule_ids[r] = i
+            print('  %d: Rule(%r, [%s], %r, %r),' % (i, r.origin, ', '.join(s.fullrepr for s in r.expansion), self.ptb.user_aliases[r], r.options ))
+        print('}')
+        print('parse_tree_builder = ParseTreeBuilder(RULES.values(), Tree)')
+
+def main(fobj, start):
+    lark_inst = Lark(fobj, parser="lalr", lexer="contextual", start=start)
+
+    lexer_atoms = ContextualLexerAtoms(lark_inst.parser.lexer)
+    parser_atoms = ParserAtoms(lark_inst.parser.parser)
+    tree_builder_atoms = TreeBuilderAtoms(lark_inst)
+
+    print('# The file was automatically generated by Lark v%s' % lark.__version__)
+
+    for pyfile in EXTRACT_STANDALONE_FILES:
+        with open(os.path.join(_larkdir, pyfile)) as f:
+            print (extract_sections(f)['standalone'])
+
+    with open(os.path.join(_larkdir, 'grammar.py')) as grammar_py:
+        print(grammar_py.read())
+
+    print('Shift = 0')
+    print('Reduce = 1')
+    lexer_atoms.print_python()
+    tree_builder_atoms.print_python()
+    parser_atoms.print_python()
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print("Lark Stand-alone Generator Tool")
+        print("Usage: python -m lark.tools.standalone <grammar-file> [<start>]")
+        sys.exit(1)
+
+    if len(sys.argv) == 3:
+        fn, start = sys.argv[1:]
+    elif len(sys.argv) == 2:
+        fn, start = sys.argv[1], 'start'
+    else:
+        assert False, sys.argv
+
+    with codecs.open(fn, encoding='utf8') as f:
+        main(f, start)

venv/lib/python3.6/site-packages/lark/tree.py

@@ -0,0 +1,168 @@
+try:
+    from future_builtins import filter
+except ImportError:
+    pass
+
+from copy import deepcopy
+
+###{standalone
+class Meta:
+    pass
+
+class Tree(object):
+    def __init__(self, data, children, meta=None):
+        self.data = data
+        self.children = children
+        self._meta = meta
+
+    @property
+    def meta(self):
+        if self._meta is None:
+            self._meta = Meta()
+        return self._meta
+
+    def __repr__(self):
+        return 'Tree(%s, %s)' % (self.data, self.children)
+
+    def _pretty_label(self):
+        return self.data
+
+    def _pretty(self, level, indent_str):
+        if len(self.children) == 1 and not isinstance(self.children[0], Tree):
+            return [ indent_str*level, self._pretty_label(), '\t', '%s' % (self.children[0],), '\n']
+
+        l = [ indent_str*level, self._pretty_label(), '\n' ]
+        for n in self.children:
+            if isinstance(n, Tree):
+                l += n._pretty(level+1, indent_str)
+            else:
+                l += [ indent_str*(level+1), '%s' % (n,), '\n' ]
+
+        return l
+
+    def pretty(self, indent_str='  '):
+        return ''.join(self._pretty(0, indent_str))
+    def __eq__(self, other):
+        try:
+            return self.data == other.data and self.children == other.children
+        except AttributeError:
+            return False
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __hash__(self):
+        return hash((self.data, tuple(self.children)))
+###}
+
+    def expand_kids_by_index(self, *indices):
+        "Expand (inline) children at the given indices"
+        for i in sorted(indices, reverse=True): # reverse so that changing tail won't affect indices
+            kid = self.children[i]
+            self.children[i:i+1] = kid.children
+
+    def find_pred(self, pred):
+        "Find all nodes where pred(tree) == True"
+        return filter(pred, self.iter_subtrees())
+
+    def find_data(self, data):
+        "Find all nodes where tree.data == data"
+        return self.find_pred(lambda t: t.data == data)
+
+    def scan_values(self, pred):
+        for c in self.children:
+            if isinstance(c, Tree):
+                for t in c.scan_values(pred):
+                    yield t
+            else:
+                if pred(c):
+                    yield c
+
+    def iter_subtrees(self):
+        # TODO: Re-write as a more efficient version
+
+        visited = set()
+        q = [self]
+
+        l = []
+        while q:
+            subtree = q.pop()
+            l.append( subtree )
+            if id(subtree) in visited:
+                continue    # already been here from another branch
+            visited.add(id(subtree))
+            q += [c for c in subtree.children if isinstance(c, Tree)]
+
+        seen = set()
+        for x in reversed(l):
+            if id(x) not in seen:
+                yield x
+                seen.add(id(x))
+
+
+    def __deepcopy__(self, memo):
+        return type(self)(self.data, deepcopy(self.children, memo))
+
+    def copy(self):
+        return type(self)(self.data, self.children)
+    def set(self, data, children):
+        self.data = data
+        self.children = children
+
+    # XXX Deprecated! Here for backwards compatibility <0.6.0
+    @property
+    def line(self):
+        return self.meta.line
+    @property
+    def column(self):
+        return self.meta.column
+    @property
+    def end_line(self):
+        return self.meta.end_line
+    @property
+    def end_column(self):
+        return self.meta.end_column
+
+
+class SlottedTree(Tree):
+    __slots__ = 'data', 'children', 'rule', '_meta'
+
+
+def pydot__tree_to_png(tree, filename, rankdir="LR"):
+    """Creates a colorful image that represents the tree (data+children, without meta)
+
+    Possible values for `rankdir` are "TB", "LR", "BT", "RL", corresponding to
+    directed graphs drawn from top to bottom, from left to right, from bottom to
+    top, and from right to left, respectively. See:
+    https://www.graphviz.org/doc/info/attrs.html#k:rankdir
+    """
+
+    import pydot
+    graph = pydot.Dot(graph_type='digraph', rankdir=rankdir)
+
+    i = [0]
+
+    def new_leaf(leaf):
+        node = pydot.Node(i[0], label=repr(leaf))
+        i[0] += 1
+        graph.add_node(node)
+        return node
+
+    def _to_pydot(subtree):
+        color = hash(subtree.data) & 0xffffff
+        color |= 0x808080
+
+        subnodes = [_to_pydot(child) if isinstance(child, Tree) else new_leaf(child)
+                    for child in subtree.children]
+        node = pydot.Node(i[0], style="filled", fillcolor="#%x"%color, label=subtree.data)
+        i[0] += 1
+        graph.add_node(node)
+
+        for subnode in subnodes:
+            graph.add_edge(pydot.Edge(node, subnode))
+
+        return node
+
+    _to_pydot(tree)
+    graph.write_png(filename)
+

venv/lib/python3.6/site-packages/lark/utils.py

@@ -0,0 +1,120 @@
+import sys
+from collections import deque
+
+Py36 = (sys.version_info[:2] >= (3, 6))
+
+class fzset(frozenset):
+    def __repr__(self):
+        return '{%s}' % ', '.join(map(repr, self))
+
+
+def classify_bool(seq, pred):
+    true_elems = []
+    false_elems = []
+
+    for elem in seq:
+        if pred(elem):
+            true_elems.append(elem)
+        else:
+            false_elems.append(elem)
+
+    return true_elems, false_elems
+
+def classify(seq, key=None, value=None):
+    d = {}
+    for item in seq:
+        k = key(item) if (key is not None) else item
+        v = value(item) if (value is not None) else item
+        if k in d:
+            d[k].append(v)
+        else:
+            d[k] = [v]
+    return d
+
+def bfs(initial, expand):
+    open_q = deque(list(initial))
+    visited = set(open_q)
+    while open_q:
+        node = open_q.popleft()
+        yield node
+        for next_node in expand(node):
+            if next_node not in visited:
+                visited.add(next_node)
+                open_q.append(next_node)
+
+
+
+
+###{standalone
+try:
+    STRING_TYPE = basestring
+except NameError:   # Python 3
+    STRING_TYPE = str
+
+
+import types
+from functools import wraps, partial
+from contextlib import contextmanager
+
+Str = type(u'')
+
+def smart_decorator(f, create_decorator):
+    if isinstance(f, types.FunctionType):
+        return wraps(f)(create_decorator(f, True))
+
+    elif isinstance(f, (type, types.BuiltinFunctionType)):
+        return wraps(f)(create_decorator(f, False))
+
+    elif isinstance(f, types.MethodType):
+        return wraps(f)(create_decorator(f.__func__, True))
+
+    elif isinstance(f, partial):
+        # wraps does not work for partials in 2.7: https://bugs.python.org/issue3445
+        return create_decorator(f.__func__, True)
+
+    else:
+        return create_decorator(f.__func__.__call__, True)
+
+
+###}
+
+
+try:
+    from contextlib import suppress     # Python 3
+except ImportError:
+    @contextmanager
+    def suppress(*excs):
+        '''Catch and dismiss the provided exception
+
+        >>> x = 'hello'
+        >>> with suppress(IndexError):
+        ...     x = x[10]
+        >>> x
+        'hello'
+        '''
+        try:
+            yield
+        except excs:
+            pass
+
+
+
+
+try:
+    compare = cmp
+except NameError:
+    def compare(a, b):
+        if a == b:
+            return 0
+        elif a > b:
+            return 1
+        return -1
+
+
+import sre_parse
+import sre_constants
+def get_regexp_width(regexp):
+    try:
+        return sre_parse.parse(regexp).getwidth()
+    except sre_constants.error:
+        raise ValueError(regexp)

venv/lib/python3.6/site-packages/lark/visitors.py

@@ -0,0 +1,259 @@
+from functools import wraps
+
+from .utils import smart_decorator
+from .tree import Tree
+
+###{standalone
+from inspect import getmembers, getmro
+
+class Discard(Exception):
+    pass
+
+# Transformers
+
+class Transformer:
+    """Visits the tree recursively, starting with the leaves and finally the root (bottom-up)
+
+    Calls its methods (provided by user via inheritance) according to tree.data
+    The returned value replaces the old one in the structure.
+
+    Can be used to implement map or reduce.
+    """
+
+    def _call_userfunc(self, tree, new_children=None):
+        # Assumes tree is already transformed
+        children = new_children if new_children is not None else tree.children
+        try:
+            f = getattr(self, tree.data)
+        except AttributeError:
+            return self.__default__(tree.data, children, tree.meta)
+        else:
+            if getattr(f, 'meta', False):
+                return f(children, tree.meta)
+            elif getattr(f, 'inline', False):
+                return f(*children)
+            elif getattr(f, 'whole_tree', False):
+                if new_children is not None:
+                    raise NotImplementedError("Doesn't work with the base Transformer class")
+                return f(tree)
+            else:
+                return f(children)
+
+    def _transform_children(self, children):
+        for c in children:
+            try:
+                yield self._transform_tree(c) if isinstance(c, Tree) else c
+            except Discard:
+                pass
+
+    def _transform_tree(self, tree):
+        children = list(self._transform_children(tree.children))
+        return self._call_userfunc(tree, children)
+
+    def transform(self, tree):
+        return self._transform_tree(tree)
+
+    def __mul__(self, other):
+        return TransformerChain(self, other)
+
+    def __default__(self, data, children, meta):
+        "Default operation on tree (for override)"
+        return Tree(data, children, meta)
+
+    @classmethod
+    def _apply_decorator(cls, decorator, **kwargs):
+        mro = getmro(cls)
+        assert mro[0] is cls
+        libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)}
+        for name, value in getmembers(cls):
+            if name.startswith('_') or name in libmembers:
+                continue
+
+            if isinstance(cls.__dict__[name], (staticmethod, classmethod)):
+                kwargs['static'] = True
+            setattr(cls, name, decorator(value, **kwargs))
+        return cls
+
+
+class InlineTransformer(Transformer):   # XXX Deprecated
+    def _call_userfunc(self, tree, new_children=None):
+        # Assumes tree is already transformed
+        children = new_children if new_children is not None else tree.children
+        try:
+            f = getattr(self, tree.data)
+        except AttributeError:
+            return self.__default__(tree.data, children, tree.meta)
+        else:
+            return f(*children)
+
+
+class TransformerChain(object):
+    def __init__(self, *transformers):
+        self.transformers = transformers
+
+    def transform(self, tree):
+        for t in self.transformers:
+            tree = t.transform(tree)
+        return tree
+
+    def __mul__(self, other):
+        return TransformerChain(*self.transformers + (other,))
+
+
+class Transformer_InPlace(Transformer):
+    "Non-recursive. Changes the tree in-place instead of returning new instances"
+    def _transform_tree(self, tree):           # Cancel recursion
+        return self._call_userfunc(tree)
+
+    def transform(self, tree):
+        for subtree in tree.iter_subtrees():
+            subtree.children = list(self._transform_children(subtree.children))
+
+        return self._transform_tree(tree)
+
+
+class Transformer_InPlaceRecursive(Transformer):
+    "Recursive. Changes the tree in-place instead of returning new instances"
+    def _transform_tree(self, tree):
+        tree.children = list(self._transform_children(tree.children))
+        return self._call_userfunc(tree)
+
+
+
+# Visitors
+
+class VisitorBase:
+    def _call_userfunc(self, tree):
+        return getattr(self, tree.data, self.__default__)(tree)
+
+    def __default__(self, tree):
+        "Default operation on tree (for override)"
+        return tree
+
+
+class Visitor(VisitorBase):
+    """Bottom-up visitor, non-recursive
+
+    Visits the tree, starting with the leaves and finally the root (bottom-up)
+    Calls its methods (provided by user via inheritance) according to tree.data
+    """
+
+
+    def visit(self, tree):
+        for subtree in tree.iter_subtrees():
+            self._call_userfunc(subtree)
+        return tree
+
+class Visitor_Recursive(VisitorBase):
+    """Bottom-up visitor, recursive
+
+    Visits the tree, starting with the leaves and finally the root (bottom-up)
+    Calls its methods (provided by user via inheritance) according to tree.data
+    """
+
+    def visit(self, tree):
+        for child in tree.children:
+            if isinstance(child, Tree):
+                self.visit(child)
+
+        f = getattr(self, tree.data, self.__default__)
+        f(tree)
+        return tree
+
+
+
+def visit_children_decor(func):
+    "See Interpreter"
+    @wraps(func)
+    def inner(cls, tree):
+        values = cls.visit_children(tree)
+        return func(cls, values)
+    return inner
+
+
+class Interpreter:
+    """Top-down visitor, recursive
+
+    Visits the tree, starting with the root and finally the leaves (top-down)
+    Calls its methods (provided by user via inheritance) according to tree.data
+
+    Unlike Transformer and Visitor, the Interpreter doesn't automatically visit its sub-branches.
+    The user has to explicitly call visit_children, or use the @visit_children_decor
+    """
+    def visit(self, tree):
+        return getattr(self, tree.data)(tree)
+
+    def visit_children(self, tree):
+        return [self.visit(child) if isinstance(child, Tree) else child
+                for child in tree.children]
+
+    def __getattr__(self, name):
+        return self.__default__
+
+    def __default__(self, tree):
+        return self.visit_children(tree)
+
+
+
+
+# Decorators
+
+def _apply_decorator(obj, decorator, **kwargs):
+    try:
+        _apply = obj._apply_decorator
+    except AttributeError:
+        return decorator(obj, **kwargs)
+    else:
+        return _apply(decorator, **kwargs)
+
+
+
+def _inline_args__func(func):
+    @wraps(func)
+    def create_decorator(_f, with_self):
+        if with_self:
+            def f(self, children):
+                return _f(self, *children)
+        else:
+            def f(self, children):
+                return _f(*children)
+        return f
+
+    return smart_decorator(func, create_decorator)
+
+
+def inline_args(obj):   # XXX Deprecated
+    return _apply_decorator(obj, _inline_args__func)
+
+
+
+def _visitor_args_func_dec(func, inline=False, meta=False, whole_tree=False, static=False):
+    assert [whole_tree, meta, inline].count(True) <= 1
+    def create_decorator(_f, with_self):
+        if with_self:
+            def f(self, *args, **kwargs):
+                return _f(self, *args, **kwargs)
+        else:
+            def f(self, *args, **kwargs):
+                return _f(*args, **kwargs)
+        return f
+
+    if static:
+        f = wraps(func)(create_decorator(func, False))
+    else:
+        f = smart_decorator(func, create_decorator)
+    f.inline = inline
+    f.meta = meta
+    f.whole_tree = whole_tree
+    return f
+
+def v_args(inline=False, meta=False, tree=False):
+    "A convenience decorator factory, for modifying the behavior of user-supplied visitor methods"
+    if [tree, meta, inline].count(True) > 1:
+        raise ValueError("Visitor functions can either accept tree, or meta, or be inlined. These cannot be combined.")
+    def _visitor_args_dec(obj):
+        return _apply_decorator(obj, _visitor_args_func_dec, inline=inline, meta=meta, whole_tree=tree)
+    return _visitor_args_dec
+
+
+###}

venv/lib/python3.6/site-packages/lark_parser-0.6.5-py3.6.egg-info/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 1.1
+Name: lark-parser
+Version: 0.6.5
+Summary: a modern parsing library
+Home-page: https://github.com/erezsh/lark
+Author: Erez Shinan
+Author-email: erezshin@gmail.com
+License: MIT
+Download-URL: https://github.com/erezsh/lark/tarball/master
+Description: 
+        Lark is a modern general-purpose parsing library for Python.
+        
+        With Lark, you can parse any context-free grammar, efficiently, with very little code.
+        
+        Main Features:
+         - Builds a parse-tree (AST) automagically, based on the structure of the grammar
+         - Earley parser
+            - Can parse all context-free grammars
+            - Full support for ambiguous grammars
+         - LALR(1) parser
+            - Fast and light, competitive with PLY
+            - Can generate a stand-alone parser
+         - CYK parser, for highly ambiguous grammars
+         - EBNF grammar
+         - Unicode fully supported
+         - Python 2 & 3 compatible
+         - Automatic line & column tracking
+         - Standard library of terminals (strings, numbers, names, etc.)
+         - Import grammars from Nearley.js
+         - Extensive test suite
+         - And much more!
+        
+Keywords: Earley LALR parser parsing ast
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: General
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: License :: OSI Approved :: MIT License

venv/lib/python3.6/site-packages/lark_parser-0.6.5-py3.6.egg-info/SOURCES.txt

@@ -0,0 +1,80 @@
+LICENSE
+MANIFEST.in
+README.md
+setup.cfg
+setup.py
+docs/classes.md
+docs/comparison_memory.png
+docs/comparison_runtime.png
+docs/features.md
+docs/grammar.md
+docs/how_to_use.md
+docs/index.md
+docs/json_tutorial.md
+docs/lark_cheatsheet.pdf
+docs/philosophy.md
+docs/recipes.md
+docs/tree_construction.md
+examples/__init__.py
+examples/calc.py
+examples/comments.py
+examples/conf_earley.py
+examples/conf_lalr.py
+examples/custom_lexer.py
+examples/error_handling.py
+examples/error_reporting_lalr.py
+examples/fruitflies.png
+examples/fruitflies.py
+examples/indented_tree.py
+examples/json_parser.py
+examples/lark.lark
+examples/lark_grammar.py
+examples/python2.lark
+examples/python3.lark
+examples/python_parser.py
+examples/qscintilla_json.py
+examples/reconstruct_json.py
+examples/turtle_dsl.py
+lark/__init__.py
+lark/common.py
+lark/exceptions.py
+lark/grammar.py
+lark/indenter.py
+lark/lark.py
+lark/lexer.py
+lark/load_grammar.py
+lark/parse_tree_builder.py
+lark/parser_frontends.py
+lark/reconstruct.py
+lark/reconstruct2.py
+lark/tree.py
+lark/utils.py
+lark/visitors.py
+lark/grammars/common.lark
+lark/parsers/__init__.py
+lark/parsers/ablalr.py
+lark/parsers/cyk.py
+lark/parsers/earley.py
+lark/parsers/grammar_analysis.py
+lark/parsers/lalr_analysis.py
+lark/parsers/lalr_parser.py
+lark/parsers/resolve_ambig.py
+lark/parsers/xearley.py
+lark/tools/__init__.py
+lark/tools/nearley.py
+lark/tools/standalone.py
+lark_parser.egg-info/PKG-INFO
+lark_parser.egg-info/SOURCES.txt
+lark_parser.egg-info/dependency_links.txt
+lark_parser.egg-info/not-zip-safe
+lark_parser.egg-info/top_level.txt
+tests/__init__.py
+tests/__main__.py
+tests/test_parser.py
+tests/test_reconstructor.py
+tests/test_tools.py
+tests/test_trees.py
+tests/test_nearley/__init__.py
+tests/test_nearley/test_nearley.py
+tests/test_nearley/grammars/include_unicode.ne
+tests/test_nearley/grammars/unicode.ne

venv/lib/python3.6/site-packages/lark_parser-0.6.5-py3.6.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

venv/lib/python3.6/site-packages/lark_parser-0.6.5-py3.6.egg-info/installed-files.txt

@@ -0,0 +1,60 @@
+../lark/__init__.py
+../lark/__pycache__/__init__.cpython-36.pyc
+../lark/__pycache__/common.cpython-36.pyc
+../lark/__pycache__/exceptions.cpython-36.pyc
+../lark/__pycache__/grammar.cpython-36.pyc
+../lark/__pycache__/indenter.cpython-36.pyc
+../lark/__pycache__/lark.cpython-36.pyc
+../lark/__pycache__/lexer.cpython-36.pyc
+../lark/__pycache__/load_grammar.cpython-36.pyc
+../lark/__pycache__/parse_tree_builder.cpython-36.pyc
+../lark/__pycache__/parser_frontends.cpython-36.pyc
+../lark/__pycache__/reconstruct.cpython-36.pyc
+../lark/__pycache__/reconstruct2.cpython-36.pyc
+../lark/__pycache__/tree.cpython-36.pyc
+../lark/__pycache__/utils.cpython-36.pyc
+../lark/__pycache__/visitors.cpython-36.pyc
+../lark/common.py
+../lark/exceptions.py
+../lark/grammar.py
+../lark/grammars/common.lark
+../lark/indenter.py
+../lark/lark.py
+../lark/lexer.py
+../lark/load_grammar.py
+../lark/parse_tree_builder.py
+../lark/parser_frontends.py
+../lark/parsers/__init__.py
+../lark/parsers/__pycache__/__init__.cpython-36.pyc
+../lark/parsers/__pycache__/ablalr.cpython-36.pyc
+../lark/parsers/__pycache__/cyk.cpython-36.pyc
+../lark/parsers/__pycache__/earley.cpython-36.pyc
+../lark/parsers/__pycache__/grammar_analysis.cpython-36.pyc
+../lark/parsers/__pycache__/lalr_analysis.cpython-36.pyc
+../lark/parsers/__pycache__/lalr_parser.cpython-36.pyc
+../lark/parsers/__pycache__/resolve_ambig.cpython-36.pyc
+../lark/parsers/__pycache__/xearley.cpython-36.pyc
+../lark/parsers/ablalr.py
+../lark/parsers/cyk.py
+../lark/parsers/earley.py
+../lark/parsers/grammar_analysis.py
+../lark/parsers/lalr_analysis.py
+../lark/parsers/lalr_parser.py
+../lark/parsers/resolve_ambig.py
+../lark/parsers/xearley.py
+../lark/reconstruct.py
+../lark/reconstruct2.py
+../lark/tools/__init__.py
+../lark/tools/__pycache__/__init__.cpython-36.pyc
+../lark/tools/__pycache__/nearley.cpython-36.pyc
+../lark/tools/__pycache__/standalone.cpython-36.pyc
+../lark/tools/nearley.py
+../lark/tools/standalone.py
+../lark/tree.py
+../lark/utils.py
+../lark/visitors.py
+PKG-INFO
+SOURCES.txt
+dependency_links.txt
+not-zip-safe
+top_level.txt

venv/lib/python3.6/site-packages/lark_parser-0.6.5-py3.6.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

venv/lib/python3.6/site-packages/lark_parser-0.6.5-py3.6.egg-info/top_level.txt

@@ -0,0 +1 @@
+lark

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/EGG-INFO/PKG-INFO

@@ -0,0 +1,69 @@
+Metadata-Version: 2.1
+Name: pip
+Version: 10.0.1
+Summary: The PyPA recommended tool for installing Python packages.
+Home-page: https://pip.pypa.io/
+Author: The pip developers
+Author-email: python-virtualenv@groups.google.com
+License: MIT
+Description: pip
+        ===
+        
+        The `PyPA recommended`_ tool for installing Python packages.
+        
+        .. image:: https://img.shields.io/pypi/v/pip.svg
+           :target: https://pypi.org/project/pip/
+        
+        .. image:: https://img.shields.io/travis/pypa/pip/master.svg
+           :target: http://travis-ci.org/pypa/pip
+        
+        .. image:: https://img.shields.io/appveyor/ci/pypa/pip.svg
+           :target: https://ci.appveyor.com/project/pypa/pip/history
+        
+        .. image:: https://readthedocs.org/projects/pip/badge/?version=latest
+           :target: https://pip.pypa.io/en/latest
+        
+        * `Installation`_
+        * `Documentation`_
+        * `Changelog`_
+        * `GitHub Page`_
+        * `Issue Tracking`_
+        * `User mailing list`_
+        * `Dev mailing list`_
+        * User IRC: #pypa on Freenode.
+        * Dev IRC: #pypa-dev on Freenode.
+        
+        Code of Conduct
+        ---------------
+        
+        Everyone interacting in the pip project's codebases, issue trackers, chat
+        rooms and mailing lists is expected to follow the `PyPA Code of Conduct`_.
+        
+        .. _PyPA recommended: https://packaging.python.org/en/latest/current/
+        .. _Installation: https://pip.pypa.io/en/stable/installing.html
+        .. _Documentation: https://pip.pypa.io/en/stable/
+        .. _Changelog: https://pip.pypa.io/en/stable/news.html
+        .. _GitHub Page: https://github.com/pypa/pip
+        .. _Issue Tracking: https://github.com/pypa/pip/issues
+        .. _User mailing list: http://groups.google.com/group/python-virtualenv
+        .. _Dev mailing list: http://groups.google.com/group/pypa-dev
+        .. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/
+        
+Keywords: easy_install distutils setuptools egg virtualenv
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*
+Provides-Extra: testing

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/EGG-INFO/SOURCES.txt

@@ -0,0 +1,347 @@
+AUTHORS.txt
+LICENSE.txt
+MANIFEST.in
+NEWS.rst
+README.rst
+pyproject.toml
+setup.cfg
+setup.py
+docs/Makefile
+docs/__init__.py
+docs/conf.py
+docs/configuration.rst
+docs/cookbook.rst
+docs/development.rst
+docs/docutils.conf
+docs/index.rst
+docs/installing.rst
+docs/logic.rst
+docs/make.bat
+docs/news.rst
+docs/pipext.py
+docs/quickstart.rst
+docs/usage.rst
+docs/user_guide.rst
+docs/man/pip.rst
+docs/man/commands/check.rst
+docs/man/commands/config.rst
+docs/man/commands/download.rst
+docs/man/commands/freeze.rst
+docs/man/commands/hash.rst
+docs/man/commands/help.rst
+docs/man/commands/install.rst
+docs/man/commands/list.rst
+docs/man/commands/search.rst
+docs/man/commands/show.rst
+docs/man/commands/uninstall.rst
+docs/man/commands/wheel.rst
+docs/reference/index.rst
+docs/reference/pip.rst
+docs/reference/pip_check.rst
+docs/reference/pip_config.rst
+docs/reference/pip_download.rst
+docs/reference/pip_freeze.rst
+docs/reference/pip_hash.rst
+docs/reference/pip_install.rst
+docs/reference/pip_list.rst
+docs/reference/pip_search.rst
+docs/reference/pip_show.rst
+docs/reference/pip_uninstall.rst
+docs/reference/pip_wheel.rst
+src/pip/__init__.py
+src/pip/__main__.py
+src/pip.egg-info/PKG-INFO
+src/pip.egg-info/SOURCES.txt
+src/pip.egg-info/dependency_links.txt
+src/pip.egg-info/entry_points.txt
+src/pip.egg-info/not-zip-safe
+src/pip.egg-info/requires.txt
+src/pip.egg-info/top_level.txt
+src/pip/_internal/__init__.py
+src/pip/_internal/basecommand.py
+src/pip/_internal/baseparser.py
+src/pip/_internal/build_env.py
+src/pip/_internal/cache.py
+src/pip/_internal/cmdoptions.py
+src/pip/_internal/compat.py
+src/pip/_internal/configuration.py
+src/pip/_internal/download.py
+src/pip/_internal/exceptions.py
+src/pip/_internal/index.py
+src/pip/_internal/locations.py
+src/pip/_internal/pep425tags.py
+src/pip/_internal/resolve.py
+src/pip/_internal/status_codes.py
+src/pip/_internal/wheel.py
+src/pip/_internal/commands/__init__.py
+src/pip/_internal/commands/check.py
+src/pip/_internal/commands/completion.py
+src/pip/_internal/commands/configuration.py
+src/pip/_internal/commands/download.py
+src/pip/_internal/commands/freeze.py
+src/pip/_internal/commands/hash.py
+src/pip/_internal/commands/help.py
+src/pip/_internal/commands/install.py
+src/pip/_internal/commands/list.py
+src/pip/_internal/commands/search.py
+src/pip/_internal/commands/show.py
+src/pip/_internal/commands/uninstall.py
+src/pip/_internal/commands/wheel.py
+src/pip/_internal/models/__init__.py
+src/pip/_internal/models/index.py
+src/pip/_internal/operations/__init__.py
+src/pip/_internal/operations/check.py
+src/pip/_internal/operations/freeze.py
+src/pip/_internal/operations/prepare.py
+src/pip/_internal/req/__init__.py
+src/pip/_internal/req/req_file.py
+src/pip/_internal/req/req_install.py
+src/pip/_internal/req/req_set.py
+src/pip/_internal/req/req_uninstall.py
+src/pip/_internal/utils/__init__.py
+src/pip/_internal/utils/appdirs.py
+src/pip/_internal/utils/deprecation.py
+src/pip/_internal/utils/encoding.py
+src/pip/_internal/utils/filesystem.py
+src/pip/_internal/utils/glibc.py
+src/pip/_internal/utils/hashes.py
+src/pip/_internal/utils/logging.py
+src/pip/_internal/utils/misc.py
+src/pip/_internal/utils/outdated.py
+src/pip/_internal/utils/packaging.py
+src/pip/_internal/utils/setuptools_build.py
+src/pip/_internal/utils/temp_dir.py
+src/pip/_internal/utils/typing.py
+src/pip/_internal/utils/ui.py
+src/pip/_internal/vcs/__init__.py
+src/pip/_internal/vcs/bazaar.py
+src/pip/_internal/vcs/git.py
+src/pip/_internal/vcs/mercurial.py
+src/pip/_internal/vcs/subversion.py
+src/pip/_vendor/README.rst
+src/pip/_vendor/__init__.py
+src/pip/_vendor/appdirs.py
+src/pip/_vendor/distro.py
+src/pip/_vendor/ipaddress.py
+src/pip/_vendor/pyparsing.py
+src/pip/_vendor/retrying.py
+src/pip/_vendor/six.py
+src/pip/_vendor/vendor.txt
+src/pip/_vendor/cachecontrol/__init__.py
+src/pip/_vendor/cachecontrol/_cmd.py
+src/pip/_vendor/cachecontrol/adapter.py
+src/pip/_vendor/cachecontrol/cache.py
+src/pip/_vendor/cachecontrol/compat.py
+src/pip/_vendor/cachecontrol/controller.py
+src/pip/_vendor/cachecontrol/filewrapper.py
+src/pip/_vendor/cachecontrol/heuristics.py
+src/pip/_vendor/cachecontrol/serialize.py
+src/pip/_vendor/cachecontrol/wrapper.py
+src/pip/_vendor/cachecontrol/caches/__init__.py
+src/pip/_vendor/cachecontrol/caches/file_cache.py
+src/pip/_vendor/cachecontrol/caches/redis_cache.py
+src/pip/_vendor/certifi/__init__.py
+src/pip/_vendor/certifi/__main__.py
+src/pip/_vendor/certifi/cacert.pem
+src/pip/_vendor/certifi/core.py
+src/pip/_vendor/chardet/__init__.py
+src/pip/_vendor/chardet/big5freq.py
+src/pip/_vendor/chardet/big5prober.py
+src/pip/_vendor/chardet/chardistribution.py
+src/pip/_vendor/chardet/charsetgroupprober.py
+src/pip/_vendor/chardet/charsetprober.py
+src/pip/_vendor/chardet/codingstatemachine.py
+src/pip/_vendor/chardet/compat.py
+src/pip/_vendor/chardet/cp949prober.py
+src/pip/_vendor/chardet/enums.py
+src/pip/_vendor/chardet/escprober.py
+src/pip/_vendor/chardet/escsm.py
+src/pip/_vendor/chardet/eucjpprober.py
+src/pip/_vendor/chardet/euckrfreq.py
+src/pip/_vendor/chardet/euckrprober.py
+src/pip/_vendor/chardet/euctwfreq.py
+src/pip/_vendor/chardet/euctwprober.py
+src/pip/_vendor/chardet/gb2312freq.py
+src/pip/_vendor/chardet/gb2312prober.py
+src/pip/_vendor/chardet/hebrewprober.py
+src/pip/_vendor/chardet/jisfreq.py
+src/pip/_vendor/chardet/jpcntx.py
+src/pip/_vendor/chardet/langbulgarianmodel.py
+src/pip/_vendor/chardet/langcyrillicmodel.py
+src/pip/_vendor/chardet/langgreekmodel.py
+src/pip/_vendor/chardet/langhebrewmodel.py
+src/pip/_vendor/chardet/langhungarianmodel.py
+src/pip/_vendor/chardet/langthaimodel.py
+src/pip/_vendor/chardet/langturkishmodel.py
+src/pip/_vendor/chardet/latin1prober.py
+src/pip/_vendor/chardet/mbcharsetprober.py
+src/pip/_vendor/chardet/mbcsgroupprober.py
+src/pip/_vendor/chardet/mbcssm.py
+src/pip/_vendor/chardet/sbcharsetprober.py
+src/pip/_vendor/chardet/sbcsgroupprober.py
+src/pip/_vendor/chardet/sjisprober.py
+src/pip/_vendor/chardet/universaldetector.py
+src/pip/_vendor/chardet/utf8prober.py
+src/pip/_vendor/chardet/version.py
+src/pip/_vendor/chardet/cli/__init__.py
+src/pip/_vendor/chardet/cli/chardetect.py
+src/pip/_vendor/colorama/__init__.py
+src/pip/_vendor/colorama/ansi.py
+src/pip/_vendor/colorama/ansitowin32.py
+src/pip/_vendor/colorama/initialise.py
+src/pip/_vendor/colorama/win32.py
+src/pip/_vendor/colorama/winterm.py
+src/pip/_vendor/distlib/__init__.py
+src/pip/_vendor/distlib/compat.py
+src/pip/_vendor/distlib/database.py
+src/pip/_vendor/distlib/index.py
+src/pip/_vendor/distlib/locators.py
+src/pip/_vendor/distlib/manifest.py
+src/pip/_vendor/distlib/markers.py
+src/pip/_vendor/distlib/metadata.py
+src/pip/_vendor/distlib/resources.py
+src/pip/_vendor/distlib/scripts.py
+src/pip/_vendor/distlib/t32.exe
+src/pip/_vendor/distlib/t64.exe
+src/pip/_vendor/distlib/util.py
+src/pip/_vendor/distlib/version.py
+src/pip/_vendor/distlib/w32.exe
+src/pip/_vendor/distlib/w64.exe
+src/pip/_vendor/distlib/wheel.py
+src/pip/_vendor/distlib/_backport/__init__.py
+src/pip/_vendor/distlib/_backport/misc.py
+src/pip/_vendor/distlib/_backport/shutil.py
+src/pip/_vendor/distlib/_backport/sysconfig.cfg
+src/pip/_vendor/distlib/_backport/sysconfig.py
+src/pip/_vendor/distlib/_backport/tarfile.py
+src/pip/_vendor/html5lib/__init__.py
+src/pip/_vendor/html5lib/_ihatexml.py
+src/pip/_vendor/html5lib/_inputstream.py
+src/pip/_vendor/html5lib/_tokenizer.py
+src/pip/_vendor/html5lib/_utils.py
+src/pip/_vendor/html5lib/constants.py
+src/pip/_vendor/html5lib/html5parser.py
+src/pip/_vendor/html5lib/serializer.py
+src/pip/_vendor/html5lib/_trie/__init__.py
+src/pip/_vendor/html5lib/_trie/_base.py
+src/pip/_vendor/html5lib/_trie/datrie.py
+src/pip/_vendor/html5lib/_trie/py.py
+src/pip/_vendor/html5lib/filters/__init__.py
+src/pip/_vendor/html5lib/filters/alphabeticalattributes.py
+src/pip/_vendor/html5lib/filters/base.py
+src/pip/_vendor/html5lib/filters/inject_meta_charset.py
+src/pip/_vendor/html5lib/filters/lint.py
+src/pip/_vendor/html5lib/filters/optionaltags.py
+src/pip/_vendor/html5lib/filters/sanitizer.py
+src/pip/_vendor/html5lib/filters/whitespace.py
+src/pip/_vendor/html5lib/treeadapters/__init__.py
+src/pip/_vendor/html5lib/treeadapters/genshi.py
+src/pip/_vendor/html5lib/treeadapters/sax.py
+src/pip/_vendor/html5lib/treebuilders/__init__.py
+src/pip/_vendor/html5lib/treebuilders/base.py
+src/pip/_vendor/html5lib/treebuilders/dom.py
+src/pip/_vendor/html5lib/treebuilders/etree.py
+src/pip/_vendor/html5lib/treebuilders/etree_lxml.py
+src/pip/_vendor/html5lib/treewalkers/__init__.py
+src/pip/_vendor/html5lib/treewalkers/base.py
+src/pip/_vendor/html5lib/treewalkers/dom.py
+src/pip/_vendor/html5lib/treewalkers/etree.py
+src/pip/_vendor/html5lib/treewalkers/etree_lxml.py
+src/pip/_vendor/html5lib/treewalkers/genshi.py
+src/pip/_vendor/idna/__init__.py
+src/pip/_vendor/idna/codec.py
+src/pip/_vendor/idna/compat.py
+src/pip/_vendor/idna/core.py
+src/pip/_vendor/idna/idnadata.py
+src/pip/_vendor/idna/intranges.py
+src/pip/_vendor/idna/package_data.py
+src/pip/_vendor/idna/uts46data.py
+src/pip/_vendor/lockfile/__init__.py
+src/pip/_vendor/lockfile/linklockfile.py
+src/pip/_vendor/lockfile/mkdirlockfile.py
+src/pip/_vendor/lockfile/pidlockfile.py
+src/pip/_vendor/lockfile/sqlitelockfile.py
+src/pip/_vendor/lockfile/symlinklockfile.py
+src/pip/_vendor/msgpack/__init__.py
+src/pip/_vendor/msgpack/_version.py
+src/pip/_vendor/msgpack/exceptions.py
+src/pip/_vendor/msgpack/fallback.py
+src/pip/_vendor/packaging/__about__.py
+src/pip/_vendor/packaging/__init__.py
+src/pip/_vendor/packaging/_compat.py
+src/pip/_vendor/packaging/_structures.py
+src/pip/_vendor/packaging/markers.py
+src/pip/_vendor/packaging/requirements.py
+src/pip/_vendor/packaging/specifiers.py
+src/pip/_vendor/packaging/utils.py
+src/pip/_vendor/packaging/version.py
+src/pip/_vendor/pkg_resources/__init__.py
+src/pip/_vendor/pkg_resources/py31compat.py
+src/pip/_vendor/progress/__init__.py
+src/pip/_vendor/progress/bar.py
+src/pip/_vendor/progress/counter.py
+src/pip/_vendor/progress/helpers.py
+src/pip/_vendor/progress/spinner.py
+src/pip/_vendor/pytoml/__init__.py
+src/pip/_vendor/pytoml/core.py
+src/pip/_vendor/pytoml/parser.py
+src/pip/_vendor/pytoml/writer.py
+src/pip/_vendor/requests/__init__.py
+src/pip/_vendor/requests/__version__.py
+src/pip/_vendor/requests/_internal_utils.py
+src/pip/_vendor/requests/adapters.py
+src/pip/_vendor/requests/api.py
+src/pip/_vendor/requests/auth.py
+src/pip/_vendor/requests/certs.py
+src/pip/_vendor/requests/compat.py
+src/pip/_vendor/requests/cookies.py
+src/pip/_vendor/requests/exceptions.py
+src/pip/_vendor/requests/help.py
+src/pip/_vendor/requests/hooks.py
+src/pip/_vendor/requests/models.py
+src/pip/_vendor/requests/packages.py
+src/pip/_vendor/requests/sessions.py
+src/pip/_vendor/requests/status_codes.py
+src/pip/_vendor/requests/structures.py
+src/pip/_vendor/requests/utils.py
+src/pip/_vendor/urllib3/__init__.py
+src/pip/_vendor/urllib3/_collections.py
+src/pip/_vendor/urllib3/connection.py
+src/pip/_vendor/urllib3/connectionpool.py
+src/pip/_vendor/urllib3/exceptions.py
+src/pip/_vendor/urllib3/fields.py
+src/pip/_vendor/urllib3/filepost.py
+src/pip/_vendor/urllib3/poolmanager.py
+src/pip/_vendor/urllib3/request.py
+src/pip/_vendor/urllib3/response.py
+src/pip/_vendor/urllib3/contrib/__init__.py
+src/pip/_vendor/urllib3/contrib/appengine.py
+src/pip/_vendor/urllib3/contrib/ntlmpool.py
+src/pip/_vendor/urllib3/contrib/pyopenssl.py
+src/pip/_vendor/urllib3/contrib/securetransport.py
+src/pip/_vendor/urllib3/contrib/socks.py
+src/pip/_vendor/urllib3/contrib/_securetransport/__init__.py
+src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py
+src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py
+src/pip/_vendor/urllib3/packages/__init__.py
+src/pip/_vendor/urllib3/packages/ordered_dict.py
+src/pip/_vendor/urllib3/packages/six.py
+src/pip/_vendor/urllib3/packages/backports/__init__.py
+src/pip/_vendor/urllib3/packages/backports/makefile.py
+src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py
+src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py
+src/pip/_vendor/urllib3/util/__init__.py
+src/pip/_vendor/urllib3/util/connection.py
+src/pip/_vendor/urllib3/util/request.py
+src/pip/_vendor/urllib3/util/response.py
+src/pip/_vendor/urllib3/util/retry.py
+src/pip/_vendor/urllib3/util/selectors.py
+src/pip/_vendor/urllib3/util/ssl_.py
+src/pip/_vendor/urllib3/util/timeout.py
+src/pip/_vendor/urllib3/util/url.py
+src/pip/_vendor/urllib3/util/wait.py
+src/pip/_vendor/webencodings/__init__.py
+src/pip/_vendor/webencodings/labels.py
+src/pip/_vendor/webencodings/mklabels.py
+src/pip/_vendor/webencodings/tests.py
+src/pip/_vendor/webencodings/x_user_defined.py

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/EGG-INFO/dependency_links.txt

@@ -0,0 +1 @@
+

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/EGG-INFO/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+pip = pip._internal:main
+pip3 = pip._internal:main
+pip3.6 = pip._internal:main
+

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/EGG-INFO/not-zip-safe

@@ -0,0 +1 @@
+

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/EGG-INFO/requires.txt

@@ -0,0 +1,8 @@
+
+[testing]
+pytest
+mock
+pretend
+scripttest>=1.3
+virtualenv>=1.10
+freezegun

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/EGG-INFO/top_level.txt

@@ -0,0 +1 @@
+pip

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/__init__.py

@@ -0,0 +1 @@
+__version__ = "10.0.1"

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/__main__.py

@@ -0,0 +1,19 @@
+from __future__ import absolute_import
+
+import os
+import sys
+
+# If we are running from a wheel, add the wheel to sys.path
+# This allows the usage python pip-*.whl/pip install pip-*.whl
+if __package__ == '':
+    # __file__ is pip-*.whl/pip/__main__.py
+    # first dirname call strips of '/__main__.py', second strips off '/pip'
+    # Resulting path is the name of the wheel itself
+    # Add that to sys.path so we can import pip
+    path = os.path.dirname(os.path.dirname(__file__))
+    sys.path.insert(0, path)
+
+from pip._internal import main as _main  # noqa
+
+if __name__ == '__main__':
+    sys.exit(_main())

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/__init__.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+
+import locale
+import logging
+import os
+import optparse
+import warnings
+
+import sys
+
+# 2016-06-17 barry@debian.org: urllib3 1.14 added optional support for socks,
+# but if invoked (i.e. imported), it will issue a warning to stderr if socks
+# isn't available.  requests unconditionally imports urllib3's socks contrib
+# module, triggering this warning.  The warning breaks DEP-8 tests (because of
+# the stderr output) and is just plain annoying in normal usage.  I don't want
+# to add socks as yet another dependency for pip, nor do I want to allow-stder
+# in the DEP-8 tests, so just suppress the warning.  pdb tells me this has to
+# be done before the import of pip.vcs.
+from pip._vendor.urllib3.exceptions import DependencyWarning
+warnings.filterwarnings("ignore", category=DependencyWarning)  # noqa
+
+# We want to inject the use of SecureTransport as early as possible so that any
+# references or sessions or what have you are ensured to have it, however we
+# only want to do this in the case that we're running on macOS and the linked
+# OpenSSL is too old to handle TLSv1.2
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    # Checks for OpenSSL 1.0.1 on MacOS
+    if sys.platform == "darwin" and ssl.OPENSSL_VERSION_NUMBER < 0x1000100f:
+        try:
+            from pip._vendor.urllib3.contrib import securetransport
+        except (ImportError, OSError):
+            pass
+        else:
+            securetransport.inject_into_urllib3()
+
+from pip import __version__
+from pip._internal import cmdoptions
+from pip._internal.exceptions import CommandError, PipError
+from pip._internal.utils.misc import get_installed_distributions, get_prog
+from pip._internal.utils import deprecation
+from pip._internal.vcs import git, mercurial, subversion, bazaar  # noqa
+from pip._internal.baseparser import (
+    ConfigOptionParser, UpdatingDefaultsHelpFormatter,
+)
+from pip._internal.commands import get_summaries, get_similar_commands
+from pip._internal.commands import commands_dict
+from pip._vendor.urllib3.exceptions import InsecureRequestWarning
+
+logger = logging.getLogger(__name__)
+
+# Hide the InsecureRequestWarning from urllib3
+warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+
+
+def autocomplete():
+    """Command and option completion for the main option parser (and options)
+    and its subcommands (and options).
+
+    Enable by sourcing one of the completion shell scripts (bash, zsh or fish).
+    """
+    # Don't complete if user hasn't sourced bash_completion file.
+    if 'PIP_AUTO_COMPLETE' not in os.environ:
+        return
+    cwords = os.environ['COMP_WORDS'].split()[1:]
+    cword = int(os.environ['COMP_CWORD'])
+    try:
+        current = cwords[cword - 1]
+    except IndexError:
+        current = ''
+
+    subcommands = [cmd for cmd, summary in get_summaries()]
+    options = []
+    # subcommand
+    try:
+        subcommand_name = [w for w in cwords if w in subcommands][0]
+    except IndexError:
+        subcommand_name = None
+
+    parser = create_main_parser()
+    # subcommand options
+    if subcommand_name:
+        # special case: 'help' subcommand has no options
+        if subcommand_name == 'help':
+            sys.exit(1)
+        # special case: list locally installed dists for show and uninstall
+        should_list_installed = (
+            subcommand_name in ['show', 'uninstall'] and
+            not current.startswith('-')
+        )
+        if should_list_installed:
+            installed = []
+            lc = current.lower()
+            for dist in get_installed_distributions(local_only=True):
+                if dist.key.startswith(lc) and dist.key not in cwords[1:]:
+                    installed.append(dist.key)
+            # if there are no dists installed, fall back to option completion
+            if installed:
+                for dist in installed:
+                    print(dist)
+                sys.exit(1)
+
+        subcommand = commands_dict[subcommand_name]()
+
+        for opt in subcommand.parser.option_list_all:
+            if opt.help != optparse.SUPPRESS_HELP:
+                for opt_str in opt._long_opts + opt._short_opts:
+                    options.append((opt_str, opt.nargs))
+
+        # filter out previously specified options from available options
+        prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]]
+        options = [(x, v) for (x, v) in options if x not in prev_opts]
+        # filter options by current input
+        options = [(k, v) for k, v in options if k.startswith(current)]
+        for option in options:
+            opt_label = option[0]
+            # append '=' to options which require args
+            if option[1] and option[0][:2] == "--":
+                opt_label += '='
+            print(opt_label)
+    else:
+        # show main parser options only when necessary
+        if current.startswith('-') or current.startswith('--'):
+            opts = [i.option_list for i in parser.option_groups]
+            opts.append(parser.option_list)
+            opts = (o for it in opts for o in it)
+
+            for opt in opts:
+                if opt.help != optparse.SUPPRESS_HELP:
+                    subcommands += opt._long_opts + opt._short_opts
+
+        print(' '.join([x for x in subcommands if x.startswith(current)]))
+    sys.exit(1)
+
+
+def create_main_parser():
+    parser_kw = {
+        'usage': '\n%prog <command> [options]',
+        'add_help_option': False,
+        'formatter': UpdatingDefaultsHelpFormatter(),
+        'name': 'global',
+        'prog': get_prog(),
+    }
+
+    parser = ConfigOptionParser(**parser_kw)
+    parser.disable_interspersed_args()
+
+    pip_pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    parser.version = 'pip %s from %s (python %s)' % (
+        __version__, pip_pkg_dir, sys.version[:3],
+    )
+
+    # add the general options
+    gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser)
+    parser.add_option_group(gen_opts)
+
+    parser.main = True  # so the help formatter knows
+
+    # create command listing for description
+    command_summaries = get_summaries()
+    description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries]
+    parser.description = '\n'.join(description)
+
+    return parser
+
+
+def parseopts(args):
+    parser = create_main_parser()
+
+    # Note: parser calls disable_interspersed_args(), so the result of this
+    # call is to split the initial args into the general options before the
+    # subcommand and everything else.
+    # For example:
+    #  args: ['--timeout=5', 'install', '--user', 'INITools']
+    #  general_options: ['--timeout==5']
+    #  args_else: ['install', '--user', 'INITools']
+    general_options, args_else = parser.parse_args(args)
+
+    # --version
+    if general_options.version:
+        sys.stdout.write(parser.version)
+        sys.stdout.write(os.linesep)
+        sys.exit()
+
+    # pip || pip help -> print_help()
+    if not args_else or (args_else[0] == 'help' and len(args_else) == 1):
+        parser.print_help()
+        sys.exit()
+
+    # the subcommand name
+    cmd_name = args_else[0]
+
+    if cmd_name not in commands_dict:
+        guess = get_similar_commands(cmd_name)
+
+        msg = ['unknown command "%s"' % cmd_name]
+        if guess:
+            msg.append('maybe you meant "%s"' % guess)
+
+        raise CommandError(' - '.join(msg))
+
+    # all the args without the subcommand
+    cmd_args = args[:]
+    cmd_args.remove(cmd_name)
+
+    return cmd_name, cmd_args
+
+
+def check_isolated(args):
+    isolated = False
+
+    if "--isolated" in args:
+        isolated = True
+
+    return isolated
+
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    # Configure our deprecation warnings to be sent through loggers
+    deprecation.install_warning_logger()
+
+    autocomplete()
+
+    try:
+        cmd_name, cmd_args = parseopts(args)
+    except PipError as exc:
+        sys.stderr.write("ERROR: %s" % exc)
+        sys.stderr.write(os.linesep)
+        sys.exit(1)
+
+    # Needed for locale.getpreferredencoding(False) to work
+    # in pip._internal.utils.encoding.auto_decode
+    try:
+        locale.setlocale(locale.LC_ALL, '')
+    except locale.Error as e:
+        # setlocale can apparently crash if locale are uninitialized
+        logger.debug("Ignoring error %s when setting locale", e)
+    command = commands_dict[cmd_name](isolated=check_isolated(cmd_args))
+    return command.main(cmd_args)

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/basecommand.py

@@ -0,0 +1,373 @@
+"""Base Command class, and related routines"""
+from __future__ import absolute_import
+
+import logging
+import logging.config
+import optparse
+import os
+import sys
+import warnings
+
+from pip._internal import cmdoptions
+from pip._internal.baseparser import (
+    ConfigOptionParser, UpdatingDefaultsHelpFormatter,
+)
+from pip._internal.compat import WINDOWS
+from pip._internal.download import PipSession
+from pip._internal.exceptions import (
+    BadCommand, CommandError, InstallationError, PreviousBuildDirError,
+    UninstallationError,
+)
+from pip._internal.index import PackageFinder
+from pip._internal.locations import running_under_virtualenv
+from pip._internal.req.req_file import parse_requirements
+from pip._internal.req.req_install import InstallRequirement
+from pip._internal.status_codes import (
+    ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR,
+    VIRTUALENV_NOT_FOUND,
+)
+from pip._internal.utils import deprecation
+from pip._internal.utils.logging import IndentingFormatter
+from pip._internal.utils.misc import get_prog, normalize_path
+from pip._internal.utils.outdated import pip_version_check
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Optional
+
+__all__ = ['Command']
+
+logger = logging.getLogger(__name__)
+
+
+class Command(object):
+    name = None  # type: Optional[str]
+    usage = None  # type: Optional[str]
+    hidden = False  # type: bool
+    ignore_require_venv = False  # type: bool
+    log_streams = ("ext://sys.stdout", "ext://sys.stderr")
+
+    def __init__(self, isolated=False):
+        parser_kw = {
+            'usage': self.usage,
+            'prog': '%s %s' % (get_prog(), self.name),
+            'formatter': UpdatingDefaultsHelpFormatter(),
+            'add_help_option': False,
+            'name': self.name,
+            'description': self.__doc__,
+            'isolated': isolated,
+        }
+
+        self.parser = ConfigOptionParser(**parser_kw)
+
+        # Commands should add options to this option group
+        optgroup_name = '%s Options' % self.name.capitalize()
+        self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name)
+
+        # Add the general options
+        gen_opts = cmdoptions.make_option_group(
+            cmdoptions.general_group,
+            self.parser,
+        )
+        self.parser.add_option_group(gen_opts)
+
+    def _build_session(self, options, retries=None, timeout=None):
+        session = PipSession(
+            cache=(
+                normalize_path(os.path.join(options.cache_dir, "http"))
+                if options.cache_dir else None
+            ),
+            retries=retries if retries is not None else options.retries,
+            insecure_hosts=options.trusted_hosts,
+        )
+
+        # Handle custom ca-bundles from the user
+        if options.cert:
+            session.verify = options.cert
+
+        # Handle SSL client certificate
+        if options.client_cert:
+            session.cert = options.client_cert
+
+        # Handle timeouts
+        if options.timeout or timeout:
+            session.timeout = (
+                timeout if timeout is not None else options.timeout
+            )
+
+        # Handle configured proxies
+        if options.proxy:
+            session.proxies = {
+                "http": options.proxy,
+                "https": options.proxy,
+            }
+
+        # Determine if we can prompt the user for authentication or not
+        session.auth.prompting = not options.no_input
+
+        return session
+
+    def parse_args(self, args):
+        # factored out for testability
+        return self.parser.parse_args(args)
+
+    def main(self, args):
+        options, args = self.parse_args(args)
+
+        # Set verbosity so that it can be used elsewhere.
+        self.verbosity = options.verbose - options.quiet
+
+        if self.verbosity >= 1:
+            level = "DEBUG"
+        elif self.verbosity == -1:
+            level = "WARNING"
+        elif self.verbosity == -2:
+            level = "ERROR"
+        elif self.verbosity <= -3:
+            level = "CRITICAL"
+        else:
+            level = "INFO"
+
+        # The root logger should match the "console" level *unless* we
+        # specified "--log" to send debug logs to a file.
+        root_level = level
+        if options.log:
+            root_level = "DEBUG"
+
+        logger_class = "pip._internal.utils.logging.ColorizedStreamHandler"
+        handler_class = "pip._internal.utils.logging.BetterRotatingFileHandler"
+
+        logging.config.dictConfig({
+            "version": 1,
+            "disable_existing_loggers": False,
+            "filters": {
+                "exclude_warnings": {
+                    "()": "pip._internal.utils.logging.MaxLevelFilter",
+                    "level": logging.WARNING,
+                },
+            },
+            "formatters": {
+                "indent": {
+                    "()": IndentingFormatter,
+                    "format": "%(message)s",
+                },
+            },
+            "handlers": {
+                "console": {
+                    "level": level,
+                    "class": logger_class,
+                    "no_color": options.no_color,
+                    "stream": self.log_streams[0],
+                    "filters": ["exclude_warnings"],
+                    "formatter": "indent",
+                },
+                "console_errors": {
+                    "level": "WARNING",
+                    "class": logger_class,
+                    "no_color": options.no_color,
+                    "stream": self.log_streams[1],
+                    "formatter": "indent",
+                },
+                "user_log": {
+                    "level": "DEBUG",
+                    "class": handler_class,
+                    "filename": options.log or "/dev/null",
+                    "delay": True,
+                    "formatter": "indent",
+                },
+            },
+            "root": {
+                "level": root_level,
+                "handlers": list(filter(None, [
+                    "console",
+                    "console_errors",
+                    "user_log" if options.log else None,
+                ])),
+            },
+            # Disable any logging besides WARNING unless we have DEBUG level
+            # logging enabled. These use both pip._vendor and the bare names
+            # for the case where someone unbundles our libraries.
+            "loggers": {
+                name: {
+                    "level": (
+                        "WARNING" if level in ["INFO", "ERROR"] else "DEBUG"
+                    )
+                } for name in [
+                    "pip._vendor", "distlib", "requests", "urllib3"
+                ]
+            },
+        })
+
+        if sys.version_info[:2] == (3, 3):
+            warnings.warn(
+                "Python 3.3 supported has been deprecated and support for it "
+                "will be dropped in the future. Please upgrade your Python.",
+                deprecation.RemovedInPip11Warning,
+            )
+
+        # TODO: try to get these passing down from the command?
+        #      without resorting to os.environ to hold these.
+
+        if options.no_input:
+            os.environ['PIP_NO_INPUT'] = '1'
+
+        if options.exists_action:
+            os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action)
+
+        if options.require_venv and not self.ignore_require_venv:
+            # If a venv is required check if it can really be found
+            if not running_under_virtualenv():
+                logger.critical(
+                    'Could not find an activated virtualenv (required).'
+                )
+                sys.exit(VIRTUALENV_NOT_FOUND)
+
+        original_root_handlers = set(logging.root.handlers)
+
+        try:
+            status = self.run(options, args)
+            # FIXME: all commands should return an exit status
+            # and when it is done, isinstance is not needed anymore
+            if isinstance(status, int):
+                return status
+        except PreviousBuildDirError as exc:
+            logger.critical(str(exc))
+            logger.debug('Exception information:', exc_info=True)
+
+            return PREVIOUS_BUILD_DIR_ERROR
+        except (InstallationError, UninstallationError, BadCommand) as exc:
+            logger.critical(str(exc))
+            logger.debug('Exception information:', exc_info=True)
+
+            return ERROR
+        except CommandError as exc:
+            logger.critical('ERROR: %s', exc)
+            logger.debug('Exception information:', exc_info=True)
+
+            return ERROR
+        except KeyboardInterrupt:
+            logger.critical('Operation cancelled by user')
+            logger.debug('Exception information:', exc_info=True)
+
+            return ERROR
+        except:
+            logger.critical('Exception:', exc_info=True)
+
+            return UNKNOWN_ERROR
+        finally:
+            # Check if we're using the latest version of pip available
+            if (not options.disable_pip_version_check and not
+                    getattr(options, "no_index", False)):
+                with self._build_session(
+                        options,
+                        retries=0,
+                        timeout=min(5, options.timeout)) as session:
+                    pip_version_check(session, options)
+            # Avoid leaking loggers
+            for handler in set(logging.root.handlers) - original_root_handlers:
+                # this method benefit from the Logger class internal lock
+                logging.root.removeHandler(handler)
+
+        return SUCCESS
+
+
+class RequirementCommand(Command):
+
+    @staticmethod
+    def populate_requirement_set(requirement_set, args, options, finder,
+                                 session, name, wheel_cache):
+        """
+        Marshal cmd line args into a requirement set.
+        """
+        # NOTE: As a side-effect, options.require_hashes and
+        #       requirement_set.require_hashes may be updated
+
+        for filename in options.constraints:
+            for req_to_add in parse_requirements(
+                    filename,
+                    constraint=True, finder=finder, options=options,
+                    session=session, wheel_cache=wheel_cache):
+                req_to_add.is_direct = True
+                requirement_set.add_requirement(req_to_add)
+
+        for req in args:
+            req_to_add = InstallRequirement.from_line(
+                req, None, isolated=options.isolated_mode,
+                wheel_cache=wheel_cache
+            )
+            req_to_add.is_direct = True
+            requirement_set.add_requirement(req_to_add)
+
+        for req in options.editables:
+            req_to_add = InstallRequirement.from_editable(
+                req,
+                isolated=options.isolated_mode,
+                wheel_cache=wheel_cache
+            )
+            req_to_add.is_direct = True
+            requirement_set.add_requirement(req_to_add)
+
+        for filename in options.requirements:
+            for req_to_add in parse_requirements(
+                    filename,
+                    finder=finder, options=options, session=session,
+                    wheel_cache=wheel_cache):
+                req_to_add.is_direct = True
+                requirement_set.add_requirement(req_to_add)
+        # If --require-hashes was a line in a requirements file, tell
+        # RequirementSet about it:
+        requirement_set.require_hashes = options.require_hashes
+
+        if not (args or options.editables or options.requirements):
+            opts = {'name': name}
+            if options.find_links:
+                raise CommandError(
+                    'You must give at least one requirement to %(name)s '
+                    '(maybe you meant "pip %(name)s %(links)s"?)' %
+                    dict(opts, links=' '.join(options.find_links)))
+            else:
+                raise CommandError(
+                    'You must give at least one requirement to %(name)s '
+                    '(see "pip help %(name)s")' % opts)
+
+        # On Windows, any operation modifying pip should be run as:
+        #     python -m pip ...
+        # See https://github.com/pypa/pip/issues/1299 for more discussion
+        should_show_use_python_msg = (
+            WINDOWS and
+            requirement_set.has_requirement("pip") and
+            os.path.basename(sys.argv[0]).startswith("pip")
+        )
+        if should_show_use_python_msg:
+            new_command = [
+                sys.executable, "-m", "pip"
+            ] + sys.argv[1:]
+            raise CommandError(
+                'To modify pip, please run the following command:\n{}'
+                .format(" ".join(new_command))
+            )
+
+    def _build_package_finder(self, options, session,
+                              platform=None, python_versions=None,
+                              abi=None, implementation=None):
+        """
+        Create a package finder appropriate to this requirement command.
+        """
+        index_urls = [options.index_url] + options.extra_index_urls
+        if options.no_index:
+            logger.debug('Ignoring indexes: %s', ','.join(index_urls))
+            index_urls = []
+
+        return PackageFinder(
+            find_links=options.find_links,
+            format_control=options.format_control,
+            index_urls=index_urls,
+            trusted_hosts=options.trusted_hosts,
+            allow_all_prereleases=options.pre,
+            process_dependency_links=options.process_dependency_links,
+            session=session,
+            platform=platform,
+            versions=python_versions,
+            abi=abi,
+            implementation=implementation,
+        )

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/baseparser.py

@@ -0,0 +1,240 @@
+"""Base option parser setup"""
+from __future__ import absolute_import
+
+import logging
+import optparse
+import sys
+import textwrap
+from distutils.util import strtobool
+
+from pip._vendor.six import string_types
+
+from pip._internal.compat import get_terminal_size
+from pip._internal.configuration import Configuration, ConfigurationError
+
+logger = logging.getLogger(__name__)
+
+
+class PrettyHelpFormatter(optparse.IndentedHelpFormatter):
+    """A prettier/less verbose help formatter for optparse."""
+
+    def __init__(self, *args, **kwargs):
+        # help position must be aligned with __init__.parseopts.description
+        kwargs['max_help_position'] = 30
+        kwargs['indent_increment'] = 1
+        kwargs['width'] = get_terminal_size()[0] - 2
+        optparse.IndentedHelpFormatter.__init__(self, *args, **kwargs)
+
+    def format_option_strings(self, option):
+        return self._format_option_strings(option, ' <%s>', ', ')
+
+    def _format_option_strings(self, option, mvarfmt=' <%s>', optsep=', '):
+        """
+        Return a comma-separated list of option strings and metavars.
+
+        :param option:  tuple of (short opt, long opt), e.g: ('-f', '--format')
+        :param mvarfmt: metavar format string - evaluated as mvarfmt % metavar
+        :param optsep:  separator
+        """
+        opts = []
+
+        if option._short_opts:
+            opts.append(option._short_opts[0])
+        if option._long_opts:
+            opts.append(option._long_opts[0])
+        if len(opts) > 1:
+            opts.insert(1, optsep)
+
+        if option.takes_value():
+            metavar = option.metavar or option.dest.lower()
+            opts.append(mvarfmt % metavar.lower())
+
+        return ''.join(opts)
+
+    def format_heading(self, heading):
+        if heading == 'Options':
+            return ''
+        return heading + ':\n'
+
+    def format_usage(self, usage):
+        """
+        Ensure there is only one newline between usage and the first heading
+        if there is no description.
+        """
+        msg = '\nUsage: %s\n' % self.indent_lines(textwrap.dedent(usage), "  ")
+        return msg
+
+    def format_description(self, description):
+        # leave full control over description to us
+        if description:
+            if hasattr(self.parser, 'main'):
+                label = 'Commands'
+            else:
+                label = 'Description'
+            # some doc strings have initial newlines, some don't
+            description = description.lstrip('\n')
+            # some doc strings have final newlines and spaces, some don't
+            description = description.rstrip()
+            # dedent, then reindent
+            description = self.indent_lines(textwrap.dedent(description), "  ")
+            description = '%s:\n%s\n' % (label, description)
+            return description
+        else:
+            return ''
+
+    def format_epilog(self, epilog):
+        # leave full control over epilog to us
+        if epilog:
+            return epilog
+        else:
+            return ''
+
+    def indent_lines(self, text, indent):
+        new_lines = [indent + line for line in text.split('\n')]
+        return "\n".join(new_lines)
+
+
+class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter):
+    """Custom help formatter for use in ConfigOptionParser.
+
+    This is updates the defaults before expanding them, allowing
+    them to show up correctly in the help listing.
+    """
+
+    def expand_default(self, option):
+        if self.parser is not None:
+            self.parser._update_defaults(self.parser.defaults)
+        return optparse.IndentedHelpFormatter.expand_default(self, option)
+
+
+class CustomOptionParser(optparse.OptionParser):
+
+    def insert_option_group(self, idx, *args, **kwargs):
+        """Insert an OptionGroup at a given position."""
+        group = self.add_option_group(*args, **kwargs)
+
+        self.option_groups.pop()
+        self.option_groups.insert(idx, group)
+
+        return group
+
+    @property
+    def option_list_all(self):
+        """Get a list of all options, including those in option groups."""
+        res = self.option_list[:]
+        for i in self.option_groups:
+            res.extend(i.option_list)
+
+        return res
+
+
+class ConfigOptionParser(CustomOptionParser):
+    """Custom option parser which updates its defaults by checking the
+    configuration files and environmental variables"""
+
+    def __init__(self, *args, **kwargs):
+        self.name = kwargs.pop('name')
+
+        isolated = kwargs.pop("isolated", False)
+        self.config = Configuration(isolated)
+
+        assert self.name
+        optparse.OptionParser.__init__(self, *args, **kwargs)
+
+    def check_default(self, option, key, val):
+        try:
+            return option.check_value(key, val)
+        except optparse.OptionValueError as exc:
+            print("An error occurred during configuration: %s" % exc)
+            sys.exit(3)
+
+    def _get_ordered_configuration_items(self):
+        # Configuration gives keys in an unordered manner. Order them.
+        override_order = ["global", self.name, ":env:"]
+
+        # Pool the options into different groups
+        section_items = {name: [] for name in override_order}
+        for section_key, val in self.config.items():
+            # ignore empty values
+            if not val:
+                logger.debug(
+                    "Ignoring configuration key '%s' as it's value is empty.",
+                    section_key
+                )
+                continue
+
+            section, key = section_key.split(".", 1)
+            if section in override_order:
+                section_items[section].append((key, val))
+
+        # Yield each group in their override order
+        for section in override_order:
+            for key, val in section_items[section]:
+                yield key, val
+
+    def _update_defaults(self, defaults):
+        """Updates the given defaults with values from the config files and
+        the environ. Does a little special handling for certain types of
+        options (lists)."""
+
+        # Accumulate complex default state.
+        self.values = optparse.Values(self.defaults)
+        late_eval = set()
+        # Then set the options with those values
+        for key, val in self._get_ordered_configuration_items():
+            # '--' because configuration supports only long names
+            option = self.get_option('--' + key)
+
+            # Ignore options not present in this parser. E.g. non-globals put
+            # in [global] by users that want them to apply to all applicable
+            # commands.
+            if option is None:
+                continue
+
+            if option.action in ('store_true', 'store_false', 'count'):
+                val = strtobool(val)
+            elif option.action == 'append':
+                val = val.split()
+                val = [self.check_default(option, key, v) for v in val]
+            elif option.action == 'callback':
+                late_eval.add(option.dest)
+                opt_str = option.get_opt_string()
+                val = option.convert_value(opt_str, val)
+                # From take_action
+                args = option.callback_args or ()
+                kwargs = option.callback_kwargs or {}
+                option.callback(option, opt_str, val, self, *args, **kwargs)
+            else:
+                val = self.check_default(option, key, val)
+
+            defaults[option.dest] = val
+
+        for key in late_eval:
+            defaults[key] = getattr(self.values, key)
+        self.values = None
+        return defaults
+
+    def get_default_values(self):
+        """Overriding to make updating the defaults after instantiation of
+        the option parser possible, _update_defaults() does the dirty work."""
+        if not self.process_default_values:
+            # Old, pre-Optik 1.5 behaviour.
+            return optparse.Values(self.defaults)
+
+        # Load the configuration, or error out in case of an error
+        try:
+            self.config.load()
+        except ConfigurationError as err:
+            self.exit(2, err.args[0])
+
+        defaults = self._update_defaults(self.defaults.copy())  # ours
+        for option in self._get_all_options():
+            default = defaults.get(option.dest)
+            if isinstance(default, string_types):
+                opt_str = option.get_opt_string()
+                defaults[option.dest] = option.check_value(opt_str, default)
+        return optparse.Values(defaults)
+
+    def error(self, msg):
+        self.print_usage(sys.stderr)
+        self.exit(2, "%s\n" % msg)

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/build_env.py

@@ -0,0 +1,92 @@
+"""Build Environment used for isolation during sdist building
+"""
+
+import os
+from distutils.sysconfig import get_python_lib
+from sysconfig import get_paths
+
+from pip._internal.utils.temp_dir import TempDirectory
+
+
+class BuildEnvironment(object):
+    """Creates and manages an isolated environment to install build deps
+    """
+
+    def __init__(self, no_clean):
+        self._temp_dir = TempDirectory(kind="build-env")
+        self._no_clean = no_clean
+
+    @property
+    def path(self):
+        return self._temp_dir.path
+
+    def __enter__(self):
+        self._temp_dir.create()
+
+        self.save_path = os.environ.get('PATH', None)
+        self.save_pythonpath = os.environ.get('PYTHONPATH', None)
+        self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None)
+
+        install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix'
+        install_dirs = get_paths(install_scheme, vars={
+            'base': self.path,
+            'platbase': self.path,
+        })
+
+        scripts = install_dirs['scripts']
+        if self.save_path:
+            os.environ['PATH'] = scripts + os.pathsep + self.save_path
+        else:
+            os.environ['PATH'] = scripts + os.pathsep + os.defpath
+
+        # Note: prefer distutils' sysconfig to get the
+        # library paths so PyPy is correctly supported.
+        purelib = get_python_lib(plat_specific=0, prefix=self.path)
+        platlib = get_python_lib(plat_specific=1, prefix=self.path)
+        if purelib == platlib:
+            lib_dirs = purelib
+        else:
+            lib_dirs = purelib + os.pathsep + platlib
+        if self.save_pythonpath:
+            os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \
+                self.save_pythonpath
+        else:
+            os.environ['PYTHONPATH'] = lib_dirs
+
+        os.environ['PYTHONNOUSERSITE'] = '1'
+
+        return self.path
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if not self._no_clean:
+            self._temp_dir.cleanup()
+
+        def restore_var(varname, old_value):
+            if old_value is None:
+                os.environ.pop(varname, None)
+            else:
+                os.environ[varname] = old_value
+
+        restore_var('PATH', self.save_path)
+        restore_var('PYTHONPATH', self.save_pythonpath)
+        restore_var('PYTHONNOUSERSITE', self.save_nousersite)
+
+    def cleanup(self):
+        self._temp_dir.cleanup()
+
+
+class NoOpBuildEnvironment(BuildEnvironment):
+    """A no-op drop-in replacement for BuildEnvironment
+    """
+
+    def __init__(self, no_clean):
+        pass
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def cleanup(self):
+        pass

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/cache.py

@@ -0,0 +1,202 @@
+"""Cache Management
+"""
+
+import errno
+import hashlib
+import logging
+import os
+
+from pip._vendor.packaging.utils import canonicalize_name
+
+from pip._internal import index
+from pip._internal.compat import expanduser
+from pip._internal.download import path_to_url
+from pip._internal.utils.temp_dir import TempDirectory
+from pip._internal.wheel import InvalidWheelFilename, Wheel
+
+logger = logging.getLogger(__name__)
+
+
+class Cache(object):
+    """An abstract class - provides cache directories for data from links
+
+
+        :param cache_dir: The root of the cache.
+        :param format_control: A pip.index.FormatControl object to limit
+            binaries being read from the cache.
+        :param allowed_formats: which formats of files the cache should store.
+            ('binary' and 'source' are the only allowed values)
+    """
+
+    def __init__(self, cache_dir, format_control, allowed_formats):
+        super(Cache, self).__init__()
+        self.cache_dir = expanduser(cache_dir) if cache_dir else None
+        self.format_control = format_control
+        self.allowed_formats = allowed_formats
+
+        _valid_formats = {"source", "binary"}
+        assert self.allowed_formats.union(_valid_formats) == _valid_formats
+
+    def _get_cache_path_parts(self, link):
+        """Get parts of part that must be os.path.joined with cache_dir
+        """
+
+        # We want to generate an url to use as our cache key, we don't want to
+        # just re-use the URL because it might have other items in the fragment
+        # and we don't care about those.
+        key_parts = [link.url_without_fragment]
+        if link.hash_name is not None and link.hash is not None:
+            key_parts.append("=".join([link.hash_name, link.hash]))
+        key_url = "#".join(key_parts)
+
+        # Encode our key url with sha224, we'll use this because it has similar
+        # security properties to sha256, but with a shorter total output (and
+        # thus less secure). However the differences don't make a lot of
+        # difference for our use case here.
+        hashed = hashlib.sha224(key_url.encode()).hexdigest()
+
+        # We want to nest the directories some to prevent having a ton of top
+        # level directories where we might run out of sub directories on some
+        # FS.
+        parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]]
+
+        return parts
+
+    def _get_candidates(self, link, package_name):
+        can_not_cache = (
+            not self.cache_dir or
+            not package_name or
+            not link
+        )
+        if can_not_cache:
+            return []
+
+        canonical_name = canonicalize_name(package_name)
+        formats = index.fmt_ctl_formats(
+            self.format_control, canonical_name
+        )
+        if not self.allowed_formats.intersection(formats):
+            return []
+
+        root = self.get_path_for_link(link)
+        try:
+            return os.listdir(root)
+        except OSError as err:
+            if err.errno in {errno.ENOENT, errno.ENOTDIR}:
+                return []
+            raise
+
+    def get_path_for_link(self, link):
+        """Return a directory to store cached items in for link.
+        """
+        raise NotImplementedError()
+
+    def get(self, link, package_name):
+        """Returns a link to a cached item if it exists, otherwise returns the
+        passed link.
+        """
+        raise NotImplementedError()
+
+    def _link_for_candidate(self, link, candidate):
+        root = self.get_path_for_link(link)
+        path = os.path.join(root, candidate)
+
+        return index.Link(path_to_url(path))
+
+    def cleanup(self):
+        pass
+
+
+class SimpleWheelCache(Cache):
+    """A cache of wheels for future installs.
+    """
+
+    def __init__(self, cache_dir, format_control):
+        super(SimpleWheelCache, self).__init__(
+            cache_dir, format_control, {"binary"}
+        )
+
+    def get_path_for_link(self, link):
+        """Return a directory to store cached wheels for link
+
+        Because there are M wheels for any one sdist, we provide a directory
+        to cache them in, and then consult that directory when looking up
+        cache hits.
+
+        We only insert things into the cache if they have plausible version
+        numbers, so that we don't contaminate the cache with things that were
+        not unique. E.g. ./package might have dozens of installs done for it
+        and build a version of 0.0...and if we built and cached a wheel, we'd
+        end up using the same wheel even if the source has been edited.
+
+        :param link: The link of the sdist for which this will cache wheels.
+        """
+        parts = self._get_cache_path_parts(link)
+
+        # Store wheels within the root cache_dir
+        return os.path.join(self.cache_dir, "wheels", *parts)
+
+    def get(self, link, package_name):
+        candidates = []
+
+        for wheel_name in self._get_candidates(link, package_name):
+            try:
+                wheel = Wheel(wheel_name)
+            except InvalidWheelFilename:
+                continue
+            if not wheel.supported():
+                # Built for a different python/arch/etc
+                continue
+            candidates.append((wheel.support_index_min(), wheel_name))
+
+        if not candidates:
+            return link
+
+        return self._link_for_candidate(link, min(candidates)[1])
+
+
+class EphemWheelCache(SimpleWheelCache):
+    """A SimpleWheelCache that creates it's own temporary cache directory
+    """
+
+    def __init__(self, format_control):
+        self._temp_dir = TempDirectory(kind="ephem-wheel-cache")
+        self._temp_dir.create()
+
+        super(EphemWheelCache, self).__init__(
+            self._temp_dir.path, format_control
+        )
+
+    def cleanup(self):
+        self._temp_dir.cleanup()
+
+
+class WheelCache(Cache):
+    """Wraps EphemWheelCache and SimpleWheelCache into a single Cache
+
+    This Cache allows for gracefully degradation, using the ephem wheel cache
+    when a certain link is not found in the simple wheel cache first.
+    """
+
+    def __init__(self, cache_dir, format_control):
+        super(WheelCache, self).__init__(
+            cache_dir, format_control, {'binary'}
+        )
+        self._wheel_cache = SimpleWheelCache(cache_dir, format_control)
+        self._ephem_cache = EphemWheelCache(format_control)
+
+    def get_path_for_link(self, link):
+        return self._wheel_cache.get_path_for_link(link)
+
+    def get_ephem_path_for_link(self, link):
+        return self._ephem_cache.get_path_for_link(link)
+
+    def get(self, link, package_name):
+        retval = self._wheel_cache.get(link, package_name)
+        if retval is link:
+            retval = self._ephem_cache.get(link, package_name)
+        return retval
+
+    def cleanup(self):
+        self._wheel_cache.cleanup()
+        self._ephem_cache.cleanup()

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/cmdoptions.py

@@ -0,0 +1,609 @@
+"""
+shared options and groups
+
+The principle here is to define options once, but *not* instantiate them
+globally. One reason being that options with action='append' can carry state
+between parses. pip parses general options twice internally, and shouldn't
+pass on state. To be consistent, all options will follow this design.
+
+"""
+from __future__ import absolute_import
+
+import warnings
+from functools import partial
+from optparse import SUPPRESS_HELP, Option, OptionGroup
+
+from pip._internal.index import (
+    FormatControl, fmt_ctl_handle_mutual_exclude, fmt_ctl_no_binary,
+)
+from pip._internal.locations import USER_CACHE_DIR, src_prefix
+from pip._internal.models import PyPI
+from pip._internal.utils.hashes import STRONG_HASHES
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.ui import BAR_TYPES
+
+if MYPY_CHECK_RUNNING:
+    from typing import Any
+
+
+def make_option_group(group, parser):
+    """
+    Return an OptionGroup object
+    group  -- assumed to be dict with 'name' and 'options' keys
+    parser -- an optparse Parser
+    """
+    option_group = OptionGroup(parser, group['name'])
+    for option in group['options']:
+        option_group.add_option(option())
+    return option_group
+
+
+def check_install_build_global(options, check_options=None):
+    """Disable wheels if per-setup.py call options are set.
+
+    :param options: The OptionParser options to update.
+    :param check_options: The options to check, if not supplied defaults to
+        options.
+    """
+    if check_options is None:
+        check_options = options
+
+    def getname(n):
+        return getattr(check_options, n, None)
+    names = ["build_options", "global_options", "install_options"]
+    if any(map(getname, names)):
+        control = options.format_control
+        fmt_ctl_no_binary(control)
+        warnings.warn(
+            'Disabling all use of wheels due to the use of --build-options '
+            '/ --global-options / --install-options.', stacklevel=2,
+        )
+
+
+###########
+# options #
+###########
+
+help_ = partial(
+    Option,
+    '-h', '--help',
+    dest='help',
+    action='help',
+    help='Show help.',
+)  # type: Any
+
+isolated_mode = partial(
+    Option,
+    "--isolated",
+    dest="isolated_mode",
+    action="store_true",
+    default=False,
+    help=(
+        "Run pip in an isolated mode, ignoring environment variables and user "
+        "configuration."
+    ),
+)
+
+require_virtualenv = partial(
+    Option,
+    # Run only if inside a virtualenv, bail if not.
+    '--require-virtualenv', '--require-venv',
+    dest='require_venv',
+    action='store_true',
+    default=False,
+    help=SUPPRESS_HELP
+)  # type: Any
+
+verbose = partial(
+    Option,
+    '-v', '--verbose',
+    dest='verbose',
+    action='count',
+    default=0,
+    help='Give more output. Option is additive, and can be used up to 3 times.'
+)
+
+no_color = partial(
+    Option,
+    '--no-color',
+    dest='no_color',
+    action='store_true',
+    default=False,
+    help="Suppress colored output",
+)
+
+version = partial(
+    Option,
+    '-V', '--version',
+    dest='version',
+    action='store_true',
+    help='Show version and exit.',
+)  # type: Any
+
+quiet = partial(
+    Option,
+    '-q', '--quiet',
+    dest='quiet',
+    action='count',
+    default=0,
+    help=(
+        'Give less output. Option is additive, and can be used up to 3'
+        ' times (corresponding to WARNING, ERROR, and CRITICAL logging'
+        ' levels).'
+    ),
+)  # type: Any
+
+progress_bar = partial(
+    Option,
+    '--progress-bar',
+    dest='progress_bar',
+    type='choice',
+    choices=list(BAR_TYPES.keys()),
+    default='on',
+    help=(
+        'Specify type of progress to be displayed [' +
+        '|'.join(BAR_TYPES.keys()) + '] (default: %default)'
+    ),
+)  # type: Any
+
+log = partial(
+    Option,
+    "--log", "--log-file", "--local-log",
+    dest="log",
+    metavar="path",
+    help="Path to a verbose appending log."
+)  # type: Any
+
+no_input = partial(
+    Option,
+    # Don't ask for input
+    '--no-input',
+    dest='no_input',
+    action='store_true',
+    default=False,
+    help=SUPPRESS_HELP
+)  # type: Any
+
+proxy = partial(
+    Option,
+    '--proxy',
+    dest='proxy',
+    type='str',
+    default='',
+    help="Specify a proxy in the form [user:passwd@]proxy.server:port."
+)  # type: Any
+
+retries = partial(
+    Option,
+    '--retries',
+    dest='retries',
+    type='int',
+    default=5,
+    help="Maximum number of retries each connection should attempt "
+         "(default %default times).",
+)  # type: Any
+
+timeout = partial(
+    Option,
+    '--timeout', '--default-timeout',
+    metavar='sec',
+    dest='timeout',
+    type='float',
+    default=15,
+    help='Set the socket timeout (default %default seconds).',
+)  # type: Any
+
+skip_requirements_regex = partial(
+    Option,
+    # A regex to be used to skip requirements
+    '--skip-requirements-regex',
+    dest='skip_requirements_regex',
+    type='str',
+    default='',
+    help=SUPPRESS_HELP,
+)  # type: Any
+
+
+def exists_action():
+    return Option(
+        # Option when path already exist
+        '--exists-action',
+        dest='exists_action',
+        type='choice',
+        choices=['s', 'i', 'w', 'b', 'a'],
+        default=[],
+        action='append',
+        metavar='action',
+        help="Default action when a path already exists: "
+             "(s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort).",
+    )
+
+
+cert = partial(
+    Option,
+    '--cert',
+    dest='cert',
+    type='str',
+    metavar='path',
+    help="Path to alternate CA bundle.",
+)  # type: Any
+
+client_cert = partial(
+    Option,
+    '--client-cert',
+    dest='client_cert',
+    type='str',
+    default=None,
+    metavar='path',
+    help="Path to SSL client certificate, a single file containing the "
+         "private key and the certificate in PEM format.",
+)  # type: Any
+
+index_url = partial(
+    Option,
+    '-i', '--index-url', '--pypi-url',
+    dest='index_url',
+    metavar='URL',
+    default=PyPI.simple_url,
+    help="Base URL of Python Package Index (default %default). "
+         "This should point to a repository compliant with PEP 503 "
+         "(the simple repository API) or a local directory laid out "
+         "in the same format.",
+)  # type: Any
+
+
+def extra_index_url():
+    return Option(
+        '--extra-index-url',
+        dest='extra_index_urls',
+        metavar='URL',
+        action='append',
+        default=[],
+        help="Extra URLs of package indexes to use in addition to "
+             "--index-url. Should follow the same rules as "
+             "--index-url.",
+    )
+
+
+no_index = partial(
+    Option,
+    '--no-index',
+    dest='no_index',
+    action='store_true',
+    default=False,
+    help='Ignore package index (only looking at --find-links URLs instead).',
+)  # type: Any
+
+
+def find_links():
+    return Option(
+        '-f', '--find-links',
+        dest='find_links',
+        action='append',
+        default=[],
+        metavar='url',
+        help="If a url or path to an html file, then parse for links to "
+             "archives. If a local path or file:// url that's a directory, "
+             "then look for archives in the directory listing.",
+    )
+
+
+def trusted_host():
+    return Option(
+        "--trusted-host",
+        dest="trusted_hosts",
+        action="append",
+        metavar="HOSTNAME",
+        default=[],
+        help="Mark this host as trusted, even though it does not have valid "
+             "or any HTTPS.",
+    )
+
+
+# Remove after 1.5
+process_dependency_links = partial(
+    Option,
+    "--process-dependency-links",
+    dest="process_dependency_links",
+    action="store_true",
+    default=False,
+    help="Enable the processing of dependency links.",
+)  # type: Any
+
+
+def constraints():
+    return Option(
+        '-c', '--constraint',
+        dest='constraints',
+        action='append',
+        default=[],
+        metavar='file',
+        help='Constrain versions using the given constraints file. '
+        'This option can be used multiple times.'
+    )
+
+
+def requirements():
+    return Option(
+        '-r', '--requirement',
+        dest='requirements',
+        action='append',
+        default=[],
+        metavar='file',
+        help='Install from the given requirements file. '
+        'This option can be used multiple times.'
+    )
+
+
+def editable():
+    return Option(
+        '-e', '--editable',
+        dest='editables',
+        action='append',
+        default=[],
+        metavar='path/url',
+        help=('Install a project in editable mode (i.e. setuptools '
+              '"develop mode") from a local project path or a VCS url.'),
+    )
+
+
+src = partial(
+    Option,
+    '--src', '--source', '--source-dir', '--source-directory',
+    dest='src_dir',
+    metavar='dir',
+    default=src_prefix,
+    help='Directory to check out editable projects into. '
+    'The default in a virtualenv is "<venv path>/src". '
+    'The default for global installs is "<current dir>/src".'
+)  # type: Any
+
+
+def _get_format_control(values, option):
+    """Get a format_control object."""
+    return getattr(values, option.dest)
+
+
+def _handle_no_binary(option, opt_str, value, parser):
+    existing = getattr(parser.values, option.dest)
+    fmt_ctl_handle_mutual_exclude(
+        value, existing.no_binary, existing.only_binary,
+    )
+
+
+def _handle_only_binary(option, opt_str, value, parser):
+    existing = getattr(parser.values, option.dest)
+    fmt_ctl_handle_mutual_exclude(
+        value, existing.only_binary, existing.no_binary,
+    )
+
+
+def no_binary():
+    return Option(
+        "--no-binary", dest="format_control", action="callback",
+        callback=_handle_no_binary, type="str",
+        default=FormatControl(set(), set()),
+        help="Do not use binary packages. Can be supplied multiple times, and "
+             "each time adds to the existing value. Accepts either :all: to "
+             "disable all binary packages, :none: to empty the set, or one or "
+             "more package names with commas between them. Note that some "
+             "packages are tricky to compile and may fail to install when "
+             "this option is used on them.",
+    )
+
+
+def only_binary():
+    return Option(
+        "--only-binary", dest="format_control", action="callback",
+        callback=_handle_only_binary, type="str",
+        default=FormatControl(set(), set()),
+        help="Do not use source packages. Can be supplied multiple times, and "
+             "each time adds to the existing value. Accepts either :all: to "
+             "disable all source packages, :none: to empty the set, or one or "
+             "more package names with commas between them. Packages without "
+             "binary distributions will fail to install when this option is "
+             "used on them.",
+    )
+
+
+cache_dir = partial(
+    Option,
+    "--cache-dir",
+    dest="cache_dir",
+    default=USER_CACHE_DIR,
+    metavar="dir",
+    help="Store the cache data in <dir>."
+)
+
+no_cache = partial(
+    Option,
+    "--no-cache-dir",
+    dest="cache_dir",
+    action="store_false",
+    help="Disable the cache.",
+)
+
+no_deps = partial(
+    Option,
+    '--no-deps', '--no-dependencies',
+    dest='ignore_dependencies',
+    action='store_true',
+    default=False,
+    help="Don't install package dependencies.",
+)  # type: Any
+
+build_dir = partial(
+    Option,
+    '-b', '--build', '--build-dir', '--build-directory',
+    dest='build_dir',
+    metavar='dir',
+    help='Directory to unpack packages into and build in. Note that '
+         'an initial build still takes place in a temporary directory. '
+         'The location of temporary directories can be controlled by setting '
+         'the TMPDIR environment variable (TEMP on Windows) appropriately. '
+         'When passed, build directories are not cleaned in case of failures.'
+)  # type: Any
+
+ignore_requires_python = partial(
+    Option,
+    '--ignore-requires-python',
+    dest='ignore_requires_python',
+    action='store_true',
+    help='Ignore the Requires-Python information.'
+)  # type: Any
+
+no_build_isolation = partial(
+    Option,
+    '--no-build-isolation',
+    dest='build_isolation',
+    action='store_false',
+    default=True,
+    help='Disable isolation when building a modern source distribution. '
+         'Build dependencies specified by PEP 518 must be already installed '
+         'if this option is used.'
+)  # type: Any
+
+install_options = partial(
+    Option,
+    '--install-option',
+    dest='install_options',
+    action='append',
+    metavar='options',
+    help="Extra arguments to be supplied to the setup.py install "
+         "command (use like --install-option=\"--install-scripts=/usr/local/"
+         "bin\"). Use multiple --install-option options to pass multiple "
+         "options to setup.py install. If you are using an option with a "
+         "directory path, be sure to use absolute path.",
+)  # type: Any
+
+global_options = partial(
+    Option,
+    '--global-option',
+    dest='global_options',
+    action='append',
+    metavar='options',
+    help="Extra global options to be supplied to the setup.py "
+         "call before the install command.",
+)  # type: Any
+
+no_clean = partial(
+    Option,
+    '--no-clean',
+    action='store_true',
+    default=False,
+    help="Don't clean up build directories)."
+)  # type: Any
+
+pre = partial(
+    Option,
+    '--pre',
+    action='store_true',
+    default=False,
+    help="Include pre-release and development versions. By default, "
+         "pip only finds stable versions.",
+)  # type: Any
+
+disable_pip_version_check = partial(
+    Option,
+    "--disable-pip-version-check",
+    dest="disable_pip_version_check",
+    action="store_true",
+    default=False,
+    help="Don't periodically check PyPI to determine whether a new version "
+         "of pip is available for download. Implied with --no-index.",
+)  # type: Any
+
+
+# Deprecated, Remove later
+always_unzip = partial(
+    Option,
+    '-Z', '--always-unzip',
+    dest='always_unzip',
+    action='store_true',
+    help=SUPPRESS_HELP,
+)  # type: Any
+
+
+def _merge_hash(option, opt_str, value, parser):
+    """Given a value spelled "algo:digest", append the digest to a list
+    pointed to in a dict by the algo name."""
+    if not parser.values.hashes:
+        parser.values.hashes = {}
+    try:
+        algo, digest = value.split(':', 1)
+    except ValueError:
+        parser.error('Arguments to %s must be a hash name '
+                     'followed by a value, like --hash=sha256:abcde...' %
+                     opt_str)
+    if algo not in STRONG_HASHES:
+        parser.error('Allowed hash algorithms for %s are %s.' %
+                     (opt_str, ', '.join(STRONG_HASHES)))
+    parser.values.hashes.setdefault(algo, []).append(digest)
+
+
+hash = partial(
+    Option,
+    '--hash',
+    # Hash values eventually end up in InstallRequirement.hashes due to
+    # __dict__ copying in process_line().
+    dest='hashes',
+    action='callback',
+    callback=_merge_hash,
+    type='string',
+    help="Verify that the package's archive matches this "
+         'hash before installing. Example: --hash=sha256:abcdef...',
+)  # type: Any
+
+
+require_hashes = partial(
+    Option,
+    '--require-hashes',
+    dest='require_hashes',
+    action='store_true',
+    default=False,
+    help='Require a hash to check each requirement against, for '
+         'repeatable installs. This option is implied when any package in a '
+         'requirements file has a --hash option.',
+)  # type: Any
+
+
+##########
+# groups #
+##########
+
+general_group = {
+    'name': 'General Options',
+    'options': [
+        help_,
+        isolated_mode,
+        require_virtualenv,
+        verbose,
+        version,
+        quiet,
+        log,
+        no_input,
+        proxy,
+        retries,
+        timeout,
+        skip_requirements_regex,
+        exists_action,
+        trusted_host,
+        cert,
+        client_cert,
+        cache_dir,
+        no_cache,
+        disable_pip_version_check,
+        no_color,
+    ]
+}
+
+index_group = {
+    'name': 'Package Index Options',
+    'options': [
+        index_url,
+        extra_index_url,
+        no_index,
+        find_links,
+        process_dependency_links,
+    ]
+}

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/__init__.py

@@ -0,0 +1,79 @@
+"""
+Package containing all pip commands
+"""
+from __future__ import absolute_import
+
+from pip._internal.commands.completion import CompletionCommand
+from pip._internal.commands.configuration import ConfigurationCommand
+from pip._internal.commands.download import DownloadCommand
+from pip._internal.commands.freeze import FreezeCommand
+from pip._internal.commands.hash import HashCommand
+from pip._internal.commands.help import HelpCommand
+from pip._internal.commands.list import ListCommand
+from pip._internal.commands.check import CheckCommand
+from pip._internal.commands.search import SearchCommand
+from pip._internal.commands.show import ShowCommand
+from pip._internal.commands.install import InstallCommand
+from pip._internal.commands.uninstall import UninstallCommand
+from pip._internal.commands.wheel import WheelCommand
+
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import List, Type
+    from pip._internal.basecommand import Command
+
+commands_order = [
+    InstallCommand,
+    DownloadCommand,
+    UninstallCommand,
+    FreezeCommand,
+    ListCommand,
+    ShowCommand,
+    CheckCommand,
+    ConfigurationCommand,
+    SearchCommand,
+    WheelCommand,
+    HashCommand,
+    CompletionCommand,
+    HelpCommand,
+]  # type: List[Type[Command]]
+
+commands_dict = {c.name: c for c in commands_order}
+
+
+def get_summaries(ordered=True):
+    """Yields sorted (command name, command summary) tuples."""
+
+    if ordered:
+        cmditems = _sort_commands(commands_dict, commands_order)
+    else:
+        cmditems = commands_dict.items()
+
+    for name, command_class in cmditems:
+        yield (name, command_class.summary)
+
+
+def get_similar_commands(name):
+    """Command name auto-correct."""
+    from difflib import get_close_matches
+
+    name = name.lower()
+
+    close_commands = get_close_matches(name, commands_dict.keys())
+
+    if close_commands:
+        return close_commands[0]
+    else:
+        return False
+
+
+def _sort_commands(cmddict, order):
+    def keyfn(key):
+        try:
+            return order.index(key[1])
+        except ValueError:
+            # unordered items should come last
+            return 0xff
+
+    return sorted(cmddict.items(), key=keyfn)

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/check.py

@@ -0,0 +1,42 @@
+import logging
+
+from pip._internal.basecommand import Command
+from pip._internal.operations.check import (
+    check_package_set, create_package_set_from_installed,
+)
+from pip._internal.utils.misc import get_installed_distributions
+
+logger = logging.getLogger(__name__)
+
+
+class CheckCommand(Command):
+    """Verify installed packages have compatible dependencies."""
+    name = 'check'
+    usage = """
+      %prog [options]"""
+    summary = 'Verify installed packages have compatible dependencies.'
+
+    def run(self, options, args):
+        package_set = create_package_set_from_installed()
+        missing, conflicting = check_package_set(package_set)
+
+        for project_name in missing:
+            version = package_set[project_name].version
+            for dependency in missing[project_name]:
+                logger.info(
+                    "%s %s requires %s, which is not installed.",
+                    project_name, version, dependency[0],
+                )
+
+        for project_name in conflicting:
+            version = package_set[project_name].version
+            for dep_name, dep_version, req in conflicting[project_name]:
+                logger.info(
+                    "%s %s has requirement %s, but you have %s %s.",
+                    project_name, version, req, dep_name, dep_version,
+                )
+
+        if missing or conflicting:
+            return 1
+        else:
+            logger.info("No broken requirements found.")

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/completion.py

@@ -0,0 +1,94 @@
+from __future__ import absolute_import
+
+import sys
+import textwrap
+
+from pip._internal.basecommand import Command
+from pip._internal.utils.misc import get_prog
+
+BASE_COMPLETION = """
+# pip %(shell)s completion start%(script)s# pip %(shell)s completion end
+"""
+
+COMPLETION_SCRIPTS = {
+    'bash': """
+        _pip_completion()
+        {
+            COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\
+                           COMP_CWORD=$COMP_CWORD \\
+                           PIP_AUTO_COMPLETE=1 $1 ) )
+        }
+        complete -o default -F _pip_completion %(prog)s
+    """,
+    'zsh': """
+        function _pip_completion {
+          local words cword
+          read -Ac words
+          read -cn cword
+          reply=( $( COMP_WORDS="$words[*]" \\
+                     COMP_CWORD=$(( cword-1 )) \\
+                     PIP_AUTO_COMPLETE=1 $words[1] ) )
+        }
+        compctl -K _pip_completion %(prog)s
+    """,
+    'fish': """
+        function __fish_complete_pip
+            set -lx COMP_WORDS (commandline -o) ""
+            set -lx COMP_CWORD ( \\
+                math (contains -i -- (commandline -t) $COMP_WORDS)-1 \\
+            )
+            set -lx PIP_AUTO_COMPLETE 1
+            string split \\  -- (eval $COMP_WORDS[1])
+        end
+        complete -fa "(__fish_complete_pip)" -c %(prog)s
+    """,
+}
+
+
+class CompletionCommand(Command):
+    """A helper command to be used for command completion."""
+    name = 'completion'
+    summary = 'A helper command used for command completion.'
+    ignore_require_venv = True
+
+    def __init__(self, *args, **kw):
+        super(CompletionCommand, self).__init__(*args, **kw)
+
+        cmd_opts = self.cmd_opts
+
+        cmd_opts.add_option(
+            '--bash', '-b',
+            action='store_const',
+            const='bash',
+            dest='shell',
+            help='Emit completion code for bash')
+        cmd_opts.add_option(
+            '--zsh', '-z',
+            action='store_const',
+            const='zsh',
+            dest='shell',
+            help='Emit completion code for zsh')
+        cmd_opts.add_option(
+            '--fish', '-f',
+            action='store_const',
+            const='fish',
+            dest='shell',
+            help='Emit completion code for fish')
+
+        self.parser.insert_option_group(0, cmd_opts)
+
+    def run(self, options, args):
+        """Prints the completion code of the given shell"""
+        shells = COMPLETION_SCRIPTS.keys()
+        shell_options = ['--' + shell for shell in sorted(shells)]
+        if options.shell in shells:
+            script = textwrap.dedent(
+                COMPLETION_SCRIPTS.get(options.shell, '') % {
+                    'prog': get_prog(),
+                }
+            )
+            print(BASE_COMPLETION % {'script': script, 'shell': options.shell})
+        else:
+            sys.stderr.write(
+                'ERROR: You must pass %s\n' % ' or '.join(shell_options)
+            )

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/configuration.py

@@ -0,0 +1,227 @@
+import logging
+import os
+import subprocess
+
+from pip._internal.basecommand import Command
+from pip._internal.configuration import Configuration, kinds
+from pip._internal.exceptions import PipError
+from pip._internal.locations import venv_config_file
+from pip._internal.status_codes import ERROR, SUCCESS
+from pip._internal.utils.misc import get_prog
+
+logger = logging.getLogger(__name__)
+
+
+class ConfigurationCommand(Command):
+    """Manage local and global configuration.
+
+        Subcommands:
+
+        list: List the active configuration (or from the file specified)
+        edit: Edit the configuration file in an editor
+        get: Get the value associated with name
+        set: Set the name=value
+        unset: Unset the value associated with name
+
+        If none of --user, --global and --venv are passed, a virtual
+        environment configuration file is used if one is active and the file
+        exists. Otherwise, all modifications happen on the to the user file by
+        default.
+    """
+
+    name = 'config'
+    usage = """
+        %prog [<file-option>] list
+        %prog [<file-option>] [--editor <editor-path>] edit
+
+        %prog [<file-option>] get name
+        %prog [<file-option>] set name value
+        %prog [<file-option>] unset name
+    """
+
+    summary = "Manage local and global configuration."
+
+    def __init__(self, *args, **kwargs):
+        super(ConfigurationCommand, self).__init__(*args, **kwargs)
+
+        self.configuration = None
+
+        self.cmd_opts.add_option(
+            '--editor',
+            dest='editor',
+            action='store',
+            default=None,
+            help=(
+                'Editor to use to edit the file. Uses VISUAL or EDITOR '
+                'environment variables if not provided.'
+            )
+        )
+
+        self.cmd_opts.add_option(
+            '--global',
+            dest='global_file',
+            action='store_true',
+            default=False,
+            help='Use the system-wide configuration file only'
+        )
+
+        self.cmd_opts.add_option(
+            '--user',
+            dest='user_file',
+            action='store_true',
+            default=False,
+            help='Use the user configuration file only'
+        )
+
+        self.cmd_opts.add_option(
+            '--venv',
+            dest='venv_file',
+            action='store_true',
+            default=False,
+            help='Use the virtualenv configuration file only'
+        )
+
+        self.parser.insert_option_group(0, self.cmd_opts)
+
+    def run(self, options, args):
+        handlers = {
+            "list": self.list_values,
+            "edit": self.open_in_editor,
+            "get": self.get_name,
+            "set": self.set_name_value,
+            "unset": self.unset_name
+        }
+
+        # Determine action
+        if not args or args[0] not in handlers:
+            logger.error("Need an action ({}) to perform.".format(
+                ", ".join(sorted(handlers)))
+            )
+            return ERROR
+
+        action = args[0]
+
+        # Determine which configuration files are to be loaded
+        #    Depends on whether the command is modifying.
+        try:
+            load_only = self._determine_file(
+                options, need_value=(action in ["get", "set", "unset", "edit"])
+            )
+        except PipError as e:
+            logger.error(e.args[0])
+            return ERROR
+
+        # Load a new configuration
+        self.configuration = Configuration(
+            isolated=options.isolated_mode, load_only=load_only
+        )
+        self.configuration.load()
+
+        # Error handling happens here, not in the action-handlers.
+        try:
+            handlers[action](options, args[1:])
+        except PipError as e:
+            logger.error(e.args[0])
+            return ERROR
+
+        return SUCCESS
+
+    def _determine_file(self, options, need_value):
+        file_options = {
+            kinds.USER: options.user_file,
+            kinds.GLOBAL: options.global_file,
+            kinds.VENV: options.venv_file
+        }
+
+        if sum(file_options.values()) == 0:
+            if not need_value:
+                return None
+            # Default to user, unless there's a virtualenv file.
+            elif os.path.exists(venv_config_file):
+                return kinds.VENV
+            else:
+                return kinds.USER
+        elif sum(file_options.values()) == 1:
+            # There's probably a better expression for this.
+            return [key for key in file_options if file_options[key]][0]
+
+        raise PipError(
+            "Need exactly one file to operate upon "
+            "(--user, --venv, --global) to perform."
+        )
+
+    def list_values(self, options, args):
+        self._get_n_args(args, "list", n=0)
+
+        for key, value in sorted(self.configuration.items()):
+            logger.info("%s=%r", key, value)
+
+    def get_name(self, options, args):
+        key = self._get_n_args(args, "get [name]", n=1)
+        value = self.configuration.get_value(key)
+
+        logger.info("%s", value)
+
+    def set_name_value(self, options, args):
+        key, value = self._get_n_args(args, "set [name] [value]", n=2)
+        self.configuration.set_value(key, value)
+
+        self._save_configuration()
+
+    def unset_name(self, options, args):
+        key = self._get_n_args(args, "unset [name]", n=1)
+        self.configuration.unset_value(key)
+
+        self._save_configuration()
+
+    def open_in_editor(self, options, args):
+        editor = self._determine_editor(options)
+
+        fname = self.configuration.get_file_to_edit()
+        if fname is None:
+            raise PipError("Could not determine appropriate file.")
+
+        try:
+            subprocess.check_call([editor, fname])
+        except subprocess.CalledProcessError as e:
+            raise PipError(
+                "Editor Subprocess exited with exit code {}"
+                .format(e.returncode)
+            )
+
+    def _get_n_args(self, args, example, n):
+        """Helper to make sure the command got the right number of arguments
+        """
+        if len(args) != n:
+            msg = (
+                'Got unexpected number of arguments, expected {}. '
+                '(example: "{} config {}")'
+            ).format(n, get_prog(), example)
+            raise PipError(msg)
+
+        if n == 1:
+            return args[0]
+        else:
+            return args
+
+    def _save_configuration(self):
+        # We successfully ran a modifying command. Need to save the
+        # configuration.
+        try:
+            self.configuration.save()
+        except Exception:
+            logger.error(
+                "Unable to save configuration. Please report this as a bug.",
+                exc_info=1
+            )
+            raise PipError("Internal Error.")
+
+    def _determine_editor(self, options):
+        if options.editor is not None:
+            return options.editor
+        elif "VISUAL" in os.environ:
+            return os.environ["VISUAL"]
+        elif "EDITOR" in os.environ:
+            return os.environ["EDITOR"]
+        else:
+            raise PipError("Could not determine editor to use.")

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/download.py

@@ -0,0 +1,233 @@
+from __future__ import absolute_import
+
+import logging
+import os
+
+from pip._internal import cmdoptions
+from pip._internal.basecommand import RequirementCommand
+from pip._internal.exceptions import CommandError
+from pip._internal.index import FormatControl
+from pip._internal.operations.prepare import RequirementPreparer
+from pip._internal.req import RequirementSet
+from pip._internal.resolve import Resolver
+from pip._internal.utils.filesystem import check_path_owner
+from pip._internal.utils.misc import ensure_dir, normalize_path
+from pip._internal.utils.temp_dir import TempDirectory
+
+logger = logging.getLogger(__name__)
+
+
+class DownloadCommand(RequirementCommand):
+    """
+    Download packages from:
+
+    - PyPI (and other indexes) using requirement specifiers.
+    - VCS project urls.
+    - Local project directories.
+    - Local or remote source archives.
+
+    pip also supports downloading from "requirements files", which provide
+    an easy way to specify a whole environment to be downloaded.
+    """
+    name = 'download'
+
+    usage = """
+      %prog [options] <requirement specifier> [package-index-options] ...
+      %prog [options] -r <requirements file> [package-index-options] ...
+      %prog [options] <vcs project url> ...
+      %prog [options] <local project path> ...
+      %prog [options] <archive url/path> ..."""
+
+    summary = 'Download packages.'
+
+    def __init__(self, *args, **kw):
+        super(DownloadCommand, self).__init__(*args, **kw)
+
+        cmd_opts = self.cmd_opts
+
+        cmd_opts.add_option(cmdoptions.constraints())
+        cmd_opts.add_option(cmdoptions.requirements())
+        cmd_opts.add_option(cmdoptions.build_dir())
+        cmd_opts.add_option(cmdoptions.no_deps())
+        cmd_opts.add_option(cmdoptions.global_options())
+        cmd_opts.add_option(cmdoptions.no_binary())
+        cmd_opts.add_option(cmdoptions.only_binary())
+        cmd_opts.add_option(cmdoptions.src())
+        cmd_opts.add_option(cmdoptions.pre())
+        cmd_opts.add_option(cmdoptions.no_clean())
+        cmd_opts.add_option(cmdoptions.require_hashes())
+        cmd_opts.add_option(cmdoptions.progress_bar())
+        cmd_opts.add_option(cmdoptions.no_build_isolation())
+
+        cmd_opts.add_option(
+            '-d', '--dest', '--destination-dir', '--destination-directory',
+            dest='download_dir',
+            metavar='dir',
+            default=os.curdir,
+            help=("Download packages into <dir>."),
+        )
+
+        cmd_opts.add_option(
+            '--platform',
+            dest='platform',
+            metavar='platform',
+            default=None,
+            help=("Only download wheels compatible with <platform>. "
+                  "Defaults to the platform of the running system."),
+        )
+
+        cmd_opts.add_option(
+            '--python-version',
+            dest='python_version',
+            metavar='python_version',
+            default=None,
+            help=("Only download wheels compatible with Python "
+                  "interpreter version <version>. If not specified, then the "
+                  "current system interpreter minor version is used. A major "
+                  "version (e.g. '2') can be specified to match all "
+                  "minor revs of that major version.  A minor version "
+                  "(e.g. '34') can also be specified."),
+        )
+
+        cmd_opts.add_option(
+            '--implementation',
+            dest='implementation',
+            metavar='implementation',
+            default=None,
+            help=("Only download wheels compatible with Python "
+                  "implementation <implementation>, e.g. 'pp', 'jy', 'cp', "
+                  " or 'ip'. If not specified, then the current "
+                  "interpreter implementation is used.  Use 'py' to force "
+                  "implementation-agnostic wheels."),
+        )
+
+        cmd_opts.add_option(
+            '--abi',
+            dest='abi',
+            metavar='abi',
+            default=None,
+            help=("Only download wheels compatible with Python "
+                  "abi <abi>, e.g. 'pypy_41'.  If not specified, then the "
+                  "current interpreter abi tag is used.  Generally "
+                  "you will need to specify --implementation, "
+                  "--platform, and --python-version when using "
+                  "this option."),
+        )
+
+        index_opts = cmdoptions.make_option_group(
+            cmdoptions.index_group,
+            self.parser,
+        )
+
+        self.parser.insert_option_group(0, index_opts)
+        self.parser.insert_option_group(0, cmd_opts)
+
+    def run(self, options, args):
+        options.ignore_installed = True
+        # editable doesn't really make sense for `pip download`, but the bowels
+        # of the RequirementSet code require that property.
+        options.editables = []
+
+        if options.python_version:
+            python_versions = [options.python_version]
+        else:
+            python_versions = None
+
+        dist_restriction_set = any([
+            options.python_version,
+            options.platform,
+            options.abi,
+            options.implementation,
+        ])
+        binary_only = FormatControl(set(), {':all:'})
+        no_sdist_dependencies = (
+            options.format_control != binary_only and
+            not options.ignore_dependencies
+        )
+        if dist_restriction_set and no_sdist_dependencies:
+            raise CommandError(
+                "When restricting platform and interpreter constraints using "
+                "--python-version, --platform, --abi, or --implementation, "
+                "either --no-deps must be set, or --only-binary=:all: must be "
+                "set and --no-binary must not be set (or must be set to "
+                ":none:)."
+            )
+
+        options.src_dir = os.path.abspath(options.src_dir)
+        options.download_dir = normalize_path(options.download_dir)
+
+        ensure_dir(options.download_dir)
+
+        with self._build_session(options) as session:
+            finder = self._build_package_finder(
+                options=options,
+                session=session,
+                platform=options.platform,
+                python_versions=python_versions,
+                abi=options.abi,
+                implementation=options.implementation,
+            )
+            build_delete = (not (options.no_clean or options.build_dir))
+            if options.cache_dir and not check_path_owner(options.cache_dir):
+                logger.warning(
+                    "The directory '%s' or its parent directory is not owned "
+                    "by the current user and caching wheels has been "
+                    "disabled. check the permissions and owner of that "
+                    "directory. If executing pip with sudo, you may want "
+                    "sudo's -H flag.",
+                    options.cache_dir,
+                )
+                options.cache_dir = None
+
+            with TempDirectory(
+                options.build_dir, delete=build_delete, kind="download"
+            ) as directory:
+
+                requirement_set = RequirementSet(
+                    require_hashes=options.require_hashes,
+                )
+                self.populate_requirement_set(
+                    requirement_set,
+                    args,
+                    options,
+                    finder,
+                    session,
+                    self.name,
+                    None
+                )
+
+                preparer = RequirementPreparer(
+                    build_dir=directory.path,
+                    src_dir=options.src_dir,
+                    download_dir=options.download_dir,
+                    wheel_download_dir=None,
+                    progress_bar=options.progress_bar,
+                    build_isolation=options.build_isolation,
+                )
+
+                resolver = Resolver(
+                    preparer=preparer,
+                    finder=finder,
+                    session=session,
+                    wheel_cache=None,
+                    use_user_site=False,
+                    upgrade_strategy="to-satisfy-only",
+                    force_reinstall=False,
+                    ignore_dependencies=options.ignore_dependencies,
+                    ignore_requires_python=False,
+                    ignore_installed=True,
+                    isolated=options.isolated_mode,
+                )
+                resolver.resolve(requirement_set)
+
+                downloaded = ' '.join([
+                    req.name for req in requirement_set.successfully_downloaded
+                ])
+                if downloaded:
+                    logger.info('Successfully downloaded %s', downloaded)
+
+                # Clean up
+                if not options.no_clean:
+                    requirement_set.cleanup_files()
+
+        return requirement_set

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/freeze.py

@@ -0,0 +1,96 @@
+from __future__ import absolute_import
+
+import sys
+
+from pip._internal import index
+from pip._internal.basecommand import Command
+from pip._internal.cache import WheelCache
+from pip._internal.compat import stdlib_pkgs
+from pip._internal.operations.freeze import freeze
+
+DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel'}
+
+
+class FreezeCommand(Command):
+    """
+    Output installed packages in requirements format.
+
+    packages are listed in a case-insensitive sorted order.
+    """
+    name = 'freeze'
+    usage = """
+      %prog [options]"""
+    summary = 'Output installed packages in requirements format.'
+    log_streams = ("ext://sys.stderr", "ext://sys.stderr")
+
+    def __init__(self, *args, **kw):
+        super(FreezeCommand, self).__init__(*args, **kw)
+
+        self.cmd_opts.add_option(
+            '-r', '--requirement',
+            dest='requirements',
+            action='append',
+            default=[],
+            metavar='file',
+            help="Use the order in the given requirements file and its "
+                 "comments when generating output. This option can be "
+                 "used multiple times.")
+        self.cmd_opts.add_option(
+            '-f', '--find-links',
+            dest='find_links',
+            action='append',
+            default=[],
+            metavar='URL',
+            help='URL for finding packages, which will be added to the '
+                 'output.')
+        self.cmd_opts.add_option(
+            '-l', '--local',
+            dest='local',
+            action='store_true',
+            default=False,
+            help='If in a virtualenv that has global access, do not output '
+                 'globally-installed packages.')
+        self.cmd_opts.add_option(
+            '--user',
+            dest='user',
+            action='store_true',
+            default=False,
+            help='Only output packages installed in user-site.')
+        self.cmd_opts.add_option(
+            '--all',
+            dest='freeze_all',
+            action='store_true',
+            help='Do not skip these packages in the output:'
+                 ' %s' % ', '.join(DEV_PKGS))
+        self.cmd_opts.add_option(
+            '--exclude-editable',
+            dest='exclude_editable',
+            action='store_true',
+            help='Exclude editable package from output.')
+
+        self.parser.insert_option_group(0, self.cmd_opts)
+
+    def run(self, options, args):
+        format_control = index.FormatControl(set(), set())
+        wheel_cache = WheelCache(options.cache_dir, format_control)
+        skip = set(stdlib_pkgs)
+        if not options.freeze_all:
+            skip.update(DEV_PKGS)
+
+        freeze_kwargs = dict(
+            requirement=options.requirements,
+            find_links=options.find_links,
+            local_only=options.local,
+            user_only=options.user,
+            skip_regex=options.skip_requirements_regex,
+            isolated=options.isolated_mode,
+            wheel_cache=wheel_cache,
+            skip=skip,
+            exclude_editable=options.exclude_editable,
+        )
+
+        try:
+            for line in freeze(**freeze_kwargs):
+                sys.stdout.write(line + '\n')
+        finally:
+            wheel_cache.cleanup()

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/hash.py

@@ -0,0 +1,57 @@
+from __future__ import absolute_import
+
+import hashlib
+import logging
+import sys
+
+from pip._internal.basecommand import Command
+from pip._internal.status_codes import ERROR
+from pip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES
+from pip._internal.utils.misc import read_chunks
+
+logger = logging.getLogger(__name__)
+
+
+class HashCommand(Command):
+    """
+    Compute a hash of a local package archive.
+
+    These can be used with --hash in a requirements file to do repeatable
+    installs.
+
+    """
+    name = 'hash'
+    usage = '%prog [options] <file> ...'
+    summary = 'Compute hashes of package archives.'
+    ignore_require_venv = True
+
+    def __init__(self, *args, **kw):
+        super(HashCommand, self).__init__(*args, **kw)
+        self.cmd_opts.add_option(
+            '-a', '--algorithm',
+            dest='algorithm',
+            choices=STRONG_HASHES,
+            action='store',
+            default=FAVORITE_HASH,
+            help='The hash algorithm to use: one of %s' %
+                 ', '.join(STRONG_HASHES))
+        self.parser.insert_option_group(0, self.cmd_opts)
+
+    def run(self, options, args):
+        if not args:
+            self.parser.print_usage(sys.stderr)
+            return ERROR
+
+        algorithm = options.algorithm
+        for path in args:
+            logger.info('%s:\n--hash=%s:%s',
+                        path, algorithm, _hash_of_file(path, algorithm))
+
+
+def _hash_of_file(path, algorithm):
+    """Return the hash digest of a file."""
+    with open(path, 'rb') as archive:
+        hash = hashlib.new(algorithm)
+        for chunk in read_chunks(archive):
+            hash.update(chunk)
+    return hash.hexdigest()

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/help.py

@@ -0,0 +1,36 @@
+from __future__ import absolute_import
+
+from pip._internal.basecommand import SUCCESS, Command
+from pip._internal.exceptions import CommandError
+
+
+class HelpCommand(Command):
+    """Show help for commands"""
+    name = 'help'
+    usage = """
+      %prog <command>"""
+    summary = 'Show help for commands.'
+    ignore_require_venv = True
+
+    def run(self, options, args):
+        from pip._internal.commands import commands_dict, get_similar_commands
+
+        try:
+            # 'pip help' with no args is handled by pip.__init__.parseopt()
+            cmd_name = args[0]  # the command we need help for
+        except IndexError:
+            return SUCCESS
+
+        if cmd_name not in commands_dict:
+            guess = get_similar_commands(cmd_name)
+
+            msg = ['unknown command "%s"' % cmd_name]
+            if guess:
+                msg.append('maybe you meant "%s"' % guess)
+
+            raise CommandError(' - '.join(msg))
+
+        command = commands_dict[cmd_name]()
+        command.parser.print_help()
+
+        return SUCCESS

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/install.py

@@ -0,0 +1,502 @@
+from __future__ import absolute_import
+
+import errno
+import logging
+import operator
+import os
+import shutil
+from optparse import SUPPRESS_HELP
+
+from pip._internal import cmdoptions
+from pip._internal.basecommand import RequirementCommand
+from pip._internal.cache import WheelCache
+from pip._internal.exceptions import (
+    CommandError, InstallationError, PreviousBuildDirError,
+)
+from pip._internal.locations import distutils_scheme, virtualenv_no_global
+from pip._internal.operations.check import check_install_conflicts
+from pip._internal.operations.prepare import RequirementPreparer
+from pip._internal.req import RequirementSet, install_given_reqs
+from pip._internal.resolve import Resolver
+from pip._internal.status_codes import ERROR
+from pip._internal.utils.filesystem import check_path_owner
+from pip._internal.utils.misc import ensure_dir, get_installed_version
+from pip._internal.utils.temp_dir import TempDirectory
+from pip._internal.wheel import WheelBuilder
+
+try:
+    import wheel
+except ImportError:
+    wheel = None
+
+
+logger = logging.getLogger(__name__)
+
+
+class InstallCommand(RequirementCommand):
+    """
+    Install packages from:
+
+    - PyPI (and other indexes) using requirement specifiers.
+    - VCS project urls.
+    - Local project directories.
+    - Local or remote source archives.
+
+    pip also supports installing from "requirements files", which provide
+    an easy way to specify a whole environment to be installed.
+    """
+    name = 'install'
+
+    usage = """
+      %prog [options] <requirement specifier> [package-index-options] ...
+      %prog [options] -r <requirements file> [package-index-options] ...
+      %prog [options] [-e] <vcs project url> ...
+      %prog [options] [-e] <local project path> ...
+      %prog [options] <archive url/path> ..."""
+
+    summary = 'Install packages.'
+
+    def __init__(self, *args, **kw):
+        super(InstallCommand, self).__init__(*args, **kw)
+
+        cmd_opts = self.cmd_opts
+
+        cmd_opts.add_option(cmdoptions.requirements())
+        cmd_opts.add_option(cmdoptions.constraints())
+        cmd_opts.add_option(cmdoptions.no_deps())
+        cmd_opts.add_option(cmdoptions.pre())
+
+        cmd_opts.add_option(cmdoptions.editable())
+        cmd_opts.add_option(
+            '-t', '--target',
+            dest='target_dir',
+            metavar='dir',
+            default=None,
+            help='Install packages into <dir>. '
+                 'By default this will not replace existing files/folders in '
+                 '<dir>. Use --upgrade to replace existing packages in <dir> '
+                 'with new versions.'
+        )
+        cmd_opts.add_option(
+            '--user',
+            dest='use_user_site',
+            action='store_true',
+            help="Install to the Python user install directory for your "
+                 "platform. Typically ~/.local/, or %APPDATA%\\Python on "
+                 "Windows. (See the Python documentation for site.USER_BASE "
+                 "for full details.)")
+        cmd_opts.add_option(
+            '--no-user',
+            dest='use_user_site',
+            action='store_false',
+            help=SUPPRESS_HELP)
+        cmd_opts.add_option(
+            '--root',
+            dest='root_path',
+            metavar='dir',
+            default=None,
+            help="Install everything relative to this alternate root "
+                 "directory.")
+        cmd_opts.add_option(
+            '--prefix',
+            dest='prefix_path',
+            metavar='dir',
+            default=None,
+            help="Installation prefix where lib, bin and other top-level "
+                 "folders are placed")
+
+        cmd_opts.add_option(cmdoptions.build_dir())
+
+        cmd_opts.add_option(cmdoptions.src())
+
+        cmd_opts.add_option(
+            '-U', '--upgrade',
+            dest='upgrade',
+            action='store_true',
+            help='Upgrade all specified packages to the newest available '
+                 'version. The handling of dependencies depends on the '
+                 'upgrade-strategy used.'
+        )
+
+        cmd_opts.add_option(
+            '--upgrade-strategy',
+            dest='upgrade_strategy',
+            default='only-if-needed',
+            choices=['only-if-needed', 'eager'],
+            help='Determines how dependency upgrading should be handled '
+                 '[default: %default]. '
+                 '"eager" - dependencies are upgraded regardless of '
+                 'whether the currently installed version satisfies the '
+                 'requirements of the upgraded package(s). '
+                 '"only-if-needed" -  are upgraded only when they do not '
+                 'satisfy the requirements of the upgraded package(s).'
+        )
+
+        cmd_opts.add_option(
+            '--force-reinstall',
+            dest='force_reinstall',
+            action='store_true',
+            help='Reinstall all packages even if they are already '
+                 'up-to-date.')
+
+        cmd_opts.add_option(
+            '-I', '--ignore-installed',
+            dest='ignore_installed',
+            action='store_true',
+            help='Ignore the installed packages (reinstalling instead).')
+
+        cmd_opts.add_option(cmdoptions.ignore_requires_python())
+        cmd_opts.add_option(cmdoptions.no_build_isolation())
+
+        cmd_opts.add_option(cmdoptions.install_options())
+        cmd_opts.add_option(cmdoptions.global_options())
+
+        cmd_opts.add_option(
+            "--compile",
+            action="store_true",
+            dest="compile",
+            default=True,
+            help="Compile Python source files to bytecode",
+        )
+
+        cmd_opts.add_option(
+            "--no-compile",
+            action="store_false",
+            dest="compile",
+            help="Do not compile Python source files to bytecode",
+        )
+
+        cmd_opts.add_option(
+            "--no-warn-script-location",
+            action="store_false",
+            dest="warn_script_location",
+            default=True,
+            help="Do not warn when installing scripts outside PATH",
+        )
+        cmd_opts.add_option(
+            "--no-warn-conflicts",
+            action="store_false",
+            dest="warn_about_conflicts",
+            default=True,
+            help="Do not warn about broken dependencies",
+        )
+
+        cmd_opts.add_option(cmdoptions.no_binary())
+        cmd_opts.add_option(cmdoptions.only_binary())
+        cmd_opts.add_option(cmdoptions.no_clean())
+        cmd_opts.add_option(cmdoptions.require_hashes())
+        cmd_opts.add_option(cmdoptions.progress_bar())
+
+        index_opts = cmdoptions.make_option_group(
+            cmdoptions.index_group,
+            self.parser,
+        )
+
+        self.parser.insert_option_group(0, index_opts)
+        self.parser.insert_option_group(0, cmd_opts)
+
+    def run(self, options, args):
+        cmdoptions.check_install_build_global(options)
+
+        upgrade_strategy = "to-satisfy-only"
+        if options.upgrade:
+            upgrade_strategy = options.upgrade_strategy
+
+        if options.build_dir:
+            options.build_dir = os.path.abspath(options.build_dir)
+
+        options.src_dir = os.path.abspath(options.src_dir)
+        install_options = options.install_options or []
+        if options.use_user_site:
+            if options.prefix_path:
+                raise CommandError(
+                    "Can not combine '--user' and '--prefix' as they imply "
+                    "different installation locations"
+                )
+            if virtualenv_no_global():
+                raise InstallationError(
+                    "Can not perform a '--user' install. User site-packages "
+                    "are not visible in this virtualenv."
+                )
+            install_options.append('--user')
+            install_options.append('--prefix=')
+
+        target_temp_dir = TempDirectory(kind="target")
+        if options.target_dir:
+            options.ignore_installed = True
+            options.target_dir = os.path.abspath(options.target_dir)
+            if (os.path.exists(options.target_dir) and not
+                    os.path.isdir(options.target_dir)):
+                raise CommandError(
+                    "Target path exists but is not a directory, will not "
+                    "continue."
+                )
+
+            # Create a target directory for using with the target option
+            target_temp_dir.create()
+            install_options.append('--home=' + target_temp_dir.path)
+
+        global_options = options.global_options or []
+
+        with self._build_session(options) as session:
+            finder = self._build_package_finder(options, session)
+            build_delete = (not (options.no_clean or options.build_dir))
+            wheel_cache = WheelCache(options.cache_dir, options.format_control)
+
+            if options.cache_dir and not check_path_owner(options.cache_dir):
+                logger.warning(
+                    "The directory '%s' or its parent directory is not owned "
+                    "by the current user and caching wheels has been "
+                    "disabled. check the permissions and owner of that "
+                    "directory. If executing pip with sudo, you may want "
+                    "sudo's -H flag.",
+                    options.cache_dir,
+                )
+                options.cache_dir = None
+
+            with TempDirectory(
+                options.build_dir, delete=build_delete, kind="install"
+            ) as directory:
+                requirement_set = RequirementSet(
+                    require_hashes=options.require_hashes,
+                )
+
+                try:
+                    self.populate_requirement_set(
+                        requirement_set, args, options, finder, session,
+                        self.name, wheel_cache
+                    )
+                    preparer = RequirementPreparer(
+                        build_dir=directory.path,
+                        src_dir=options.src_dir,
+                        download_dir=None,
+                        wheel_download_dir=None,
+                        progress_bar=options.progress_bar,
+                        build_isolation=options.build_isolation,
+                    )
+
+                    resolver = Resolver(
+                        preparer=preparer,
+                        finder=finder,
+                        session=session,
+                        wheel_cache=wheel_cache,
+                        use_user_site=options.use_user_site,
+                        upgrade_strategy=upgrade_strategy,
+                        force_reinstall=options.force_reinstall,
+                        ignore_dependencies=options.ignore_dependencies,
+                        ignore_requires_python=options.ignore_requires_python,
+                        ignore_installed=options.ignore_installed,
+                        isolated=options.isolated_mode,
+                    )
+                    resolver.resolve(requirement_set)
+
+                    # If caching is disabled or wheel is not installed don't
+                    # try to build wheels.
+                    if wheel and options.cache_dir:
+                        # build wheels before install.
+                        wb = WheelBuilder(
+                            finder, preparer, wheel_cache,
+                            build_options=[], global_options=[],
+                        )
+                        # Ignore the result: a failed wheel will be
+                        # installed from the sdist/vcs whatever.
+                        wb.build(
+                            requirement_set.requirements.values(),
+                            session=session, autobuilding=True
+                        )
+
+                    to_install = resolver.get_installation_order(
+                        requirement_set
+                    )
+
+                    # Consistency Checking of the package set we're installing.
+                    should_warn_about_conflicts = (
+                        not options.ignore_dependencies and
+                        options.warn_about_conflicts
+                    )
+                    if should_warn_about_conflicts:
+                        self._warn_about_conflicts(to_install)
+
+                    # Don't warn about script install locations if
+                    # --target has been specified
+                    warn_script_location = options.warn_script_location
+                    if options.target_dir:
+                        warn_script_location = False
+
+                    installed = install_given_reqs(
+                        to_install,
+                        install_options,
+                        global_options,
+                        root=options.root_path,
+                        home=target_temp_dir.path,
+                        prefix=options.prefix_path,
+                        pycompile=options.compile,
+                        warn_script_location=warn_script_location,
+                        use_user_site=options.use_user_site,
+                    )
+
+                    possible_lib_locations = get_lib_location_guesses(
+                        user=options.use_user_site,
+                        home=target_temp_dir.path,
+                        root=options.root_path,
+                        prefix=options.prefix_path,
+                        isolated=options.isolated_mode,
+                    )
+                    reqs = sorted(installed, key=operator.attrgetter('name'))
+                    items = []
+                    for req in reqs:
+                        item = req.name
+                        try:
+                            installed_version = get_installed_version(
+                                req.name, possible_lib_locations
+                            )
+                            if installed_version:
+                                item += '-' + installed_version
+                        except Exception:
+                            pass
+                        items.append(item)
+                    installed = ' '.join(items)
+                    if installed:
+                        logger.info('Successfully installed %s', installed)
+                except EnvironmentError as error:
+                    show_traceback = (self.verbosity >= 1)
+
+                    message = create_env_error_message(
+                        error, show_traceback, options.use_user_site,
+                    )
+                    logger.error(message, exc_info=show_traceback)
+
+                    return ERROR
+                except PreviousBuildDirError:
+                    options.no_clean = True
+                    raise
+                finally:
+                    # Clean up
+                    if not options.no_clean:
+                        requirement_set.cleanup_files()
+                        wheel_cache.cleanup()
+
+        if options.target_dir:
+            self._handle_target_dir(
+                options.target_dir, target_temp_dir, options.upgrade
+            )
+        return requirement_set
+
+    def _handle_target_dir(self, target_dir, target_temp_dir, upgrade):
+        ensure_dir(target_dir)
+
+        # Checking both purelib and platlib directories for installed
+        # packages to be moved to target directory
+        lib_dir_list = []
+
+        with target_temp_dir:
+            # Checking both purelib and platlib directories for installed
+            # packages to be moved to target directory
+            scheme = distutils_scheme('', home=target_temp_dir.path)
+            purelib_dir = scheme['purelib']
+            platlib_dir = scheme['platlib']
+            data_dir = scheme['data']
+
+            if os.path.exists(purelib_dir):
+                lib_dir_list.append(purelib_dir)
+            if os.path.exists(platlib_dir) and platlib_dir != purelib_dir:
+                lib_dir_list.append(platlib_dir)
+            if os.path.exists(data_dir):
+                lib_dir_list.append(data_dir)
+
+            for lib_dir in lib_dir_list:
+                for item in os.listdir(lib_dir):
+                    if lib_dir == data_dir:
+                        ddir = os.path.join(data_dir, item)
+                        if any(s.startswith(ddir) for s in lib_dir_list[:-1]):
+                            continue
+                    target_item_dir = os.path.join(target_dir, item)
+                    if os.path.exists(target_item_dir):
+                        if not upgrade:
+                            logger.warning(
+                                'Target directory %s already exists. Specify '
+                                '--upgrade to force replacement.',
+                                target_item_dir
+                            )
+                            continue
+                        if os.path.islink(target_item_dir):
+                            logger.warning(
+                                'Target directory %s already exists and is '
+                                'a link. Pip will not automatically replace '
+                                'links, please remove if replacement is '
+                                'desired.',
+                                target_item_dir
+                            )
+                            continue
+                        if os.path.isdir(target_item_dir):
+                            shutil.rmtree(target_item_dir)
+                        else:
+                            os.remove(target_item_dir)
+
+                    shutil.move(
+                        os.path.join(lib_dir, item),
+                        target_item_dir
+                    )
+
+    def _warn_about_conflicts(self, to_install):
+        package_set, _dep_info = check_install_conflicts(to_install)
+        missing, conflicting = _dep_info
+
+        # NOTE: There is some duplication here from pip check
+        for project_name in missing:
+            version = package_set[project_name][0]
+            for dependency in missing[project_name]:
+                logger.critical(
+                    "%s %s requires %s, which is not installed.",
+                    project_name, version, dependency[1],
+                )
+
+        for project_name in conflicting:
+            version = package_set[project_name][0]
+            for dep_name, dep_version, req in conflicting[project_name]:
+                logger.critical(
+                    "%s %s has requirement %s, but you'll have %s %s which is "
+                    "incompatible.",
+                    project_name, version, req, dep_name, dep_version,
+                )
+
+
+def get_lib_location_guesses(*args, **kwargs):
+    scheme = distutils_scheme('', *args, **kwargs)
+    return [scheme['purelib'], scheme['platlib']]
+
+
+def create_env_error_message(error, show_traceback, using_user_site):
+    """Format an error message for an EnvironmentError
+
+    It may occur anytime during the execution of the install command.
+    """
+    parts = []
+
+    # Mention the error if we are not going to show a traceback
+    parts.append("Could not install packages due to an EnvironmentError")
+    if not show_traceback:
+        parts.append(": ")
+        parts.append(str(error))
+    else:
+        parts.append(".")
+
+    # Spilt the error indication from a helper message (if any)
+    parts[-1] += "\n"
+
+    # Suggest useful actions to the user:
+    #  (1) using user site-packages or (2) verifying the permissions
+    if error.errno == errno.EACCES:
+        user_option_part = "Consider using the `--user` option"
+        permissions_part = "Check the permissions"
+
+        if not using_user_site:
+            parts.extend([
+                user_option_part, " or ",
+                permissions_part.lower(),
+            ])
+        else:
+            parts.append(permissions_part)
+        parts.append(".\n")
+
+    return "".join(parts).strip() + "\n"

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/list.py

@@ -0,0 +1,343 @@
+from __future__ import absolute_import
+
+import json
+import logging
+import warnings
+
+from pip._vendor import six
+from pip._vendor.six.moves import zip_longest
+
+from pip._internal.basecommand import Command
+from pip._internal.cmdoptions import index_group, make_option_group
+from pip._internal.exceptions import CommandError
+from pip._internal.index import PackageFinder
+from pip._internal.utils.deprecation import RemovedInPip11Warning
+from pip._internal.utils.misc import (
+    dist_is_editable, get_installed_distributions,
+)
+from pip._internal.utils.packaging import get_installer
+
+logger = logging.getLogger(__name__)
+
+
+class ListCommand(Command):
+    """
+    List installed packages, including editables.
+
+    Packages are listed in a case-insensitive sorted order.
+    """
+    name = 'list'
+    usage = """
+      %prog [options]"""
+    summary = 'List installed packages.'
+
+    def __init__(self, *args, **kw):
+        super(ListCommand, self).__init__(*args, **kw)
+
+        cmd_opts = self.cmd_opts
+
+        cmd_opts.add_option(
+            '-o', '--outdated',
+            action='store_true',
+            default=False,
+            help='List outdated packages')
+        cmd_opts.add_option(
+            '-u', '--uptodate',
+            action='store_true',
+            default=False,
+            help='List uptodate packages')
+        cmd_opts.add_option(
+            '-e', '--editable',
+            action='store_true',
+            default=False,
+            help='List editable projects.')
+        cmd_opts.add_option(
+            '-l', '--local',
+            action='store_true',
+            default=False,
+            help=('If in a virtualenv that has global access, do not list '
+                  'globally-installed packages.'),
+        )
+        self.cmd_opts.add_option(
+            '--user',
+            dest='user',
+            action='store_true',
+            default=False,
+            help='Only output packages installed in user-site.')
+
+        cmd_opts.add_option(
+            '--pre',
+            action='store_true',
+            default=False,
+            help=("Include pre-release and development versions. By default, "
+                  "pip only finds stable versions."),
+        )
+
+        cmd_opts.add_option(
+            '--format',
+            action='store',
+            dest='list_format',
+            default="columns",
+            choices=('legacy', 'columns', 'freeze', 'json'),
+            help="Select the output format among: columns (default), freeze, "
+                 "json, or legacy.",
+        )
+
+        cmd_opts.add_option(
+            '--not-required',
+            action='store_true',
+            dest='not_required',
+            help="List packages that are not dependencies of "
+                 "installed packages.",
+        )
+
+        cmd_opts.add_option(
+            '--exclude-editable',
+            action='store_false',
+            dest='include_editable',
+            help='Exclude editable package from output.',
+        )
+        cmd_opts.add_option(
+            '--include-editable',
+            action='store_true',
+            dest='include_editable',
+            help='Include editable package from output.',
+            default=True,
+        )
+        index_opts = make_option_group(index_group, self.parser)
+
+        self.parser.insert_option_group(0, index_opts)
+        self.parser.insert_option_group(0, cmd_opts)
+
+    def _build_package_finder(self, options, index_urls, session):
+        """
+        Create a package finder appropriate to this list command.
+        """
+        return PackageFinder(
+            find_links=options.find_links,
+            index_urls=index_urls,
+            allow_all_prereleases=options.pre,
+            trusted_hosts=options.trusted_hosts,
+            process_dependency_links=options.process_dependency_links,
+            session=session,
+        )
+
+    def run(self, options, args):
+        if options.list_format == "legacy":
+            warnings.warn(
+                "The legacy format has been deprecated and will be removed "
+                "in the future.",
+                RemovedInPip11Warning,
+            )
+
+        if options.outdated and options.uptodate:
+            raise CommandError(
+                "Options --outdated and --uptodate cannot be combined.")
+
+        packages = get_installed_distributions(
+            local_only=options.local,
+            user_only=options.user,
+            editables_only=options.editable,
+            include_editables=options.include_editable,
+        )
+
+        if options.outdated:
+            packages = self.get_outdated(packages, options)
+        elif options.uptodate:
+            packages = self.get_uptodate(packages, options)
+
+        if options.not_required:
+            packages = self.get_not_required(packages, options)
+
+        self.output_package_listing(packages, options)
+
+    def get_outdated(self, packages, options):
+        return [
+            dist for dist in self.iter_packages_latest_infos(packages, options)
+            if dist.latest_version > dist.parsed_version
+        ]
+
+    def get_uptodate(self, packages, options):
+        return [
+            dist for dist in self.iter_packages_latest_infos(packages, options)
+            if dist.latest_version == dist.parsed_version
+        ]
+
+    def get_not_required(self, packages, options):
+        dep_keys = set()
+        for dist in packages:
+            dep_keys.update(requirement.key for requirement in dist.requires())
+        return {pkg for pkg in packages if pkg.key not in dep_keys}
+
+    def iter_packages_latest_infos(self, packages, options):
+        index_urls = [options.index_url] + options.extra_index_urls
+        if options.no_index:
+            logger.debug('Ignoring indexes: %s', ','.join(index_urls))
+            index_urls = []
+
+        dependency_links = []
+        for dist in packages:
+            if dist.has_metadata('dependency_links.txt'):
+                dependency_links.extend(
+                    dist.get_metadata_lines('dependency_links.txt'),
+                )
+
+        with self._build_session(options) as session:
+            finder = self._build_package_finder(options, index_urls, session)
+            finder.add_dependency_links(dependency_links)
+
+            for dist in packages:
+                typ = 'unknown'
+                all_candidates = finder.find_all_candidates(dist.key)
+                if not options.pre:
+                    # Remove prereleases
+                    all_candidates = [candidate for candidate in all_candidates
+                                      if not candidate.version.is_prerelease]
+
+                if not all_candidates:
+                    continue
+                best_candidate = max(all_candidates,
+                                     key=finder._candidate_sort_key)
+                remote_version = best_candidate.version
+                if best_candidate.location.is_wheel:
+                    typ = 'wheel'
+                else:
+                    typ = 'sdist'
+                # This is dirty but makes the rest of the code much cleaner
+                dist.latest_version = remote_version
+                dist.latest_filetype = typ
+                yield dist
+
+    def output_legacy(self, dist, options):
+        if options.verbose >= 1:
+            return '%s (%s, %s, %s)' % (
+                dist.project_name,
+                dist.version,
+                dist.location,
+                get_installer(dist),
+            )
+        elif dist_is_editable(dist):
+            return '%s (%s, %s)' % (
+                dist.project_name,
+                dist.version,
+                dist.location,
+            )
+        else:
+            return '%s (%s)' % (dist.project_name, dist.version)
+
+    def output_legacy_latest(self, dist, options):
+        return '%s - Latest: %s [%s]' % (
+            self.output_legacy(dist, options),
+            dist.latest_version,
+            dist.latest_filetype,
+        )
+
+    def output_package_listing(self, packages, options):
+        packages = sorted(
+            packages,
+            key=lambda dist: dist.project_name.lower(),
+        )
+        if options.list_format == 'columns' and packages:
+            data, header = format_for_columns(packages, options)
+            self.output_package_listing_columns(data, header)
+        elif options.list_format == 'freeze':
+            for dist in packages:
+                if options.verbose >= 1:
+                    logger.info("%s==%s (%s)", dist.project_name,
+                                dist.version, dist.location)
+                else:
+                    logger.info("%s==%s", dist.project_name, dist.version)
+        elif options.list_format == 'json':
+            logger.info(format_for_json(packages, options))
+        elif options.list_format == "legacy":
+            for dist in packages:
+                if options.outdated:
+                    logger.info(self.output_legacy_latest(dist, options))
+                else:
+                    logger.info(self.output_legacy(dist, options))
+
+    def output_package_listing_columns(self, data, header):
+        # insert the header first: we need to know the size of column names
+        if len(data) > 0:
+            data.insert(0, header)
+
+        pkg_strings, sizes = tabulate(data)
+
+        # Create and add a separator.
+        if len(data) > 0:
+            pkg_strings.insert(1, " ".join(map(lambda x: '-' * x, sizes)))
+
+        for val in pkg_strings:
+            logger.info(val)
+
+
+def tabulate(vals):
+    # From pfmoore on GitHub:
+    # https://github.com/pypa/pip/issues/3651#issuecomment-216932564
+    assert len(vals) > 0
+
+    sizes = [0] * max(len(x) for x in vals)
+    for row in vals:
+        sizes = [max(s, len(str(c))) for s, c in zip_longest(sizes, row)]
+
+    result = []
+    for row in vals:
+        display = " ".join([str(c).ljust(s) if c is not None else ''
+                            for s, c in zip_longest(sizes, row)])
+        result.append(display)
+
+    return result, sizes
+
+
+def format_for_columns(pkgs, options):
+    """
+    Convert the package data into something usable
+    by output_package_listing_columns.
+    """
+    running_outdated = options.outdated
+    # Adjust the header for the `pip list --outdated` case.
+    if running_outdated:
+        header = ["Package", "Version", "Latest", "Type"]
+    else:
+        header = ["Package", "Version"]
+
+    data = []
+    if options.verbose >= 1 or any(dist_is_editable(x) for x in pkgs):
+        header.append("Location")
+    if options.verbose >= 1:
+        header.append("Installer")
+
+    for proj in pkgs:
+        # if we're working on the 'outdated' list, separate out the
+        # latest_version and type
+        row = [proj.project_name, proj.version]
+
+        if running_outdated:
+            row.append(proj.latest_version)
+            row.append(proj.latest_filetype)
+
+        if options.verbose >= 1 or dist_is_editable(proj):
+            row.append(proj.location)
+        if options.verbose >= 1:
+            row.append(get_installer(proj))
+
+        data.append(row)
+
+    return data, header
+
+
+def format_for_json(packages, options):
+    data = []
+    for dist in packages:
+        info = {
+            'name': dist.project_name,
+            'version': six.text_type(dist.version),
+        }
+        if options.verbose >= 1:
+            info['location'] = dist.location
+            info['installer'] = get_installer(dist)
+        if options.outdated:
+            info['latest_version'] = six.text_type(dist.latest_version)
+            info['latest_filetype'] = dist.latest_filetype
+        data.append(info)
+    return json.dumps(data)

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/search.py

@@ -0,0 +1,135 @@
+from __future__ import absolute_import
+
+import logging
+import sys
+import textwrap
+from collections import OrderedDict
+
+from pip._vendor import pkg_resources
+from pip._vendor.packaging.version import parse as parse_version
+# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
+#       why we ignore the type on this import
+from pip._vendor.six.moves import xmlrpc_client  # type: ignore
+
+from pip._internal.basecommand import SUCCESS, Command
+from pip._internal.compat import get_terminal_size
+from pip._internal.download import PipXmlrpcTransport
+from pip._internal.exceptions import CommandError
+from pip._internal.models import PyPI
+from pip._internal.status_codes import NO_MATCHES_FOUND
+from pip._internal.utils.logging import indent_log
+
+logger = logging.getLogger(__name__)
+
+
+class SearchCommand(Command):
+    """Search for PyPI packages whose name or summary contains <query>."""
+    name = 'search'
+    usage = """
+      %prog [options] <query>"""
+    summary = 'Search PyPI for packages.'
+    ignore_require_venv = True
+
+    def __init__(self, *args, **kw):
+        super(SearchCommand, self).__init__(*args, **kw)
+        self.cmd_opts.add_option(
+            '-i', '--index',
+            dest='index',
+            metavar='URL',
+            default=PyPI.pypi_url,
+            help='Base URL of Python Package Index (default %default)')
+
+        self.parser.insert_option_group(0, self.cmd_opts)
+
+    def run(self, options, args):
+        if not args:
+            raise CommandError('Missing required argument (search query).')
+        query = args
+        pypi_hits = self.search(query, options)
+        hits = transform_hits(pypi_hits)
+
+        terminal_width = None
+        if sys.stdout.isatty():
+            terminal_width = get_terminal_size()[0]
+
+        print_results(hits, terminal_width=terminal_width)
+        if pypi_hits:
+            return SUCCESS
+        return NO_MATCHES_FOUND
+
+    def search(self, query, options):
+        index_url = options.index
+        with self._build_session(options) as session:
+            transport = PipXmlrpcTransport(index_url, session)
+            pypi = xmlrpc_client.ServerProxy(index_url, transport)
+            hits = pypi.search({'name': query, 'summary': query}, 'or')
+            return hits
+
+
+def transform_hits(hits):
+    """
+    The list from pypi is really a list of versions. We want a list of
+    packages with the list of versions stored inline. This converts the
+    list from pypi into one we can use.
+    """
+    packages = OrderedDict()
+    for hit in hits:
+        name = hit['name']
+        summary = hit['summary']
+        version = hit['version']
+
+        if name not in packages.keys():
+            packages[name] = {
+                'name': name,
+                'summary': summary,
+                'versions': [version],
+            }
+        else:
+            packages[name]['versions'].append(version)
+
+            # if this is the highest version, replace summary and score
+            if version == highest_version(packages[name]['versions']):
+                packages[name]['summary'] = summary
+
+    return list(packages.values())
+
+
+def print_results(hits, name_column_width=None, terminal_width=None):
+    if not hits:
+        return
+    if name_column_width is None:
+        name_column_width = max([
+            len(hit['name']) + len(highest_version(hit.get('versions', ['-'])))
+            for hit in hits
+        ]) + 4
+
+    installed_packages = [p.project_name for p in pkg_resources.working_set]
+    for hit in hits:
+        name = hit['name']
+        summary = hit['summary'] or ''
+        latest = highest_version(hit.get('versions', ['-']))
+        if terminal_width is not None:
+            target_width = terminal_width - name_column_width - 5
+            if target_width > 10:
+                # wrap and indent summary to fit terminal
+                summary = textwrap.wrap(summary, target_width)
+                summary = ('\n' + ' ' * (name_column_width + 3)).join(summary)
+
+        line = '%-*s - %s' % (name_column_width,
+                              '%s (%s)' % (name, latest), summary)
+        try:
+            logger.info(line)
+            if name in installed_packages:
+                dist = pkg_resources.get_distribution(name)
+                with indent_log():
+                    if dist.version == latest:
+                        logger.info('INSTALLED: %s (latest)', dist.version)
+                    else:
+                        logger.info('INSTALLED: %s', dist.version)
+                        logger.info('LATEST:    %s', latest)
+        except UnicodeEncodeError:
+            pass
+
+
+def highest_version(versions):
+    return max(versions, key=parse_version)

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/show.py

@@ -0,0 +1,164 @@
+from __future__ import absolute_import
+
+import logging
+import os
+from email.parser import FeedParser  # type: ignore
+
+from pip._vendor import pkg_resources
+from pip._vendor.packaging.utils import canonicalize_name
+
+from pip._internal.basecommand import Command
+from pip._internal.status_codes import ERROR, SUCCESS
+
+logger = logging.getLogger(__name__)
+
+
+class ShowCommand(Command):
+    """Show information about one or more installed packages."""
+    name = 'show'
+    usage = """
+      %prog [options] <package> ..."""
+    summary = 'Show information about installed packages.'
+    ignore_require_venv = True
+
+    def __init__(self, *args, **kw):
+        super(ShowCommand, self).__init__(*args, **kw)
+        self.cmd_opts.add_option(
+            '-f', '--files',
+            dest='files',
+            action='store_true',
+            default=False,
+            help='Show the full list of installed files for each package.')
+
+        self.parser.insert_option_group(0, self.cmd_opts)
+
+    def run(self, options, args):
+        if not args:
+            logger.warning('ERROR: Please provide a package name or names.')
+            return ERROR
+        query = args
+
+        results = search_packages_info(query)
+        if not print_results(
+                results, list_files=options.files, verbose=options.verbose):
+            return ERROR
+        return SUCCESS
+
+
+def search_packages_info(query):
+    """
+    Gather details from installed distributions. Print distribution name,
+    version, location, and installed files. Installed files requires a
+    pip generated 'installed-files.txt' in the distributions '.egg-info'
+    directory.
+    """
+    installed = {}
+    for p in pkg_resources.working_set:
+        installed[canonicalize_name(p.project_name)] = p
+
+    query_names = [canonicalize_name(name) for name in query]
+
+    for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
+        package = {
+            'name': dist.project_name,
+            'version': dist.version,
+            'location': dist.location,
+            'requires': [dep.project_name for dep in dist.requires()],
+        }
+        file_list = None
+        metadata = None
+        if isinstance(dist, pkg_resources.DistInfoDistribution):
+            # RECORDs should be part of .dist-info metadatas
+            if dist.has_metadata('RECORD'):
+                lines = dist.get_metadata_lines('RECORD')
+                paths = [l.split(',')[0] for l in lines]
+                paths = [os.path.join(dist.location, p) for p in paths]
+                file_list = [os.path.relpath(p, dist.location) for p in paths]
+
+            if dist.has_metadata('METADATA'):
+                metadata = dist.get_metadata('METADATA')
+        else:
+            # Otherwise use pip's log for .egg-info's
+            if dist.has_metadata('installed-files.txt'):
+                paths = dist.get_metadata_lines('installed-files.txt')
+                paths = [os.path.join(dist.egg_info, p) for p in paths]
+                file_list = [os.path.relpath(p, dist.location) for p in paths]
+
+            if dist.has_metadata('PKG-INFO'):
+                metadata = dist.get_metadata('PKG-INFO')
+
+        if dist.has_metadata('entry_points.txt'):
+            entry_points = dist.get_metadata_lines('entry_points.txt')
+            package['entry_points'] = entry_points
+
+        if dist.has_metadata('INSTALLER'):
+            for line in dist.get_metadata_lines('INSTALLER'):
+                if line.strip():
+                    package['installer'] = line.strip()
+                    break
+
+        # @todo: Should pkg_resources.Distribution have a
+        # `get_pkg_info` method?
+        feed_parser = FeedParser()
+        feed_parser.feed(metadata)
+        pkg_info_dict = feed_parser.close()
+        for key in ('metadata-version', 'summary',
+                    'home-page', 'author', 'author-email', 'license'):
+            package[key] = pkg_info_dict.get(key)
+
+        # It looks like FeedParser cannot deal with repeated headers
+        classifiers = []
+        for line in metadata.splitlines():
+            if line.startswith('Classifier: '):
+                classifiers.append(line[len('Classifier: '):])
+        package['classifiers'] = classifiers
+
+        if file_list:
+            package['files'] = sorted(file_list)
+        yield package
+
+
+def print_results(distributions, list_files=False, verbose=False):
+    """
+    Print the informations from installed distributions found.
+    """
+    results_printed = False
+    for i, dist in enumerate(distributions):
+        results_printed = True
+        if i > 0:
+            logger.info("---")
+
+        name = dist.get('name', '')
+        required_by = [
+            pkg.project_name for pkg in pkg_resources.working_set
+            if name in [required.name for required in pkg.requires()]
+        ]
+
+        logger.info("Name: %s", name)
+        logger.info("Version: %s", dist.get('version', ''))
+        logger.info("Summary: %s", dist.get('summary', ''))
+        logger.info("Home-page: %s", dist.get('home-page', ''))
+        logger.info("Author: %s", dist.get('author', ''))
+        logger.info("Author-email: %s", dist.get('author-email', ''))
+        logger.info("License: %s", dist.get('license', ''))
+        logger.info("Location: %s", dist.get('location', ''))
+        logger.info("Requires: %s", ', '.join(dist.get('requires', [])))
+        logger.info("Required-by: %s", ', '.join(required_by))
+
+        if verbose:
+            logger.info("Metadata-Version: %s",
+                        dist.get('metadata-version', ''))
+            logger.info("Installer: %s", dist.get('installer', ''))
+            logger.info("Classifiers:")
+            for classifier in dist.get('classifiers', []):
+                logger.info("  %s", classifier)
+            logger.info("Entry-points:")
+            for entry in dist.get('entry_points', []):
+                logger.info("  %s", entry.strip())
+        if list_files:
+            logger.info("Files:")
+            for line in dist.get('files', []):
+                logger.info("  %s", line.strip())
+            if "files" not in dist:
+                logger.info("Cannot locate installed-files.txt")
+    return results_printed

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/uninstall.py

@@ -0,0 +1,71 @@
+from __future__ import absolute_import
+
+from pip._vendor.packaging.utils import canonicalize_name
+
+from pip._internal.basecommand import Command
+from pip._internal.exceptions import InstallationError
+from pip._internal.req import InstallRequirement, parse_requirements
+
+
+class UninstallCommand(Command):
+    """
+    Uninstall packages.
+
+    pip is able to uninstall most installed packages. Known exceptions are:
+
+    - Pure distutils packages installed with ``python setup.py install``, which
+      leave behind no metadata to determine what files were installed.
+    - Script wrappers installed by ``python setup.py develop``.
+    """
+    name = 'uninstall'
+    usage = """
+      %prog [options] <package> ...
+      %prog [options] -r <requirements file> ..."""
+    summary = 'Uninstall packages.'
+
+    def __init__(self, *args, **kw):
+        super(UninstallCommand, self).__init__(*args, **kw)
+        self.cmd_opts.add_option(
+            '-r', '--requirement',
+            dest='requirements',
+            action='append',
+            default=[],
+            metavar='file',
+            help='Uninstall all the packages listed in the given requirements '
+                 'file.  This option can be used multiple times.',
+        )
+        self.cmd_opts.add_option(
+            '-y', '--yes',
+            dest='yes',
+            action='store_true',
+            help="Don't ask for confirmation of uninstall deletions.")
+
+        self.parser.insert_option_group(0, self.cmd_opts)
+
+    def run(self, options, args):
+        with self._build_session(options) as session:
+            reqs_to_uninstall = {}
+            for name in args:
+                req = InstallRequirement.from_line(
+                    name, isolated=options.isolated_mode,
+                )
+                if req.name:
+                    reqs_to_uninstall[canonicalize_name(req.name)] = req
+            for filename in options.requirements:
+                for req in parse_requirements(
+                        filename,
+                        options=options,
+                        session=session):
+                    if req.name:
+                        reqs_to_uninstall[canonicalize_name(req.name)] = req
+            if not reqs_to_uninstall:
+                raise InstallationError(
+                    'You must give at least one requirement to %(name)s (see '
+                    '"pip help %(name)s")' % dict(name=self.name)
+                )
+            for req in reqs_to_uninstall.values():
+                uninstall_pathset = req.uninstall(
+                    auto_confirm=options.yes, verbose=self.verbosity > 0,
+                )
+                if uninstall_pathset:
+                    uninstall_pathset.commit()

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/commands/wheel.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+import logging
+import os
+
+from pip._internal import cmdoptions
+from pip._internal.basecommand import RequirementCommand
+from pip._internal.cache import WheelCache
+from pip._internal.exceptions import CommandError, PreviousBuildDirError
+from pip._internal.operations.prepare import RequirementPreparer
+from pip._internal.req import RequirementSet
+from pip._internal.resolve import Resolver
+from pip._internal.utils.temp_dir import TempDirectory
+from pip._internal.wheel import WheelBuilder
+
+logger = logging.getLogger(__name__)
+
+
+class WheelCommand(RequirementCommand):
+    """
+    Build Wheel archives for your requirements and dependencies.
+
+    Wheel is a built-package format, and offers the advantage of not
+    recompiling your software during every install. For more details, see the
+    wheel docs: https://wheel.readthedocs.io/en/latest/
+
+    Requirements: setuptools>=0.8, and wheel.
+
+    'pip wheel' uses the bdist_wheel setuptools extension from the wheel
+    package to build individual wheels.
+
+    """
+
+    name = 'wheel'
+    usage = """
+      %prog [options] <requirement specifier> ...
+      %prog [options] -r <requirements file> ...
+      %prog [options] [-e] <vcs project url> ...
+      %prog [options] [-e] <local project path> ...
+      %prog [options] <archive url/path> ..."""
+
+    summary = 'Build wheels from your requirements.'
+
+    def __init__(self, *args, **kw):
+        super(WheelCommand, self).__init__(*args, **kw)
+
+        cmd_opts = self.cmd_opts
+
+        cmd_opts.add_option(
+            '-w', '--wheel-dir',
+            dest='wheel_dir',
+            metavar='dir',
+            default=os.curdir,
+            help=("Build wheels into <dir>, where the default is the "
+                  "current working directory."),
+        )
+        cmd_opts.add_option(cmdoptions.no_binary())
+        cmd_opts.add_option(cmdoptions.only_binary())
+        cmd_opts.add_option(
+            '--build-option',
+            dest='build_options',
+            metavar='options',
+            action='append',
+            help="Extra arguments to be supplied to 'setup.py bdist_wheel'.",
+        )
+        cmd_opts.add_option(cmdoptions.no_build_isolation())
+        cmd_opts.add_option(cmdoptions.constraints())
+        cmd_opts.add_option(cmdoptions.editable())
+        cmd_opts.add_option(cmdoptions.requirements())
+        cmd_opts.add_option(cmdoptions.src())
+        cmd_opts.add_option(cmdoptions.ignore_requires_python())
+        cmd_opts.add_option(cmdoptions.no_deps())
+        cmd_opts.add_option(cmdoptions.build_dir())
+        cmd_opts.add_option(cmdoptions.progress_bar())
+
+        cmd_opts.add_option(
+            '--global-option',
+            dest='global_options',
+            action='append',
+            metavar='options',
+            help="Extra global options to be supplied to the setup.py "
+            "call before the 'bdist_wheel' command.")
+
+        cmd_opts.add_option(
+            '--pre',
+            action='store_true',
+            default=False,
+            help=("Include pre-release and development versions. By default, "
+                  "pip only finds stable versions."),
+        )
+
+        cmd_opts.add_option(cmdoptions.no_clean())
+        cmd_opts.add_option(cmdoptions.require_hashes())
+
+        index_opts = cmdoptions.make_option_group(
+            cmdoptions.index_group,
+            self.parser,
+        )
+
+        self.parser.insert_option_group(0, index_opts)
+        self.parser.insert_option_group(0, cmd_opts)
+
+    def run(self, options, args):
+        cmdoptions.check_install_build_global(options)
+
+        index_urls = [options.index_url] + options.extra_index_urls
+        if options.no_index:
+            logger.debug('Ignoring indexes: %s', ','.join(index_urls))
+            index_urls = []
+
+        if options.build_dir:
+            options.build_dir = os.path.abspath(options.build_dir)
+
+        options.src_dir = os.path.abspath(options.src_dir)
+
+        with self._build_session(options) as session:
+            finder = self._build_package_finder(options, session)
+            build_delete = (not (options.no_clean or options.build_dir))
+            wheel_cache = WheelCache(options.cache_dir, options.format_control)
+
+            with TempDirectory(
+                options.build_dir, delete=build_delete, kind="wheel"
+            ) as directory:
+                requirement_set = RequirementSet(
+                    require_hashes=options.require_hashes,
+                )
+
+                try:
+                    self.populate_requirement_set(
+                        requirement_set, args, options, finder, session,
+                        self.name, wheel_cache
+                    )
+
+                    preparer = RequirementPreparer(
+                        build_dir=directory.path,
+                        src_dir=options.src_dir,
+                        download_dir=None,
+                        wheel_download_dir=options.wheel_dir,
+                        progress_bar=options.progress_bar,
+                        build_isolation=options.build_isolation,
+                    )
+
+                    resolver = Resolver(
+                        preparer=preparer,
+                        finder=finder,
+                        session=session,
+                        wheel_cache=wheel_cache,
+                        use_user_site=False,
+                        upgrade_strategy="to-satisfy-only",
+                        force_reinstall=False,
+                        ignore_dependencies=options.ignore_dependencies,
+                        ignore_requires_python=options.ignore_requires_python,
+                        ignore_installed=True,
+                        isolated=options.isolated_mode,
+                    )
+                    resolver.resolve(requirement_set)
+
+                    # build wheels
+                    wb = WheelBuilder(
+                        finder, preparer, wheel_cache,
+                        build_options=options.build_options or [],
+                        global_options=options.global_options or [],
+                        no_clean=options.no_clean,
+                    )
+                    wheels_built_successfully = wb.build(
+                        requirement_set.requirements.values(), session=session,
+                    )
+                    if not wheels_built_successfully:
+                        raise CommandError(
+                            "Failed to build one or more wheels"
+                        )
+                except PreviousBuildDirError:
+                    options.no_clean = True
+                    raise
+                finally:
+                    if not options.no_clean:
+                        requirement_set.cleanup_files()
+                        wheel_cache.cleanup()

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/compat.py

@@ -0,0 +1,235 @@
+"""Stuff that differs in different Python versions and platform
+distributions."""
+from __future__ import absolute_import, division
+
+import codecs
+import locale
+import logging
+import os
+import shutil
+import sys
+
+from pip._vendor.six import text_type
+
+try:
+    import ipaddress
+except ImportError:
+    try:
+        from pip._vendor import ipaddress  # type: ignore
+    except ImportError:
+        import ipaddr as ipaddress  # type: ignore
+        ipaddress.ip_address = ipaddress.IPAddress
+        ipaddress.ip_network = ipaddress.IPNetwork
+
+
+__all__ = [
+    "ipaddress", "uses_pycache", "console_to_str", "native_str",
+    "get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size",
+]
+
+
+logger = logging.getLogger(__name__)
+
+if sys.version_info >= (3, 4):
+    uses_pycache = True
+    from importlib.util import cache_from_source
+else:
+    import imp
+
+    try:
+        cache_from_source = imp.cache_from_source  # type: ignore
+    except AttributeError:
+        # does not use __pycache__
+        cache_from_source = None
+
+    uses_pycache = cache_from_source is not None
+
+
+if sys.version_info >= (3, 5):
+    backslashreplace_decode = "backslashreplace"
+else:
+    # In version 3.4 and older, backslashreplace exists
+    # but does not support use for decoding.
+    # We implement our own replace handler for this
+    # situation, so that we can consistently use
+    # backslash replacement for all versions.
+    def backslashreplace_decode_fn(err):
+        raw_bytes = (err.object[i] for i in range(err.start, err.end))
+        if sys.version_info[0] == 2:
+            # Python 2 gave us characters - convert to numeric bytes
+            raw_bytes = (ord(b) for b in raw_bytes)
+        return u"".join(u"\\x%x" % c for c in raw_bytes), err.end
+    codecs.register_error(
+        "backslashreplace_decode",
+        backslashreplace_decode_fn,
+    )
+    backslashreplace_decode = "backslashreplace_decode"
+
+
+def console_to_str(data):
+    """Return a string, safe for output, of subprocess output.
+
+    We assume the data is in the locale preferred encoding.
+    If it won't decode properly, we warn the user but decode as
+    best we can.
+
+    We also ensure that the output can be safely written to
+    standard output without encoding errors.
+    """
+
+    # First, get the encoding we assume. This is the preferred
+    # encoding for the locale, unless that is not found, or
+    # it is ASCII, in which case assume UTF-8
+    encoding = locale.getpreferredencoding()
+    if (not encoding) or codecs.lookup(encoding).name == "ascii":
+        encoding = "utf-8"
+
+    # Now try to decode the data - if we fail, warn the user and
+    # decode with replacement.
+    try:
+        s = data.decode(encoding)
+    except UnicodeDecodeError:
+        logger.warning(
+            "Subprocess output does not appear to be encoded as %s",
+            encoding,
+        )
+        s = data.decode(encoding, errors=backslashreplace_decode)
+
+    # Make sure we can print the output, by encoding it to the output
+    # encoding with replacement of unencodable characters, and then
+    # decoding again.
+    # We use stderr's encoding because it's less likely to be
+    # redirected and if we don't find an encoding we skip this
+    # step (on the assumption that output is wrapped by something
+    # that won't fail).
+    # The double getattr is to deal with the possibility that we're
+    # being called in a situation where sys.__stderr__ doesn't exist,
+    # or doesn't have an encoding attribute. Neither of these cases
+    # should occur in normal pip use, but there's no harm in checking
+    # in case people use pip in (unsupported) unusual situations.
+    output_encoding = getattr(getattr(sys, "__stderr__", None),
+                              "encoding", None)
+
+    if output_encoding:
+        s = s.encode(output_encoding, errors="backslashreplace")
+        s = s.decode(output_encoding)
+
+    return s
+
+
+if sys.version_info >= (3,):
+    def native_str(s, replace=False):
+        if isinstance(s, bytes):
+            return s.decode('utf-8', 'replace' if replace else 'strict')
+        return s
+
+else:
+    def native_str(s, replace=False):
+        # Replace is ignored -- unicode to UTF-8 can't fail
+        if isinstance(s, text_type):
+            return s.encode('utf-8')
+        return s
+
+
+def get_path_uid(path):
+    """
+    Return path's uid.
+
+    Does not follow symlinks:
+        https://github.com/pypa/pip/pull/935#discussion_r5307003
+
+    Placed this function in compat due to differences on AIX and
+    Jython, that should eventually go away.
+
+    :raises OSError: When path is a symlink or can't be read.
+    """
+    if hasattr(os, 'O_NOFOLLOW'):
+        fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
+        file_uid = os.fstat(fd).st_uid
+        os.close(fd)
+    else:  # AIX and Jython
+        # WARNING: time of check vulnerability, but best we can do w/o NOFOLLOW
+        if not os.path.islink(path):
+            # older versions of Jython don't have `os.fstat`
+            file_uid = os.stat(path).st_uid
+        else:
+            # raise OSError for parity with os.O_NOFOLLOW above
+            raise OSError(
+                "%s is a symlink; Will not return uid for symlinks" % path
+            )
+    return file_uid
+
+
+def expanduser(path):
+    """
+    Expand ~ and ~user constructions.
+
+    Includes a workaround for http://bugs.python.org/issue14768
+    """
+    expanded = os.path.expanduser(path)
+    if path.startswith('~/') and expanded.startswith('//'):
+        expanded = expanded[1:]
+    return expanded
+
+
+# packages in the stdlib that may have installation metadata, but should not be
+# considered 'installed'.  this theoretically could be determined based on
+# dist.location (py27:`sysconfig.get_paths()['stdlib']`,
+# py26:sysconfig.get_config_vars('LIBDEST')), but fear platform variation may
+# make this ineffective, so hard-coding
+stdlib_pkgs = {"python", "wsgiref", "argparse"}
+
+
+# windows detection, covers cpython and ironpython
+WINDOWS = (sys.platform.startswith("win") or
+           (sys.platform == 'cli' and os.name == 'nt'))
+
+
+def samefile(file1, file2):
+    """Provide an alternative for os.path.samefile on Windows/Python2"""
+    if hasattr(os.path, 'samefile'):
+        return os.path.samefile(file1, file2)
+    else:
+        path1 = os.path.normcase(os.path.abspath(file1))
+        path2 = os.path.normcase(os.path.abspath(file2))
+        return path1 == path2
+
+
+if hasattr(shutil, 'get_terminal_size'):
+    def get_terminal_size():
+        """
+        Returns a tuple (x, y) representing the width(x) and the height(y)
+        in characters of the terminal window.
+        """
+        return tuple(shutil.get_terminal_size())
+else:
+    def get_terminal_size():
+        """
+        Returns a tuple (x, y) representing the width(x) and the height(y)
+        in characters of the terminal window.
+        """
+        def ioctl_GWINSZ(fd):
+            try:
+                import fcntl
+                import termios
+                import struct
+                cr = struct.unpack_from(
+                    'hh',
+                    fcntl.ioctl(fd, termios.TIOCGWINSZ, '12345678')
+                )
+            except:
+                return None
+            if cr == (0, 0):
+                return None
+            return cr
+        cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
+        if not cr:
+            try:
+                fd = os.open(os.ctermid(), os.O_RDONLY)
+                cr = ioctl_GWINSZ(fd)
+                os.close(fd)
+            except:
+                pass
+        if not cr:
+            cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
+        return int(cr[1]), int(cr[0])

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/configuration.py

@@ -0,0 +1,378 @@
+"""Configuration management setup
+
+Some terminology:
+- name
+  As written in config files.
+- value
+  Value associated with a name
+- key
+  Name combined with it's section (section.name)
+- variant
+  A single word describing where the configuration key-value pair came from
+"""
+
+import locale
+import logging
+import os
+
+from pip._vendor import six
+from pip._vendor.six.moves import configparser
+
+from pip._internal.exceptions import ConfigurationError
+from pip._internal.locations import (
+    legacy_config_file, new_config_file, running_under_virtualenv,
+    site_config_files, venv_config_file,
+)
+from pip._internal.utils.misc import ensure_dir, enum
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple
+
+    RawConfigParser = configparser.RawConfigParser  # Shorthand
+    Kind = NewType("Kind", str)
+
+logger = logging.getLogger(__name__)
+
+
+# NOTE: Maybe use the optionx attribute to normalize keynames.
+def _normalize_name(name):
+    # type: (str) -> str
+    """Make a name consistent regardless of source (environment or file)
+    """
+    name = name.lower().replace('_', '-')
+    if name.startswith('--'):
+        name = name[2:]  # only prefer long opts
+    return name
+
+
+def _disassemble_key(name):
+    # type: (str) -> List[str]
+    return name.split(".", 1)
+
+
+# The kinds of configurations there are.
+kinds = enum(
+    USER="user",        # User Specific
+    GLOBAL="global",    # System Wide
+    VENV="venv",        # Virtual Environment Specific
+    ENV="env",          # from PIP_CONFIG_FILE
+    ENV_VAR="env-var",  # from Environment Variables
+)
+
+
+class Configuration(object):
+    """Handles management of configuration.
+
+    Provides an interface to accessing and managing configuration files.
+
+    This class converts provides an API that takes "section.key-name" style
+    keys and stores the value associated with it as "key-name" under the
+    section "section".
+
+    This allows for a clean interface wherein the both the section and the
+    key-name are preserved in an easy to manage form in the configuration files
+    and the data stored is also nice.
+    """
+
+    def __init__(self, isolated, load_only=None):
+        # type: (bool, Kind) -> None
+        super(Configuration, self).__init__()
+
+        _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.VENV, None]
+        if load_only not in _valid_load_only:
+            raise ConfigurationError(
+                "Got invalid value for load_only - should be one of {}".format(
+                    ", ".join(map(repr, _valid_load_only[:-1]))
+                )
+            )
+        self.isolated = isolated  # type: bool
+        self.load_only = load_only  # type: Optional[Kind]
+
+        # The order here determines the override order.
+        self._override_order = [
+            kinds.GLOBAL, kinds.USER, kinds.VENV, kinds.ENV, kinds.ENV_VAR
+        ]
+
+        self._ignore_env_names = ["version", "help"]
+
+        # Because we keep track of where we got the data from
+        self._parsers = {
+            variant: [] for variant in self._override_order
+        }  # type: Dict[Kind, List[Tuple[str, RawConfigParser]]]
+        self._config = {
+            variant: {} for variant in self._override_order
+        }  # type: Dict[Kind, Dict[str, Any]]
+        self._modified_parsers = []  # type: List[Tuple[str, RawConfigParser]]
+
+    def load(self):
+        # type: () -> None
+        """Loads configuration from configuration files and environment
+        """
+        self._load_config_files()
+        if not self.isolated:
+            self._load_environment_vars()
+
+    def get_file_to_edit(self):
+        # type: () -> Optional[str]
+        """Returns the file with highest priority in configuration
+        """
+        assert self.load_only is not None, \
+            "Need to be specified a file to be editing"
+
+        try:
+            return self._get_parser_to_modify()[0]
+        except IndexError:
+            return None
+
+    def items(self):
+        # type: () -> Iterable[Tuple[str, Any]]
+        """Returns key-value pairs like dict.items() representing the loaded
+        configuration
+        """
+        return self._dictionary.items()
+
+    def get_value(self, key):
+        # type: (str) -> Any
+        """Get a value from the configuration.
+        """
+        try:
+            return self._dictionary[key]
+        except KeyError:
+            raise ConfigurationError("No such key - {}".format(key))
+
+    def set_value(self, key, value):
+        # type: (str, Any) -> None
+        """Modify a value in the configuration.
+        """
+        self._ensure_have_load_only()
+
+        fname, parser = self._get_parser_to_modify()
+
+        if parser is not None:
+            section, name = _disassemble_key(key)
+
+            # Modify the parser and the configuration
+            if not parser.has_section(section):
+                parser.add_section(section)
+            parser.set(section, name, value)
+
+        self._config[self.load_only][key] = value
+        self._mark_as_modified(fname, parser)
+
+    def unset_value(self, key):
+        # type: (str) -> None
+        """Unset a value in the configuration.
+        """
+        self._ensure_have_load_only()
+
+        if key not in self._config[self.load_only]:
+            raise ConfigurationError("No such key - {}".format(key))
+
+        fname, parser = self._get_parser_to_modify()
+
+        if parser is not None:
+            section, name = _disassemble_key(key)
+
+            # Remove the key in the parser
+            modified_something = False
+            if parser.has_section(section):
+                # Returns whether the option was removed or not
+                modified_something = parser.remove_option(section, name)
+
+            if modified_something:
+                # name removed from parser, section may now be empty
+                section_iter = iter(parser.items(section))
+                try:
+                    val = six.next(section_iter)
+                except StopIteration:
+                    val = None
+
+                if val is None:
+                    parser.remove_section(section)
+
+                self._mark_as_modified(fname, parser)
+            else:
+                raise ConfigurationError(
+                    "Fatal Internal error [id=1]. Please report as a bug."
+                )
+
+        del self._config[self.load_only][key]
+
+    def save(self):
+        # type: () -> None
+        """Save the currentin-memory state.
+        """
+        self._ensure_have_load_only()
+
+        for fname, parser in self._modified_parsers:
+            logger.info("Writing to %s", fname)
+
+            # Ensure directory exists.
+            ensure_dir(os.path.dirname(fname))
+
+            with open(fname, "w") as f:
+                parser.write(f)  # type: ignore
+
+    #
+    # Private routines
+    #
+
+    def _ensure_have_load_only(self):
+        # type: () -> None
+        if self.load_only is None:
+            raise ConfigurationError("Needed a specific file to be modifying.")
+        logger.debug("Will be working with %s variant only", self.load_only)
+
+    @property
+    def _dictionary(self):
+        # type: () -> Dict[str, Any]
+        """A dictionary representing the loaded configuration.
+        """
+        # NOTE: Dictionaries are not populated if not loaded. So, conditionals
+        #       are not needed here.
+        retval = {}
+
+        for variant in self._override_order:
+            retval.update(self._config[variant])
+
+        return retval
+
+    def _load_config_files(self):
+        # type: () -> None
+        """Loads configuration from configuration files
+        """
+        config_files = dict(self._iter_config_files())
+        if config_files[kinds.ENV][0:1] == [os.devnull]:
+            logger.debug(
+                "Skipping loading configuration files due to "
+                "environment's PIP_CONFIG_FILE being os.devnull"
+            )
+            return
+
+        for variant, files in config_files.items():
+            for fname in files:
+                # If there's specific variant set in `load_only`, load only
+                # that variant, not the others.
+                if self.load_only is not None and variant != self.load_only:
+                    logger.debug(
+                        "Skipping file '%s' (variant: %s)", fname, variant
+                    )
+                    continue
+
+                parser = self._load_file(variant, fname)
+
+                # Keeping track of the parsers used
+                self._parsers[variant].append((fname, parser))
+
+    def _load_file(self, variant, fname):
+        # type: (Kind, str) -> RawConfigParser
+        logger.debug("For variant '%s', will try loading '%s'", variant, fname)
+        parser = self._construct_parser(fname)
+
+        for section in parser.sections():
+            items = parser.items(section)
+            self._config[variant].update(self._normalized_keys(section, items))
+
+        return parser
+
+    def _construct_parser(self, fname):
+        # type: (str) -> RawConfigParser
+        parser = configparser.RawConfigParser()
+        # If there is no such file, don't bother reading it but create the
+        # parser anyway, to hold the data.
+        # Doing this is useful when modifying and saving files, where we don't
+        # need to construct a parser.
+        if os.path.exists(fname):
+            try:
+                parser.read(fname)
+            except UnicodeDecodeError:
+                raise ConfigurationError((
+                    "ERROR: "
+                    "Configuration file contains invalid %s characters.\n"
+                    "Please fix your configuration, located at %s\n"
+                ) % (locale.getpreferredencoding(False), fname))
+        return parser
+
+    def _load_environment_vars(self):
+        # type: () -> None
+        """Loads configuration from environment variables
+        """
+        self._config[kinds.ENV_VAR].update(
+            self._normalized_keys(":env:", self._get_environ_vars())
+        )
+
+    def _normalized_keys(self, section, items):
+        # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any]
+        """Normalizes items to construct a dictionary with normalized keys.
+
+        This routine is where the names become keys and are made the same
+        regardless of source - configuration files or environment.
+        """
+        normalized = {}
+        for name, val in items:
+            key = section + "." + _normalize_name(name)
+            normalized[key] = val
+        return normalized
+
+    def _get_environ_vars(self):
+        # type: () -> Iterable[Tuple[str, str]]
+        """Returns a generator with all environmental vars with prefix PIP_"""
+        for key, val in os.environ.items():
+            should_be_yielded = (
+                key.startswith("PIP_") and
+                key[4:].lower() not in self._ignore_env_names
+            )
+            if should_be_yielded:
+                yield key[4:].lower(), val
+
+    # XXX: This is patched in the tests.
+    def _iter_config_files(self):
+        # type: () -> Iterable[Tuple[Kind, List[str]]]
+        """Yields variant and configuration files associated with it.
+
+        This should be treated like items of a dictionary.
+        """
+        # SMELL: Move the conditions out of this function
+
+        # environment variables have the lowest priority
+        config_file = os.environ.get('PIP_CONFIG_FILE', None)
+        if config_file is not None:
+            yield kinds.ENV, [config_file]
+        else:
+            yield kinds.ENV, []
+
+        # at the base we have any global configuration
+        yield kinds.GLOBAL, list(site_config_files)
+
+        # per-user configuration next
+        should_load_user_config = not self.isolated and not (
+            config_file and os.path.exists(config_file)
+        )
+        if should_load_user_config:
+            # The legacy config file is overridden by the new config file
+            yield kinds.USER, [legacy_config_file, new_config_file]
+
+        # finally virtualenv configuration first trumping others
+        if running_under_virtualenv():
+            yield kinds.VENV, [venv_config_file]
+
+    def _get_parser_to_modify(self):
+        # type: () -> Tuple[str, RawConfigParser]
+        # Determine which parser to modify
+        parsers = self._parsers[self.load_only]
+        if not parsers:
+            # This should not happen if everything works correctly.
+            raise ConfigurationError(
+                "Fatal Internal error [id=2]. Please report as a bug."
+            )
+
+        # Use the highest priority parser.
+        return parsers[-1]
+
+    # XXX: This is patched in the tests.
+    def _mark_as_modified(self, fname, parser):
+        # type: (str, RawConfigParser) -> None
+        file_parser_tuple = (fname, parser)
+        if file_parser_tuple not in self._modified_parsers:
+            self._modified_parsers.append(file_parser_tuple)

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/download.py

@@ -0,0 +1,922 @@
+from __future__ import absolute_import
+
+import cgi
+import email.utils
+import getpass
+import json
+import logging
+import mimetypes
+import os
+import platform
+import re
+import shutil
+import sys
+
+from pip._vendor import requests, six, urllib3
+from pip._vendor.cachecontrol import CacheControlAdapter
+from pip._vendor.cachecontrol.caches import FileCache
+from pip._vendor.lockfile import LockError
+from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
+from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
+from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
+from pip._vendor.requests.structures import CaseInsensitiveDict
+from pip._vendor.requests.utils import get_netrc_auth
+# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
+#       why we ignore the type on this import
+from pip._vendor.six.moves import xmlrpc_client  # type: ignore
+from pip._vendor.six.moves.urllib import parse as urllib_parse
+from pip._vendor.six.moves.urllib import request as urllib_request
+from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote
+from pip._vendor.urllib3.util import IS_PYOPENSSL
+
+import pip
+from pip._internal.compat import WINDOWS
+from pip._internal.exceptions import HashMismatch, InstallationError
+from pip._internal.locations import write_delete_marker_file
+from pip._internal.models import PyPI
+from pip._internal.utils.encoding import auto_decode
+from pip._internal.utils.filesystem import check_path_owner
+from pip._internal.utils.glibc import libc_ver
+from pip._internal.utils.logging import indent_log
+from pip._internal.utils.misc import (
+    ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume,
+    display_path, format_size, get_installed_version, rmtree, splitext,
+    unpack_file,
+)
+from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM
+from pip._internal.utils.temp_dir import TempDirectory
+from pip._internal.utils.ui import DownloadProgressProvider
+from pip._internal.vcs import vcs
+
+try:
+    import ssl  # noqa
+except ImportError:
+    ssl = None
+
+HAS_TLS = (ssl is not None) or IS_PYOPENSSL
+
+__all__ = ['get_file_content',
+           'is_url', 'url_to_path', 'path_to_url',
+           'is_archive_file', 'unpack_vcs_link',
+           'unpack_file_url', 'is_vcs_url', 'is_file_url',
+           'unpack_http_url', 'unpack_url']
+
+
+logger = logging.getLogger(__name__)
+
+
+def user_agent():
+    """
+    Return a string representing the user agent.
+    """
+    data = {
+        "installer": {"name": "pip", "version": pip.__version__},
+        "python": platform.python_version(),
+        "implementation": {
+            "name": platform.python_implementation(),
+        },
+    }
+
+    if data["implementation"]["name"] == 'CPython':
+        data["implementation"]["version"] = platform.python_version()
+    elif data["implementation"]["name"] == 'PyPy':
+        if sys.pypy_version_info.releaselevel == 'final':
+            pypy_version_info = sys.pypy_version_info[:3]
+        else:
+            pypy_version_info = sys.pypy_version_info
+        data["implementation"]["version"] = ".".join(
+            [str(x) for x in pypy_version_info]
+        )
+    elif data["implementation"]["name"] == 'Jython':
+        # Complete Guess
+        data["implementation"]["version"] = platform.python_version()
+    elif data["implementation"]["name"] == 'IronPython':
+        # Complete Guess
+        data["implementation"]["version"] = platform.python_version()
+
+    if sys.platform.startswith("linux"):
+        from pip._vendor import distro
+        distro_infos = dict(filter(
+            lambda x: x[1],
+            zip(["name", "version", "id"], distro.linux_distribution()),
+        ))
+        libc = dict(filter(
+            lambda x: x[1],
+            zip(["lib", "version"], libc_ver()),
+        ))
+        if libc:
+            distro_infos["libc"] = libc
+        if distro_infos:
+            data["distro"] = distro_infos
+
+    if sys.platform.startswith("darwin") and platform.mac_ver()[0]:
+        data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]}
+
+    if platform.system():
+        data.setdefault("system", {})["name"] = platform.system()
+
+    if platform.release():
+        data.setdefault("system", {})["release"] = platform.release()
+
+    if platform.machine():
+        data["cpu"] = platform.machine()
+
+    if HAS_TLS:
+        data["openssl_version"] = ssl.OPENSSL_VERSION
+
+    setuptools_version = get_installed_version("setuptools")
+    if setuptools_version is not None:
+        data["setuptools_version"] = setuptools_version
+
+    return "{data[installer][name]}/{data[installer][version]} {json}".format(
+        data=data,
+        json=json.dumps(data, separators=(",", ":"), sort_keys=True),
+    )
+
+
+class MultiDomainBasicAuth(AuthBase):
+
+    def __init__(self, prompting=True):
+        self.prompting = prompting
+        self.passwords = {}
+
+    def __call__(self, req):
+        parsed = urllib_parse.urlparse(req.url)
+
+        # Get the netloc without any embedded credentials
+        netloc = parsed.netloc.rsplit("@", 1)[-1]
+
+        # Set the url of the request to the url without any credentials
+        req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:])
+
+        # Use any stored credentials that we have for this netloc
+        username, password = self.passwords.get(netloc, (None, None))
+
+        # Extract credentials embedded in the url if we have none stored
+        if username is None:
+            username, password = self.parse_credentials(parsed.netloc)
+
+        # Get creds from netrc if we still don't have them
+        if username is None and password is None:
+            netrc_auth = get_netrc_auth(req.url)
+            username, password = netrc_auth if netrc_auth else (None, None)
+
+        if username or password:
+            # Store the username and password
+            self.passwords[netloc] = (username, password)
+
+            # Send the basic auth with this request
+            req = HTTPBasicAuth(username or "", password or "")(req)
+
+        # Attach a hook to handle 401 responses
+        req.register_hook("response", self.handle_401)
+
+        return req
+
+    def handle_401(self, resp, **kwargs):
+        # We only care about 401 responses, anything else we want to just
+        #   pass through the actual response
+        if resp.status_code != 401:
+            return resp
+
+        # We are not able to prompt the user so simply return the response
+        if not self.prompting:
+            return resp
+
+        parsed = urllib_parse.urlparse(resp.url)
+
+        # Prompt the user for a new username and password
+        username = six.moves.input("User for %s: " % parsed.netloc)
+        password = getpass.getpass("Password: ")
+
+        # Store the new username and password to use for future requests
+        if username or password:
+            self.passwords[parsed.netloc] = (username, password)
+
+        # Consume content and release the original connection to allow our new
+        #   request to reuse the same one.
+        resp.content
+        resp.raw.release_conn()
+
+        # Add our new username and password to the request
+        req = HTTPBasicAuth(username or "", password or "")(resp.request)
+
+        # Send our new request
+        new_resp = resp.connection.send(req, **kwargs)
+        new_resp.history.append(resp)
+
+        return new_resp
+
+    def parse_credentials(self, netloc):
+        if "@" in netloc:
+            userinfo = netloc.rsplit("@", 1)[0]
+            if ":" in userinfo:
+                user, pwd = userinfo.split(":", 1)
+                return (urllib_unquote(user), urllib_unquote(pwd))
+            return urllib_unquote(userinfo), None
+        return None, None
+
+
+class LocalFSAdapter(BaseAdapter):
+
+    def send(self, request, stream=None, timeout=None, verify=None, cert=None,
+             proxies=None):
+        pathname = url_to_path(request.url)
+
+        resp = Response()
+        resp.status_code = 200
+        resp.url = request.url
+
+        try:
+            stats = os.stat(pathname)
+        except OSError as exc:
+            resp.status_code = 404
+            resp.raw = exc
+        else:
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
+            resp.headers = CaseInsensitiveDict({
+                "Content-Type": content_type,
+                "Content-Length": stats.st_size,
+                "Last-Modified": modified,
+            })
+
+            resp.raw = open(pathname, "rb")
+            resp.close = resp.raw.close
+
+        return resp
+
+    def close(self):
+        pass
+
+
+class SafeFileCache(FileCache):
+    """
+    A file based cache which is safe to use even when the target directory may
+    not be accessible or writable.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(SafeFileCache, self).__init__(*args, **kwargs)
+
+        # Check to ensure that the directory containing our cache directory
+        # is owned by the user current executing pip. If it does not exist
+        # we will check the parent directory until we find one that does exist.
+        # If it is not owned by the user executing pip then we will disable
+        # the cache and log a warning.
+        if not check_path_owner(self.directory):
+            logger.warning(
+                "The directory '%s' or its parent directory is not owned by "
+                "the current user and the cache has been disabled. Please "
+                "check the permissions and owner of that directory. If "
+                "executing pip with sudo, you may want sudo's -H flag.",
+                self.directory,
+            )
+
+            # Set our directory to None to disable the Cache
+            self.directory = None
+
+    def get(self, *args, **kwargs):
+        # If we don't have a directory, then the cache should be a no-op.
+        if self.directory is None:
+            return
+
+        try:
+            return super(SafeFileCache, self).get(*args, **kwargs)
+        except (LockError, OSError, IOError):
+            # We intentionally silence this error, if we can't access the cache
+            # then we can just skip caching and process the request as if
+            # caching wasn't enabled.
+            pass
+
+    def set(self, *args, **kwargs):
+        # If we don't have a directory, then the cache should be a no-op.
+        if self.directory is None:
+            return
+
+        try:
+            return super(SafeFileCache, self).set(*args, **kwargs)
+        except (LockError, OSError, IOError):
+            # We intentionally silence this error, if we can't access the cache
+            # then we can just skip caching and process the request as if
+            # caching wasn't enabled.
+            pass
+
+    def delete(self, *args, **kwargs):
+        # If we don't have a directory, then the cache should be a no-op.
+        if self.directory is None:
+            return
+
+        try:
+            return super(SafeFileCache, self).delete(*args, **kwargs)
+        except (LockError, OSError, IOError):
+            # We intentionally silence this error, if we can't access the cache
+            # then we can just skip caching and process the request as if
+            # caching wasn't enabled.
+            pass
+
+
+class InsecureHTTPAdapter(HTTPAdapter):
+
+    def cert_verify(self, conn, url, verify, cert):
+        conn.cert_reqs = 'CERT_NONE'
+        conn.ca_certs = None
+
+
+class PipSession(requests.Session):
+
+    timeout = None
+
+    def __init__(self, *args, **kwargs):
+        retries = kwargs.pop("retries", 0)
+        cache = kwargs.pop("cache", None)
+        insecure_hosts = kwargs.pop("insecure_hosts", [])
+
+        super(PipSession, self).__init__(*args, **kwargs)
+
+        # Attach our User Agent to the request
+        self.headers["User-Agent"] = user_agent()
+
+        # Attach our Authentication handler to the session
+        self.auth = MultiDomainBasicAuth()
+
+        # Create our urllib3.Retry instance which will allow us to customize
+        # how we handle retries.
+        retries = urllib3.Retry(
+            # Set the total number of retries that a particular request can
+            # have.
+            total=retries,
+
+            # A 503 error from PyPI typically means that the Fastly -> Origin
+            # connection got interrupted in some way. A 503 error in general
+            # is typically considered a transient error so we'll go ahead and
+            # retry it.
+            # A 500 may indicate transient error in Amazon S3
+            # A 520 or 527 - may indicate transient error in CloudFlare
+            status_forcelist=[500, 503, 520, 527],
+
+            # Add a small amount of back off between failed requests in
+            # order to prevent hammering the service.
+            backoff_factor=0.25,
+        )
+
+        # We want to _only_ cache responses on securely fetched origins. We do
+        # this because we can't validate the response of an insecurely fetched
+        # origin, and we don't want someone to be able to poison the cache and
+        # require manual eviction from the cache to fix it.
+        if cache:
+            secure_adapter = CacheControlAdapter(
+                cache=SafeFileCache(cache, use_dir_lock=True),
+                max_retries=retries,
+            )
+        else:
+            secure_adapter = HTTPAdapter(max_retries=retries)
+
+        # Our Insecure HTTPAdapter disables HTTPS validation. It does not
+        # support caching (see above) so we'll use it for all http:// URLs as
+        # well as any https:// host that we've marked as ignoring TLS errors
+        # for.
+        insecure_adapter = InsecureHTTPAdapter(max_retries=retries)
+
+        self.mount("https://", secure_adapter)
+        self.mount("http://", insecure_adapter)
+
+        # Enable file:// urls
+        self.mount("file://", LocalFSAdapter())
+
+        # We want to use a non-validating adapter for any requests which are
+        # deemed insecure.
+        for host in insecure_hosts:
+            self.mount("https://{}/".format(host), insecure_adapter)
+
+    def request(self, method, url, *args, **kwargs):
+        # Allow setting a default timeout on a session
+        kwargs.setdefault("timeout", self.timeout)
+
+        # Dispatch the actual request
+        return super(PipSession, self).request(method, url, *args, **kwargs)
+
+
+def get_file_content(url, comes_from=None, session=None):
+    """Gets the content of a file; it may be a filename, file: URL, or
+    http: URL.  Returns (location, content).  Content is unicode.
+
+    :param url:         File path or url.
+    :param comes_from:  Origin description of requirements.
+    :param session:     Instance of pip.download.PipSession.
+    """
+    if session is None:
+        raise TypeError(
+            "get_file_content() missing 1 required keyword argument: 'session'"
+        )
+
+    match = _scheme_re.search(url)
+    if match:
+        scheme = match.group(1).lower()
+        if (scheme == 'file' and comes_from and
+                comes_from.startswith('http')):
+            raise InstallationError(
+                'Requirements file %s references URL %s, which is local'
+                % (comes_from, url))
+        if scheme == 'file':
+            path = url.split(':', 1)[1]
+            path = path.replace('\\', '/')
+            match = _url_slash_drive_re.match(path)
+            if match:
+                path = match.group(1) + ':' + path.split('|', 1)[1]
+            path = urllib_parse.unquote(path)
+            if path.startswith('/'):
+                path = '/' + path.lstrip('/')
+            url = path
+        else:
+            # FIXME: catch some errors
+            resp = session.get(url)
+            resp.raise_for_status()
+            return resp.url, resp.text
+    try:
+        with open(url, 'rb') as f:
+            content = auto_decode(f.read())
+    except IOError as exc:
+        raise InstallationError(
+            'Could not open requirements file: %s' % str(exc)
+        )
+    return url, content
+
+
+_scheme_re = re.compile(r'^(http|https|file):', re.I)
+_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)
+
+
+def is_url(name):
+    """Returns true if the name looks like a URL"""
+    if ':' not in name:
+        return False
+    scheme = name.split(':', 1)[0].lower()
+    return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes
+
+
+def url_to_path(url):
+    """
+    Convert a file: URL to a path.
+    """
+    assert url.startswith('file:'), (
+        "You can only turn file: urls into filenames (not %r)" % url)
+
+    _, netloc, path, _, _ = urllib_parse.urlsplit(url)
+
+    # if we have a UNC path, prepend UNC share notation
+    if netloc:
+        netloc = '\\\\' + netloc
+
+    path = urllib_request.url2pathname(netloc + path)
+    return path
+
+
+def path_to_url(path):
+    """
+    Convert a path to a file: URL.  The path will be made absolute and have
+    quoted path parts.
+    """
+    path = os.path.normpath(os.path.abspath(path))
+    url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path))
+    return url
+
+
+def is_archive_file(name):
+    """Return True if `name` is a considered as an archive file."""
+    ext = splitext(name)[1].lower()
+    if ext in ARCHIVE_EXTENSIONS:
+        return True
+    return False
+
+
+def unpack_vcs_link(link, location):
+    vcs_backend = _get_used_vcs_backend(link)
+    vcs_backend.unpack(location)
+
+
+def _get_used_vcs_backend(link):
+    for backend in vcs.backends:
+        if link.scheme in backend.schemes:
+            vcs_backend = backend(link.url)
+            return vcs_backend
+
+
+def is_vcs_url(link):
+    return bool(_get_used_vcs_backend(link))
+
+
+def is_file_url(link):
+    return link.url.lower().startswith('file:')
+
+
+def is_dir_url(link):
+    """Return whether a file:// Link points to a directory.
+
+    ``link`` must not have any other scheme but file://. Call is_file_url()
+    first.
+
+    """
+    link_path = url_to_path(link.url_without_fragment)
+    return os.path.isdir(link_path)
+
+
+def _progress_indicator(iterable, *args, **kwargs):
+    return iterable
+
+
+def _download_url(resp, link, content_file, hashes, progress_bar):
+    try:
+        total_length = int(resp.headers['content-length'])
+    except (ValueError, KeyError, TypeError):
+        total_length = 0
+
+    cached_resp = getattr(resp, "from_cache", False)
+    if logger.getEffectiveLevel() > logging.INFO:
+        show_progress = False
+    elif cached_resp:
+        show_progress = False
+    elif total_length > (40 * 1000):
+        show_progress = True
+    elif not total_length:
+        show_progress = True
+    else:
+        show_progress = False
+
+    show_url = link.show_url
+
+    def resp_read(chunk_size):
+        try:
+            # Special case for urllib3.
+            for chunk in resp.raw.stream(
+                    chunk_size,
+                    # We use decode_content=False here because we don't
+                    # want urllib3 to mess with the raw bytes we get
+                    # from the server. If we decompress inside of
+                    # urllib3 then we cannot verify the checksum
+                    # because the checksum will be of the compressed
+                    # file. This breakage will only occur if the
+                    # server adds a Content-Encoding header, which
+                    # depends on how the server was configured:
+                    # - Some servers will notice that the file isn't a
+                    #   compressible file and will leave the file alone
+                    #   and with an empty Content-Encoding
+                    # - Some servers will notice that the file is
+                    #   already compressed and will leave the file
+                    #   alone and will add a Content-Encoding: gzip
+                    #   header
+                    # - Some servers won't notice anything at all and
+                    #   will take a file that's already been compressed
+                    #   and compress it again and set the
+                    #   Content-Encoding: gzip header
+                    #
+                    # By setting this not to decode automatically we
+                    # hope to eliminate problems with the second case.
+                    decode_content=False):
+                yield chunk
+        except AttributeError:
+            # Standard file-like object.
+            while True:
+                chunk = resp.raw.read(chunk_size)
+                if not chunk:
+                    break
+                yield chunk
+
+    def written_chunks(chunks):
+        for chunk in chunks:
+            content_file.write(chunk)
+            yield chunk
+
+    progress_indicator = _progress_indicator
+
+    if link.netloc == PyPI.netloc:
+        url = show_url
+    else:
+        url = link.url_without_fragment
+
+    if show_progress:  # We don't show progress on cached responses
+        progress_indicator = DownloadProgressProvider(progress_bar,
+                                                      max=total_length)
+        if total_length:
+            logger.info("Downloading %s (%s)", url, format_size(total_length))
+        else:
+            logger.info("Downloading %s", url)
+    elif cached_resp:
+        logger.info("Using cached %s", url)
+    else:
+        logger.info("Downloading %s", url)
+
+    logger.debug('Downloading from URL %s', link)
+
+    downloaded_chunks = written_chunks(
+        progress_indicator(
+            resp_read(CONTENT_CHUNK_SIZE),
+            CONTENT_CHUNK_SIZE
+        )
+    )
+    if hashes:
+        hashes.check_against_chunks(downloaded_chunks)
+    else:
+        consume(downloaded_chunks)
+
+
+def _copy_file(filename, location, link):
+    copy = True
+    download_location = os.path.join(location, link.filename)
+    if os.path.exists(download_location):
+        response = ask_path_exists(
+            'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' %
+            display_path(download_location), ('i', 'w', 'b', 'a'))
+        if response == 'i':
+            copy = False
+        elif response == 'w':
+            logger.warning('Deleting %s', display_path(download_location))
+            os.remove(download_location)
+        elif response == 'b':
+            dest_file = backup_dir(download_location)
+            logger.warning(
+                'Backing up %s to %s',
+                display_path(download_location),
+                display_path(dest_file),
+            )
+            shutil.move(download_location, dest_file)
+        elif response == 'a':
+            sys.exit(-1)
+    if copy:
+        shutil.copy(filename, download_location)
+        logger.info('Saved %s', display_path(download_location))
+
+
+def unpack_http_url(link, location, download_dir=None,
+                    session=None, hashes=None, progress_bar="on"):
+    if session is None:
+        raise TypeError(
+            "unpack_http_url() missing 1 required keyword argument: 'session'"
+        )
+
+    with TempDirectory(kind="unpack") as temp_dir:
+        # If a download dir is specified, is the file already downloaded there?
+        already_downloaded_path = None
+        if download_dir:
+            already_downloaded_path = _check_download_dir(link,
+                                                          download_dir,
+                                                          hashes)
+
+        if already_downloaded_path:
+            from_path = already_downloaded_path
+            content_type = mimetypes.guess_type(from_path)[0]
+        else:
+            # let's download to a tmp dir
+            from_path, content_type = _download_http_url(link,
+                                                         session,
+                                                         temp_dir.path,
+                                                         hashes,
+                                                         progress_bar)
+
+        # unpack the archive to the build dir location. even when only
+        # downloading archives, they have to be unpacked to parse dependencies
+        unpack_file(from_path, location, content_type, link)
+
+        # a download dir is specified; let's copy the archive there
+        if download_dir and not already_downloaded_path:
+            _copy_file(from_path, download_dir, link)
+
+        if not already_downloaded_path:
+            os.unlink(from_path)
+
+
+def unpack_file_url(link, location, download_dir=None, hashes=None):
+    """Unpack link into location.
+
+    If download_dir is provided and link points to a file, make a copy
+    of the link file inside download_dir.
+    """
+    link_path = url_to_path(link.url_without_fragment)
+
+    # If it's a url to a local directory
+    if is_dir_url(link):
+        if os.path.isdir(location):
+            rmtree(location)
+        shutil.copytree(link_path, location, symlinks=True)
+        if download_dir:
+            logger.info('Link is a directory, ignoring download_dir')
+        return
+
+    # If --require-hashes is off, `hashes` is either empty, the
+    # link's embedded hash, or MissingHashes; it is required to
+    # match. If --require-hashes is on, we are satisfied by any
+    # hash in `hashes` matching: a URL-based or an option-based
+    # one; no internet-sourced hash will be in `hashes`.
+    if hashes:
+        hashes.check_against_path(link_path)
+
+    # If a download dir is specified, is the file already there and valid?
+    already_downloaded_path = None
+    if download_dir:
+        already_downloaded_path = _check_download_dir(link,
+                                                      download_dir,
+                                                      hashes)
+
+    if already_downloaded_path:
+        from_path = already_downloaded_path
+    else:
+        from_path = link_path
+
+    content_type = mimetypes.guess_type(from_path)[0]
+
+    # unpack the archive to the build dir location. even when only downloading
+    # archives, they have to be unpacked to parse dependencies
+    unpack_file(from_path, location, content_type, link)
+
+    # a download dir is specified and not already downloaded
+    if download_dir and not already_downloaded_path:
+        _copy_file(from_path, download_dir, link)
+
+
+def _copy_dist_from_dir(link_path, location):
+    """Copy distribution files in `link_path` to `location`.
+
+    Invoked when user requests to install a local directory. E.g.:
+
+        pip install .
+        pip install ~/dev/git-repos/python-prompt-toolkit
+
+    """
+
+    # Note: This is currently VERY SLOW if you have a lot of data in the
+    # directory, because it copies everything with `shutil.copytree`.
+    # What it should really do is build an sdist and install that.
+    # See https://github.com/pypa/pip/issues/2195
+
+    if os.path.isdir(location):
+        rmtree(location)
+
+    # build an sdist
+    setup_py = 'setup.py'
+    sdist_args = [sys.executable]
+    sdist_args.append('-c')
+    sdist_args.append(SETUPTOOLS_SHIM % setup_py)
+    sdist_args.append('sdist')
+    sdist_args += ['--dist-dir', location]
+    logger.info('Running setup.py sdist for %s', link_path)
+
+    with indent_log():
+        call_subprocess(sdist_args, cwd=link_path, show_stdout=False)
+
+    # unpack sdist into `location`
+    sdist = os.path.join(location, os.listdir(location)[0])
+    logger.info('Unpacking sdist %s into %s', sdist, location)
+    unpack_file(sdist, location, content_type=None, link=None)
+
+
+class PipXmlrpcTransport(xmlrpc_client.Transport):
+    """Provide a `xmlrpclib.Transport` implementation via a `PipSession`
+    object.
+    """
+
+    def __init__(self, index_url, session, use_datetime=False):
+        xmlrpc_client.Transport.__init__(self, use_datetime)
+        index_parts = urllib_parse.urlparse(index_url)
+        self._scheme = index_parts.scheme
+        self._session = session
+
+    def request(self, host, handler, request_body, verbose=False):
+        parts = (self._scheme, host, handler, None, None, None)
+        url = urllib_parse.urlunparse(parts)
+        try:
+            headers = {'Content-Type': 'text/xml'}
+            response = self._session.post(url, data=request_body,
+                                          headers=headers, stream=True)
+            response.raise_for_status()
+            self.verbose = verbose
+            return self.parse_response(response.raw)
+        except requests.HTTPError as exc:
+            logger.critical(
+                "HTTP error %s while getting %s",
+                exc.response.status_code, url,
+            )
+            raise
+
+
+def unpack_url(link, location, download_dir=None,
+               only_download=False, session=None, hashes=None,
+               progress_bar="on"):
+    """Unpack link.
+       If link is a VCS link:
+         if only_download, export into download_dir and ignore location
+          else unpack into location
+       for other types of link:
+         - unpack into location
+         - if download_dir, copy the file into download_dir
+         - if only_download, mark location for deletion
+
+    :param hashes: A Hashes object, one of whose embedded hashes must match,
+        or HashMismatch will be raised. If the Hashes is empty, no matches are
+        required, and unhashable types of requirements (like VCS ones, which
+        would ordinarily raise HashUnsupported) are allowed.
+    """
+    # non-editable vcs urls
+    if is_vcs_url(link):
+        unpack_vcs_link(link, location)
+
+    # file urls
+    elif is_file_url(link):
+        unpack_file_url(link, location, download_dir, hashes=hashes)
+
+    # http urls
+    else:
+        if session is None:
+            session = PipSession()
+
+        unpack_http_url(
+            link,
+            location,
+            download_dir,
+            session,
+            hashes=hashes,
+            progress_bar=progress_bar
+        )
+    if only_download:
+        write_delete_marker_file(location)
+
+
+def _download_http_url(link, session, temp_dir, hashes, progress_bar):
+    """Download link url into temp_dir using provided session"""
+    target_url = link.url.split('#', 1)[0]
+    try:
+        resp = session.get(
+            target_url,
+            # We use Accept-Encoding: identity here because requests
+            # defaults to accepting compressed responses. This breaks in
+            # a variety of ways depending on how the server is configured.
+            # - Some servers will notice that the file isn't a compressible
+            #   file and will leave the file alone and with an empty
+            #   Content-Encoding
+            # - Some servers will notice that the file is already
+            #   compressed and will leave the file alone and will add a
+            #   Content-Encoding: gzip header
+            # - Some servers won't notice anything at all and will take
+            #   a file that's already been compressed and compress it again
+            #   and set the Content-Encoding: gzip header
+            # By setting this to request only the identity encoding We're
+            # hoping to eliminate the third case. Hopefully there does not
+            # exist a server which when given a file will notice it is
+            # already compressed and that you're not asking for a
+            # compressed file and will then decompress it before sending
+            # because if that's the case I don't think it'll ever be
+            # possible to make this work.
+            headers={"Accept-Encoding": "identity"},
+            stream=True,
+        )
+        resp.raise_for_status()
+    except requests.HTTPError as exc:
+        logger.critical(
+            "HTTP error %s while getting %s", exc.response.status_code, link,
+        )
+        raise
+
+    content_type = resp.headers.get('content-type', '')
+    filename = link.filename  # fallback
+    # Have a look at the Content-Disposition header for a better guess
+    content_disposition = resp.headers.get('content-disposition')
+    if content_disposition:
+        type, params = cgi.parse_header(content_disposition)
+        # We use ``or`` here because we don't want to use an "empty" value
+        # from the filename param.
+        filename = params.get('filename') or filename
+    ext = splitext(filename)[1]
+    if not ext:
+        ext = mimetypes.guess_extension(content_type)
+        if ext:
+            filename += ext
+    if not ext and link.url != resp.url:
+        ext = os.path.splitext(resp.url)[1]
+        if ext:
+            filename += ext
+    file_path = os.path.join(temp_dir, filename)
+    with open(file_path, 'wb') as content_file:
+        _download_url(resp, link, content_file, hashes, progress_bar)
+    return file_path, content_type
+
+
+def _check_download_dir(link, download_dir, hashes):
+    """ Check download_dir for previously downloaded file with correct hash
+        If a correct file is found return its path else None
+    """
+    download_path = os.path.join(download_dir, link.filename)
+    if os.path.exists(download_path):
+        # If already downloaded, does its hash match?
+        logger.info('File was already downloaded %s', download_path)
+        if hashes:
+            try:
+                hashes.check_against_path(download_path)
+            except HashMismatch:
+                logger.warning(
+                    'Previously-downloaded file %s has bad hash. '
+                    'Re-downloading.',
+                    download_path
+                )
+                os.unlink(download_path)
+                return None
+        return download_path
+    return None

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/exceptions.py

@@ -0,0 +1,249 @@
+"""Exceptions used throughout package"""
+from __future__ import absolute_import
+
+from itertools import chain, groupby, repeat
+
+from pip._vendor.six import iteritems
+
+
+class PipError(Exception):
+    """Base pip exception"""
+
+
+class ConfigurationError(PipError):
+    """General exception in configuration"""
+
+
+class InstallationError(PipError):
+    """General exception during installation"""
+
+
+class UninstallationError(PipError):
+    """General exception during uninstallation"""
+
+
+class DistributionNotFound(InstallationError):
+    """Raised when a distribution cannot be found to satisfy a requirement"""
+
+
+class RequirementsFileParseError(InstallationError):
+    """Raised when a general error occurs parsing a requirements file line."""
+
+
+class BestVersionAlreadyInstalled(PipError):
+    """Raised when the most up-to-date version of a package is already
+    installed."""
+
+
+class BadCommand(PipError):
+    """Raised when virtualenv or a command is not found"""
+
+
+class CommandError(PipError):
+    """Raised when there is an error in command-line arguments"""
+
+
+class PreviousBuildDirError(PipError):
+    """Raised when there's a previous conflicting build directory"""
+
+
+class InvalidWheelFilename(InstallationError):
+    """Invalid wheel filename."""
+
+
+class UnsupportedWheel(InstallationError):
+    """Unsupported wheel."""
+
+
+class HashErrors(InstallationError):
+    """Multiple HashError instances rolled into one for reporting"""
+
+    def __init__(self):
+        self.errors = []
+
+    def append(self, error):
+        self.errors.append(error)
+
+    def __str__(self):
+        lines = []
+        self.errors.sort(key=lambda e: e.order)
+        for cls, errors_of_cls in groupby(self.errors, lambda e: e.__class__):
+            lines.append(cls.head)
+            lines.extend(e.body() for e in errors_of_cls)
+        if lines:
+            return '\n'.join(lines)
+
+    def __nonzero__(self):
+        return bool(self.errors)
+
+    def __bool__(self):
+        return self.__nonzero__()
+
+
+class HashError(InstallationError):
+    """
+    A failure to verify a package against known-good hashes
+
+    :cvar order: An int sorting hash exception classes by difficulty of
+        recovery (lower being harder), so the user doesn't bother fretting
+        about unpinned packages when he has deeper issues, like VCS
+        dependencies, to deal with. Also keeps error reports in a
+        deterministic order.
+    :cvar head: A section heading for display above potentially many
+        exceptions of this kind
+    :ivar req: The InstallRequirement that triggered this error. This is
+        pasted on after the exception is instantiated, because it's not
+        typically available earlier.
+
+    """
+    req = None
+    head = ''
+
+    def body(self):
+        """Return a summary of me for display under the heading.
+
+        This default implementation simply prints a description of the
+        triggering requirement.
+
+        :param req: The InstallRequirement that provoked this error, with
+            populate_link() having already been called
+
+        """
+        return '    %s' % self._requirement_name()
+
+    def __str__(self):
+        return '%s\n%s' % (self.head, self.body())
+
+    def _requirement_name(self):
+        """Return a description of the requirement that triggered me.
+
+        This default implementation returns long description of the req, with
+        line numbers
+
+        """
+        return str(self.req) if self.req else 'unknown package'
+
+
+class VcsHashUnsupported(HashError):
+    """A hash was provided for a version-control-system-based requirement, but
+    we don't have a method for hashing those."""
+
+    order = 0
+    head = ("Can't verify hashes for these requirements because we don't "
+            "have a way to hash version control repositories:")
+
+
+class DirectoryUrlHashUnsupported(HashError):
+    """A hash was provided for a version-control-system-based requirement, but
+    we don't have a method for hashing those."""
+
+    order = 1
+    head = ("Can't verify hashes for these file:// requirements because they "
+            "point to directories:")
+
+
+class HashMissing(HashError):
+    """A hash was needed for a requirement but is absent."""
+
+    order = 2
+    head = ('Hashes are required in --require-hashes mode, but they are '
+            'missing from some requirements. Here is a list of those '
+            'requirements along with the hashes their downloaded archives '
+            'actually had. Add lines like these to your requirements files to '
+            'prevent tampering. (If you did not enable --require-hashes '
+            'manually, note that it turns on automatically when any package '
+            'has a hash.)')
+
+    def __init__(self, gotten_hash):
+        """
+        :param gotten_hash: The hash of the (possibly malicious) archive we
+            just downloaded
+        """
+        self.gotten_hash = gotten_hash
+
+    def body(self):
+        # Dodge circular import.
+        from pip._internal.utils.hashes import FAVORITE_HASH
+
+        package = None
+        if self.req:
+            # In the case of URL-based requirements, display the original URL
+            # seen in the requirements file rather than the package name,
+            # so the output can be directly copied into the requirements file.
+            package = (self.req.original_link if self.req.original_link
+                       # In case someone feeds something downright stupid
+                       # to InstallRequirement's constructor.
+                       else getattr(self.req, 'req', None))
+        return '    %s --hash=%s:%s' % (package or 'unknown package',
+                                        FAVORITE_HASH,
+                                        self.gotten_hash)
+
+
+class HashUnpinned(HashError):
+    """A requirement had a hash specified but was not pinned to a specific
+    version."""
+
+    order = 3
+    head = ('In --require-hashes mode, all requirements must have their '
+            'versions pinned with ==. These do not:')
+
+
+class HashMismatch(HashError):
+    """
+    Distribution file hash values don't match.
+
+    :ivar package_name: The name of the package that triggered the hash
+        mismatch. Feel free to write to this after the exception is raise to
+        improve its error message.
+
+    """
+    order = 4
+    head = ('THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS '
+            'FILE. If you have updated the package versions, please update '
+            'the hashes. Otherwise, examine the package contents carefully; '
+            'someone may have tampered with them.')
+
+    def __init__(self, allowed, gots):
+        """
+        :param allowed: A dict of algorithm names pointing to lists of allowed
+            hex digests
+        :param gots: A dict of algorithm names pointing to hashes we
+            actually got from the files under suspicion
+        """
+        self.allowed = allowed
+        self.gots = gots
+
+    def body(self):
+        return '    %s:\n%s' % (self._requirement_name(),
+                                self._hash_comparison())
+
+    def _hash_comparison(self):
+        """
+        Return a comparison of actual and expected hash values.
+
+        Example::
+
+               Expected sha256 abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde
+                            or 123451234512345123451234512345123451234512345
+                    Got        bcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdef
+
+        """
+        def hash_then_or(hash_name):
+            # For now, all the decent hashes have 6-char names, so we can get
+            # away with hard-coding space literals.
+            return chain([hash_name], repeat('    or'))
+
+        lines = []
+        for hash_name, expecteds in iteritems(self.allowed):
+            prefix = hash_then_or(hash_name)
+            lines.extend(('        Expected %s %s' % (next(prefix), e))
+                         for e in expecteds)
+            lines.append('             Got        %s\n' %
+                         self.gots[hash_name].hexdigest())
+            prefix = '    or'
+        return '\n'.join(lines)
+
+
+class UnsupportedPythonVersion(InstallationError):
+    """Unsupported python version according to Requires-Python package
+    metadata."""

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/locations.py

@@ -0,0 +1,194 @@
+"""Locations where we look for configs, install stuff, etc"""
+from __future__ import absolute_import
+
+import os
+import os.path
+import platform
+import site
+import sys
+import sysconfig
+from distutils import sysconfig as distutils_sysconfig
+from distutils.command.install import SCHEME_KEYS, install  # type: ignore
+
+from pip._internal.compat import WINDOWS, expanduser
+from pip._internal.utils import appdirs
+
+# Application Directories
+USER_CACHE_DIR = appdirs.user_cache_dir("pip")
+
+
+DELETE_MARKER_MESSAGE = '''\
+This file is placed here by pip to indicate the source was put
+here by pip.
+
+Once this package is successfully installed this source code will be
+deleted (unless you remove this file).
+'''
+PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt'
+
+
+def write_delete_marker_file(directory):
+    """
+    Write the pip delete marker file into this directory.
+    """
+    filepath = os.path.join(directory, PIP_DELETE_MARKER_FILENAME)
+    with open(filepath, 'w') as marker_fp:
+        marker_fp.write(DELETE_MARKER_MESSAGE)
+
+
+def running_under_virtualenv():
+    """
+    Return True if we're running inside a virtualenv, False otherwise.
+
+    """
+    if hasattr(sys, 'real_prefix'):
+        return True
+    elif sys.prefix != getattr(sys, "base_prefix", sys.prefix):
+        return True
+
+    return False
+
+
+def virtualenv_no_global():
+    """
+    Return True if in a venv and no system site packages.
+    """
+    # this mirrors the logic in virtualenv.py for locating the
+    # no-global-site-packages.txt file
+    site_mod_dir = os.path.dirname(os.path.abspath(site.__file__))
+    no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt')
+    if running_under_virtualenv() and os.path.isfile(no_global_file):
+        return True
+
+
+if running_under_virtualenv():
+    src_prefix = os.path.join(sys.prefix, 'src')
+else:
+    # FIXME: keep src in cwd for now (it is not a temporary folder)
+    try:
+        src_prefix = os.path.join(os.getcwd(), 'src')
+    except OSError:
+        # In case the current working directory has been renamed or deleted
+        sys.exit(
+            "The folder you are executing pip from can no longer be found."
+        )
+
+# under macOS + virtualenv sys.prefix is not properly resolved
+# it is something like /path/to/python/bin/..
+# Note: using realpath due to tmp dirs on OSX being symlinks
+src_prefix = os.path.abspath(src_prefix)
+
+# FIXME doesn't account for venv linked to global site-packages
+
+site_packages = sysconfig.get_path("purelib")
+# This is because of a bug in PyPy's sysconfig module, see
+# https://bitbucket.org/pypy/pypy/issues/2506/sysconfig-returns-incorrect-paths
+# for more information.
+if platform.python_implementation().lower() == "pypy":
+    site_packages = distutils_sysconfig.get_python_lib()
+try:
+    # Use getusersitepackages if this is present, as it ensures that the
+    # value is initialised properly.
+    user_site = site.getusersitepackages()
+except AttributeError:
+    user_site = site.USER_SITE
+user_dir = expanduser('~')
+if WINDOWS:
+    bin_py = os.path.join(sys.prefix, 'Scripts')
+    bin_user = os.path.join(user_site, 'Scripts')
+    # buildout uses 'bin' on Windows too?
+    if not os.path.exists(bin_py):
+        bin_py = os.path.join(sys.prefix, 'bin')
+        bin_user = os.path.join(user_site, 'bin')
+
+    config_basename = 'pip.ini'
+
+    legacy_storage_dir = os.path.join(user_dir, 'pip')
+    legacy_config_file = os.path.join(
+        legacy_storage_dir,
+        config_basename,
+    )
+else:
+    bin_py = os.path.join(sys.prefix, 'bin')
+    bin_user = os.path.join(user_site, 'bin')
+
+    config_basename = 'pip.conf'
+
+    legacy_storage_dir = os.path.join(user_dir, '.pip')
+    legacy_config_file = os.path.join(
+        legacy_storage_dir,
+        config_basename,
+    )
+    # Forcing to use /usr/local/bin for standard macOS framework installs
+    # Also log to ~/Library/Logs/ for use with the Console.app log viewer
+    if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/':
+        bin_py = '/usr/local/bin'
+
+site_config_files = [
+    os.path.join(path, config_basename)
+    for path in appdirs.site_config_dirs('pip')
+]
+
+venv_config_file = os.path.join(sys.prefix, config_basename)
+new_config_file = os.path.join(appdirs.user_config_dir("pip"), config_basename)
+
+
+def distutils_scheme(dist_name, user=False, home=None, root=None,
+                     isolated=False, prefix=None):
+    """
+    Return a distutils install scheme
+    """
+    from distutils.dist import Distribution
+
+    scheme = {}
+
+    if isolated:
+        extra_dist_args = {"script_args": ["--no-user-cfg"]}
+    else:
+        extra_dist_args = {}
+    dist_args = {'name': dist_name}
+    dist_args.update(extra_dist_args)
+
+    d = Distribution(dist_args)
+    d.parse_config_files()
+    i = d.get_command_obj('install', create=True)
+    # NOTE: setting user or home has the side-effect of creating the home dir
+    # or user base for installations during finalize_options()
+    # ideally, we'd prefer a scheme class that has no side-effects.
+    assert not (user and prefix), "user={} prefix={}".format(user, prefix)
+    i.user = user or i.user
+    if user:
+        i.prefix = ""
+    i.prefix = prefix or i.prefix
+    i.home = home or i.home
+    i.root = root or i.root
+    i.finalize_options()
+    for key in SCHEME_KEYS:
+        scheme[key] = getattr(i, 'install_' + key)
+
+    # install_lib specified in setup.cfg should install *everything*
+    # into there (i.e. it takes precedence over both purelib and
+    # platlib).  Note, i.install_lib is *always* set after
+    # finalize_options(); we only want to override here if the user
+    # has explicitly requested it hence going back to the config
+    if 'install_lib' in d.get_option_dict('install'):
+        scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib))
+
+    if running_under_virtualenv():
+        scheme['headers'] = os.path.join(
+            sys.prefix,
+            'include',
+            'site',
+            'python' + sys.version[:3],
+            dist_name,
+        )
+
+        if root is not None:
+            path_no_drive = os.path.splitdrive(
+                os.path.abspath(scheme["headers"]))[1]
+            scheme["headers"] = os.path.join(
+                root,
+                path_no_drive[1:],
+            )
+
+    return scheme

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/models/__init__.py

@@ -0,0 +1,4 @@
+from pip._internal.models.index import Index, PyPI
+
+
+__all__ = ["Index", "PyPI"]

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/models/index.py

@@ -0,0 +1,15 @@
+from pip._vendor.six.moves.urllib import parse as urllib_parse
+
+
+class Index(object):
+    def __init__(self, url):
+        self.url = url
+        self.netloc = urllib_parse.urlsplit(url).netloc
+        self.simple_url = self.url_to_path('simple')
+        self.pypi_url = self.url_to_path('pypi')
+
+    def url_to_path(self, path):
+        return urllib_parse.urljoin(self.url, path)
+
+
+PyPI = Index('https://pypi.org/')

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/operations/check.py

@@ -0,0 +1,106 @@
+"""Validation of dependencies of packages
+"""
+
+from collections import namedtuple
+
+from pip._vendor.packaging.utils import canonicalize_name
+
+from pip._internal.operations.prepare import make_abstract_dist
+
+from pip._internal.utils.misc import get_installed_distributions
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from pip._internal.req.req_install import InstallRequirement
+    from typing import Any, Dict, Iterator, Set, Tuple, List
+
+    # Shorthands
+    PackageSet = Dict[str, 'PackageDetails']
+    Missing = Tuple[str, Any]
+    Conflicting = Tuple[str, str, Any]
+
+    MissingDict = Dict[str, List[Missing]]
+    ConflictingDict = Dict[str, List[Conflicting]]
+    CheckResult = Tuple[MissingDict, ConflictingDict]
+
+PackageDetails = namedtuple('PackageDetails', ['version', 'requires'])
+
+
+def create_package_set_from_installed(**kwargs):
+    # type: (**Any) -> PackageSet
+    """Converts a list of distributions into a PackageSet.
+    """
+    # Default to using all packages installed on the system
+    if kwargs == {}:
+        kwargs = {"local_only": False, "skip": ()}
+    retval = {}
+    for dist in get_installed_distributions(**kwargs):
+        name = canonicalize_name(dist.project_name)
+        retval[name] = PackageDetails(dist.version, dist.requires())
+    return retval
+
+
+def check_package_set(package_set):
+    # type: (PackageSet) -> CheckResult
+    """Check if a package set is consistent
+    """
+    missing = dict()
+    conflicting = dict()
+
+    for package_name in package_set:
+        # Info about dependencies of package_name
+        missing_deps = set()  # type: Set[Missing]
+        conflicting_deps = set()  # type: Set[Conflicting]
+
+        for req in package_set[package_name].requires:
+            name = canonicalize_name(req.project_name)  # type: str
+
+            # Check if it's missing
+            if name not in package_set:
+                missed = True
+                if req.marker is not None:
+                    missed = req.marker.evaluate()
+                if missed:
+                    missing_deps.add((name, req))
+                continue
+
+            # Check if there's a conflict
+            version = package_set[name].version  # type: str
+            if not req.specifier.contains(version, prereleases=True):
+                conflicting_deps.add((name, version, req))
+
+        def str_key(x):
+            return str(x)
+
+        if missing_deps:
+            missing[package_name] = sorted(missing_deps, key=str_key)
+        if conflicting_deps:
+            conflicting[package_name] = sorted(conflicting_deps, key=str_key)
+
+    return missing, conflicting
+
+
+def check_install_conflicts(to_install):
+    # type: (List[InstallRequirement]) -> Tuple[PackageSet, CheckResult]
+    """For checking if the dependency graph would be consistent after \
+    installing given requirements
+    """
+    # Start from the current state
+    state = create_package_set_from_installed()
+    _simulate_installation_of(to_install, state)
+    return state, check_package_set(state)
+
+
+# NOTE from @pradyunsg
+# This required a minor update in dependency link handling logic over at
+# operations.prepare.IsSDist.dist() to get it working
+def _simulate_installation_of(to_install, state):
+    # type: (List[InstallRequirement], PackageSet) -> None
+    """Computes the version of packages after installing to_install.
+    """
+
+    # Modify it as installing requirement_set would (assuming no errors)
+    for inst_req in to_install:
+        dist = make_abstract_dist(inst_req).dist(finder=None)
+        name = canonicalize_name(dist.key)
+        state[name] = PackageDetails(dist.version, dist.requires())

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/operations/freeze.py

@@ -0,0 +1,252 @@
+from __future__ import absolute_import
+
+import collections
+import logging
+import os
+import re
+import warnings
+
+from pip._vendor import pkg_resources, six
+from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.pkg_resources import RequirementParseError
+
+from pip._internal.exceptions import InstallationError
+from pip._internal.req import InstallRequirement
+from pip._internal.req.req_file import COMMENT_RE
+from pip._internal.utils.deprecation import RemovedInPip11Warning
+from pip._internal.utils.misc import (
+    dist_is_editable, get_installed_distributions,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def freeze(
+        requirement=None,
+        find_links=None, local_only=None, user_only=None, skip_regex=None,
+        isolated=False,
+        wheel_cache=None,
+        exclude_editable=False,
+        skip=()):
+    find_links = find_links or []
+    skip_match = None
+
+    if skip_regex:
+        skip_match = re.compile(skip_regex).search
+
+    dependency_links = []
+
+    for dist in pkg_resources.working_set:
+        if dist.has_metadata('dependency_links.txt'):
+            dependency_links.extend(
+                dist.get_metadata_lines('dependency_links.txt')
+            )
+    for link in find_links:
+        if '#egg=' in link:
+            dependency_links.append(link)
+    for link in find_links:
+        yield '-f %s' % link
+    installations = {}
+    for dist in get_installed_distributions(local_only=local_only,
+                                            skip=(),
+                                            user_only=user_only):
+        try:
+            req = FrozenRequirement.from_dist(
+                dist,
+                dependency_links
+            )
+        except RequirementParseError:
+            logger.warning(
+                "Could not parse requirement: %s",
+                dist.project_name
+            )
+            continue
+        if exclude_editable and req.editable:
+            continue
+        installations[req.name] = req
+
+    if requirement:
+        # the options that don't get turned into an InstallRequirement
+        # should only be emitted once, even if the same option is in multiple
+        # requirements files, so we need to keep track of what has been emitted
+        # so that we don't emit it again if it's seen again
+        emitted_options = set()
+        # keep track of which files a requirement is in so that we can
+        # give an accurate warning if a requirement appears multiple times.
+        req_files = collections.defaultdict(list)
+        for req_file_path in requirement:
+            with open(req_file_path) as req_file:
+                for line in req_file:
+                    if (not line.strip() or
+                            line.strip().startswith('#') or
+                            (skip_match and skip_match(line)) or
+                            line.startswith((
+                                '-r', '--requirement',
+                                '-Z', '--always-unzip',
+                                '-f', '--find-links',
+                                '-i', '--index-url',
+                                '--pre',
+                                '--trusted-host',
+                                '--process-dependency-links',
+                                '--extra-index-url'))):
+                        line = line.rstrip()
+                        if line not in emitted_options:
+                            emitted_options.add(line)
+                            yield line
+                        continue
+
+                    if line.startswith('-e') or line.startswith('--editable'):
+                        if line.startswith('-e'):
+                            line = line[2:].strip()
+                        else:
+                            line = line[len('--editable'):].strip().lstrip('=')
+                        line_req = InstallRequirement.from_editable(
+                            line,
+                            isolated=isolated,
+                            wheel_cache=wheel_cache,
+                        )
+                    else:
+                        line_req = InstallRequirement.from_line(
+                            COMMENT_RE.sub('', line).strip(),
+                            isolated=isolated,
+                            wheel_cache=wheel_cache,
+                        )
+
+                    if not line_req.name:
+                        logger.info(
+                            "Skipping line in requirement file [%s] because "
+                            "it's not clear what it would install: %s",
+                            req_file_path, line.strip(),
+                        )
+                        logger.info(
+                            "  (add #egg=PackageName to the URL to avoid"
+                            " this warning)"
+                        )
+                    elif line_req.name not in installations:
+                        # either it's not installed, or it is installed
+                        # but has been processed already
+                        if not req_files[line_req.name]:
+                            logger.warning(
+                                "Requirement file [%s] contains %s, but that "
+                                "package is not installed",
+                                req_file_path,
+                                COMMENT_RE.sub('', line).strip(),
+                            )
+                        else:
+                            req_files[line_req.name].append(req_file_path)
+                    else:
+                        yield str(installations[line_req.name]).rstrip()
+                        del installations[line_req.name]
+                        req_files[line_req.name].append(req_file_path)
+
+        # Warn about requirements that were included multiple times (in a
+        # single requirements file or in different requirements files).
+        for name, files in six.iteritems(req_files):
+            if len(files) > 1:
+                logger.warning("Requirement %s included multiple times [%s]",
+                               name, ', '.join(sorted(set(files))))
+
+        yield(
+            '## The following requirements were added by '
+            'pip freeze:'
+        )
+    for installation in sorted(
+            installations.values(), key=lambda x: x.name.lower()):
+        if canonicalize_name(installation.name) not in skip:
+            yield str(installation).rstrip()
+
+
+class FrozenRequirement(object):
+    def __init__(self, name, req, editable, comments=()):
+        self.name = name
+        self.req = req
+        self.editable = editable
+        self.comments = comments
+
+    _rev_re = re.compile(r'-r(\d+)$')
+    _date_re = re.compile(r'-(20\d\d\d\d\d\d)$')
+
+    @classmethod
+    def from_dist(cls, dist, dependency_links):
+        location = os.path.normcase(os.path.abspath(dist.location))
+        comments = []
+        from pip._internal.vcs import vcs, get_src_requirement
+        if dist_is_editable(dist) and vcs.get_backend_name(location):
+            editable = True
+            try:
+                req = get_src_requirement(dist, location)
+            except InstallationError as exc:
+                logger.warning(
+                    "Error when trying to get requirement for VCS system %s, "
+                    "falling back to uneditable format", exc
+                )
+                req = None
+            if req is None:
+                logger.warning(
+                    'Could not determine repository location of %s', location
+                )
+                comments.append(
+                    '## !! Could not determine repository location'
+                )
+                req = dist.as_requirement()
+                editable = False
+        else:
+            editable = False
+            req = dist.as_requirement()
+            specs = req.specs
+            assert len(specs) == 1 and specs[0][0] in ["==", "==="], \
+                'Expected 1 spec with == or ===; specs = %r; dist = %r' % \
+                (specs, dist)
+            version = specs[0][1]
+            ver_match = cls._rev_re.search(version)
+            date_match = cls._date_re.search(version)
+            if ver_match or date_match:
+                svn_backend = vcs.get_backend('svn')
+                if svn_backend:
+                    svn_location = svn_backend().get_location(
+                        dist,
+                        dependency_links,
+                    )
+                if not svn_location:
+                    logger.warning(
+                        'Warning: cannot find svn location for %s', req,
+                    )
+                    comments.append(
+                        '## FIXME: could not find svn URL in dependency_links '
+                        'for this package:'
+                    )
+                else:
+                    warnings.warn(
+                        "SVN editable detection based on dependency links "
+                        "will be dropped in the future.",
+                        RemovedInPip11Warning,
+                    )
+                    comments.append(
+                        '# Installing as editable to satisfy requirement %s:' %
+                        req
+                    )
+                    if ver_match:
+                        rev = ver_match.group(1)
+                    else:
+                        rev = '{%s}' % date_match.group(1)
+                    editable = True
+                    req = '%s@%s#egg=%s' % (
+                        svn_location,
+                        rev,
+                        cls.egg_name(dist)
+                    )
+        return cls(dist.project_name, req, editable, comments)
+
+    @staticmethod
+    def egg_name(dist):
+        name = dist.egg_name()
+        match = re.search(r'-py\d\.\d$', name)
+        if match:
+            name = name[:match.start()]
+        return name
+
+    def __str__(self):
+        req = self.req
+        if self.editable:
+            req = '-e %s' % req
+        return '\n'.join(list(self.comments) + [str(req)]) + '\n'

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/operations/prepare.py

@@ -0,0 +1,380 @@
+"""Prepares a distribution for installation
+"""
+
+import itertools
+import logging
+import os
+import sys
+from copy import copy
+
+from pip._vendor import pkg_resources, requests
+
+from pip._internal.build_env import NoOpBuildEnvironment
+from pip._internal.compat import expanduser
+from pip._internal.download import (
+    is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path,
+)
+from pip._internal.exceptions import (
+    DirectoryUrlHashUnsupported, HashUnpinned, InstallationError,
+    PreviousBuildDirError, VcsHashUnsupported,
+)
+from pip._internal.index import FormatControl
+from pip._internal.req.req_install import InstallRequirement
+from pip._internal.utils.hashes import MissingHashes
+from pip._internal.utils.logging import indent_log
+from pip._internal.utils.misc import (
+    call_subprocess, display_path, normalize_path,
+)
+from pip._internal.utils.ui import open_spinner
+from pip._internal.vcs import vcs
+
+logger = logging.getLogger(__name__)
+
+
+def make_abstract_dist(req):
+    """Factory to make an abstract dist object.
+
+    Preconditions: Either an editable req with a source_dir, or satisfied_by or
+    a wheel link, or a non-editable req with a source_dir.
+
+    :return: A concrete DistAbstraction.
+    """
+    if req.editable:
+        return IsSDist(req)
+    elif req.link and req.link.is_wheel:
+        return IsWheel(req)
+    else:
+        return IsSDist(req)
+
+
+def _install_build_reqs(finder, prefix, build_requirements):
+    # NOTE: What follows is not a very good thing.
+    #       Eventually, this should move into the BuildEnvironment class and
+    #       that should handle all the isolation and sub-process invocation.
+    finder = copy(finder)
+    finder.format_control = FormatControl(set(), set([":all:"]))
+    urls = [
+        finder.find_requirement(
+            InstallRequirement.from_line(r), upgrade=False).url
+        for r in build_requirements
+    ]
+    args = [
+        sys.executable, '-m', 'pip', 'install', '--ignore-installed',
+        '--no-user', '--prefix', prefix,
+    ] + list(urls)
+
+    with open_spinner("Installing build dependencies") as spinner:
+        call_subprocess(args, show_stdout=False, spinner=spinner)
+
+
+class DistAbstraction(object):
+    """Abstracts out the wheel vs non-wheel Resolver.resolve() logic.
+
+    The requirements for anything installable are as follows:
+     - we must be able to determine the requirement name
+       (or we can't correctly handle the non-upgrade case).
+     - we must be able to generate a list of run-time dependencies
+       without installing any additional packages (or we would
+       have to either burn time by doing temporary isolated installs
+       or alternatively violate pips 'don't start installing unless
+       all requirements are available' rule - neither of which are
+       desirable).
+     - for packages with setup requirements, we must also be able
+       to determine their requirements without installing additional
+       packages (for the same reason as run-time dependencies)
+     - we must be able to create a Distribution object exposing the
+       above metadata.
+    """
+
+    def __init__(self, req):
+        self.req = req
+
+    def dist(self, finder):
+        """Return a setuptools Dist object."""
+        raise NotImplementedError(self.dist)
+
+    def prep_for_dist(self, finder):
+        """Ensure that we can get a Dist for this requirement."""
+        raise NotImplementedError(self.dist)
+
+
+class IsWheel(DistAbstraction):
+
+    def dist(self, finder):
+        return list(pkg_resources.find_distributions(
+            self.req.source_dir))[0]
+
+    def prep_for_dist(self, finder, build_isolation):
+        # FIXME:https://github.com/pypa/pip/issues/1112
+        pass
+
+
+class IsSDist(DistAbstraction):
+
+    def dist(self, finder):
+        dist = self.req.get_dist()
+        # FIXME: shouldn't be globally added.
+        if finder and dist.has_metadata('dependency_links.txt'):
+            finder.add_dependency_links(
+                dist.get_metadata_lines('dependency_links.txt')
+            )
+        return dist
+
+    def prep_for_dist(self, finder, build_isolation):
+        # Before calling "setup.py egg_info", we need to set-up the build
+        # environment.
+        build_requirements, isolate = self.req.get_pep_518_info()
+        should_isolate = build_isolation and isolate
+
+        minimum_requirements = ('setuptools', 'wheel')
+        missing_requirements = set(minimum_requirements) - set(
+            pkg_resources.Requirement(r).key
+            for r in build_requirements
+        )
+        if missing_requirements:
+            def format_reqs(rs):
+                return ' and '.join(map(repr, sorted(rs)))
+            logger.warning(
+                "Missing build time requirements in pyproject.toml for %s: "
+                "%s.", self.req, format_reqs(missing_requirements)
+            )
+            logger.warning(
+                "This version of pip does not implement PEP 517 so it cannot "
+                "build a wheel without %s.", format_reqs(minimum_requirements)
+            )
+
+        if should_isolate:
+            with self.req.build_env:
+                pass
+            _install_build_reqs(finder, self.req.build_env.path,
+                                build_requirements)
+        else:
+            self.req.build_env = NoOpBuildEnvironment(no_clean=False)
+
+        self.req.run_egg_info()
+        self.req.assert_source_matches_version()
+
+
+class Installed(DistAbstraction):
+
+    def dist(self, finder):
+        return self.req.satisfied_by
+
+    def prep_for_dist(self, finder):
+        pass
+
+
+class RequirementPreparer(object):
+    """Prepares a Requirement
+    """
+
+    def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir,
+                 progress_bar, build_isolation):
+        super(RequirementPreparer, self).__init__()
+
+        self.src_dir = src_dir
+        self.build_dir = build_dir
+
+        # Where still packed archives should be written to. If None, they are
+        # not saved, and are deleted immediately after unpacking.
+        self.download_dir = download_dir
+
+        # Where still-packed .whl files should be written to. If None, they are
+        # written to the download_dir parameter. Separate to download_dir to
+        # permit only keeping wheel archives for pip wheel.
+        if wheel_download_dir:
+            wheel_download_dir = normalize_path(wheel_download_dir)
+        self.wheel_download_dir = wheel_download_dir
+
+        # NOTE
+        # download_dir and wheel_download_dir overlap semantically and may
+        # be combined if we're willing to have non-wheel archives present in
+        # the wheelhouse output by 'pip wheel'.
+
+        self.progress_bar = progress_bar
+
+        # Is build isolation allowed?
+        self.build_isolation = build_isolation
+
+    @property
+    def _download_should_save(self):
+        # TODO: Modify to reduce indentation needed
+        if self.download_dir:
+            self.download_dir = expanduser(self.download_dir)
+            if os.path.exists(self.download_dir):
+                return True
+            else:
+                logger.critical('Could not find download directory')
+                raise InstallationError(
+                    "Could not find or access download directory '%s'"
+                    % display_path(self.download_dir))
+        return False
+
+    def prepare_linked_requirement(self, req, session, finder,
+                                   upgrade_allowed, require_hashes):
+        """Prepare a requirement that would be obtained from req.link
+        """
+        # TODO: Breakup into smaller functions
+        if req.link and req.link.scheme == 'file':
+            path = url_to_path(req.link.url)
+            logger.info('Processing %s', display_path(path))
+        else:
+            logger.info('Collecting %s', req)
+
+        with indent_log():
+            # @@ if filesystem packages are not marked
+            # editable in a req, a non deterministic error
+            # occurs when the script attempts to unpack the
+            # build directory
+            req.ensure_has_source_dir(self.build_dir)
+            # If a checkout exists, it's unwise to keep going.  version
+            # inconsistencies are logged later, but do not fail the
+            # installation.
+            # FIXME: this won't upgrade when there's an existing
+            # package unpacked in `req.source_dir`
+            # package unpacked in `req.source_dir`
+            if os.path.exists(os.path.join(req.source_dir, 'setup.py')):
+                raise PreviousBuildDirError(
+                    "pip can't proceed with requirements '%s' due to a"
+                    " pre-existing build directory (%s). This is "
+                    "likely due to a previous installation that failed"
+                    ". pip is being responsible and not assuming it "
+                    "can delete this. Please delete it and try again."
+                    % (req, req.source_dir)
+                )
+            req.populate_link(finder, upgrade_allowed, require_hashes)
+
+            # We can't hit this spot and have populate_link return None.
+            # req.satisfied_by is None here (because we're
+            # guarded) and upgrade has no impact except when satisfied_by
+            # is not None.
+            # Then inside find_requirement existing_applicable -> False
+            # If no new versions are found, DistributionNotFound is raised,
+            # otherwise a result is guaranteed.
+            assert req.link
+            link = req.link
+
+            # Now that we have the real link, we can tell what kind of
+            # requirements we have and raise some more informative errors
+            # than otherwise. (For example, we can raise VcsHashUnsupported
+            # for a VCS URL rather than HashMissing.)
+            if require_hashes:
+                # We could check these first 2 conditions inside
+                # unpack_url and save repetition of conditions, but then
+                # we would report less-useful error messages for
+                # unhashable requirements, complaining that there's no
+                # hash provided.
+                if is_vcs_url(link):
+                    raise VcsHashUnsupported()
+                elif is_file_url(link) and is_dir_url(link):
+                    raise DirectoryUrlHashUnsupported()
+                if not req.original_link and not req.is_pinned:
+                    # Unpinned packages are asking for trouble when a new
+                    # version is uploaded. This isn't a security check, but
+                    # it saves users a surprising hash mismatch in the
+                    # future.
+                    #
+                    # file:/// URLs aren't pinnable, so don't complain
+                    # about them not being pinned.
+                    raise HashUnpinned()
+
+            hashes = req.hashes(trust_internet=not require_hashes)
+            if require_hashes and not hashes:
+                # Known-good hashes are missing for this requirement, so
+                # shim it with a facade object that will provoke hash
+                # computation and then raise a HashMissing exception
+                # showing the user what the hash should be.
+                hashes = MissingHashes()
+
+            try:
+                download_dir = self.download_dir
+                # We always delete unpacked sdists after pip ran.
+                autodelete_unpacked = True
+                if req.link.is_wheel and self.wheel_download_dir:
+                    # when doing 'pip wheel` we download wheels to a
+                    # dedicated dir.
+                    download_dir = self.wheel_download_dir
+                if req.link.is_wheel:
+                    if download_dir:
+                        # When downloading, we only unpack wheels to get
+                        # metadata.
+                        autodelete_unpacked = True
+                    else:
+                        # When installing a wheel, we use the unpacked
+                        # wheel.
+                        autodelete_unpacked = False
+                unpack_url(
+                    req.link, req.source_dir,
+                    download_dir, autodelete_unpacked,
+                    session=session, hashes=hashes,
+                    progress_bar=self.progress_bar
+                )
+            except requests.HTTPError as exc:
+                logger.critical(
+                    'Could not install requirement %s because of error %s',
+                    req,
+                    exc,
+                )
+                raise InstallationError(
+                    'Could not install requirement %s because of HTTP '
+                    'error %s for URL %s' %
+                    (req, exc, req.link)
+                )
+            abstract_dist = make_abstract_dist(req)
+            abstract_dist.prep_for_dist(finder, self.build_isolation)
+            if self._download_should_save:
+                # Make a .zip of the source_dir we already created.
+                if req.link.scheme in vcs.all_schemes:
+                    req.archive(self.download_dir)
+        return abstract_dist
+
+    def prepare_editable_requirement(self, req, require_hashes, use_user_site,
+                                     finder):
+        """Prepare an editable requirement
+        """
+        assert req.editable, "cannot prepare a non-editable req as editable"
+
+        logger.info('Obtaining %s', req)
+
+        with indent_log():
+            if require_hashes:
+                raise InstallationError(
+                    'The editable requirement %s cannot be installed when '
+                    'requiring hashes, because there is no single file to '
+                    'hash.' % req
+                )
+            req.ensure_has_source_dir(self.src_dir)
+            req.update_editable(not self._download_should_save)
+
+            abstract_dist = make_abstract_dist(req)
+            abstract_dist.prep_for_dist(finder, self.build_isolation)
+
+            if self._download_should_save:
+                req.archive(self.download_dir)
+            req.check_if_exists(use_user_site)
+
+        return abstract_dist
+
+    def prepare_installed_requirement(self, req, require_hashes, skip_reason):
+        """Prepare an already-installed requirement
+        """
+        assert req.satisfied_by, "req should have been satisfied but isn't"
+        assert skip_reason is not None, (
+            "did not get skip reason skipped but req.satisfied_by "
+            "is set to %r" % (req.satisfied_by,)
+        )
+        logger.info(
+            'Requirement %s: %s (%s)',
+            skip_reason, req, req.satisfied_by.version
+        )
+        with indent_log():
+            if require_hashes:
+                logger.debug(
+                    'Since it is already installed, we are trusting this '
+                    'package without checking its hash. To ensure a '
+                    'completely repeatable environment, install into an '
+                    'empty virtualenv.'
+                )
+            abstract_dist = Installed(req)
+
+        return abstract_dist

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/pep425tags.py

@@ -0,0 +1,317 @@
+"""Generate and work with PEP 425 Compatibility Tags."""
+from __future__ import absolute_import
+
+import distutils.util
+import logging
+import platform
+import re
+import sys
+import sysconfig
+import warnings
+from collections import OrderedDict
+
+import pip._internal.utils.glibc
+
+logger = logging.getLogger(__name__)
+
+_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)')
+
+
+def get_config_var(var):
+    try:
+        return sysconfig.get_config_var(var)
+    except IOError as e:  # Issue #1074
+        warnings.warn("{}".format(e), RuntimeWarning)
+        return None
+
+
+def get_abbr_impl():
+    """Return abbreviated implementation name."""
+    if hasattr(sys, 'pypy_version_info'):
+        pyimpl = 'pp'
+    elif sys.platform.startswith('java'):
+        pyimpl = 'jy'
+    elif sys.platform == 'cli':
+        pyimpl = 'ip'
+    else:
+        pyimpl = 'cp'
+    return pyimpl
+
+
+def get_impl_ver():
+    """Return implementation version."""
+    impl_ver = get_config_var("py_version_nodot")
+    if not impl_ver or get_abbr_impl() == 'pp':
+        impl_ver = ''.join(map(str, get_impl_version_info()))
+    return impl_ver
+
+
+def get_impl_version_info():
+    """Return sys.version_info-like tuple for use in decrementing the minor
+    version."""
+    if get_abbr_impl() == 'pp':
+        # as per https://github.com/pypa/pip/issues/2882
+        return (sys.version_info[0], sys.pypy_version_info.major,
+                sys.pypy_version_info.minor)
+    else:
+        return sys.version_info[0], sys.version_info[1]
+
+
+def get_impl_tag():
+    """
+    Returns the Tag for this specific implementation.
+    """
+    return "{}{}".format(get_abbr_impl(), get_impl_ver())
+
+
+def get_flag(var, fallback, expected=True, warn=True):
+    """Use a fallback method for determining SOABI flags if the needed config
+    var is unset or unavailable."""
+    val = get_config_var(var)
+    if val is None:
+        if warn:
+            logger.debug("Config variable '%s' is unset, Python ABI tag may "
+                         "be incorrect", var)
+        return fallback()
+    return val == expected
+
+
+def get_abi_tag():
+    """Return the ABI tag based on SOABI (if available) or emulate SOABI
+    (CPython 2, PyPy)."""
+    soabi = get_config_var('SOABI')
+    impl = get_abbr_impl()
+    if not soabi and impl in {'cp', 'pp'} and hasattr(sys, 'maxunicode'):
+        d = ''
+        m = ''
+        u = ''
+        if get_flag('Py_DEBUG',
+                    lambda: hasattr(sys, 'gettotalrefcount'),
+                    warn=(impl == 'cp')):
+            d = 'd'
+        if get_flag('WITH_PYMALLOC',
+                    lambda: impl == 'cp',
+                    warn=(impl == 'cp')):
+            m = 'm'
+        if get_flag('Py_UNICODE_SIZE',
+                    lambda: sys.maxunicode == 0x10ffff,
+                    expected=4,
+                    warn=(impl == 'cp' and
+                          sys.version_info < (3, 3))) \
+                and sys.version_info < (3, 3):
+            u = 'u'
+        abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u)
+    elif soabi and soabi.startswith('cpython-'):
+        abi = 'cp' + soabi.split('-')[1]
+    elif soabi:
+        abi = soabi.replace('.', '_').replace('-', '_')
+    else:
+        abi = None
+    return abi
+
+
+def _is_running_32bit():
+    return sys.maxsize == 2147483647
+
+
+def get_platform():
+    """Return our platform name 'win32', 'linux_x86_64'"""
+    if sys.platform == 'darwin':
+        # distutils.util.get_platform() returns the release based on the value
+        # of MACOSX_DEPLOYMENT_TARGET on which Python was built, which may
+        # be significantly older than the user's current machine.
+        release, _, machine = platform.mac_ver()
+        split_ver = release.split('.')
+
+        if machine == "x86_64" and _is_running_32bit():
+            machine = "i386"
+        elif machine == "ppc64" and _is_running_32bit():
+            machine = "ppc"
+
+        return 'macosx_{}_{}_{}'.format(split_ver[0], split_ver[1], machine)
+
+    # XXX remove distutils dependency
+    result = distutils.util.get_platform().replace('.', '_').replace('-', '_')
+    if result == "linux_x86_64" and _is_running_32bit():
+        # 32 bit Python program (running on a 64 bit Linux): pip should only
+        # install and run 32 bit compiled extensions in that case.
+        result = "linux_i686"
+
+    return result
+
+
+def is_manylinux1_compatible():
+    # Only Linux, and only x86-64 / i686
+    if get_platform() not in {"linux_x86_64", "linux_i686"}:
+        return False
+
+    # Check for presence of _manylinux module
+    try:
+        import _manylinux
+        return bool(_manylinux.manylinux1_compatible)
+    except (ImportError, AttributeError):
+        # Fall through to heuristic check below
+        pass
+
+    # Check glibc version. CentOS 5 uses glibc 2.5.
+    return pip._internal.utils.glibc.have_compatible_glibc(2, 5)
+
+
+def get_darwin_arches(major, minor, machine):
+    """Return a list of supported arches (including group arches) for
+    the given major, minor and machine architecture of an macOS machine.
+    """
+    arches = []
+
+    def _supports_arch(major, minor, arch):
+        # Looking at the application support for macOS versions in the chart
+        # provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears
+        # our timeline looks roughly like:
+        #
+        # 10.0 - Introduces ppc support.
+        # 10.4 - Introduces ppc64, i386, and x86_64 support, however the ppc64
+        #        and x86_64 support is CLI only, and cannot be used for GUI
+        #        applications.
+        # 10.5 - Extends ppc64 and x86_64 support to cover GUI applications.
+        # 10.6 - Drops support for ppc64
+        # 10.7 - Drops support for ppc
+        #
+        # Given that we do not know if we're installing a CLI or a GUI
+        # application, we must be conservative and assume it might be a GUI
+        # application and behave as if ppc64 and x86_64 support did not occur
+        # until 10.5.
+        #
+        # Note: The above information is taken from the "Application support"
+        #       column in the chart not the "Processor support" since I believe
+        #       that we care about what instruction sets an application can use
+        #       not which processors the OS supports.
+        if arch == 'ppc':
+            return (major, minor) <= (10, 5)
+        if arch == 'ppc64':
+            return (major, minor) == (10, 5)
+        if arch == 'i386':
+            return (major, minor) >= (10, 4)
+        if arch == 'x86_64':
+            return (major, minor) >= (10, 5)
+        if arch in groups:
+            for garch in groups[arch]:
+                if _supports_arch(major, minor, garch):
+                    return True
+        return False
+
+    groups = OrderedDict([
+        ("fat", ("i386", "ppc")),
+        ("intel", ("x86_64", "i386")),
+        ("fat64", ("x86_64", "ppc64")),
+        ("fat32", ("x86_64", "i386", "ppc")),
+    ])
+
+    if _supports_arch(major, minor, machine):
+        arches.append(machine)
+
+    for garch in groups:
+        if machine in groups[garch] and _supports_arch(major, minor, garch):
+            arches.append(garch)
+
+    arches.append('universal')
+
+    return arches
+
+
+def get_supported(versions=None, noarch=False, platform=None,
+                  impl=None, abi=None):
+    """Return a list of supported tags for each version specified in
+    `versions`.
+
+    :param versions: a list of string versions, of the form ["33", "32"],
+        or None. The first version will be assumed to support our ABI.
+    :param platform: specify the exact platform you want valid
+        tags for, or None. If None, use the local system platform.
+    :param impl: specify the exact implementation you want valid
+        tags for, or None. If None, use the local interpreter impl.
+    :param abi: specify the exact abi you want valid
+        tags for, or None. If None, use the local interpreter abi.
+    """
+    supported = []
+
+    # Versions must be given with respect to the preference
+    if versions is None:
+        versions = []
+        version_info = get_impl_version_info()
+        major = version_info[:-1]
+        # Support all previous minor Python versions.
+        for minor in range(version_info[-1], -1, -1):
+            versions.append(''.join(map(str, major + (minor,))))
+
+    impl = impl or get_abbr_impl()
+
+    abis = []
+
+    abi = abi or get_abi_tag()
+    if abi:
+        abis[0:0] = [abi]
+
+    abi3s = set()
+    import imp
+    for suffix in imp.get_suffixes():
+        if suffix[0].startswith('.abi'):
+            abi3s.add(suffix[0].split('.', 2)[1])
+
+    abis.extend(sorted(list(abi3s)))
+
+    abis.append('none')
+
+    if not noarch:
+        arch = platform or get_platform()
+        if arch.startswith('macosx'):
+            # support macosx-10.6-intel on macosx-10.9-x86_64
+            match = _osx_arch_pat.match(arch)
+            if match:
+                name, major, minor, actual_arch = match.groups()
+                tpl = '{}_{}_%i_%s'.format(name, major)
+                arches = []
+                for m in reversed(range(int(minor) + 1)):
+                    for a in get_darwin_arches(int(major), m, actual_arch):
+                        arches.append(tpl % (m, a))
+            else:
+                # arch pattern didn't match (?!)
+                arches = [arch]
+        elif platform is None and is_manylinux1_compatible():
+            arches = [arch.replace('linux', 'manylinux1'), arch]
+        else:
+            arches = [arch]
+
+        # Current version, current API (built specifically for our Python):
+        for abi in abis:
+            for arch in arches:
+                supported.append(('%s%s' % (impl, versions[0]), abi, arch))
+
+        # abi3 modules compatible with older version of Python
+        for version in versions[1:]:
+            # abi3 was introduced in Python 3.2
+            if version in {'31', '30'}:
+                break
+            for abi in abi3s:   # empty set if not Python 3
+                for arch in arches:
+                    supported.append(("%s%s" % (impl, version), abi, arch))
+
+        # Has binaries, does not use the Python API:
+        for arch in arches:
+            supported.append(('py%s' % (versions[0][0]), 'none', arch))
+
+    # No abi / arch, but requires our implementation:
+    supported.append(('%s%s' % (impl, versions[0]), 'none', 'any'))
+    # Tagged specifically as being cross-version compatible
+    # (with just the major version specified)
+    supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any'))
+
+    # No abi / arch, generic Python
+    for i, version in enumerate(versions):
+        supported.append(('py%s' % (version,), 'none', 'any'))
+        if i == 0:
+            supported.append(('py%s' % (version[0]), 'none', 'any'))
+
+    return supported
+
+
+implementation_tag = get_impl_tag()

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/req/__init__.py

@@ -0,0 +1,69 @@
+from __future__ import absolute_import
+
+import logging
+
+from .req_install import InstallRequirement
+from .req_set import RequirementSet
+from .req_file import parse_requirements
+from pip._internal.utils.logging import indent_log
+
+
+__all__ = [
+    "RequirementSet", "InstallRequirement",
+    "parse_requirements", "install_given_reqs",
+]
+
+logger = logging.getLogger(__name__)
+
+
+def install_given_reqs(to_install, install_options, global_options=(),
+                       *args, **kwargs):
+    """
+    Install everything in the given list.
+
+    (to be called after having downloaded and unpacked the packages)
+    """
+
+    if to_install:
+        logger.info(
+            'Installing collected packages: %s',
+            ', '.join([req.name for req in to_install]),
+        )
+
+    with indent_log():
+        for requirement in to_install:
+            if requirement.conflicts_with:
+                logger.info(
+                    'Found existing installation: %s',
+                    requirement.conflicts_with,
+                )
+                with indent_log():
+                    uninstalled_pathset = requirement.uninstall(
+                        auto_confirm=True
+                    )
+            try:
+                requirement.install(
+                    install_options,
+                    global_options,
+                    *args,
+                    **kwargs
+                )
+            except:
+                should_rollback = (
+                    requirement.conflicts_with and
+                    not requirement.install_succeeded
+                )
+                # if install did not succeed, rollback previous uninstall
+                if should_rollback:
+                    uninstalled_pathset.rollback()
+                raise
+            else:
+                should_commit = (
+                    requirement.conflicts_with and
+                    requirement.install_succeeded
+                )
+                if should_commit:
+                    uninstalled_pathset.commit()
+            requirement.remove_temporary_source()
+
+    return to_install

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/req/req_file.py

@@ -0,0 +1,338 @@
+"""
+Requirements file parsing
+"""
+
+from __future__ import absolute_import
+
+import optparse
+import os
+import re
+import shlex
+import sys
+
+from pip._vendor.six.moves import filterfalse
+from pip._vendor.six.moves.urllib import parse as urllib_parse
+
+from pip._internal import cmdoptions
+from pip._internal.download import get_file_content
+from pip._internal.exceptions import RequirementsFileParseError
+from pip._internal.req.req_install import InstallRequirement
+
+__all__ = ['parse_requirements']
+
+SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
+COMMENT_RE = re.compile(r'(^|\s)+#.*$')
+
+# Matches environment variable-style values in '${MY_VARIABLE_1}' with the
+# variable name consisting of only uppercase letters, digits or the '_'
+# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
+# 2013 Edition.
+ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})')
+
+SUPPORTED_OPTIONS = [
+    cmdoptions.constraints,
+    cmdoptions.editable,
+    cmdoptions.requirements,
+    cmdoptions.no_index,
+    cmdoptions.index_url,
+    cmdoptions.find_links,
+    cmdoptions.extra_index_url,
+    cmdoptions.always_unzip,
+    cmdoptions.no_binary,
+    cmdoptions.only_binary,
+    cmdoptions.pre,
+    cmdoptions.process_dependency_links,
+    cmdoptions.trusted_host,
+    cmdoptions.require_hashes,
+]
+
+# options to be passed to requirements
+SUPPORTED_OPTIONS_REQ = [
+    cmdoptions.install_options,
+    cmdoptions.global_options,
+    cmdoptions.hash,
+]
+
+# the 'dest' string values
+SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ]
+
+
+def parse_requirements(filename, finder=None, comes_from=None, options=None,
+                       session=None, constraint=False, wheel_cache=None):
+    """Parse a requirements file and yield InstallRequirement instances.
+
+    :param filename:    Path or url of requirements file.
+    :param finder:      Instance of pip.index.PackageFinder.
+    :param comes_from:  Origin description of requirements.
+    :param options:     cli options.
+    :param session:     Instance of pip.download.PipSession.
+    :param constraint:  If true, parsing a constraint file rather than
+        requirements file.
+    :param wheel_cache: Instance of pip.wheel.WheelCache
+    """
+    if session is None:
+        raise TypeError(
+            "parse_requirements() missing 1 required keyword argument: "
+            "'session'"
+        )
+
+    _, content = get_file_content(
+        filename, comes_from=comes_from, session=session
+    )
+
+    lines_enum = preprocess(content, options)
+
+    for line_number, line in lines_enum:
+        req_iter = process_line(line, filename, line_number, finder,
+                                comes_from, options, session, wheel_cache,
+                                constraint=constraint)
+        for req in req_iter:
+            yield req
+
+
+def preprocess(content, options):
+    """Split, filter, and join lines, and return a line iterator
+
+    :param content: the content of the requirements file
+    :param options: cli options
+    """
+    lines_enum = enumerate(content.splitlines(), start=1)
+    lines_enum = join_lines(lines_enum)
+    lines_enum = ignore_comments(lines_enum)
+    lines_enum = skip_regex(lines_enum, options)
+    lines_enum = expand_env_variables(lines_enum)
+    return lines_enum
+
+
+def process_line(line, filename, line_number, finder=None, comes_from=None,
+                 options=None, session=None, wheel_cache=None,
+                 constraint=False):
+    """Process a single requirements line; This can result in creating/yielding
+    requirements, or updating the finder.
+
+    For lines that contain requirements, the only options that have an effect
+    are from SUPPORTED_OPTIONS_REQ, and they are scoped to the
+    requirement. Other options from SUPPORTED_OPTIONS may be present, but are
+    ignored.
+
+    For lines that do not contain requirements, the only options that have an
+    effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may
+    be present, but are ignored. These lines may contain multiple options
+    (although our docs imply only one is supported), and all our parsed and
+    affect the finder.
+
+    :param constraint: If True, parsing a constraints file.
+    :param options: OptionParser options that we may update
+    """
+    parser = build_parser(line)
+    defaults = parser.get_default_values()
+    defaults.index_url = None
+    if finder:
+        # `finder.format_control` will be updated during parsing
+        defaults.format_control = finder.format_control
+    args_str, options_str = break_args_options(line)
+    if sys.version_info < (2, 7, 3):
+        # Prior to 2.7.3, shlex cannot deal with unicode entries
+        options_str = options_str.encode('utf8')
+    opts, _ = parser.parse_args(shlex.split(options_str), defaults)
+
+    # preserve for the nested code path
+    line_comes_from = '%s %s (line %s)' % (
+        '-c' if constraint else '-r', filename, line_number,
+    )
+
+    # yield a line requirement
+    if args_str:
+        isolated = options.isolated_mode if options else False
+        if options:
+            cmdoptions.check_install_build_global(options, opts)
+        # get the options that apply to requirements
+        req_options = {}
+        for dest in SUPPORTED_OPTIONS_REQ_DEST:
+            if dest in opts.__dict__ and opts.__dict__[dest]:
+                req_options[dest] = opts.__dict__[dest]
+        yield InstallRequirement.from_line(
+            args_str, line_comes_from, constraint=constraint,
+            isolated=isolated, options=req_options, wheel_cache=wheel_cache
+        )
+
+    # yield an editable requirement
+    elif opts.editables:
+        isolated = options.isolated_mode if options else False
+        yield InstallRequirement.from_editable(
+            opts.editables[0], comes_from=line_comes_from,
+            constraint=constraint, isolated=isolated, wheel_cache=wheel_cache
+        )
+
+    # parse a nested requirements file
+    elif opts.requirements or opts.constraints:
+        if opts.requirements:
+            req_path = opts.requirements[0]
+            nested_constraint = False
+        else:
+            req_path = opts.constraints[0]
+            nested_constraint = True
+        # original file is over http
+        if SCHEME_RE.search(filename):
+            # do a url join so relative paths work
+            req_path = urllib_parse.urljoin(filename, req_path)
+        # original file and nested file are paths
+        elif not SCHEME_RE.search(req_path):
+            # do a join so relative paths work
+            req_path = os.path.join(os.path.dirname(filename), req_path)
+        # TODO: Why not use `comes_from='-r {} (line {})'` here as well?
+        parser = parse_requirements(
+            req_path, finder, comes_from, options, session,
+            constraint=nested_constraint, wheel_cache=wheel_cache
+        )
+        for req in parser:
+            yield req
+
+    # percolate hash-checking option upward
+    elif opts.require_hashes:
+        options.require_hashes = opts.require_hashes
+
+    # set finder options
+    elif finder:
+        if opts.index_url:
+            finder.index_urls = [opts.index_url]
+        if opts.no_index is True:
+            finder.index_urls = []
+        if opts.extra_index_urls:
+            finder.index_urls.extend(opts.extra_index_urls)
+        if opts.find_links:
+            # FIXME: it would be nice to keep track of the source
+            # of the find_links: support a find-links local path
+            # relative to a requirements file.
+            value = opts.find_links[0]
+            req_dir = os.path.dirname(os.path.abspath(filename))
+            relative_to_reqs_file = os.path.join(req_dir, value)
+            if os.path.exists(relative_to_reqs_file):
+                value = relative_to_reqs_file
+            finder.find_links.append(value)
+        if opts.pre:
+            finder.allow_all_prereleases = True
+        if opts.process_dependency_links:
+            finder.process_dependency_links = True
+        if opts.trusted_hosts:
+            finder.secure_origins.extend(
+                ("*", host, "*") for host in opts.trusted_hosts)
+
+
+def break_args_options(line):
+    """Break up the line into an args and options string.  We only want to shlex
+    (and then optparse) the options, not the args.  args can contain markers
+    which are corrupted by shlex.
+    """
+    tokens = line.split(' ')
+    args = []
+    options = tokens[:]
+    for token in tokens:
+        if token.startswith('-') or token.startswith('--'):
+            break
+        else:
+            args.append(token)
+            options.pop(0)
+    return ' '.join(args), ' '.join(options)
+
+
+def build_parser(line):
+    """
+    Return a parser for parsing requirement lines
+    """
+    parser = optparse.OptionParser(add_help_option=False)
+
+    option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ
+    for option_factory in option_factories:
+        option = option_factory()
+        parser.add_option(option)
+
+    # By default optparse sys.exits on parsing errors. We want to wrap
+    # that in our own exception.
+    def parser_exit(self, msg):
+        # add offending line
+        msg = 'Invalid requirement: %s\n%s' % (line, msg)
+        raise RequirementsFileParseError(msg)
+    parser.exit = parser_exit
+
+    return parser
+
+
+def join_lines(lines_enum):
+    """Joins a line ending in '\' with the previous line (except when following
+    comments).  The joined line takes on the index of the first line.
+    """
+    primary_line_number = None
+    new_line = []
+    for line_number, line in lines_enum:
+        if not line.endswith('\\') or COMMENT_RE.match(line):
+            if COMMENT_RE.match(line):
+                # this ensures comments are always matched later
+                line = ' ' + line
+            if new_line:
+                new_line.append(line)
+                yield primary_line_number, ''.join(new_line)
+                new_line = []
+            else:
+                yield line_number, line
+        else:
+            if not new_line:
+                primary_line_number = line_number
+            new_line.append(line.strip('\\'))
+
+    # last line contains \
+    if new_line:
+        yield primary_line_number, ''.join(new_line)
+
+    # TODO: handle space after '\'.
+
+
+def ignore_comments(lines_enum):
+    """
+    Strips comments and filter empty lines.
+    """
+    for line_number, line in lines_enum:
+        line = COMMENT_RE.sub('', line)
+        line = line.strip()
+        if line:
+            yield line_number, line
+
+
+def skip_regex(lines_enum, options):
+    """
+    Skip lines that match '--skip-requirements-regex' pattern
+
+    Note: the regex pattern is only built once
+    """
+    skip_regex = options.skip_requirements_regex if options else None
+    if skip_regex:
+        pattern = re.compile(skip_regex)
+        lines_enum = filterfalse(lambda e: pattern.search(e[1]), lines_enum)
+    return lines_enum
+
+
+def expand_env_variables(lines_enum):
+    """Replace all environment variables that can be retrieved via `os.getenv`.
+
+    The only allowed format for environment variables defined in the
+    requirement file is `${MY_VARIABLE_1}` to ensure two things:
+
+    1. Strings that contain a `$` aren't accidentally (partially) expanded.
+    2. Ensure consistency across platforms for requirement files.
+
+    These points are the result of a discusssion on the `github pull
+    request #3514 <https://github.com/pypa/pip/pull/3514>`_.
+
+    Valid characters in variable names follow the `POSIX standard
+    <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
+    to uppercase letter, digits and the `_` (underscore).
+    """
+    for line_number, line in lines_enum:
+        for env_var, var_name in ENV_VAR_RE.findall(line):
+            value = os.getenv(var_name)
+            if not value:
+                continue
+
+            line = line.replace(env_var, value)
+
+        yield line_number, line

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/req/req_set.py

@@ -0,0 +1,164 @@
+from __future__ import absolute_import
+
+import logging
+from collections import OrderedDict
+
+from pip._internal.exceptions import InstallationError
+from pip._internal.utils.logging import indent_log
+from pip._internal.wheel import Wheel
+
+logger = logging.getLogger(__name__)
+
+
+class RequirementSet(object):
+
+    def __init__(self, require_hashes=False):
+        """Create a RequirementSet.
+
+        :param wheel_cache: The pip wheel cache, for passing to
+            InstallRequirement.
+        """
+
+        self.requirements = OrderedDict()
+        self.require_hashes = require_hashes
+
+        # Mapping of alias: real_name
+        self.requirement_aliases = {}
+        self.unnamed_requirements = []
+        self.successfully_downloaded = []
+        self.reqs_to_cleanup = []
+
+    def __str__(self):
+        reqs = [req for req in self.requirements.values()
+                if not req.comes_from]
+        reqs.sort(key=lambda req: req.name.lower())
+        return ' '.join([str(req.req) for req in reqs])
+
+    def __repr__(self):
+        reqs = [req for req in self.requirements.values()]
+        reqs.sort(key=lambda req: req.name.lower())
+        reqs_str = ', '.join([str(req.req) for req in reqs])
+        return ('<%s object; %d requirement(s): %s>'
+                % (self.__class__.__name__, len(reqs), reqs_str))
+
+    def add_requirement(self, install_req, parent_req_name=None,
+                        extras_requested=None):
+        """Add install_req as a requirement to install.
+
+        :param parent_req_name: The name of the requirement that needed this
+            added. The name is used because when multiple unnamed requirements
+            resolve to the same name, we could otherwise end up with dependency
+            links that point outside the Requirements set. parent_req must
+            already be added. Note that None implies that this is a user
+            supplied requirement, vs an inferred one.
+        :param extras_requested: an iterable of extras used to evaluate the
+            environment markers.
+        :return: Additional requirements to scan. That is either [] if
+            the requirement is not applicable, or [install_req] if the
+            requirement is applicable and has just been added.
+        """
+        name = install_req.name
+        if not install_req.match_markers(extras_requested):
+            logger.info("Ignoring %s: markers '%s' don't match your "
+                        "environment", install_req.name,
+                        install_req.markers)
+            return [], None
+
+        # This check has to come after we filter requirements with the
+        # environment markers.
+        if install_req.link and install_req.link.is_wheel:
+            wheel = Wheel(install_req.link.filename)
+            if not wheel.supported():
+                raise InstallationError(
+                    "%s is not a supported wheel on this platform." %
+                    wheel.filename
+                )
+
+        # This next bit is really a sanity check.
+        assert install_req.is_direct == (parent_req_name is None), (
+            "a direct req shouldn't have a parent and also, "
+            "a non direct req should have a parent"
+        )
+
+        if not name:
+            # url or path requirement w/o an egg fragment
+            self.unnamed_requirements.append(install_req)
+            return [install_req], None
+        else:
+            try:
+                existing_req = self.get_requirement(name)
+            except KeyError:
+                existing_req = None
+            if (parent_req_name is None and existing_req and not
+                    existing_req.constraint and
+                    existing_req.extras == install_req.extras and not
+                    existing_req.req.specifier == install_req.req.specifier):
+                raise InstallationError(
+                    'Double requirement given: %s (already in %s, name=%r)'
+                    % (install_req, existing_req, name))
+            if not existing_req:
+                # Add requirement
+                self.requirements[name] = install_req
+                # FIXME: what about other normalizations?  E.g., _ vs. -?
+                if name.lower() != name:
+                    self.requirement_aliases[name.lower()] = name
+                result = [install_req]
+            else:
+                # Assume there's no need to scan, and that we've already
+                # encountered this for scanning.
+                result = []
+                if not install_req.constraint and existing_req.constraint:
+                    if (install_req.link and not (existing_req.link and
+                       install_req.link.path == existing_req.link.path)):
+                        self.reqs_to_cleanup.append(install_req)
+                        raise InstallationError(
+                            "Could not satisfy constraints for '%s': "
+                            "installation from path or url cannot be "
+                            "constrained to a version" % name,
+                        )
+                    # If we're now installing a constraint, mark the existing
+                    # object for real installation.
+                    existing_req.constraint = False
+                    existing_req.extras = tuple(
+                        sorted(set(existing_req.extras).union(
+                               set(install_req.extras))))
+                    logger.debug("Setting %s extras to: %s",
+                                 existing_req, existing_req.extras)
+                    # And now we need to scan this.
+                    result = [existing_req]
+                # Canonicalise to the already-added object for the backref
+                # check below.
+                install_req = existing_req
+
+            # We return install_req here to allow for the caller to add it to
+            # the dependency information for the parent package.
+            return result, install_req
+
+    def has_requirement(self, project_name):
+        name = project_name.lower()
+        if (name in self.requirements and
+           not self.requirements[name].constraint or
+           name in self.requirement_aliases and
+           not self.requirements[self.requirement_aliases[name]].constraint):
+            return True
+        return False
+
+    @property
+    def has_requirements(self):
+        return list(req for req in self.requirements.values() if not
+                    req.constraint) or self.unnamed_requirements
+
+    def get_requirement(self, project_name):
+        for name in project_name, project_name.lower():
+            if name in self.requirements:
+                return self.requirements[name]
+            if name in self.requirement_aliases:
+                return self.requirements[self.requirement_aliases[name]]
+        raise KeyError("No project with the name %r" % project_name)
+
+    def cleanup_files(self):
+        """Clean up files, remove builds."""
+        logger.debug('Cleaning up...')
+        with indent_log():
+            for req in self.reqs_to_cleanup:
+                req.remove_temporary_source()

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/req/req_uninstall.py

@@ -0,0 +1,455 @@
+from __future__ import absolute_import
+
+import csv
+import functools
+import logging
+import os
+import sys
+import sysconfig
+
+from pip._vendor import pkg_resources
+
+from pip._internal.compat import WINDOWS, cache_from_source, uses_pycache
+from pip._internal.exceptions import UninstallationError
+from pip._internal.locations import bin_py, bin_user
+from pip._internal.utils.logging import indent_log
+from pip._internal.utils.misc import (
+    FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local,
+    normalize_path, renames,
+)
+from pip._internal.utils.temp_dir import TempDirectory
+
+logger = logging.getLogger(__name__)
+
+
+def _script_names(dist, script_name, is_gui):
+    """Create the fully qualified name of the files created by
+    {console,gui}_scripts for the given ``dist``.
+    Returns the list of file names
+    """
+    if dist_in_usersite(dist):
+        bin_dir = bin_user
+    else:
+        bin_dir = bin_py
+    exe_name = os.path.join(bin_dir, script_name)
+    paths_to_remove = [exe_name]
+    if WINDOWS:
+        paths_to_remove.append(exe_name + '.exe')
+        paths_to_remove.append(exe_name + '.exe.manifest')
+        if is_gui:
+            paths_to_remove.append(exe_name + '-script.pyw')
+        else:
+            paths_to_remove.append(exe_name + '-script.py')
+    return paths_to_remove
+
+
+def _unique(fn):
+    @functools.wraps(fn)
+    def unique(*args, **kw):
+        seen = set()
+        for item in fn(*args, **kw):
+            if item not in seen:
+                seen.add(item)
+                yield item
+    return unique
+
+
+@_unique
+def uninstallation_paths(dist):
+    """
+    Yield all the uninstallation paths for dist based on RECORD-without-.pyc
+
+    Yield paths to all the files in RECORD. For each .py file in RECORD, add
+    the .pyc in the same directory.
+
+    UninstallPathSet.add() takes care of the __pycache__ .pyc.
+    """
+    r = csv.reader(FakeFile(dist.get_metadata_lines('RECORD')))
+    for row in r:
+        path = os.path.join(dist.location, row[0])
+        yield path
+        if path.endswith('.py'):
+            dn, fn = os.path.split(path)
+            base = fn[:-3]
+            path = os.path.join(dn, base + '.pyc')
+            yield path
+
+
+def compact(paths):
+    """Compact a path set to contain the minimal number of paths
+    necessary to contain all paths in the set. If /a/path/ and
+    /a/path/to/a/file.txt are both in the set, leave only the
+    shorter path."""
+
+    sep = os.path.sep
+    short_paths = set()
+    for path in sorted(paths, key=len):
+        should_add = any(
+            path.startswith(shortpath.rstrip("*")) and
+            path[len(shortpath.rstrip("*").rstrip(sep))] == sep
+            for shortpath in short_paths
+        )
+        if not should_add:
+            short_paths.add(path)
+    return short_paths
+
+
+def compress_for_output_listing(paths):
+    """Returns a tuple of 2 sets of which paths to display to user
+
+    The first set contains paths that would be deleted. Files of a package
+    are not added and the top-level directory of the package has a '*' added
+    at the end - to signify that all it's contents are removed.
+
+    The second set contains files that would have been skipped in the above
+    folders.
+    """
+
+    will_remove = list(paths)
+    will_skip = set()
+
+    # Determine folders and files
+    folders = set()
+    files = set()
+    for path in will_remove:
+        if path.endswith(".pyc"):
+            continue
+        if path.endswith("__init__.py") or ".dist-info" in path:
+            folders.add(os.path.dirname(path))
+        files.add(path)
+
+    folders = compact(folders)
+
+    # This walks the tree using os.walk to not miss extra folders
+    # that might get added.
+    for folder in folders:
+        for dirpath, _, dirfiles in os.walk(folder):
+            for fname in dirfiles:
+                if fname.endswith(".pyc"):
+                    continue
+
+                file_ = os.path.normcase(os.path.join(dirpath, fname))
+                if os.path.isfile(file_) and file_ not in files:
+                    # We are skipping this file. Add it to the set.
+                    will_skip.add(file_)
+
+    will_remove = files | {
+        os.path.join(folder, "*") for folder in folders
+    }
+
+    return will_remove, will_skip
+
+
+class UninstallPathSet(object):
+    """A set of file paths to be removed in the uninstallation of a
+    requirement."""
+    def __init__(self, dist):
+        self.paths = set()
+        self._refuse = set()
+        self.pth = {}
+        self.dist = dist
+        self.save_dir = TempDirectory(kind="uninstall")
+        self._moved_paths = []
+
+    def _permitted(self, path):
+        """
+        Return True if the given path is one we are permitted to
+        remove/modify, False otherwise.
+
+        """
+        return is_local(path)
+
+    def add(self, path):
+        head, tail = os.path.split(path)
+
+        # we normalize the head to resolve parent directory symlinks, but not
+        # the tail, since we only want to uninstall symlinks, not their targets
+        path = os.path.join(normalize_path(head), os.path.normcase(tail))
+
+        if not os.path.exists(path):
+            return
+        if self._permitted(path):
+            self.paths.add(path)
+        else:
+            self._refuse.add(path)
+
+        # __pycache__ files can show up after 'installed-files.txt' is created,
+        # due to imports
+        if os.path.splitext(path)[1] == '.py' and uses_pycache:
+            self.add(cache_from_source(path))
+
+    def add_pth(self, pth_file, entry):
+        pth_file = normalize_path(pth_file)
+        if self._permitted(pth_file):
+            if pth_file not in self.pth:
+                self.pth[pth_file] = UninstallPthEntries(pth_file)
+            self.pth[pth_file].add(entry)
+        else:
+            self._refuse.add(pth_file)
+
+    def _stash(self, path):
+        return os.path.join(
+            self.save_dir.path, os.path.splitdrive(path)[1].lstrip(os.path.sep)
+        )
+
+    def remove(self, auto_confirm=False, verbose=False):
+        """Remove paths in ``self.paths`` with confirmation (unless
+        ``auto_confirm`` is True)."""
+
+        if not self.paths:
+            logger.info(
+                "Can't uninstall '%s'. No files were found to uninstall.",
+                self.dist.project_name,
+            )
+            return
+
+        dist_name_version = (
+            self.dist.project_name + "-" + self.dist.version
+        )
+        logger.info('Uninstalling %s:', dist_name_version)
+
+        with indent_log():
+            if auto_confirm or self._allowed_to_proceed(verbose):
+                self.save_dir.create()
+
+                for path in sorted(compact(self.paths)):
+                    new_path = self._stash(path)
+                    logger.debug('Removing file or directory %s', path)
+                    self._moved_paths.append(path)
+                    renames(path, new_path)
+                for pth in self.pth.values():
+                    pth.remove()
+
+                logger.info('Successfully uninstalled %s', dist_name_version)
+
+    def _allowed_to_proceed(self, verbose):
+        """Display which files would be deleted and prompt for confirmation
+        """
+
+        def _display(msg, paths):
+            if not paths:
+                return
+
+            logger.info(msg)
+            with indent_log():
+                for path in sorted(compact(paths)):
+                    logger.info(path)
+
+        if not verbose:
+            will_remove, will_skip = compress_for_output_listing(self.paths)
+        else:
+            # In verbose mode, display all the files that are going to be
+            # deleted.
+            will_remove = list(self.paths)
+            will_skip = set()
+
+        _display('Would remove:', will_remove)
+        _display('Would not remove (might be manually added):', will_skip)
+        _display('Would not remove (outside of prefix):', self._refuse)
+
+        return ask('Proceed (y/n)? ', ('y', 'n')) == 'y'
+
+    def rollback(self):
+        """Rollback the changes previously made by remove()."""
+        if self.save_dir.path is None:
+            logger.error(
+                "Can't roll back %s; was not uninstalled",
+                self.dist.project_name,
+            )
+            return False
+        logger.info('Rolling back uninstall of %s', self.dist.project_name)
+        for path in self._moved_paths:
+            tmp_path = self._stash(path)
+            logger.debug('Replacing %s', path)
+            renames(tmp_path, path)
+        for pth in self.pth.values():
+            pth.rollback()
+
+    def commit(self):
+        """Remove temporary save dir: rollback will no longer be possible."""
+        self.save_dir.cleanup()
+        self._moved_paths = []
+
+    @classmethod
+    def from_dist(cls, dist):
+        dist_path = normalize_path(dist.location)
+        if not dist_is_local(dist):
+            logger.info(
+                "Not uninstalling %s at %s, outside environment %s",
+                dist.key,
+                dist_path,
+                sys.prefix,
+            )
+            return cls(dist)
+
+        if dist_path in {p for p in {sysconfig.get_path("stdlib"),
+                                     sysconfig.get_path("platstdlib")}
+                         if p}:
+            logger.info(
+                "Not uninstalling %s at %s, as it is in the standard library.",
+                dist.key,
+                dist_path,
+            )
+            return cls(dist)
+
+        paths_to_remove = cls(dist)
+        develop_egg_link = egg_link_path(dist)
+        develop_egg_link_egg_info = '{}.egg-info'.format(
+            pkg_resources.to_filename(dist.project_name))
+        egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info)
+        # Special case for distutils installed package
+        distutils_egg_info = getattr(dist._provider, 'path', None)
+
+        # Uninstall cases order do matter as in the case of 2 installs of the
+        # same package, pip needs to uninstall the currently detected version
+        if (egg_info_exists and dist.egg_info.endswith('.egg-info') and
+                not dist.egg_info.endswith(develop_egg_link_egg_info)):
+            # if dist.egg_info.endswith(develop_egg_link_egg_info), we
+            # are in fact in the develop_egg_link case
+            paths_to_remove.add(dist.egg_info)
+            if dist.has_metadata('installed-files.txt'):
+                for installed_file in dist.get_metadata(
+                        'installed-files.txt').splitlines():
+                    path = os.path.normpath(
+                        os.path.join(dist.egg_info, installed_file)
+                    )
+                    paths_to_remove.add(path)
+            # FIXME: need a test for this elif block
+            # occurs with --single-version-externally-managed/--record outside
+            # of pip
+            elif dist.has_metadata('top_level.txt'):
+                if dist.has_metadata('namespace_packages.txt'):
+                    namespaces = dist.get_metadata('namespace_packages.txt')
+                else:
+                    namespaces = []
+                for top_level_pkg in [
+                        p for p
+                        in dist.get_metadata('top_level.txt').splitlines()
+                        if p and p not in namespaces]:
+                    path = os.path.join(dist.location, top_level_pkg)
+                    paths_to_remove.add(path)
+                    paths_to_remove.add(path + '.py')
+                    paths_to_remove.add(path + '.pyc')
+                    paths_to_remove.add(path + '.pyo')
+
+        elif distutils_egg_info:
+            raise UninstallationError(
+                "Cannot uninstall {!r}. It is a distutils installed project "
+                "and thus we cannot accurately determine which files belong "
+                "to it which would lead to only a partial uninstall.".format(
+                    dist.project_name,
+                )
+            )
+
+        elif dist.location.endswith('.egg'):
+            # package installed by easy_install
+            # We cannot match on dist.egg_name because it can slightly vary
+            # i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg
+            paths_to_remove.add(dist.location)
+            easy_install_egg = os.path.split(dist.location)[1]
+            easy_install_pth = os.path.join(os.path.dirname(dist.location),
+                                            'easy-install.pth')
+            paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg)
+
+        elif egg_info_exists and dist.egg_info.endswith('.dist-info'):
+            for path in uninstallation_paths(dist):
+                paths_to_remove.add(path)
+
+        elif develop_egg_link:
+            # develop egg
+            with open(develop_egg_link, 'r') as fh:
+                link_pointer = os.path.normcase(fh.readline().strip())
+            assert (link_pointer == dist.location), (
+                'Egg-link %s does not match installed location of %s '
+                '(at %s)' % (link_pointer, dist.project_name, dist.location)
+            )
+            paths_to_remove.add(develop_egg_link)
+            easy_install_pth = os.path.join(os.path.dirname(develop_egg_link),
+                                            'easy-install.pth')
+            paths_to_remove.add_pth(easy_install_pth, dist.location)
+
+        else:
+            logger.debug(
+                'Not sure how to uninstall: %s - Check: %s',
+                dist, dist.location,
+            )
+
+        # find distutils scripts= scripts
+        if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'):
+            for script in dist.metadata_listdir('scripts'):
+                if dist_in_usersite(dist):
+                    bin_dir = bin_user
+                else:
+                    bin_dir = bin_py
+                paths_to_remove.add(os.path.join(bin_dir, script))
+                if WINDOWS:
+                    paths_to_remove.add(os.path.join(bin_dir, script) + '.bat')
+
+        # find console_scripts
+        _scripts_to_remove = []
+        console_scripts = dist.get_entry_map(group='console_scripts')
+        for name in console_scripts.keys():
+            _scripts_to_remove.extend(_script_names(dist, name, False))
+        # find gui_scripts
+        gui_scripts = dist.get_entry_map(group='gui_scripts')
+        for name in gui_scripts.keys():
+            _scripts_to_remove.extend(_script_names(dist, name, True))
+
+        for s in _scripts_to_remove:
+            paths_to_remove.add(s)
+
+        return paths_to_remove
+
+
+class UninstallPthEntries(object):
+    def __init__(self, pth_file):
+        if not os.path.isfile(pth_file):
+            raise UninstallationError(
+                "Cannot remove entries from nonexistent file %s" % pth_file
+            )
+        self.file = pth_file
+        self.entries = set()
+        self._saved_lines = None
+
+    def add(self, entry):
+        entry = os.path.normcase(entry)
+        # On Windows, os.path.normcase converts the entry to use
+        # backslashes.  This is correct for entries that describe absolute
+        # paths outside of site-packages, but all the others use forward
+        # slashes.
+        if WINDOWS and not os.path.splitdrive(entry)[0]:
+            entry = entry.replace('\\', '/')
+        self.entries.add(entry)
+
+    def remove(self):
+        logger.debug('Removing pth entries from %s:', self.file)
+        with open(self.file, 'rb') as fh:
+            # windows uses '\r\n' with py3k, but uses '\n' with py2.x
+            lines = fh.readlines()
+            self._saved_lines = lines
+        if any(b'\r\n' in line for line in lines):
+            endline = '\r\n'
+        else:
+            endline = '\n'
+        # handle missing trailing newline
+        if lines and not lines[-1].endswith(endline.encode("utf-8")):
+            lines[-1] = lines[-1] + endline.encode("utf-8")
+        for entry in self.entries:
+            try:
+                logger.debug('Removing entry: %s', entry)
+                lines.remove((entry + endline).encode("utf-8"))
+            except ValueError:
+                pass
+        with open(self.file, 'wb') as fh:
+            fh.writelines(lines)
+
+    def rollback(self):
+        if self._saved_lines is None:
+            logger.error(
+                'Cannot roll back changes to %s, none were made', self.file
+            )
+            return False
+        logger.debug('Rolling %s back to previous state', self.file)
+        with open(self.file, 'wb') as fh:
+            fh.writelines(self._saved_lines)
+        return True

venv/lib/python3.6/site-packages/pip-10.0.1-py3.6.egg/pip/_internal/resolve.py

@@ -0,0 +1,354 @@
+"""Dependency Resolution
+
+The dependency resolution in pip is performed as follows:
+
+for top-level requirements:
+    a. only one spec allowed per project, regardless of conflicts or not.
+       otherwise a "double requirement" exception is raised
+    b. they override sub-dependency requirements.
+for sub-dependencies
+    a. "first found, wins" (where the order is breadth first)
+"""
+
+import logging
+from collections import defaultdict
+from itertools import chain
+
+from pip._internal.exceptions import (
+    BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors,
+    UnsupportedPythonVersion,
+)
+
+from pip._internal.req.req_install import InstallRequirement
+from pip._internal.utils.logging import indent_log
+from pip._internal.utils.misc import dist_in_usersite, ensure_dir
+from pip._internal.utils.packaging import check_dist_requires_python
+
+logger = logging.getLogger(__name__)
+
+
+class Resolver(object):
+    """Resolves which packages need to be installed/uninstalled to perform \
+    the requested operation without breaking the requirements of any package.
+    """
+
+    _allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"}
+
+    def __init__(self, preparer, session, finder, wheel_cache, use_user_site,
+                 ignore_dependencies, ignore_installed, ignore_requires_python,
+                 force_reinstall, isolated, upgrade_strategy):
+        super(Resolver, self).__init__()
+        assert upgrade_strategy in self._allowed_strategies
+
+        self.preparer = preparer
+        self.finder = finder
+        self.session = session
+
+        # NOTE: This would eventually be replaced with a cache that can give
+        #       information about both sdist and wheels transparently.
+        self.wheel_cache = wheel_cache
+
+        self.require_hashes = None  # This is set in resolve
+
+        self.upgrade_strategy = upgrade_strategy
+        self.force_reinstall = force_reinstall
+        self.isolated = isolated
+        self.ignore_dependencies = ignore_dependencies
+        self.ignore_installed = ignore_installed
+        self.ignore_requires_python = ignore_requires_python
+        self.use_user_site = use_user_site
+
+        self._discovered_dependencies = defaultdict(list)
+
+    def resolve(self, requirement_set):
+        """Resolve what operations need to be done
+
+        As a side-effect of this method, the packages (and their dependencies)
+        are downloaded, unpacked and prepared for installation. This
+        preparation is done by ``pip.operations.prepare``.
+
+        Once PyPI has static dependency metadata available, it would be
+        possible to move the preparation to become a step separated from
+        dependency resolution.
+        """
+        # make the wheelhouse
+        if self.preparer.wheel_download_dir:
+            ensure_dir(self.preparer.wheel_download_dir)
+
+        # If any top-level requirement has a hash specified, enter
+        # hash-checking mode, which requires hashes from all.
+        root_reqs = (
+            requirement_set.unnamed_requirements +
+            list(requirement_set.requirements.values())
+        )
+        self.require_hashes = (
+            requirement_set.require_hashes or
+            any(req.has_hash_options for req in root_reqs)
+        )
+
+        # Display where finder is looking for packages
+        locations = self.finder.get_formatted_locations()
+        if locations:
+            logger.info(locations)
+
+        # Actually prepare the files, and collect any exceptions. Most hash
+        # exceptions cannot be checked ahead of time, because
+        # req.populate_link() needs to be called before we can make decisions
+        # based on link type.
+        discovered_reqs = []
+        hash_errors = HashErrors()
+        for req in chain(root_reqs, discovered_reqs):
+            try:
+                discovered_reqs.extend(
+                    self._resolve_one(requirement_set, req)
+                )
+            except HashError as exc:
+                exc.req = req
+                hash_errors.append(exc)
+
+        if hash_errors:
+            raise hash_errors
+
+    def _is_upgrade_allowed(self, req):
+        if self.upgrade_strategy == "to-satisfy-only":
+            return False
+        elif self.upgrade_strategy == "eager":
+            return True
+        else:
+            assert self.upgrade_strategy == "only-if-needed"
+            return req.is_direct
+
+    def _set_req_to_reinstall(self, req):
+        """
+        Set a requirement to be installed.
+        """
+        # Don't uninstall the conflict if doing a user install and the
+        # conflict is not a user install.
+        if not self.use_user_site or dist_in_usersite(req.satisfied_by):
+            req.conflicts_with = req.satisfied_by
+        req.satisfied_by = None
+
+    # XXX: Stop passing requirement_set for options
+    def _check_skip_installed(self, req_to_install):
+        """Check if req_to_install should be skipped.
+
+        This will check if the req is installed, and whether we should upgrade
+        or reinstall it, taking into account all the relevant user options.
+
+        After calling this req_to_install will only have satisfied_by set to
+        None if the req_to_install is to be upgraded/reinstalled etc. Any
+        other value will be a dist recording the current thing installed that
+        satisfies the requirement.
+
+        Note that for vcs urls and the like we can't assess skipping in this
+        routine - we simply identify that we need to pull the thing down,
+        then later on it is pulled down and introspected to assess upgrade/
+        reinstalls etc.
+
+        :return: A text reason for why it was skipped, or None.
+        """
+        if self.ignore_installed:
+            return None
+
+        req_to_install.check_if_exists(self.use_user_site)
+        if not req_to_install.satisfied_by:
+            return None
+
+        if self.force_reinstall:
+            self._set_req_to_reinstall(req_to_install)
+            return None
+
+        if not self._is_upgrade_allowed(req_to_install):
+            if self.upgrade_strategy == "only-if-needed":
+                return 'not upgraded as not directly required'
+            return 'already satisfied'
+
+        # Check for the possibility of an upgrade.  For link-based
+        # requirements we have to pull the tree down and inspect to assess
+        # the version #, so it's handled way down.
+        if not req_to_install.link:
+            try:
+                self.finder.find_requirement(req_to_install, upgrade=True)
+            except BestVersionAlreadyInstalled:
+                # Then the best version is installed.
+                return 'already up-to-date'
+            except DistributionNotFound:
+                # No distribution found, so we squash the error.  It will
+                # be raised later when we re-try later to do the install.
+                # Why don't we just raise here?
+                pass
+
+        self._set_req_to_reinstall(req_to_install)
+        return None
+
+    def _get_abstract_dist_for(self, req):
+        """Takes a InstallRequirement and returns a single AbstractDist \
+        representing a prepared variant of the same.
+        """
+        assert self.require_hashes is not None, (
+            "require_hashes should have been set in Resolver.resolve()"
+        )
+
+        if req.editable:
+            return self.preparer.prepare_editable_requirement(
+                req, self.require_hashes, self.use_user_site, self.finder,
+            )
+
+        # satisfied_by is only evaluated by calling _check_skip_installed,
+        # so it must be None here.
+        assert req.satisfied_by is None
+        skip_reason = self._check_skip_installed(req)
+
+        if req.satisfied_by:
+            return self.preparer.prepare_installed_requirement(
+                req, self.require_hashes, skip_reason
+            )
+
+        upgrade_allowed = self._is_upgrade_allowed(req)
+        abstract_dist = self.preparer.prepare_linked_requirement(
+            req, self.session, self.finder, upgrade_allowed,
+            self.require_hashes
+        )
+
+        # NOTE
+        # The following portion is for determining if a certain package is
+        # going to be re-installed/upgraded or not and reporting to the user.
+        # This should probably get cleaned up in a future refactor.
+
+        # req.req is only avail after unpack for URL
+        # pkgs repeat check_if_exists to uninstall-on-upgrade
+        # (#14)
+        if not self.ignore_installed:
+            req.check_if_exists(self.use_user_site)
+
+        if req.satisfied_by:
+            should_modify = (
+                self.upgrade_strategy != "to-satisfy-only" or
+                self.force_reinstall or
+                self.ignore_installed or
+                req.link.scheme == 'file'
+            )
+            if should_modify:
+                self._set_req_to_reinstall(req)
+            else:
+                logger.info(
+                    'Requirement already satisfied (use --upgrade to upgrade):'
+                    ' %s', req,
+                )
+
+        return abstract_dist
+
+    def _resolve_one(self, requirement_set, req_to_install):
+        """Prepare a single requirements file.
+
+        :return: A list of additional InstallRequirements to also install.
+        """
+        # Tell user what we are doing for this requirement:
+        # obtain (editable), skipping, processing (local url), collecting
+        # (remote url or package name)
+        if req_to_install.constraint or req_to_install.prepared:
+            return []
+
+        req_to_install.prepared = True
+
+        # register tmp src for cleanup in case something goes wrong
+        requirement_set.reqs_to_cleanup.append(req_to_install)
+
+        abstract_dist = self._get_abstract_dist_for(req_to_install)
+
+        # Parse and return dependencies
+        dist = abstract_dist.dist(self.finder)
+        try:
+            check_dist_requires_python(dist)
+        except UnsupportedPythonVersion as err:
+            if self.ignore_requires_python:
+                logger.warning(err.args[0])
+            else:
+                raise
+
+        more_reqs = []
+
+        def add_req(subreq, extras_requested):
+            sub_install_req = InstallRequirement.from_req(
+                str(subreq),
+                req_to_install,
+                isolated=self.isolated,
+                wheel_cache=self.wheel_cache,
+            )
+            parent_req_name = req_to_install.name
+            to_scan_again, add_to_parent = requirement_set.add_requirement(
+                sub_install_req,
+                parent_req_name=parent_req_name,
+                extras_requested=extras_requested,
+            )
+            if parent_req_name and add_to_parent:
+                self._discovered_dependencies[parent_req_name].append(
+                    add_to_parent
+                )
+            more_reqs.extend(to_scan_again)
+
+        with indent_log():
+            # We add req_to_install before its dependencies, so that we
+            # can refer to it when adding dependencies.
+            if not requirement_set.has_requirement(req_to_install.name):
+                # 'unnamed' requirements will get added here
+                req_to_install.is_direct = True
+                requirement_set.add_requirement(
+                    req_to_install, parent_req_name=None,
+                )
+
+            if not self.ignore_dependencies:
+                if req_to_install.extras:
+                    logger.debug(
+                        "Installing extra requirements: %r",
+                        ','.join(req_to_install.extras),
+                    )
+                missing_requested = sorted(
+                    set(req_to_install.extras) - set(dist.extras)
+                )
+                for missing in missing_requested:
+                    logger.warning(
+                        '%s does not provide the extra \'%s\'',
+                        dist, missing
+                    )
+
+                available_requested = sorted(
+                    set(dist.extras) & set(req_to_install.extras)
+                )
+                for subreq in dist.requires(available_requested):
+                    add_req(subreq, extras_requested=available_requested)
+
+            if not req_to_install.editable and not req_to_install.satisfied_by:
+                # XXX: --no-install leads this to report 'Successfully
+                # downloaded' for only non-editable reqs, even though we took
+                # action on them.
+                requirement_set.successfully_downloaded.append(req_to_install)
+
+        return more_reqs
+
+    def get_installation_order(self, req_set):
+        """Create the installation order.
+
+        The installation order is topological - requirements are installed
+        before the requiring thing. We break cycles at an arbitrary point,
+        and make no other guarantees.
+        """
+        # The current implementation, which we may change at any point
+        # installs the user specified things in the order given, except when
+        # dependencies must come earlier to achieve topological order.
+        order = []
+        ordered_reqs = set()
+
+        def schedule(req):
+            if req.satisfied_by or req in ordered_reqs:
+                return
+            if req.constraint:
+                return
+            ordered_reqs.add(req)
+            for dep in self._discovered_dependencies[req.name]:
+                schedule(dep)
+            order.append(req)
+
+        for install_req in req_set.requirements.values():
+            schedule(install_req)
+        return order