From f197479dc8b496e236b1db5321a79be97a37bfdf Mon Sep 17 00:00:00 2001 From: tbeswick Date: Fri, 11 Oct 2019 14:30:30 +1300 Subject: [PATCH] Fixed LF getting converted to CRLF in last commit. --- src/pip/_internal/vcs/git.py | 744 ++++++------- src/pip/_internal/vcs/mercurial.py | 308 +++--- src/pip/_internal/vcs/versioncontrol.py | 1344 +++++++++++------------ 3 files changed, 1198 insertions(+), 1198 deletions(-) diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 6b804114002..92b84571406 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -1,372 +1,372 @@ -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - -from __future__ import absolute_import - -import logging -import os.path -import re - -from pip._vendor.packaging.version import parse as parse_version -from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._vendor.six.moves.urllib import request as urllib_request - -from pip._internal.exceptions import BadCommand -from pip._internal.utils.misc import display_path -from pip._internal.utils.subprocess import make_command -from pip._internal.utils.temp_dir import TempDirectory -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.vcs.versioncontrol import ( - RemoteNotFoundError, - VersionControl, - find_path_to_setup_from_repo_root, - vcs, -) - -if MYPY_CHECK_RUNNING: - from typing import Optional, Tuple - from pip._internal.utils.misc import HiddenText - from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions - - -urlsplit = urllib_parse.urlsplit -urlunsplit = urllib_parse.urlunsplit - - -logger = logging.getLogger(__name__) - - -HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') - - -def looks_like_hash(sha): - return bool(HASH_REGEX.match(sha)) - - -class Git(VersionControl): - name = 'git' - dirname = '.git' - repo_name = 'clone' - schemes = ( - 'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file', - ) - # Prevent the user's environment variables from interfering with pip: - # https://github.com/pypa/pip/issues/1130 - unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') - default_arg_rev = 'HEAD' - - @staticmethod - def get_base_rev_args(rev): - return [rev] - - def get_git_version(self): - VERSION_PFX = 'git version ' - version = self.run_command(['version'], show_stdout=False) - if version.startswith(VERSION_PFX): - version = version[len(VERSION_PFX):].split()[0] - else: - version = '' - # get first 3 positions of the git version because - # on windows it is x.y.z.windows.t, and this parses as - # LegacyVersion which always smaller than a Version. - version = '.'.join(version.split('.')[:3]) - return parse_version(version) - - @classmethod - def get_current_branch(cls, location): - """ - Return the current branch, or None if HEAD isn't at a branch - (e.g. detached HEAD). - """ - # git-symbolic-ref exits with empty stdout if "HEAD" is a detached - # HEAD rather than a symbolic ref. In addition, the -q causes the - # command to exit with status code 1 instead of 128 in this case - # and to suppress the message to stderr. - args = ['symbolic-ref', '-q', 'HEAD'] - output = cls.run_command( - args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, - ) - ref = output.strip() - - if ref.startswith('refs/heads/'): - return ref[len('refs/heads/'):] - - return None - - def export(self, location, url): - # type: (str, HiddenText) -> None - """Export the Git repository at the url to the destination location""" - if not location.endswith('/'): - location = location + '/' - - with TempDirectory(kind="export") as temp_dir: - self.unpack(temp_dir.path, url=url) - self.run_command( - ['checkout-index', '-a', '-f', '--prefix', location], - show_stdout=False, cwd=temp_dir.path - ) - - @classmethod - def get_revision_sha(cls, dest, rev): - """ - Return (sha_or_none, is_branch), where sha_or_none is a commit hash - if the revision names a remote branch or tag, otherwise None. - - Args: - dest: the repository directory. - rev: the revision name. - """ - # Pass rev to pre-filter the list. - output = cls.run_command(['show-ref', rev], cwd=dest, - show_stdout=False, on_returncode='ignore') - refs = {} - for line in output.strip().splitlines(): - try: - sha, ref = line.split() - except ValueError: - # Include the offending line to simplify troubleshooting if - # this error ever occurs. - raise ValueError('unexpected show-ref line: {!r}'.format(line)) - - refs[ref] = sha - - branch_ref = 'refs/remotes/origin/{}'.format(rev) - tag_ref = 'refs/tags/{}'.format(rev) - - sha = refs.get(branch_ref) - if sha is not None: - return (sha, True) - - sha = refs.get(tag_ref) - - return (sha, False) - - @classmethod - def resolve_revision(cls, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> RevOptions - """ - Resolve a revision to a new RevOptions object with the SHA1 of the - branch, tag, or ref if found. - - Args: - rev_options: a RevOptions object. - """ - rev = rev_options.arg_rev - # The arg_rev property's implementation for Git ensures that the - # rev return value is always non-None. - assert rev is not None - - sha, is_branch = cls.get_revision_sha(dest, rev) - - if sha is not None: - rev_options = rev_options.make_new(sha) - rev_options.branch_name = rev if is_branch else None - - return rev_options - - # Do not show a warning for the common case of something that has - # the form of a Git commit hash. - if not looks_like_hash(rev): - logger.warning( - "Did not find branch or tag '%s', assuming revision or ref.", - rev, - ) - - if not rev.startswith('refs/'): - return rev_options - - # If it looks like a ref, we have to fetch it explicitly. - cls.run_command( - make_command('fetch', '-q', url, rev_options.to_args()), - cwd=dest, - ) - # Change the revision to the SHA of the ref we fetched - sha = cls.get_revision(dest, rev='FETCH_HEAD') - rev_options = rev_options.make_new(sha) - - return rev_options - - @classmethod - def is_commit_id_equal(cls, dest, name): - """ - Return whether the current commit hash equals the given name. - - Args: - dest: the repository directory. - name: a string name. - """ - if not name: - # Then avoid an unnecessary subprocess call. - return False - - return cls.get_revision(dest) == name - - def fetch_new(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - rev_display = rev_options.to_display() - logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest)) - self.run_command(make_command('clone', '-q', url, dest)) - - if rev_options.rev: - # Then a specific revision was requested. - rev_options = self.resolve_revision(dest, url, rev_options) - branch_name = getattr(rev_options, 'branch_name', None) - if branch_name is None: - # Only do a checkout if the current commit id doesn't match - # the requested revision. - if not self.is_commit_id_equal(dest, rev_options.rev): - cmd_args = make_command( - 'checkout', '-q', rev_options.to_args(), - ) - self.run_command(cmd_args, cwd=dest) - elif self.get_current_branch(dest) != branch_name: - # Then a specific branch was requested, and that branch - # is not yet checked out. - track_branch = 'origin/{}'.format(branch_name) - cmd_args = [ - 'checkout', '-b', branch_name, '--track', track_branch, - ] - self.run_command(cmd_args, cwd=dest) - - #: repo may contain submodules - self.update_submodules(dest) - - def switch(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - self.run_command( - make_command('config', 'remote.origin.url', url), - cwd=dest, - ) - cmd_args = make_command('checkout', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - - self.update_submodules(dest) - - def update(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - # First fetch changes from the default remote - if self.get_git_version() >= parse_version('1.9.0'): - # fetch tags in addition to everything else - self.run_command(['fetch', '-q', '--tags'], cwd=dest) - else: - self.run_command(['fetch', '-q'], cwd=dest) - # Then reset to wanted revision (maybe even origin/master) - rev_options = self.resolve_revision(dest, url, rev_options) - cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - #: update submodules - self.update_submodules(dest) - - @classmethod - def get_remote_url(cls, location): - """ - Return URL of the first remote encountered. - - Raises RemoteNotFoundError if the repository does not have a remote - url configured. - """ - # We need to pass 1 for extra_ok_returncodes since the command - # exits with return code 1 if there are no matching lines. - stdout = cls.run_command( - ['config', '--get-regexp', r'remote\..*\.url'], - extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, - ) - remotes = stdout.splitlines() - try: - found_remote = remotes[0] - except IndexError: - raise RemoteNotFoundError - - for remote in remotes: - if remote.startswith('remote.origin.url '): - found_remote = remote - break - url = found_remote.split(' ')[1] - return url.strip() - - @classmethod - def get_revision(cls, location, rev=None): - if rev is None: - rev = 'HEAD' - current_rev = cls.run_command( - ['rev-parse', rev], show_stdout=False, cwd=location, - ) - return current_rev.strip() - - @classmethod - def get_subdirectory(cls, location): - """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. - """ - # find the repo root - git_dir = cls.run_command( - ['rev-parse', '--git-dir'], - show_stdout=False, cwd=location).strip() - if not os.path.isabs(git_dir): - git_dir = os.path.join(location, git_dir) - repo_root = os.path.abspath(os.path.join(git_dir, '..')) - return find_path_to_setup_from_repo_root(location, repo_root) - - @classmethod - def get_url_rev_and_auth(cls, url): - # type: (str) -> Tuple[str, Optional[str], AuthInfo] - """ - Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. - That's required because although they use SSH they sometimes don't - work with a ssh:// scheme (e.g. GitHub). But we need a scheme for - parsing. Hence we remove it again afterwards and return it as a stub. - """ - # Works around an apparent Git bug - # (see https://article.gmane.org/gmane.comp.version-control.git/146500) - scheme, netloc, path, query, fragment = urlsplit(url) - if scheme.endswith('file'): - initial_slashes = path[:-len(path.lstrip('/'))] - newpath = ( - initial_slashes + - urllib_request.url2pathname(path) - .replace('\\', '/').lstrip('/') - ) - url = urlunsplit((scheme, netloc, newpath, query, fragment)) - after_plus = scheme.find('+') + 1 - url = scheme[:after_plus] + urlunsplit( - (scheme[after_plus:], netloc, newpath, query, fragment), - ) - - if '://' not in url: - assert 'file:' not in url - url = url.replace('git+', 'git+ssh://') - url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) - url = url.replace('ssh://', '') - else: - url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) - - return url, rev, user_pass - - @classmethod - def update_submodules(cls, location): - if not os.path.exists(os.path.join(location, '.gitmodules')): - return - cls.run_command( - ['submodule', 'update', '--init', '--recursive', '-q'], - cwd=location, - ) - - @classmethod - def controls_location(cls, location): - if super(Git, cls).controls_location(location): - return True - try: - r = cls.run_command(['rev-parse'], - cwd=location, - show_stdout=False, - on_returncode='ignore', - log_failed_cmd=False) - return not r - except BadCommand: - logger.debug("could not determine if %s is under git control " - "because git is not available", location) - return False - - -vcs.register(Git) +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os.path +import re + +from pip._vendor.packaging.version import parse as parse_version +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.exceptions import BadCommand +from pip._internal.utils.misc import display_path +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.vcs.versioncontrol import ( + RemoteNotFoundError, + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions + + +urlsplit = urllib_parse.urlsplit +urlunsplit = urllib_parse.urlunsplit + + +logger = logging.getLogger(__name__) + + +HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') + + +def looks_like_hash(sha): + return bool(HASH_REGEX.match(sha)) + + +class Git(VersionControl): + name = 'git' + dirname = '.git' + repo_name = 'clone' + schemes = ( + 'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file', + ) + # Prevent the user's environment variables from interfering with pip: + # https://github.com/pypa/pip/issues/1130 + unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') + default_arg_rev = 'HEAD' + + @staticmethod + def get_base_rev_args(rev): + return [rev] + + def get_git_version(self): + VERSION_PFX = 'git version ' + version = self.run_command(['version'], show_stdout=False) + if version.startswith(VERSION_PFX): + version = version[len(VERSION_PFX):].split()[0] + else: + version = '' + # get first 3 positions of the git version because + # on windows it is x.y.z.windows.t, and this parses as + # LegacyVersion which always smaller than a Version. + version = '.'.join(version.split('.')[:3]) + return parse_version(version) + + @classmethod + def get_current_branch(cls, location): + """ + Return the current branch, or None if HEAD isn't at a branch + (e.g. detached HEAD). + """ + # git-symbolic-ref exits with empty stdout if "HEAD" is a detached + # HEAD rather than a symbolic ref. In addition, the -q causes the + # command to exit with status code 1 instead of 128 in this case + # and to suppress the message to stderr. + args = ['symbolic-ref', '-q', 'HEAD'] + output = cls.run_command( + args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + ref = output.strip() + + if ref.startswith('refs/heads/'): + return ref[len('refs/heads/'):] + + return None + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the Git repository at the url to the destination location""" + if not location.endswith('/'): + location = location + '/' + + with TempDirectory(kind="export") as temp_dir: + self.unpack(temp_dir.path, url=url) + self.run_command( + ['checkout-index', '-a', '-f', '--prefix', location], + show_stdout=False, cwd=temp_dir.path + ) + + @classmethod + def get_revision_sha(cls, dest, rev): + """ + Return (sha_or_none, is_branch), where sha_or_none is a commit hash + if the revision names a remote branch or tag, otherwise None. + + Args: + dest: the repository directory. + rev: the revision name. + """ + # Pass rev to pre-filter the list. + output = cls.run_command(['show-ref', rev], cwd=dest, + show_stdout=False, on_returncode='ignore') + refs = {} + for line in output.strip().splitlines(): + try: + sha, ref = line.split() + except ValueError: + # Include the offending line to simplify troubleshooting if + # this error ever occurs. + raise ValueError('unexpected show-ref line: {!r}'.format(line)) + + refs[ref] = sha + + branch_ref = 'refs/remotes/origin/{}'.format(rev) + tag_ref = 'refs/tags/{}'.format(rev) + + sha = refs.get(branch_ref) + if sha is not None: + return (sha, True) + + sha = refs.get(tag_ref) + + return (sha, False) + + @classmethod + def resolve_revision(cls, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> RevOptions + """ + Resolve a revision to a new RevOptions object with the SHA1 of the + branch, tag, or ref if found. + + Args: + rev_options: a RevOptions object. + """ + rev = rev_options.arg_rev + # The arg_rev property's implementation for Git ensures that the + # rev return value is always non-None. + assert rev is not None + + sha, is_branch = cls.get_revision_sha(dest, rev) + + if sha is not None: + rev_options = rev_options.make_new(sha) + rev_options.branch_name = rev if is_branch else None + + return rev_options + + # Do not show a warning for the common case of something that has + # the form of a Git commit hash. + if not looks_like_hash(rev): + logger.warning( + "Did not find branch or tag '%s', assuming revision or ref.", + rev, + ) + + if not rev.startswith('refs/'): + return rev_options + + # If it looks like a ref, we have to fetch it explicitly. + cls.run_command( + make_command('fetch', '-q', url, rev_options.to_args()), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + sha = cls.get_revision(dest, rev='FETCH_HEAD') + rev_options = rev_options.make_new(sha) + + return rev_options + + @classmethod + def is_commit_id_equal(cls, dest, name): + """ + Return whether the current commit hash equals the given name. + + Args: + dest: the repository directory. + name: a string name. + """ + if not name: + # Then avoid an unnecessary subprocess call. + return False + + return cls.get_revision(dest) == name + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest)) + self.run_command(make_command('clone', '-q', url, dest)) + + if rev_options.rev: + # Then a specific revision was requested. + rev_options = self.resolve_revision(dest, url, rev_options) + branch_name = getattr(rev_options, 'branch_name', None) + if branch_name is None: + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + cmd_args = make_command( + 'checkout', '-q', rev_options.to_args(), + ) + self.run_command(cmd_args, cwd=dest) + elif self.get_current_branch(dest) != branch_name: + # Then a specific branch was requested, and that branch + # is not yet checked out. + track_branch = 'origin/{}'.format(branch_name) + cmd_args = [ + 'checkout', '-b', branch_name, '--track', track_branch, + ] + self.run_command(cmd_args, cwd=dest) + + #: repo may contain submodules + self.update_submodules(dest) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + self.run_command( + make_command('config', 'remote.origin.url', url), + cwd=dest, + ) + cmd_args = make_command('checkout', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + self.update_submodules(dest) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + # First fetch changes from the default remote + if self.get_git_version() >= parse_version('1.9.0'): + # fetch tags in addition to everything else + self.run_command(['fetch', '-q', '--tags'], cwd=dest) + else: + self.run_command(['fetch', '-q'], cwd=dest) + # Then reset to wanted revision (maybe even origin/master) + rev_options = self.resolve_revision(dest, url, rev_options) + cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + #: update submodules + self.update_submodules(dest) + + @classmethod + def get_remote_url(cls, location): + """ + Return URL of the first remote encountered. + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + # We need to pass 1 for extra_ok_returncodes since the command + # exits with return code 1 if there are no matching lines. + stdout = cls.run_command( + ['config', '--get-regexp', r'remote\..*\.url'], + extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + remotes = stdout.splitlines() + try: + found_remote = remotes[0] + except IndexError: + raise RemoteNotFoundError + + for remote in remotes: + if remote.startswith('remote.origin.url '): + found_remote = remote + break + url = found_remote.split(' ')[1] + return url.strip() + + @classmethod + def get_revision(cls, location, rev=None): + if rev is None: + rev = 'HEAD' + current_rev = cls.run_command( + ['rev-parse', rev], show_stdout=False, cwd=location, + ) + return current_rev.strip() + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + # find the repo root + git_dir = cls.run_command( + ['rev-parse', '--git-dir'], + show_stdout=False, cwd=location).strip() + if not os.path.isabs(git_dir): + git_dir = os.path.join(location, git_dir) + repo_root = os.path.abspath(os.path.join(git_dir, '..')) + return find_path_to_setup_from_repo_root(location, repo_root) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + """ + Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. + That's required because although they use SSH they sometimes don't + work with a ssh:// scheme (e.g. GitHub). But we need a scheme for + parsing. Hence we remove it again afterwards and return it as a stub. + """ + # Works around an apparent Git bug + # (see https://article.gmane.org/gmane.comp.version-control.git/146500) + scheme, netloc, path, query, fragment = urlsplit(url) + if scheme.endswith('file'): + initial_slashes = path[:-len(path.lstrip('/'))] + newpath = ( + initial_slashes + + urllib_request.url2pathname(path) + .replace('\\', '/').lstrip('/') + ) + url = urlunsplit((scheme, netloc, newpath, query, fragment)) + after_plus = scheme.find('+') + 1 + url = scheme[:after_plus] + urlunsplit( + (scheme[after_plus:], netloc, newpath, query, fragment), + ) + + if '://' not in url: + assert 'file:' not in url + url = url.replace('git+', 'git+ssh://') + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) + url = url.replace('ssh://', '') + else: + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) + + return url, rev, user_pass + + @classmethod + def update_submodules(cls, location): + if not os.path.exists(os.path.join(location, '.gitmodules')): + return + cls.run_command( + ['submodule', 'update', '--init', '--recursive', '-q'], + cwd=location, + ) + + @classmethod + def controls_location(cls, location): + if super(Git, cls).controls_location(location): + return True + try: + r = cls.run_command(['rev-parse'], + cwd=location, + show_stdout=False, + on_returncode='ignore', + log_failed_cmd=False) + return not r + except BadCommand: + logger.debug("could not determine if %s is under git control " + "because git is not available", location) + return False + + +vcs.register(Git) diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index dd04b8bc687..bc4b19c7294 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -1,154 +1,154 @@ -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - -from __future__ import absolute_import - -import logging -import os - -from pip._vendor.six.moves import configparser - -from pip._internal.exceptions import BadCommand, InstallationError -from pip._internal.utils.misc import display_path -from pip._internal.utils.subprocess import make_command -from pip._internal.utils.temp_dir import TempDirectory -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.utils.urls import path_to_url -from pip._internal.vcs.versioncontrol import ( - VersionControl, - find_path_to_setup_from_repo_root, - vcs, -) - -if MYPY_CHECK_RUNNING: - from pip._internal.utils.misc import HiddenText - from pip._internal.vcs.versioncontrol import RevOptions - - -logger = logging.getLogger(__name__) - - -class Mercurial(VersionControl): - name = 'hg' - dirname = '.hg' - repo_name = 'clone' - schemes = ( - 'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http', - ) - - @staticmethod - def get_base_rev_args(rev): - return [rev] - - def export(self, location, url): - # type: (str, HiddenText) -> None - """Export the Hg repository at the url to the destination location""" - with TempDirectory(kind="export") as temp_dir: - self.unpack(temp_dir.path, url=url) - - self.run_command( - ['archive', location], show_stdout=False, cwd=temp_dir.path - ) - - def fetch_new(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - rev_display = rev_options.to_display() - logger.info( - 'Cloning hg %s%s to %s', - url, - rev_display, - display_path(dest), - ) - self.run_command(make_command('clone', '--noupdate', '-q', url, dest)) - self.run_command( - make_command('update', '-q', rev_options.to_args()), - cwd=dest, - ) - - def switch(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - repo_config = os.path.join(dest, self.dirname, 'hgrc') - config = configparser.RawConfigParser() - try: - config.read(repo_config) - config.set('paths', 'default', url.secret) - with open(repo_config, 'w') as config_file: - config.write(config_file) - except (OSError, configparser.NoSectionError) as exc: - logger.warning( - 'Could not switch Mercurial repository to %s: %s', url, exc, - ) - else: - cmd_args = make_command('update', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - - def update(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - self.run_command(['pull', '-q'], cwd=dest) - cmd_args = make_command('update', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - - @classmethod - def get_remote_url(cls, location): - url = cls.run_command( - ['showconfig', 'paths.default'], - show_stdout=False, cwd=location).strip() - if cls._is_local_repository(url): - url = path_to_url(url) - return url.strip() - - @classmethod - def get_revision(cls, location): - """ - Return the repository-local changeset revision number, as an integer. - """ - current_revision = cls.run_command( - ['parents', '--template={rev}'], - show_stdout=False, cwd=location).strip() - return current_revision - - @classmethod - def get_requirement_revision(cls, location): - """ - Return the changeset identification hash, as a 40-character - hexadecimal string - """ - current_rev_hash = cls.run_command( - ['parents', '--template={node}'], - show_stdout=False, cwd=location).strip() - return current_rev_hash - - @classmethod - def is_commit_id_equal(cls, dest, name): - """Always assume the versions don't match""" - return False - - @classmethod - def get_subdirectory(cls, location): - """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. - """ - # find the repo root - repo_root = cls.run_command( - ['root'], show_stdout=False, cwd=location).strip() - if not os.path.isabs(repo_root): - repo_root = os.path.abspath(os.path.join(location, repo_root)) - return find_path_to_setup_from_repo_root(location, repo_root) - - @classmethod - def controls_location(cls, location): - if super(Mercurial, cls).controls_location(location): - return True - try: - cls.run_command( - ['identify'], - cwd=location, - show_stdout=False, - on_returncode='raise', - log_failed_cmd=False) - except (BadCommand, InstallationError): - return False - - -vcs.register(Mercurial) +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os + +from pip._vendor.six.moves import configparser + +from pip._internal.exceptions import BadCommand, InstallationError +from pip._internal.utils.misc import display_path +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url +from pip._internal.vcs.versioncontrol import ( + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import RevOptions + + +logger = logging.getLogger(__name__) + + +class Mercurial(VersionControl): + name = 'hg' + dirname = '.hg' + repo_name = 'clone' + schemes = ( + 'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http', + ) + + @staticmethod + def get_base_rev_args(rev): + return [rev] + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the Hg repository at the url to the destination location""" + with TempDirectory(kind="export") as temp_dir: + self.unpack(temp_dir.path, url=url) + + self.run_command( + ['archive', location], show_stdout=False, cwd=temp_dir.path + ) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info( + 'Cloning hg %s%s to %s', + url, + rev_display, + display_path(dest), + ) + self.run_command(make_command('clone', '--noupdate', '-q', url, dest)) + self.run_command( + make_command('update', '-q', rev_options.to_args()), + cwd=dest, + ) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + repo_config = os.path.join(dest, self.dirname, 'hgrc') + config = configparser.RawConfigParser() + try: + config.read(repo_config) + config.set('paths', 'default', url.secret) + with open(repo_config, 'w') as config_file: + config.write(config_file) + except (OSError, configparser.NoSectionError) as exc: + logger.warning( + 'Could not switch Mercurial repository to %s: %s', url, exc, + ) + else: + cmd_args = make_command('update', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + self.run_command(['pull', '-q'], cwd=dest) + cmd_args = make_command('update', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + @classmethod + def get_remote_url(cls, location): + url = cls.run_command( + ['showconfig', 'paths.default'], + show_stdout=False, cwd=location).strip() + if cls._is_local_repository(url): + url = path_to_url(url) + return url.strip() + + @classmethod + def get_revision(cls, location): + """ + Return the repository-local changeset revision number, as an integer. + """ + current_revision = cls.run_command( + ['parents', '--template={rev}'], + show_stdout=False, cwd=location).strip() + return current_revision + + @classmethod + def get_requirement_revision(cls, location): + """ + Return the changeset identification hash, as a 40-character + hexadecimal string + """ + current_rev_hash = cls.run_command( + ['parents', '--template={node}'], + show_stdout=False, cwd=location).strip() + return current_rev_hash + + @classmethod + def is_commit_id_equal(cls, dest, name): + """Always assume the versions don't match""" + return False + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + # find the repo root + repo_root = cls.run_command( + ['root'], show_stdout=False, cwd=location).strip() + if not os.path.isabs(repo_root): + repo_root = os.path.abspath(os.path.join(location, repo_root)) + return find_path_to_setup_from_repo_root(location, repo_root) + + @classmethod + def controls_location(cls, location): + if super(Mercurial, cls).controls_location(location): + return True + try: + cls.run_command( + ['identify'], + cwd=location, + show_stdout=False, + on_returncode='raise', + log_failed_cmd=False) + except (BadCommand, InstallationError): + return False + + +vcs.register(Mercurial) diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 4519f29f78f..55afa908c19 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -1,672 +1,672 @@ -"""Handles all VCS (version control) support""" - -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - -from __future__ import absolute_import - -import errno -import logging -import os -import shutil -import sys - -from pip._vendor import pkg_resources -from pip._vendor.six.moves.urllib import parse as urllib_parse - -from pip._internal.exceptions import BadCommand -from pip._internal.utils.compat import samefile -from pip._internal.utils.misc import ( - ask_path_exists, - backup_dir, - display_path, - hide_url, - hide_value, - rmtree, -) -from pip._internal.utils.subprocess import call_subprocess, make_command -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.utils.urls import get_url_scheme - -if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type, Union - ) - from pip._internal.utils.ui import SpinnerInterface - from pip._internal.utils.misc import HiddenText - from pip._internal.utils.subprocess import CommandArgs - - AuthInfo = Tuple[Optional[str], Optional[str]] - - -__all__ = ['vcs'] - - -logger = logging.getLogger(__name__) - - -def is_url(name): - # type: (Union[str, Text]) -> bool - """ - Return true if the name looks like a URL. - """ - scheme = get_url_scheme(name) - if scheme is None: - return False - return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes - - -def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): - """ - Return the URL for a VCS requirement. - - Args: - repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). - project_name: the (unescaped) project name. - """ - egg_project_name = pkg_resources.to_filename(project_name) - req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) - if subdir: - req += '&subdirectory={}'.format(subdir) - - return req - - -def find_path_to_setup_from_repo_root(location, repo_root): - """ - Find the path to `setup.py` by searching up the filesystem from `location`. - Return the path to `setup.py` relative to `repo_root`. - Return None if `setup.py` is in `repo_root` or cannot be found. - """ - # find setup.py - orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem without - # finding setup.py - logger.warning( - "Could not find setup.py for directory %s (tried all " - "parent directories)", - orig_location, - ) - return None - - if samefile(repo_root, location): - return None - - return os.path.relpath(location, repo_root) - - -class RemoteNotFoundError(Exception): - pass - - -class RevOptions(object): - - """ - Encapsulates a VCS-specific revision to install, along with any VCS - install options. - - Instances of this class should be treated as if immutable. - """ - - def __init__( - self, - vc_class, # type: Type[VersionControl] - rev=None, # type: Optional[str] - extra_args=None, # type: Optional[CommandArgs] - ): - # type: (...) -> None - """ - Args: - vc_class: a VersionControl subclass. - rev: the name of the revision to install. - extra_args: a list of extra options. - """ - if extra_args is None: - extra_args = [] - - self.extra_args = extra_args - self.rev = rev - self.vc_class = vc_class - self.branch_name = None # type: Optional[str] - - def __repr__(self): - return ''.format(self.vc_class.name, self.rev) - - @property - def arg_rev(self): - # type: () -> Optional[str] - if self.rev is None: - return self.vc_class.default_arg_rev - - return self.rev - - def to_args(self): - # type: () -> CommandArgs - """ - Return the VCS-specific command arguments. - """ - args = [] # type: CommandArgs - rev = self.arg_rev - if rev is not None: - args += self.vc_class.get_base_rev_args(rev) - args += self.extra_args - - return args - - def to_display(self): - # type: () -> str - if not self.rev: - return '' - - return ' (to revision {})'.format(self.rev) - - def make_new(self, rev): - # type: (str) -> RevOptions - """ - Make a copy of the current instance, but with a new rev. - - Args: - rev: the name of the revision for the new object. - """ - return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) - - -class VcsSupport(object): - _registry = {} # type: Dict[str, VersionControl] - schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] - - def __init__(self): - # type: () -> None - # Register more schemes with urlparse for various version control - # systems - urllib_parse.uses_netloc.extend(self.schemes) - # Python >= 2.7.4, 3.3 doesn't have uses_fragment - if getattr(urllib_parse, 'uses_fragment', None): - urllib_parse.uses_fragment.extend(self.schemes) - super(VcsSupport, self).__init__() - - def __iter__(self): - return self._registry.__iter__() - - @property - def backends(self): - # type: () -> List[VersionControl] - return list(self._registry.values()) - - @property - def dirnames(self): - # type: () -> List[str] - return [backend.dirname for backend in self.backends] - - @property - def all_schemes(self): - # type: () -> List[str] - schemes = [] # type: List[str] - for backend in self.backends: - schemes.extend(backend.schemes) - return schemes - - def register(self, cls): - # type: (Type[VersionControl]) -> None - if not hasattr(cls, 'name'): - logger.warning('Cannot register VCS %s', cls.__name__) - return - if cls.name not in self._registry: - self._registry[cls.name] = cls() - logger.debug('Registered VCS backend: %s', cls.name) - - def unregister(self, name): - # type: (str) -> None - if name in self._registry: - del self._registry[name] - - def get_backend_for_dir(self, location): - # type: (str) -> Optional[VersionControl] - """ - Return a VersionControl object if a repository of that type is found - at the given directory. - """ - for vcs_backend in self._registry.values(): - if vcs_backend.controls_location(location): - logger.debug('Determine that %s uses VCS: %s', - location, vcs_backend.name) - return vcs_backend - return None - - def get_backend(self, name): - # type: (str) -> Optional[VersionControl] - """ - Return a VersionControl object or None. - """ - name = name.lower() - return self._registry.get(name) - - -vcs = VcsSupport() - - -class VersionControl(object): - name = '' - dirname = '' - repo_name = '' - # List of supported schemes for this Version Control - schemes = () # type: Tuple[str, ...] - # Iterable of environment variable names to pass to call_subprocess(). - unset_environ = () # type: Tuple[str, ...] - default_arg_rev = None # type: Optional[str] - - @classmethod - def should_add_vcs_url_prefix(cls, remote_url): - """ - Return whether the vcs prefix (e.g. "git+") should be added to a - repository's remote url when used in a requirement. - """ - return not remote_url.lower().startswith('{}:'.format(cls.name)) - - @classmethod - def get_subdirectory(cls, location): - """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. - """ - return None - - @classmethod - def get_requirement_revision(cls, repo_dir): - """ - Return the revision string that should be used in a requirement. - """ - return cls.get_revision(repo_dir) - - @classmethod - def get_src_requirement(cls, repo_dir, project_name): - """ - Return the requirement string to use to redownload the files - currently at the given repository directory. - - Args: - project_name: the (unescaped) project name. - - The return value has a form similar to the following: - - {repository_url}@{revision}#egg={project_name} - """ - repo_url = cls.get_remote_url(repo_dir) - if repo_url is None: - return None - - if cls.should_add_vcs_url_prefix(repo_url): - repo_url = '{}+{}'.format(cls.name, repo_url) - - revision = cls.get_requirement_revision(repo_dir) - subdir = cls.get_subdirectory(repo_dir) - req = make_vcs_requirement_url(repo_url, revision, project_name, - subdir=subdir) - - return req - - @staticmethod - def get_base_rev_args(rev): - """ - Return the base revision arguments for a vcs command. - - Args: - rev: the name of a revision to install. Cannot be None. - """ - raise NotImplementedError - - @classmethod - def make_rev_options(cls, rev=None, extra_args=None): - # type: (Optional[str], Optional[CommandArgs]) -> RevOptions - """ - Return a RevOptions object. - - Args: - rev: the name of a revision to install. - extra_args: a list of extra options. - """ - return RevOptions(cls, rev, extra_args=extra_args) - - @classmethod - def _is_local_repository(cls, repo): - # type: (str) -> bool - """ - posix absolute paths start with os.path.sep, - win32 ones start with drive (like c:\\folder) - """ - drive, tail = os.path.splitdrive(repo) - return repo.startswith(os.path.sep) or bool(drive) - - def export(self, location, url): - # type: (str, HiddenText) -> None - """ - Export the repository at the url to the destination location - i.e. only download the files, without vcs informations - - :param url: the repository URL starting with a vcs prefix. - """ - raise NotImplementedError - - @classmethod - def get_netloc_and_auth(cls, netloc, scheme): - """ - Parse the repository URL's netloc, and return the new netloc to use - along with auth information. - - Args: - netloc: the original repository URL netloc. - scheme: the repository URL's scheme without the vcs prefix. - - This is mainly for the Subversion class to override, so that auth - information can be provided via the --username and --password options - instead of through the URL. For other subclasses like Git without - such an option, auth information must stay in the URL. - - Returns: (netloc, (username, password)). - """ - return netloc, (None, None) - - @classmethod - def get_url_rev_and_auth(cls, url): - # type: (str) -> Tuple[str, Optional[str], AuthInfo] - """ - Parse the repository URL to use, and return the URL, revision, - and auth info to use. - - Returns: (url, rev, (username, password)). - """ - scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) - if '+' not in scheme: - raise ValueError( - "Sorry, {!r} is a malformed VCS url. " - "The format is +://, " - "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) - ) - # Remove the vcs prefix. - scheme = scheme.split('+', 1)[1] - netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) - rev = None - if '@' in path: - path, rev = path.rsplit('@', 1) - url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev, user_pass - - @staticmethod - def make_rev_args(username, password): - # type: (Optional[str], Optional[HiddenText]) -> CommandArgs - """ - Return the RevOptions "extra arguments" to use in obtain(). - """ - return [] - - def get_url_rev_options(self, url): - # type: (HiddenText) -> Tuple[HiddenText, RevOptions] - """ - Return the URL and RevOptions object to use in obtain() and in - some cases export(), as a tuple (url, rev_options). - """ - secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret) - username, secret_password = user_pass - password = None # type: Optional[HiddenText] - if secret_password is not None: - password = hide_value(secret_password) - extra_args = self.make_rev_args(username, password) - rev_options = self.make_rev_options(rev, extra_args=extra_args) - - return hide_url(secret_url), rev_options - - @staticmethod - def normalize_url(url): - # type: (str) -> str - """ - Normalize a URL for comparison by unquoting it and removing any - trailing slash. - """ - return urllib_parse.unquote(url).rstrip('/') - - @classmethod - def compare_urls(cls, url1, url2): - # type: (str, str) -> bool - """ - Compare two repo URLs for identity, ignoring incidental differences. - """ - return (cls.normalize_url(url1) == cls.normalize_url(url2)) - - def fetch_new(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - """ - Fetch a revision from a repository, in the case that this is the - first fetch from the repository. - - Args: - dest: the directory to fetch the repository to. - rev_options: a RevOptions object. - """ - raise NotImplementedError - - def switch(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - """ - Switch the repo at ``dest`` to point to ``URL``. - - Args: - rev_options: a RevOptions object. - """ - raise NotImplementedError - - def update(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - """ - Update an already-existing repo to the given ``rev_options``. - - Args: - rev_options: a RevOptions object. - """ - raise NotImplementedError - - @classmethod - def is_commit_id_equal(cls, dest, name): - """ - Return whether the id of the current commit equals the given name. - - Args: - dest: the repository directory. - name: a string name. - """ - raise NotImplementedError - - def obtain(self, dest, url): - # type: (str, HiddenText) -> None - """ - Install or update in editable mode the package represented by this - VersionControl object. - - :param dest: the repository directory in which to install or update. - :param url: the repository URL starting with a vcs prefix. - """ - url, rev_options = self.get_url_rev_options(url) - - if not os.path.exists(dest): - self.fetch_new(dest, url, rev_options) - return - - rev_display = rev_options.to_display() - if self.is_repository_directory(dest): - existing_url = self.get_remote_url(dest) - if self.compare_urls(existing_url, url.secret): - logger.debug( - '%s in %s exists, and has correct URL (%s)', - self.repo_name.title(), - display_path(dest), - url, - ) - if not self.is_commit_id_equal(dest, rev_options.rev): - logger.info( - 'Updating %s %s%s', - display_path(dest), - self.repo_name, - rev_display, - ) - self.update(dest, url, rev_options) - else: - logger.info('Skipping because already up-to-date.') - return - - logger.warning( - '%s %s in %s exists with URL %s', - self.name, - self.repo_name, - display_path(dest), - existing_url, - ) - prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', - ('s', 'i', 'w', 'b')) - else: - logger.warning( - 'Directory %s already exists, and is not a %s %s.', - dest, - self.name, - self.repo_name, - ) - # https://github.com/python/mypy/issues/1174 - prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore - ('i', 'w', 'b')) - - logger.warning( - 'The plan is to install the %s repository %s', - self.name, - url, - ) - response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) - - if response == 'a': - sys.exit(-1) - - if response == 'w': - logger.warning('Deleting %s', display_path(dest)) - rmtree(dest) - self.fetch_new(dest, url, rev_options) - return - - if response == 'b': - dest_dir = backup_dir(dest) - logger.warning( - 'Backing up %s to %s', display_path(dest), dest_dir, - ) - shutil.move(dest, dest_dir) - self.fetch_new(dest, url, rev_options) - return - - # Do nothing if the response is "i". - if response == 's': - logger.info( - 'Switching %s %s to %s%s', - self.repo_name, - display_path(dest), - url, - rev_display, - ) - self.switch(dest, url, rev_options) - - def unpack(self, location, url): - # type: (str, HiddenText) -> None - """ - Clean up current location and download the url repository - (and vcs infos) into location - - :param url: the repository URL starting with a vcs prefix. - """ - if os.path.exists(location): - rmtree(location) - self.obtain(location, url=url) - - @classmethod - def get_remote_url(cls, location): - """ - Return the url used at location - - Raises RemoteNotFoundError if the repository does not have a remote - url configured. - """ - raise NotImplementedError - - @classmethod - def get_revision(cls, location): - """ - Return the current commit id of the files at the given location. - """ - raise NotImplementedError - - @classmethod - def run_command( - cls, - cmd, # type: Union[List[str], CommandArgs] - show_stdout=True, # type: bool - cwd=None, # type: Optional[str] - on_returncode='raise', # type: str - extra_ok_returncodes=None, # type: Optional[Iterable[int]] - command_desc=None, # type: Optional[str] - extra_environ=None, # type: Optional[Mapping[str, Any]] - spinner=None, # type: Optional[SpinnerInterface] - log_failed_cmd=True - ): - # type: (...) -> Text - """ - Run a VCS subcommand - This is simply a wrapper around call_subprocess that adds the VCS - command name, and checks that the VCS is available - """ - cmd = make_command(cls.name, *cmd) - try: - return call_subprocess(cmd, show_stdout, cwd, - on_returncode=on_returncode, - extra_ok_returncodes=extra_ok_returncodes, - command_desc=command_desc, - extra_environ=extra_environ, - unset_environ=cls.unset_environ, - spinner=spinner, - log_failed_cmd=log_failed_cmd) - except OSError as e: - # errno.ENOENT = no such file or directory - # In other words, the VCS executable isn't available - if e.errno == errno.ENOENT: - raise BadCommand( - 'Cannot find command %r - do you have ' - '%r installed and in your ' - 'PATH?' % (cls.name, cls.name)) - else: - raise # re-raise exception if a different error occurred - - @classmethod - def is_repository_directory(cls, path): - # type: (str) -> bool - """ - Return whether a directory path is a repository directory. - """ - logger.debug('Checking in %s for %s (%s)...', - path, cls.dirname, cls.name) - return os.path.exists(os.path.join(path, cls.dirname)) - - @classmethod - def controls_location(cls, location): - # type: (str) -> bool - """ - Check if a location is controlled by the vcs. - - Searches up the filesystem and checks is_repository_directory(). - - It is meant to be extended to add smarter detection mechanisms for - specific vcs. For example, the Git override checks that Git is - actually available. - """ - while not cls.is_repository_directory(location): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem. - return False - return True +"""Handles all VCS (version control) support""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import errno +import logging +import os +import shutil +import sys + +from pip._vendor import pkg_resources +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.exceptions import BadCommand +from pip._internal.utils.compat import samefile +from pip._internal.utils.misc import ( + ask_path_exists, + backup_dir, + display_path, + hide_url, + hide_value, + rmtree, +) +from pip._internal.utils.subprocess import call_subprocess, make_command +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import get_url_scheme + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type, Union + ) + from pip._internal.utils.ui import SpinnerInterface + from pip._internal.utils.misc import HiddenText + from pip._internal.utils.subprocess import CommandArgs + + AuthInfo = Tuple[Optional[str], Optional[str]] + + +__all__ = ['vcs'] + + +logger = logging.getLogger(__name__) + + +def is_url(name): + # type: (Union[str, Text]) -> bool + """ + Return true if the name looks like a URL. + """ + scheme = get_url_scheme(name) + if scheme is None: + return False + return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes + + +def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): + """ + Return the URL for a VCS requirement. + + Args: + repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + project_name: the (unescaped) project name. + """ + egg_project_name = pkg_resources.to_filename(project_name) + req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) + if subdir: + req += '&subdirectory={}'.format(subdir) + + return req + + +def find_path_to_setup_from_repo_root(location, repo_root): + """ + Find the path to `setup.py` by searching up the filesystem from `location`. + Return the path to `setup.py` relative to `repo_root`. + Return None if `setup.py` is in `repo_root` or cannot be found. + """ + # find setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without + # finding setup.py + logger.warning( + "Could not find setup.py for directory %s (tried all " + "parent directories)", + orig_location, + ) + return None + + if samefile(repo_root, location): + return None + + return os.path.relpath(location, repo_root) + + +class RemoteNotFoundError(Exception): + pass + + +class RevOptions(object): + + """ + Encapsulates a VCS-specific revision to install, along with any VCS + install options. + + Instances of this class should be treated as if immutable. + """ + + def __init__( + self, + vc_class, # type: Type[VersionControl] + rev=None, # type: Optional[str] + extra_args=None, # type: Optional[CommandArgs] + ): + # type: (...) -> None + """ + Args: + vc_class: a VersionControl subclass. + rev: the name of the revision to install. + extra_args: a list of extra options. + """ + if extra_args is None: + extra_args = [] + + self.extra_args = extra_args + self.rev = rev + self.vc_class = vc_class + self.branch_name = None # type: Optional[str] + + def __repr__(self): + return ''.format(self.vc_class.name, self.rev) + + @property + def arg_rev(self): + # type: () -> Optional[str] + if self.rev is None: + return self.vc_class.default_arg_rev + + return self.rev + + def to_args(self): + # type: () -> CommandArgs + """ + Return the VCS-specific command arguments. + """ + args = [] # type: CommandArgs + rev = self.arg_rev + if rev is not None: + args += self.vc_class.get_base_rev_args(rev) + args += self.extra_args + + return args + + def to_display(self): + # type: () -> str + if not self.rev: + return '' + + return ' (to revision {})'.format(self.rev) + + def make_new(self, rev): + # type: (str) -> RevOptions + """ + Make a copy of the current instance, but with a new rev. + + Args: + rev: the name of the revision for the new object. + """ + return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) + + +class VcsSupport(object): + _registry = {} # type: Dict[str, VersionControl] + schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] + + def __init__(self): + # type: () -> None + # Register more schemes with urlparse for various version control + # systems + urllib_parse.uses_netloc.extend(self.schemes) + # Python >= 2.7.4, 3.3 doesn't have uses_fragment + if getattr(urllib_parse, 'uses_fragment', None): + urllib_parse.uses_fragment.extend(self.schemes) + super(VcsSupport, self).__init__() + + def __iter__(self): + return self._registry.__iter__() + + @property + def backends(self): + # type: () -> List[VersionControl] + return list(self._registry.values()) + + @property + def dirnames(self): + # type: () -> List[str] + return [backend.dirname for backend in self.backends] + + @property + def all_schemes(self): + # type: () -> List[str] + schemes = [] # type: List[str] + for backend in self.backends: + schemes.extend(backend.schemes) + return schemes + + def register(self, cls): + # type: (Type[VersionControl]) -> None + if not hasattr(cls, 'name'): + logger.warning('Cannot register VCS %s', cls.__name__) + return + if cls.name not in self._registry: + self._registry[cls.name] = cls() + logger.debug('Registered VCS backend: %s', cls.name) + + def unregister(self, name): + # type: (str) -> None + if name in self._registry: + del self._registry[name] + + def get_backend_for_dir(self, location): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object if a repository of that type is found + at the given directory. + """ + for vcs_backend in self._registry.values(): + if vcs_backend.controls_location(location): + logger.debug('Determine that %s uses VCS: %s', + location, vcs_backend.name) + return vcs_backend + return None + + def get_backend(self, name): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object or None. + """ + name = name.lower() + return self._registry.get(name) + + +vcs = VcsSupport() + + +class VersionControl(object): + name = '' + dirname = '' + repo_name = '' + # List of supported schemes for this Version Control + schemes = () # type: Tuple[str, ...] + # Iterable of environment variable names to pass to call_subprocess(). + unset_environ = () # type: Tuple[str, ...] + default_arg_rev = None # type: Optional[str] + + @classmethod + def should_add_vcs_url_prefix(cls, remote_url): + """ + Return whether the vcs prefix (e.g. "git+") should be added to a + repository's remote url when used in a requirement. + """ + return not remote_url.lower().startswith('{}:'.format(cls.name)) + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + return None + + @classmethod + def get_requirement_revision(cls, repo_dir): + """ + Return the revision string that should be used in a requirement. + """ + return cls.get_revision(repo_dir) + + @classmethod + def get_src_requirement(cls, repo_dir, project_name): + """ + Return the requirement string to use to redownload the files + currently at the given repository directory. + + Args: + project_name: the (unescaped) project name. + + The return value has a form similar to the following: + + {repository_url}@{revision}#egg={project_name} + """ + repo_url = cls.get_remote_url(repo_dir) + if repo_url is None: + return None + + if cls.should_add_vcs_url_prefix(repo_url): + repo_url = '{}+{}'.format(cls.name, repo_url) + + revision = cls.get_requirement_revision(repo_dir) + subdir = cls.get_subdirectory(repo_dir) + req = make_vcs_requirement_url(repo_url, revision, project_name, + subdir=subdir) + + return req + + @staticmethod + def get_base_rev_args(rev): + """ + Return the base revision arguments for a vcs command. + + Args: + rev: the name of a revision to install. Cannot be None. + """ + raise NotImplementedError + + @classmethod + def make_rev_options(cls, rev=None, extra_args=None): + # type: (Optional[str], Optional[CommandArgs]) -> RevOptions + """ + Return a RevOptions object. + + Args: + rev: the name of a revision to install. + extra_args: a list of extra options. + """ + return RevOptions(cls, rev, extra_args=extra_args) + + @classmethod + def _is_local_repository(cls, repo): + # type: (str) -> bool + """ + posix absolute paths start with os.path.sep, + win32 ones start with drive (like c:\\folder) + """ + drive, tail = os.path.splitdrive(repo) + return repo.startswith(os.path.sep) or bool(drive) + + def export(self, location, url): + # type: (str, HiddenText) -> None + """ + Export the repository at the url to the destination location + i.e. only download the files, without vcs informations + + :param url: the repository URL starting with a vcs prefix. + """ + raise NotImplementedError + + @classmethod + def get_netloc_and_auth(cls, netloc, scheme): + """ + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Args: + netloc: the original repository URL netloc. + scheme: the repository URL's scheme without the vcs prefix. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). + """ + return netloc, (None, None) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). + """ + scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + if '+' not in scheme: + raise ValueError( + "Sorry, {!r} is a malformed VCS url. " + "The format is +://, " + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + ) + # Remove the vcs prefix. + scheme = scheme.split('+', 1)[1] + netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) + rev = None + if '@' in path: + path, rev = path.rsplit('@', 1) + url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) + return url, rev, user_pass + + @staticmethod + def make_rev_args(username, password): + # type: (Optional[str], Optional[HiddenText]) -> CommandArgs + """ + Return the RevOptions "extra arguments" to use in obtain(). + """ + return [] + + def get_url_rev_options(self, url): + # type: (HiddenText) -> Tuple[HiddenText, RevOptions] + """ + Return the URL and RevOptions object to use in obtain() and in + some cases export(), as a tuple (url, rev_options). + """ + secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret) + username, secret_password = user_pass + password = None # type: Optional[HiddenText] + if secret_password is not None: + password = hide_value(secret_password) + extra_args = self.make_rev_args(username, password) + rev_options = self.make_rev_options(rev, extra_args=extra_args) + + return hide_url(secret_url), rev_options + + @staticmethod + def normalize_url(url): + # type: (str) -> str + """ + Normalize a URL for comparison by unquoting it and removing any + trailing slash. + """ + return urllib_parse.unquote(url).rstrip('/') + + @classmethod + def compare_urls(cls, url1, url2): + # type: (str, str) -> bool + """ + Compare two repo URLs for identity, ignoring incidental differences. + """ + return (cls.normalize_url(url1) == cls.normalize_url(url2)) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Fetch a revision from a repository, in the case that this is the + first fetch from the repository. + + Args: + dest: the directory to fetch the repository to. + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Switch the repo at ``dest`` to point to ``URL``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Update an already-existing repo to the given ``rev_options``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + @classmethod + def is_commit_id_equal(cls, dest, name): + """ + Return whether the id of the current commit equals the given name. + + Args: + dest: the repository directory. + name: a string name. + """ + raise NotImplementedError + + def obtain(self, dest, url): + # type: (str, HiddenText) -> None + """ + Install or update in editable mode the package represented by this + VersionControl object. + + :param dest: the repository directory in which to install or update. + :param url: the repository URL starting with a vcs prefix. + """ + url, rev_options = self.get_url_rev_options(url) + + if not os.path.exists(dest): + self.fetch_new(dest, url, rev_options) + return + + rev_display = rev_options.to_display() + if self.is_repository_directory(dest): + existing_url = self.get_remote_url(dest) + if self.compare_urls(existing_url, url.secret): + logger.debug( + '%s in %s exists, and has correct URL (%s)', + self.repo_name.title(), + display_path(dest), + url, + ) + if not self.is_commit_id_equal(dest, rev_options.rev): + logger.info( + 'Updating %s %s%s', + display_path(dest), + self.repo_name, + rev_display, + ) + self.update(dest, url, rev_options) + else: + logger.info('Skipping because already up-to-date.') + return + + logger.warning( + '%s %s in %s exists with URL %s', + self.name, + self.repo_name, + display_path(dest), + existing_url, + ) + prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', + ('s', 'i', 'w', 'b')) + else: + logger.warning( + 'Directory %s already exists, and is not a %s %s.', + dest, + self.name, + self.repo_name, + ) + # https://github.com/python/mypy/issues/1174 + prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore + ('i', 'w', 'b')) + + logger.warning( + 'The plan is to install the %s repository %s', + self.name, + url, + ) + response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) + + if response == 'a': + sys.exit(-1) + + if response == 'w': + logger.warning('Deleting %s', display_path(dest)) + rmtree(dest) + self.fetch_new(dest, url, rev_options) + return + + if response == 'b': + dest_dir = backup_dir(dest) + logger.warning( + 'Backing up %s to %s', display_path(dest), dest_dir, + ) + shutil.move(dest, dest_dir) + self.fetch_new(dest, url, rev_options) + return + + # Do nothing if the response is "i". + if response == 's': + logger.info( + 'Switching %s %s to %s%s', + self.repo_name, + display_path(dest), + url, + rev_display, + ) + self.switch(dest, url, rev_options) + + def unpack(self, location, url): + # type: (str, HiddenText) -> None + """ + Clean up current location and download the url repository + (and vcs infos) into location + + :param url: the repository URL starting with a vcs prefix. + """ + if os.path.exists(location): + rmtree(location) + self.obtain(location, url=url) + + @classmethod + def get_remote_url(cls, location): + """ + Return the url used at location + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + raise NotImplementedError + + @classmethod + def get_revision(cls, location): + """ + Return the current commit id of the files at the given location. + """ + raise NotImplementedError + + @classmethod + def run_command( + cls, + cmd, # type: Union[List[str], CommandArgs] + show_stdout=True, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True + ): + # type: (...) -> Text + """ + Run a VCS subcommand + This is simply a wrapper around call_subprocess that adds the VCS + command name, and checks that the VCS is available + """ + cmd = make_command(cls.name, *cmd) + try: + return call_subprocess(cmd, show_stdout, cwd, + on_returncode=on_returncode, + extra_ok_returncodes=extra_ok_returncodes, + command_desc=command_desc, + extra_environ=extra_environ, + unset_environ=cls.unset_environ, + spinner=spinner, + log_failed_cmd=log_failed_cmd) + except OSError as e: + # errno.ENOENT = no such file or directory + # In other words, the VCS executable isn't available + if e.errno == errno.ENOENT: + raise BadCommand( + 'Cannot find command %r - do you have ' + '%r installed and in your ' + 'PATH?' % (cls.name, cls.name)) + else: + raise # re-raise exception if a different error occurred + + @classmethod + def is_repository_directory(cls, path): + # type: (str) -> bool + """ + Return whether a directory path is a repository directory. + """ + logger.debug('Checking in %s for %s (%s)...', + path, cls.dirname, cls.name) + return os.path.exists(os.path.join(path, cls.dirname)) + + @classmethod + def controls_location(cls, location): + # type: (str) -> bool + """ + Check if a location is controlled by the vcs. + + Searches up the filesystem and checks is_repository_directory(). + + It is meant to be extended to add smarter detection mechanisms for + specific vcs. For example, the Git override checks that Git is + actually available. + """ + while not cls.is_repository_directory(location): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem. + return False + return True