diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 1432681354..8ccf025af4 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -5,6 +5,7 @@ operating systems, etc. """ import functools +import importlib import io import os import six @@ -50,6 +51,24 @@ def detach(self): class ResourceWarning(Warning): pass +# -*- coding=utf-8 -*- + + +def pip_import(module_path, subimport=None, old_path=None): + internal = 'pip._internal.{0}'.format(module_path) + old_path = old_path or module_path + pip9 = 'pip.{0}'.format(old_path) + try: + _tmp = importlib.import_module(internal) + except ImportError: + _tmp = importlib.import_module(pip9) + if subimport: + return getattr(_tmp, subimport, _tmp) + return _tmp + + +vcs = pip_import('vcs', 'VcsSupport') + class TemporaryDirectory(object): """Create and return a temporary directory. This has the same diff --git a/pipenv/core.py b/pipenv/core.py index 6731de2c5a..5ad0cd25f9 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -16,6 +16,7 @@ import dotenv import delegator from .vendor import pexpect +from first import first import pipfile from blindspin import spinner from requests.packages import urllib3 @@ -46,9 +47,11 @@ is_star, rmtree, split_argument, + extract_uri_from_vcs_dep, ) from ._compat import ( TemporaryDirectory, + vcs ) from .import pep508checker, progress from .environments import ( @@ -999,6 +1002,7 @@ def do_lock( ): """Executes the freeze functionality.""" from notpip._vendor.distlib.markers import Evaluator + from .utils import get_vcs_deps allowed_marker_keys = ['markers'] + [k for k in Evaluator.allowed_values.keys()] cached_lockfile = {} if not pre: @@ -1035,6 +1039,9 @@ def do_lock( if dev_package in project.packages: dev_packages[dev_package] = project.packages[dev_package] # Resolve dev-package dependencies, with pip-tools. + pip_freeze = delegator.run( + '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system))) + ).out deps = convert_deps_to_pip( dev_packages, project, r=False, include_index=True ) @@ -1066,24 +1073,14 @@ def do_lock( lockfile['develop'][dep['name']]['markers'] = dep['markers'] # Add refs for VCS installs. # TODO: be smarter about this. - vcs_deps = convert_deps_to_pip(project.vcs_dev_packages, project, r=False) - pip_freeze = delegator.run( - '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system))) - ).out - if vcs_deps: - for line in pip_freeze.strip().split('\n'): - # if the line doesn't match a vcs dependency in the Pipfile, - # ignore it - if not any(dep in line for dep in vcs_deps): - continue - - try: - installed = convert_deps_from_pip(line) - name = list(installed.keys())[0] - if is_vcs(installed[name]): - lockfile['develop'].update(installed) - except IndexError: - pass + vcs_dev_lines, vcs_dev_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=True) + for lf in vcs_dev_lockfiles: + try: + name = first(lf.keys()) + except AttributeError: + continue + if hasattr(lf[name], 'keys'): + lockfile['develop'].update(lf) if write: # Alert the user of progress. click.echo( @@ -1132,23 +1129,15 @@ def do_lock( lockfile['default'][dep['name']]['markers'] = dep['markers'] # Add refs for VCS installs. # TODO: be smarter about this. - vcs_deps = convert_deps_to_pip(project.vcs_packages, project, r=False) - if vcs_deps: - for line in pip_freeze.strip().split('\n'): - # if the line doesn't match a vcs dependency in the Pipfile, - # ignore it - if not any(dep in line for dep in vcs_deps): - continue + _vcs_deps, vcs_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=False) + for lf in vcs_lockfiles: + try: + name = first(lf.keys()) + except AttributeError: + continue + if hasattr(lf[name], 'keys'): + lockfile['default'].update(lf) - try: - installed = convert_deps_from_pip(line) - name = list(installed.keys())[0] - if is_vcs(installed[name]): - # Convert name to PEP 423 name. - installed = {pep423_name(name): installed[name]} - lockfile['default'].update(installed) - except IndexError: - pass # Support for --keep-outdated… if keep_outdated: for section_name, section in ( diff --git a/pipenv/project.py b/pipenv/project.py index 2294d43f73..76bcdadcd4 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -497,41 +497,41 @@ def lockfile_exists(self): def lockfile_content(self): return self.load_lockfile() - @property - def editable_packages(self): + def _get_editable_packages(self, dev=False): + section = 'dev-packages' if dev else 'packages' packages = { k: v - for k, v in self.parsed_pipfile.get('packages', {}).items() + for k, v in self.parsed_pipfile.get(section, {}).items() if is_editable(v) } return packages - @property - def editable_dev_packages(self): + def _get_vcs_packages(self, dev=False): + section = 'dev-packages' if dev else 'packages' packages = { k: v - for k, v in self.parsed_pipfile.get('dev-packages', {}).items() - if is_editable(v) + for k, v in self.parsed_pipfile.get(section, {}).items() + if is_vcs(v) or is_vcs(k) } - return packages + return packages or {} + + @property + def editable_packages(self): + return self._get_editable_packages(dev=False) + + @property + def editable_dev_packages(self): + return self._get_editable_packages(dev=True) @property def vcs_packages(self): """Returns a list of VCS packages, for not pip-tools to consume.""" - ps = {} - for k, v in self.parsed_pipfile.get('packages', {}).items(): - if is_vcs(v) or is_vcs(k): - ps.update({k: v}) - return ps + return self._get_vcs_packages(dev=False) @property def vcs_dev_packages(self): """Returns a list of VCS packages, for not pip-tools to consume.""" - ps = {} - for k, v in self.parsed_pipfile.get('dev-packages', {}).items(): - if is_vcs(v) or is_vcs(k): - ps.update({k: v}) - return ps + return self._get_vcs_packages(dev=True) @property def all_packages(self): diff --git a/pipenv/utils.py b/pipenv/utils.py index 973b1c8386..9ad18c7b18 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1358,3 +1358,71 @@ def safe_expandvars(value): if isinstance(value, six.string_types): return os.path.expandvars(value) return value + + +def extract_uri_from_vcs_dep(dep): + valid_keys = VCS_LIST + ('uri', 'file') + if hasattr(dep, 'keys'): + return first(dep[k] for k in valid_keys if k in dep) or None + return None + + +def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): + target_dir = os.path.join(src_dir, name) + target_rev = vcs_obj.make_rev_options(rev) + if not os.path.exists(target_dir): + vcs_obj.obtain(target_dir) + vcs_obj.update(target_dir, target_rev) + return vcs_obj.get_revision(target_dir) + + +def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False): + from ._compat import vcs + section = 'vcs_dev_packages' if dev else 'vcs_packages' + lines = [] + lockfiles = [] + try: + packages = getattr(project, section) + except AttributeError: + return [], [] + vcs_registry = vcs() + vcs_uri_map = { + extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')} + for k, v in packages.items() + } + for line in pip_freeze.strip().split('\n'): + # if the line doesn't match a vcs dependency in the Pipfile, + # ignore it + _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line) + if not _vcs_match: + continue + + pipfile_name = vcs_uri_map[_vcs_match]['name'] + pipfile_rev = vcs_uri_map[_vcs_match]['ref'] + src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src')) + mkdir_p(src_dir) + names = {pipfile_name} + _pip_uri = line.lstrip('-e ') + backend_name = str(_pip_uri.split('+', 1)[0]) + backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)] + __vcs = backend(url=_pip_uri) + + installed = convert_deps_from_pip(line) + if not hasattr(installed, 'keys'): + pass + lock_name = first(installed.keys()) + names.add(lock_name) + locked_rev = None + for _name in names: + locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev) + if is_vcs(installed[lock_name]): + installed[lock_name]['ref'] = locked_rev + lockfiles.append({pipfile_name: installed[lock_name]}) + pipfile_srcdir = os.path.join(src_dir, pipfile_name) + lockfile_srcdir = os.path.join(src_dir, lock_name) + lines.append(line) + if os.path.exists(pipfile_srcdir): + lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global)) + else: + lockfiles.extend(venv_resolve_deps(['-e {0}'.format(lockfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global)) + return lines, lockfiles diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 94391f5739..a19577e6b8 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -2,6 +2,10 @@ import os from flaky import flaky import delegator +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path @pytest.mark.vcs @@ -144,3 +148,25 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir): assert six_key in p.lockfile['default'] # Make sure we didn't put six in the lockfile by accident as a vcs ref assert 'six' not in p.lockfile['default'] + + +@pytest.mark.vcs +@pytest.mark.install +@pytest.mark.needs_internet +def test_get_vcs_refs(PipenvInstance, pip_src_dir): + with PipenvInstance(chdir=True) as p: + c = p.pipenv('install -e git+https://github.com/hynek/structlog.git@16.1.0#egg=structlog') + assert c.return_code == 0 + assert 'structlog' in p.pipfile['packages'] + assert 'structlog' in p.lockfile['default'] + assert 'six' in p.lockfile['default'] + assert p.lockfile['default']['structlog']['ref'] == 'a39f6906a268fb2f4c365042b31d0200468fb492' + pipfile = Path(p.pipfile_path) + new_content = pipfile.read_bytes().replace(b'16.1.0', b'18.1.0') + pipfile.write_bytes(new_content) + c = p.pipenv('lock') + assert c.return_code == 0 + assert p.lockfile['default']['structlog']['ref'] == 'a73fbd3a9c3cafb11f43168582083f839b883034' + assert 'structlog' in p.pipfile['packages'] + assert 'structlog' in p.lockfile['default'] + assert 'six' in p.lockfile['default']