Lock vcs deps to exact commit with dependencies

- Manually obtain and update VCS repository with exact commit - Always store exact commit in the lockfile - Fixes pypa#2180, pypa#1690, pypa#1611, pypa#2096 Signed-off-by: Dan Ryan <dan@danryan.co>
techalchemy · May 17, 2018 · a7d6696 · a7d6696
1 parent 94a85ac
commit a7d6696
Show file tree

Hide file tree

Showing 3 changed files with 107 additions and 85 deletions.
diff --git a/pipenv/core.py b/pipenv/core.py
@@ -1000,6 +1000,7 @@ def do_lock(
 ):
     """Executes the freeze functionality."""
     from notpip._vendor.distlib.markers import Evaluator
+    from .utils import get_vcs_deps
     allowed_marker_keys = ['markers'] + [k for k in Evaluator.allowed_values.keys()]
     cached_lockfile = {}
     if not pre:
@@ -1036,6 +1037,9 @@ def do_lock(
         if dev_package in project.packages:
             dev_packages[dev_package] = project.packages[dev_package]
     # Resolve dev-package dependencies, with pip-tools.
+    pip_freeze = delegator.run(
+        '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system)))
+    ).out
     deps = convert_deps_to_pip(
         dev_packages, project, r=False, include_index=True
     )
@@ -1067,51 +1071,14 @@ def do_lock(
             lockfile['develop'][dep['name']]['markers'] = dep['markers']
     # Add refs for VCS installs.
     # TODO: be smarter about this.
-    vcs_deps = convert_deps_to_pip(project.vcs_dev_packages, project, r=False)
-    pip_freeze = delegator.run(
-        '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system)))
-    ).out
-    vcs_registry = vcs()
-    if vcs_deps:
-        vcs_uri_map = {extract_uri_from_vcs_dep(v): k for k, v in project.vcs_dev_packages.items()}
-        for line in pip_freeze.strip().split('\n'):
-            # if the line doesn't match a vcs dependency in the Pipfile,
-            # ignore it
-            _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line)
-            if not _vcs_match:
-                continue
-
-            pipfile_name = vcs_uri_map[_vcs_match]
-            src_loc = os.path.join(project.virtualenv_location, pipfile_name)
-            pipfile_rev = project.vcs_packages[pipfile_name].get('ref', None)
-            _pip_uri = line.lstrip('-e ')
-            backend_name = str(_pip_uri.split('+', 1)[0])
-            backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
-            __vcs = backend(url=_pip_uri)
-            __target_rev = __vcs.make_rev_options(pipfile_rev)
-
-            try:
-                installed = convert_deps_from_pip(line)
-                lock_name = first(installed.keys())
-                locked_rev = None
-                lockfile_src_loc = os.path.join(project.virtualenv_location, lock_name)
-                paths = set([loc for loc in [src_loc, lockfile_src_loc] if os.path.exists(loc)])
-                paths = list(paths)
-                if paths:
-                    # If the pipfile rev and the installed rev don't match
-                    if pipfile_rev != __vcs.get_url_rev()[1]:
-                        for _p in paths:
-                            __vcs.update(_p, __target_rev)
-                            locked_rev = __vcs.get_revision(_p)
-                else:
-                    __vcs.obtain(src_loc)
-                    __vcs.update(src_loc, __target_rev)
-                    locked_rev = __vcs.get_revision(src_loc)
-                if is_vcs(installed[lock_name]):
-                    installed[lock_name]['ref'] = locked_rev
-                    lockfile['develop'].update({pipfile_name: installed[lock_name]})
-            except IndexError:
-                pass
+    vcs_dev_lines, vcs_dev_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=True)
+    for lf in vcs_dev_lockfiles:
+        try:
+            name = first(lf.keys())
+        except AttributeError:
+            continue
+        if hasattr(lf[name], 'keys'):
+            lockfile['develop'].update(lf)
     if write:
         # Alert the user of progress.
         click.echo(
@@ -1160,46 +1127,14 @@ def do_lock(
             lockfile['default'][dep['name']]['markers'] = dep['markers']
     # Add refs for VCS installs.
     # TODO: be smarter about this.
-    vcs_deps = convert_deps_to_pip(project.vcs_packages, project, r=False)
-    if vcs_deps:
-        vcs_uri_map = {extract_uri_from_vcs_dep(v): k for k, v in project.vcs_packages.items()}
-        for line in pip_freeze.strip().split('\n'):
-            # if the line doesn't match a vcs dependency in the Pipfile,
-            # ignore it
-            _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line)
-            if not _vcs_match:
-                continue
-
-            pipfile_name = vcs_uri_map[_vcs_match]
-            src_loc = os.path.join(project.virtualenv_location, pipfile_name)
-            pipfile_rev = project.vcs_packages[pipfile_name].get('ref', None)
-            _pip_uri = line.lstrip('-e ')
-            backend_name = str(_pip_uri.split('+', 1)[0])
-            backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
-            __vcs = backend(url=_pip_uri)
-            __target_rev = __vcs.make_rev_options(pipfile_rev)
-
-            try:
-                installed = convert_deps_from_pip(line)
-                lock_name = first(installed.keys())
-                locked_rev = None
-                lockfile_src_loc = os.path.join(project.virtualenv_location, lock_name)
-                paths = [loc for loc in [src_loc, lockfile_src_loc] if os.path.exists(loc)]
-                if paths:
-                    # If the pipfile rev and the installed rev don't match
-                    if pipfile_rev != __vcs.get_url_rev()[1]:
-                        for _p in paths:
-                            __vcs.update(_p, __target_rev)
-                            locked_rev = __vcs.get_revision(_p)
-                else:
-                    __vcs.obtain(src_loc)
-                    __vcs.update(src_loc, __target_rev)
-                    locked_rev = __vcs.get_revision(src_loc)
-                if is_vcs(installed[lock_name]):
-                    installed[lock_name]['ref'] = locked_rev
-                    lockfile['default'].update({pipfile_name: installed[lock_name]})
-            except IndexError:
-                pass
+    _vcs_deps, vcs_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=False)
+    for lf in vcs_lockfiles:
+        try:
+            name = first(lf.keys())
+        except AttributeError:
+            continue
+        if hasattr(lf[name], 'keys'):
+            lockfile['default'].update(lf)
 
     # Support for --keep-outdated…
     if keep_outdated:

diff --git a/pipenv/utils.py b/pipenv/utils.py
@@ -1365,3 +1365,64 @@ def extract_uri_from_vcs_dep(dep):
     if hasattr(dep, 'keys'):
         return first(dep[k] for k in valid_keys if k in dep) or None
     return None
+
+
+def install_or_update_vcs(vcs_obj, src_dir, name, rev=None):    
+    target_dir = os.path.join(src_dir, name)
+    target_rev = vcs_obj.make_rev_options(rev)
+    if not os.path.exists(target_dir):
+        vcs_obj.obtain(target_dir)
+    vcs_obj.update(target_dir, target_rev)
+    return vcs_obj.get_revision(target_dir)
+
+
+def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False):
+    from ._compat import vcs
+    section = 'vcs_dev_packages' if dev else 'vcs_packages'
+    lines = []
+    lockfiles = []
+    try:
+        packages = getattr(project, section)
+    except AttributeError:
+        return [], []
+    vcs_registry = vcs()
+    vcs_uri_map = {
+        extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')}
+        for k, v in packages.items()
+    }
+    for line in pip_freeze.strip().split('\n'):
+        # if the line doesn't match a vcs dependency in the Pipfile,
+        # ignore it
+        _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line)
+        if not _vcs_match:
+            continue
+
+        pipfile_name = vcs_uri_map[_vcs_match]['name']
+        pipfile_rev = vcs_uri_map[_vcs_match]['ref']
+        src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src'))
+        mkdir_p(src_dir)
+        names = {pipfile_name}
+        _pip_uri = line.lstrip('-e ')
+        backend_name = str(_pip_uri.split('+', 1)[0])
+        backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
+        __vcs = backend(url=_pip_uri)
+
+        installed = convert_deps_from_pip(line)
+        if not hasattr(installed, 'keys'):
+            pass
+        lock_name = first(installed.keys())
+        names.add(lock_name)
+        locked_rev = None
+        for _name in names:
+            locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev)
+        if is_vcs(installed[lock_name]):
+            installed[lock_name]['ref'] = locked_rev
+            lockfiles.append({pipfile_name: installed[lock_name]})
+        pipfile_srcdir = os.path.join(src_dir, pipfile_name)
+        lockfile_srcdir = os.path.join(src_dir, lock_name)
+        lines.append(line)
+        if os.path.exists(pipfile_srcdir):
+            lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))
+        else:
+            lockfiles.extend(venv_resolve_deps(['-e {0}'.format(lockfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))
+    return lines, lockfiles
diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py
@@ -2,6 +2,10 @@
 import os
 from flaky import flaky
 import delegator
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
 
 
 @pytest.mark.vcs
@@ -144,3 +148,25 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir):
         assert six_key in p.lockfile['default']
         # Make sure we didn't put six in the lockfile by accident as a vcs ref
         assert 'six' not in p.lockfile['default']
+
+
+@pytest.mark.vcs
+@pytest.mark.install
+@pytest.mark.needs_internet
+def test_get_vcs_refs(PipenvInstance, pip_src_dir):
+    with PipenvInstance(chdir=True) as p:
+        c = p.pipenv('install -e git+https://github.com/hynek/structlog.git@16.1.0#egg=structlog')
+        assert c.return_code == 0
+        assert 'structlog' in p.pipfile['packages']
+        assert 'structlog' in p.lockfile['default']
+        assert 'six' in p.lockfile['default']
+        assert p.lockfile['default']['structlog']['ref'] == 'a39f6906a268fb2f4c365042b31d0200468fb492'
+        pipfile = Path(p.pipfile_path)
+        new_content = pipfile.read_bytes().replace(b'16.1.0', b'18.1.0')
+        pipfile.write_bytes(new_content)
+        c = p.pipenv('lock')
+        assert c.return_code == 0
+        assert p.lockfile['default']['structlog']['ref'] == 'a73fbd3a9c3cafb11f43168582083f839b883034'
+        assert 'structlog' in p.pipfile['packages']
+        assert 'structlog' in p.lockfile['default']
+        assert 'six' in p.lockfile['default']