Update project and spinner outputs

Signed-off-by: Dan Ryan <dan@danryan.co>

Try again

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix test config to skip failed removals

Signed-off-by: Dan Ryan <dan@danryan.co>

Update piptools to handle some errors

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix test config to skip failed removals

Signed-off-by: Dan Ryan <dan@danryan.co>

Update tempfile.py

Use vistirs temporary directory implementation

Update temp_dir.py

Force pip to use weakrefs in tempdirs

Fix pip implementation to set name of tempdir

typo fix

Signed-off-by: Dan Ryan <dan@danryan.co>

fix pip tempdir implementation

Signed-off-by: Dan Ryan <dan@danryan.co>

Update tempfiles to use weakrefs

Signed-off-by: Dan Ryan <dan@danryan.co>

fix patch paths

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix pip tempdir implementation

Signed-off-by: Dan Ryan <dan@danryan.co>

Syntax error fix

Signed-off-by: Dan Ryan <dan@danryan.co>

Unconstrain windows tests

Signed-off-by: Dan Ryan <dan@danryan.co>

Update dependencies, add news

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix pythonfinder path search nesting bug

- Fixes #3121

Signed-off-by: Dan Ryan <dan@danryan.co>

Update requirementslib

- Fix subdirectory issue

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix logic error

Signed-off-by: Dan Ryan <dan@danryan.co>

conditional builds

Signed-off-by: Dan Ryan <dan@danryan.co>

.vsts-ci/linux.yml

@@ -1,3 +1,22 @@
+name: Pipenv Build Rules
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+  paths:
+    exclude:
+    - docs/*
+    - news/*
+    - README.md
+    - pipenv/*.txt
+    - CHANGELOG.rst
+    - CONTRIBUTING.md
+    - CODE_OF_CONDUCT.md
+    - .gitignore
+    - .gitattributes
+    - .editorconfig
+
 phases:
 - template: phases/test.yml
   parameters:

.vsts-ci/steps/run-tests.yml

@@ -12,7 +12,7 @@ steps:
     $env:TEMP='T:\'
     Write-Host "##vso[task.setvariable variable=TMP]T:\"
     $env:TEMP='T:\'
-    D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor -k 'test_get_vcs_refs or test_install_editable_git_tag' --junitxml=test-results.xml tests
+    D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests
   displayName: Run integration tests
 
 - task: PublishTestResults@2

.vsts-ci/windows.yml

@@ -1,3 +1,22 @@
+name: Pipenv Build Rules
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+  paths:
+    exclude:
+    - docs/*
+    - news/*
+    - README.md
+    - pipenv/*.txt
+    - CHANGELOG.rst
+    - CONTRIBUTING.md
+    - CODE_OF_CONDUCT.md
+    - .gitignore
+    - .gitattributes
+    - .editorconfig
+
 phases:
 - template: phases/test.yml
   parameters:

news/3090.bugfix.rst

@@ -0,0 +1 @@
+Fixed a bug in ``requirementslib`` which prevented successful installation from mercurial repositories.

news/3094.bugfix.rst

@@ -0,0 +1 @@
+Fixed random resource warnings when using pyenv or any other subprocess calls.

news/3102.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors.

news/3109.bugfix.rst

@@ -0,0 +1 @@
+Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures.

news/3113.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue resolving virtualenv paths for users without ``platlib`` values on their systems.

news/3114.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors.

news/3117.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors.

news/3121.bugfix.rst

@@ -0,0 +1 @@
+Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions.

news/3121.vendor.rst

@@ -0,0 +1 @@
+Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions.

pipenv/core.py

@@ -1355,7 +1355,8 @@ def pip_install(
     write_to_tmpfile = False
     if requirement:
         needs_hashes = not requirement.editable and not ignore_hashes and r is None
-        write_to_tmpfile = needs_hashes
+        has_subdir = requirement.is_vcs and requirement.req.subdirectory
+        write_to_tmpfile = needs_hashes or has_subdir
 
     if not trusted_hosts:
         trusted_hosts = []

pipenv/patched/notpip/_internal/utils/temp_dir.py

@@ -3,8 +3,10 @@
 import logging
 import os.path
 import tempfile
+import warnings
 
 from pipenv.patched.notpip._internal.utils.misc import rmtree
+from pipenv.vendor.vistir.compat import finalize, ResourceWarning
 
 logger = logging.getLogger(__name__)
 
@@ -45,6 +47,20 @@ def __init__(self, path=None, delete=None, kind="temp"):
         self.path = path
         self.delete = delete
         self.kind = kind
+        self._finalizer = None
+        if path:
+            self._register_finalizer()
+
+    def _register_finalizer(self):
+        if self.delete and self.path:
+            self._finalizer = finalize(
+                self,
+                self._cleanup,
+                self.path,
+                warn_message=None
+            )
+        else:
+            self._finalizer = None
 
     def __repr__(self):
         return "<{} {!r}>".format(self.__class__.__name__, self.path)
@@ -72,11 +88,27 @@ def create(self):
         self.path = os.path.realpath(
             tempfile.mkdtemp(prefix="pip-{}-".format(self.kind))
         )
+        self._register_finalizer()
         logger.debug("Created temporary directory: {}".format(self.path))
 
+    @classmethod
+    def _cleanup(cls, name, warn_message=None):
+        try:
+            rmtree(name)
+        except OSError:
+            pass
+        else:
+            if warn_message:
+                warnings.warn(warn_message, ResourceWarning)
+
     def cleanup(self):
         """Remove the temporary directory created and reset state
         """
-        if self.path is not None and os.path.exists(self.path):
-            rmtree(self.path)
-        self.path = None
+        if getattr(self._finalizer, "detach", None) and self._finalizer.detach():
+            if os.path.exists(self.path):
+                try:
+                    rmtree(self.path)
+                except OSError:
+                    pass
+                else:
+                    self.path = None

pipenv/patched/piptools/repositories/pypi.py

@@ -19,19 +19,18 @@
     InstallRequirement,
     SafeFileCache
 )
-os.environ["PIP_SHIMS_BASE_MODULE"] = str("notpip")
+os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip")
 from pip_shims.shims import do_import, VcsSupport, WheelCache
 from packaging.requirements import Requirement
 from packaging.specifiers import SpecifierSet, Specifier
-from packaging.markers import Op, Value, Variable, Marker
 InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999"))
 from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver
 
 
 from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR
 from ..exceptions import NoCandidateFound
 from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup,
-                     make_install_requirement, clean_requires_python)
+                        make_install_requirement, clean_requires_python)
 from .base import BaseRepository
 
 try:
@@ -243,6 +242,7 @@ def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=
         dist = None
         ireq.isolated = False
         ireq._wheel_cache = wheel_cache
+
         try:
             from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer
         except ImportError:
@@ -295,7 +295,18 @@ def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=
                 resolver = PipResolver(**resolver_kwargs)
                 resolver.require_hashes = False
                 results = resolver._resolve_one(reqset, ireq)
-                reqset.cleanup_files()
+
+        cleanup_fn = getattr(reqset, "cleanup_files", None)
+        if cleanup_fn is not None:
+            try:
+                cleanup_fn()
+            except OSError:
+                pass
+
+        if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)):
+            if ireq.editable:
+                self._source_dir = TemporaryDirectory(fs_str("source"))
+                ireq.ensure_has_source_dir(self.source_dir)
 
         if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
             # Collect setup_requires info from local eggs.

pipenv/project.py

@@ -15,7 +15,6 @@
 import pipfile.api
 import six
 import vistir
-import virtualenv as _virtualenv
 import toml
 
 from .cmdparse import Script
@@ -84,6 +83,8 @@ def default(self, obj):
 
         if isinstance(obj, (ContainerElement, TokenElement)):
             return obj.primitive_value
+        elif isinstance(obj, vistir.compat.Path):
+            obj = obj.as_posix()
         return super(_LockFileEncoder, self).default(obj)
 
     def encode(self, obj):
@@ -988,7 +989,35 @@ def _pyversion(self):
     def env_paths(self):
         location = self.virtualenv_location if self.virtualenv_location else sys.prefix
         prefix = vistir.compat.Path(location)
-        home, lib, inc, bin_ = _virtualenv.path_locations(prefix)
+        import importlib
+        try:
+            _virtualenv = importlib.import_module("virtualenv")
+        except ImportError:
+            with vistir.contextmanagers.temp_path():
+                from string import Formatter
+                formatter = Formatter()
+                import sysconfig
+                if getattr(sys, "real_prefix", None):
+                    scheme = sysconfig._get_default_scheme()
+                    sysconfig._INSTALL_SCHEMES["posix_prefix"]["purelib"]
+                    if not scheme:
+                        scheme = "posix_prefix" if not sys.platform == "win32" else "nt"
+                    is_purelib = "purelib" in sysconfig._INSTALL_SCHEMES[scheme]
+                    lib_key = "purelib" if is_purelib else "platlib"
+                    lib = sysconfig._INSTALL_SCHEMES[scheme][lib_key]
+                    fields = [field for _, field, _, _ in formatter.parse() if field]
+                    config = {
+                        "py_version_short": self._pyversion,
+                    }
+                    for field in fields:
+                        if field not in config:
+                            config[field] = prefix
+                    sys.path = [
+                        os.path.join(sysconfig._INSTALL_SCHEMES[scheme][lib_key], "site-packages"),
+                    ] + sys.path
+                    six.reload_module(importlib)
+                    _virtualenv = importlib.import_module("virtualenv")
+        home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix())
         paths = {
             "lib": lib,
             "include": inc,

pipenv/resolver.py

@@ -70,7 +70,6 @@ def resolve(packages, pre, project, sources, clear, system, requirements_dir=Non
         )
 
     from pipenv.core import project
-
     sources = (
         replace_pypi_sources(project.pipfile_sources, pypi_mirror_source)
         if pypi_mirror_source
@@ -111,8 +110,8 @@ def main():
     parsed, remaining = parser.parse_known_args()
     # sys.argv = remaining
     parsed = handle_parsed_args(parsed)
-    _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir,
-            parsed.packages)
+    _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system,
+             parsed.requirements_dir, parsed.packages)
 
 
 if __name__ == "__main__":

pipenv/utils.py

@@ -345,7 +345,6 @@ def venv_resolve_deps(
         return []
 
     req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements")
-
     cmd = [
         which("python", allow_global=allow_global),
         Path(resolver.__file__.rstrip("co")).as_posix()
@@ -364,7 +363,6 @@ def venv_resolve_deps(
         os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY)
         os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir)
         os.environ["PIP_NO_INPUT"] = fs_str("1")
-
         out = to_native_string("")
         EOF.__module__ = "pexpect.exceptions"
         with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER,
@@ -430,6 +428,8 @@ def resolve_deps(
     index_lookup = {}
     markers_lookup = {}
     python_path = which("python", allow_global=allow_global)
+    if not os.environ.get("PIP_SRC"):
+        os.environ["PIP_SRC"] = project.virtualenv_src_location
     backup_python_path = sys.executable
     results = []
     if not deps:
@@ -1116,23 +1116,19 @@ def get_vcs_deps(
         packages = getattr(project, section)
     except AttributeError:
         return [], []
-    if os.environ.get("PIP_SRC"):
-        src_dir = Path(
-            os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src"))
-        )
-        src_dir.mkdir(mode=0o775, exist_ok=True)
-    else:
-        src_dir = create_tracked_tempdir(prefix="pipenv-lock-dir")
     for pkg_name, pkg_pipfile in packages.items():
         requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile)
         name = requirement.normalized_name
         commit_hash = None
         if requirement.is_vcs:
-            with locked_repository(requirement) as repo:
-                commit_hash = repo.get_commit_hash()
-                lockfile[name] = requirement.pipfile_entry[1]
-                lockfile[name]['ref'] = commit_hash
-        reqs.append(requirement)
+            try:
+                with locked_repository(requirement) as repo:
+                    commit_hash = repo.get_commit_hash()
+                    lockfile[name] = requirement.pipfile_entry[1]
+                    lockfile[name]['ref'] = commit_hash
+                reqs.append(requirement)
+            except OSError:
+                continue
     return reqs, lockfile
 
 
@@ -1257,11 +1253,11 @@ def is_virtual_environment(path):
 @contextmanager
 def locked_repository(requirement):
     from .vendor.vistir.path import create_tracked_tempdir
-    src_dir = create_tracked_tempdir(prefix="pipenv-src")
     if not requirement.is_vcs:
         return
     original_base = os.environ.pop("PIP_SHIMS_BASE_MODULE", None)
     os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip")
+    src_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-src")
     try:
         with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo:
             yield repo

pipenv/vendor/pythonfinder/utils.py

@@ -1,6 +1,7 @@
 # -*- coding=utf-8 -*-
 from __future__ import absolute_import, print_function
 
+import itertools
 import locale
 import os
 import subprocess
@@ -21,6 +22,9 @@
 except ImportError:
     from backports.functools_lru_cache import lru_cache
 
+six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc"))
+from six.moves import Iterable
+
 
 PYTHON_IMPLEMENTATIONS = (
     "python", "ironpython", "jython", "pypy", "anaconda", "miniconda",
@@ -123,7 +127,21 @@ def filter_pythons(path):
     return filter(lambda x: path_is_python(x), path.iterdir())
 
 
+# def unnest(item):
+#     if isinstance(next((i for i in item), None), (list, tuple)):
+#         return chain(*filter(None, item))
+#     return chain(filter(None, item))
+
+
 def unnest(item):
-    if isinstance(next((i for i in item), None), (list, tuple)):
-        return chain(*filter(None, item))
-    return chain(filter(None, item))
+    if isinstance(item, Iterable) and not isinstance(item, six.string_types):
+        item, target = itertools.tee(item, 2)
+    else:
+        target = item
+    for el in target:
+        if isinstance(el, Iterable) and not isinstance(el, six.string_types):
+            el, el_copy = itertools.tee(el, 2)
+            for sub in unnest(el_copy):
+                yield sub
+        else:
+            yield el

pipenv/vendor/requirementslib/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding=utf-8 -*-
-__version__ = '1.2.1'
+__version__ = '1.2.2'
 
 import logging