Deal with vendored pip, setuptools and wheel not exposing proper whee…

…ls names.

This works but something better should be done with at least less copying.

pex/build_system/pep_517.py

@@ -5,6 +5,7 @@
 
 import json
 import os
+import shutil
 import subprocess
 from textwrap import dedent
 
@@ -43,14 +44,18 @@ def _default_build_system(
             extra_env = {}  # type: Dict[str, str]
             resolved_reqs = set()  # type: Set[str]
             resolved_dists = []  # type: List[Distribution]
+            interpreter = target.get_interpreter()
             if selected_pip_version is PipVersion.VENDORED:
                 requires = ["setuptools", str(selected_pip_version.wheel_requirement)]
-                resolved_dists.extend(
-                    Distribution.load(dist_location)
-                    for dist_location in third_party.expose(
-                        ["setuptools"], interpreter=target.get_interpreter()
-                    )
+                setuptools = next(third_party.expose(["setuptools"], interpreter=interpreter))
+                setuptools_wheel_dir = os.path.join(
+                    safe_mkdtemp(),
+                    "setuptools-{setuptools_version}-py2.py3-none-any.whl".format(
+                        setuptools_version=PipVersion.VENDORED.setuptools_version,
+                    ),
                 )
+                shutil.copytree(setuptools, setuptools_wheel_dir)
+                resolved_dists.append(Distribution.load(setuptools_wheel_dir))
                 resolved_reqs.add("setuptools")
                 extra_env.update(__PEX_UNVENDORED__="setuptools")
             else:
@@ -70,7 +75,7 @@ def _default_build_system(
             )
             build_system = try_(
                 BuildSystem.create(
-                    interpreter=target.get_interpreter(),
+                    interpreter=interpreter,
                     requires=requires,
                     resolved=resolved_dists,
                     build_backend=DEFAULT_BUILD_BACKEND,

pex/cache/data.py

@@ -18,6 +18,7 @@
     UserCodeDir,
     VenvDirs,
 )
+from pex.common import CopyMode
 from pex.dist_metadata import ProjectNameAndVersion
 from pex.typing import TYPE_CHECKING, overload
 
@@ -161,24 +162,28 @@ def record_zipapp_install(pex_info):
 
 
 def record_venv_install(
+    copy_mode,  # type: CopyMode.Value
     pex_info,  # type: PexInfo
     venv_dirs,  # type: VenvDirs
 ):
     # type: (...) -> None
 
     with _inserted_wheels(pex_info) as cursor:
-        cursor.executemany(
-            """
-            INSERT OR IGNORE INTO venv_deps (
-                venv_hash,
-                wheel_install_hash
-            ) VALUES (?, ?)
-            """,
-            tuple(
-                (venv_dirs.short_hash, wheel_install_hash)
-                for wheel_install_hash in pex_info.distributions.values()
-            ),
-        ).close()
+        if copy_mode is CopyMode.SYMLINK:
+            cursor.executemany(
+                """
+                INSERT OR IGNORE INTO venv_deps (
+                    venv_hash,
+                    wheel_install_hash
+                ) VALUES (?, ?)
+                """,
+                tuple(
+                    (venv_dirs.short_hash, wheel_install_hash)
+                    for wheel_install_hash in pex_info.distributions.values()
+                ),
+            ).close()
+        else:
+            cursor.close()
 
 
 if TYPE_CHECKING:

pex/cli/commands/cache/command.py

@@ -533,6 +533,8 @@ def prune_pip_caches(wheels):
                 prunable_wheels.add(
                     (prunable_pnav.canonicalized_project_name, prunable_pnav.canonicalized_version)
                 )
+            if not prunable_wheels:
+                return
 
             def spawn_list(pip):
                 # type: (Pip) -> SpawnedJob[Tuple[ProjectNameAndVersion, ...]]
@@ -545,10 +547,13 @@ def spawn_list(pip):
                     ),
                 )
 
-            all_pips = tuple(iter_all_pips())
+            # N.B.:We just need 1 Pip per version (really per paired cache). Whether a Pip has extra
+            # requirements installed does not affect cache management.
+            all_pip_versions = tuple({pip.version: pip for pip in iter_all_pips()}.values())
+
             pip_removes = []  # type: List[Tuple[Pip, str]]
             for pip, project_name_and_versions in zip(
-                all_pips, execute_parallel(inputs=all_pips, spawn_func=spawn_list)
+                all_pip_versions, execute_parallel(inputs=all_pip_versions, spawn_func=spawn_list)
             ):
                 for pnav in project_name_and_versions:
                     if (
@@ -564,15 +569,24 @@ def spawn_list(pip):
                             )
                         )
 
+            def parse_remove(stdout):
+                # type: (bytes) -> int
+
+                # The output from `pip cache remove` is a line like:
+                # Files removed: 42
+                _, sep, count = stdout.decode("utf-8").partition(":")
+                if sep != ":" or not count:
+                    return 0
+                try:
+                    return int(count)
+                except ValueError:
+                    return 0
+
             def spawn_remove(args):
                 # type: (Tuple[Pip, str]) -> SpawnedJob[int]
                 pip, wheel_name_glob = args
-                # Files removed: 1
                 return SpawnedJob.stdout(
-                    job=pip.spawn_cache_remove(wheel_name_glob),
-                    result_func=lambda stdout: int(
-                        stdout.decode("utf-8").rsplit(":", 1)[1].strip()
-                    ),
+                    job=pip.spawn_cache_remove(wheel_name_glob), result_func=parse_remove
                 )
 
             removes_by_pip = Counter()  # type: typing.Counter[str]

pex/pex_bootstrapper.py

@@ -599,13 +599,14 @@ def ensure_venv(
                         hermetic_scripts=pex_info.venv_hermetic_scripts,
                     )
 
-                    if copy_mode is CopyMode.SYMLINK:
-                        with TRACER.timed(
-                            "Recording venv install of {pex} {hash}".format(
-                                pex=pex.path(), hash=pex_info.pex_hash
-                            )
-                        ):
-                            record_venv_install(pex_info=pex_info, venv_dirs=venv_dirs)
+                    with TRACER.timed(
+                        "Recording venv install of {pex} {hash}".format(
+                            pex=pex.path(), hash=pex_info.pex_hash
+                        )
+                    ):
+                        record_venv_install(
+                            copy_mode=copy_mode, pex_info=pex_info, venv_dirs=venv_dirs
+                        )
 
                     # There are popular Linux distributions with shebang length limits
                     # (BINPRM_BUF_SIZE in /usr/include/linux/binfmts.h) set at 128 characters, so

pex/pip/installation.py

@@ -5,6 +5,7 @@
 
 import hashlib
 import os
+import shutil
 from collections import OrderedDict
 from textwrap import dedent
 
@@ -73,7 +74,6 @@ def _pip_installation(
 
             isolated_pip_builder = PEXBuilder(path=chroot.work_dir)
             isolated_pip_builder.info.venv = True
-            isolated_pip_builder.info.venv_site_packages_copies = True
             # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect if needed.
             isolated_pip_builder.info.venv_hermetic_scripts = False
             for dist_location in iter_distribution_locations():
@@ -123,7 +123,26 @@ def _vendored_installation(
 
     def expose_vendored():
         # type: () -> Iterator[str]
-        return third_party.expose(("pip", "setuptools"), interpreter=interpreter)
+        pip, setuptools = third_party.expose(("pip", "setuptools"), interpreter=interpreter)
+        base_dir = safe_mkdtemp()
+
+        pip_wheel_dir = os.path.join(
+            base_dir,
+            "pip-{pip_version}-py2.py3-none-any.whl".format(
+                pip_version=PipVersion.VENDORED.version
+            ),
+        )
+        shutil.copytree(pip, pip_wheel_dir)
+        yield pip_wheel_dir
+
+        setuptools_wheel_dir = os.path.join(
+            base_dir,
+            "setuptools-{setuptools_version}-py2.py3-none-any.whl".format(
+                setuptools_version=PipVersion.VENDORED.setuptools_version
+            ),
+        )
+        shutil.copytree(setuptools, setuptools_wheel_dir)
+        yield setuptools_wheel_dir
 
     if not extra_requirements:
         return _pip_installation(
@@ -214,8 +233,33 @@ def bootstrap_pip():
 
         for req in version.requirements:
             project_name = req.name
-            target_dir = os.path.join(chroot, "reqs", project_name)
-            venv.interpreter.execute(["-m", "pip", "install", "--target", target_dir, str(req)])
+            specifiers = list(req.specifier)
+            production_assert(len(specifiers) == 1)
+            specifier = specifiers[0]
+            production_assert(specifier.operator == "==")
+            project_version = specifier.version
+            target_dir = os.path.join(
+                chroot,
+                "reqs",
+                "{project_name}-{project_version}-py{py_version}-none-any.whl".format(
+                    project_name=project_name,
+                    project_version=project_version,
+                    py_version=(interpreter or PythonInterpreter.get()).version[0],
+                ),
+            )
+            venv.interpreter.execute(
+                [
+                    "-m",
+                    "pip",
+                    "install",
+                    "--no-deps",
+                    "--only-binary",
+                    str(req),
+                    "--target",
+                    target_dir,
+                    str(req),
+                ]
+            )
             yield target_dir
 
     return bootstrap_pip

pex/pip/version.py

@@ -45,34 +45,39 @@ def overridden(cls):
     def __init__(
         self,
         version,  # type: str
+        setuptools_version,  # type: str
+        wheel_version,  # type: str
+        requires_python,  # type: str
         name=None,  # type: Optional[str]
         requirement=None,  # type: Optional[str]
-        setuptools_version=None,  # type: Optional[str]
-        wheel_version=None,  # type: Optional[str]
-        requires_python=None,  # type: Optional[str]
+        setuptools_requirement=None,  # type: Optional[str]
         hidden=False,  # type: bool
     ):
         # type: (...) -> None
         super(PipVersionValue, self).__init__(name or version)
 
         def to_requirement(
             project_name,  # type: str
-            project_version=None,  # type: Optional[str]
+            project_version,  # type: str
         ):
             # type: (...) -> Requirement
             return Requirement.parse(
                 "{project_name}=={project_version}".format(
                     project_name=project_name, project_version=project_version
                 )
-                if project_version
-                else project_name
             )
 
         self.version = Version(version)
         self.requirement = (
             Requirement.parse(requirement) if requirement else to_requirement("pip", version)
         )
-        self.setuptools_requirement = to_requirement("setuptools", setuptools_version)
+        self.setuptools_version = setuptools_version
+        self.setuptools_requirement = (
+            Requirement.parse(setuptools_requirement)
+            if setuptools_requirement
+            else to_requirement("setuptools", setuptools_version)
+        )
+        self.wheel_version = wheel_version
         self.wheel_requirement = to_requirement("wheel", wheel_version)
         self.requires_python = SpecifierSet(requires_python) if requires_python else None
         self.hidden = hidden
@@ -174,6 +179,8 @@ def values(cls):
         name="20.3.4-patched",
         version="20.3.4+patched",
         requirement=vendor.PIP_SPEC.requirement,
+        setuptools_version="44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863",
+        setuptools_requirement="setuptools",
         wheel_version="0.37.1",
         requires_python="<3.12",
     )

tox.ini

@@ -156,6 +156,9 @@ deps =
     pip==20.3.4  # This version should track the version in pex/vendor/__init__.py.
     setuptools==44.0.0  # This version should track the version in pex/vendor/__init__.py.
     sphinx
+    # This is just used as a constraint - the dep is via:
+    #   spinx 5.1.1 -> jinja2>=2.3 -> MarkupSafe>=2.0
+    MarkupSafe<3
     toml==0.10.2  # This version should track the version in pex/vendor/__init__.py.
 
     PyGithub==2.4.0