From 99738cb576ca975dd0850e7459b45c0df15d2d9c Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 11 Oct 2024 17:56:15 -0700 Subject: [PATCH] Deal with vendored pip, setuptools and wheel not exposing proper wheels names. This works but something better should be done with at least less copying. --- pex/build_system/pep_517.py | 17 ++++++---- pex/cache/data.py | 29 ++++++++++------- pex/cli/commands/cache/command.py | 28 ++++++++++++----- pex/pex_bootstrapper.py | 15 ++++----- pex/pip/installation.py | 52 ++++++++++++++++++++++++++++--- pex/pip/version.py | 21 ++++++++----- tox.ini | 3 ++ 7 files changed, 122 insertions(+), 43 deletions(-) diff --git a/pex/build_system/pep_517.py b/pex/build_system/pep_517.py index 5d7ce6e82..1e530d99b 100644 --- a/pex/build_system/pep_517.py +++ b/pex/build_system/pep_517.py @@ -5,6 +5,7 @@ import json import os +import shutil import subprocess from textwrap import dedent @@ -43,14 +44,18 @@ def _default_build_system( extra_env = {} # type: Dict[str, str] resolved_reqs = set() # type: Set[str] resolved_dists = [] # type: List[Distribution] + interpreter = target.get_interpreter() if selected_pip_version is PipVersion.VENDORED: requires = ["setuptools", str(selected_pip_version.wheel_requirement)] - resolved_dists.extend( - Distribution.load(dist_location) - for dist_location in third_party.expose( - ["setuptools"], interpreter=target.get_interpreter() - ) + setuptools = next(third_party.expose(["setuptools"], interpreter=interpreter)) + setuptools_wheel_dir = os.path.join( + safe_mkdtemp(), + "setuptools-{setuptools_version}-py2.py3-none-any.whl".format( + setuptools_version=PipVersion.VENDORED.setuptools_version, + ), ) + shutil.copytree(setuptools, setuptools_wheel_dir) + resolved_dists.append(Distribution.load(setuptools_wheel_dir)) resolved_reqs.add("setuptools") extra_env.update(__PEX_UNVENDORED__="setuptools") else: @@ -70,7 +75,7 @@ def _default_build_system( ) build_system = try_( BuildSystem.create( - interpreter=target.get_interpreter(), + interpreter=interpreter, requires=requires, resolved=resolved_dists, build_backend=DEFAULT_BUILD_BACKEND, diff --git a/pex/cache/data.py b/pex/cache/data.py index 588cc111c..0c93c8a11 100644 --- a/pex/cache/data.py +++ b/pex/cache/data.py @@ -18,6 +18,7 @@ UserCodeDir, VenvDirs, ) +from pex.common import CopyMode from pex.dist_metadata import ProjectNameAndVersion from pex.typing import TYPE_CHECKING, overload @@ -161,24 +162,28 @@ def record_zipapp_install(pex_info): def record_venv_install( + copy_mode, # type: CopyMode.Value pex_info, # type: PexInfo venv_dirs, # type: VenvDirs ): # type: (...) -> None with _inserted_wheels(pex_info) as cursor: - cursor.executemany( - """ - INSERT OR IGNORE INTO venv_deps ( - venv_hash, - wheel_install_hash - ) VALUES (?, ?) - """, - tuple( - (venv_dirs.short_hash, wheel_install_hash) - for wheel_install_hash in pex_info.distributions.values() - ), - ).close() + if copy_mode is CopyMode.SYMLINK: + cursor.executemany( + """ + INSERT OR IGNORE INTO venv_deps ( + venv_hash, + wheel_install_hash + ) VALUES (?, ?) + """, + tuple( + (venv_dirs.short_hash, wheel_install_hash) + for wheel_install_hash in pex_info.distributions.values() + ), + ).close() + else: + cursor.close() if TYPE_CHECKING: diff --git a/pex/cli/commands/cache/command.py b/pex/cli/commands/cache/command.py index bdb6d5a42..d7790d701 100644 --- a/pex/cli/commands/cache/command.py +++ b/pex/cli/commands/cache/command.py @@ -533,6 +533,8 @@ def prune_pip_caches(wheels): prunable_wheels.add( (prunable_pnav.canonicalized_project_name, prunable_pnav.canonicalized_version) ) + if not prunable_wheels: + return def spawn_list(pip): # type: (Pip) -> SpawnedJob[Tuple[ProjectNameAndVersion, ...]] @@ -545,10 +547,13 @@ def spawn_list(pip): ), ) - all_pips = tuple(iter_all_pips()) + # N.B.:We just need 1 Pip per version (really per paired cache). Whether a Pip has extra + # requirements installed does not affect cache management. + all_pip_versions = tuple({pip.version: pip for pip in iter_all_pips()}.values()) + pip_removes = [] # type: List[Tuple[Pip, str]] for pip, project_name_and_versions in zip( - all_pips, execute_parallel(inputs=all_pips, spawn_func=spawn_list) + all_pip_versions, execute_parallel(inputs=all_pip_versions, spawn_func=spawn_list) ): for pnav in project_name_and_versions: if ( @@ -564,15 +569,24 @@ def spawn_list(pip): ) ) + def parse_remove(stdout): + # type: (bytes) -> int + + # The output from `pip cache remove` is a line like: + # Files removed: 42 + _, sep, count = stdout.decode("utf-8").partition(":") + if sep != ":" or not count: + return 0 + try: + return int(count) + except ValueError: + return 0 + def spawn_remove(args): # type: (Tuple[Pip, str]) -> SpawnedJob[int] pip, wheel_name_glob = args - # Files removed: 1 return SpawnedJob.stdout( - job=pip.spawn_cache_remove(wheel_name_glob), - result_func=lambda stdout: int( - stdout.decode("utf-8").rsplit(":", 1)[1].strip() - ), + job=pip.spawn_cache_remove(wheel_name_glob), result_func=parse_remove ) removes_by_pip = Counter() # type: typing.Counter[str] diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index fe70a523c..0f4d58ee5 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -599,13 +599,14 @@ def ensure_venv( hermetic_scripts=pex_info.venv_hermetic_scripts, ) - if copy_mode is CopyMode.SYMLINK: - with TRACER.timed( - "Recording venv install of {pex} {hash}".format( - pex=pex.path(), hash=pex_info.pex_hash - ) - ): - record_venv_install(pex_info=pex_info, venv_dirs=venv_dirs) + with TRACER.timed( + "Recording venv install of {pex} {hash}".format( + pex=pex.path(), hash=pex_info.pex_hash + ) + ): + record_venv_install( + copy_mode=copy_mode, pex_info=pex_info, venv_dirs=venv_dirs + ) # There are popular Linux distributions with shebang length limits # (BINPRM_BUF_SIZE in /usr/include/linux/binfmts.h) set at 128 characters, so diff --git a/pex/pip/installation.py b/pex/pip/installation.py index 561ee6556..eb66c565d 100644 --- a/pex/pip/installation.py +++ b/pex/pip/installation.py @@ -5,6 +5,7 @@ import hashlib import os +import shutil from collections import OrderedDict from textwrap import dedent @@ -73,7 +74,6 @@ def _pip_installation( isolated_pip_builder = PEXBuilder(path=chroot.work_dir) isolated_pip_builder.info.venv = True - isolated_pip_builder.info.venv_site_packages_copies = True # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect if needed. isolated_pip_builder.info.venv_hermetic_scripts = False for dist_location in iter_distribution_locations(): @@ -123,7 +123,26 @@ def _vendored_installation( def expose_vendored(): # type: () -> Iterator[str] - return third_party.expose(("pip", "setuptools"), interpreter=interpreter) + pip, setuptools = third_party.expose(("pip", "setuptools"), interpreter=interpreter) + base_dir = safe_mkdtemp() + + pip_wheel_dir = os.path.join( + base_dir, + "pip-{pip_version}-py2.py3-none-any.whl".format( + pip_version=PipVersion.VENDORED.version + ), + ) + shutil.copytree(pip, pip_wheel_dir) + yield pip_wheel_dir + + setuptools_wheel_dir = os.path.join( + base_dir, + "setuptools-{setuptools_version}-py2.py3-none-any.whl".format( + setuptools_version=PipVersion.VENDORED.setuptools_version + ), + ) + shutil.copytree(setuptools, setuptools_wheel_dir) + yield setuptools_wheel_dir if not extra_requirements: return _pip_installation( @@ -214,8 +233,33 @@ def bootstrap_pip(): for req in version.requirements: project_name = req.name - target_dir = os.path.join(chroot, "reqs", project_name) - venv.interpreter.execute(["-m", "pip", "install", "--target", target_dir, str(req)]) + specifiers = list(req.specifier) + production_assert(len(specifiers) == 1) + specifier = specifiers[0] + production_assert(specifier.operator == "==") + project_version = specifier.version + target_dir = os.path.join( + chroot, + "reqs", + "{project_name}-{project_version}-py{py_version}-none-any.whl".format( + project_name=project_name, + project_version=project_version, + py_version=(interpreter or PythonInterpreter.get()).version[0], + ), + ) + venv.interpreter.execute( + [ + "-m", + "pip", + "install", + "--no-deps", + "--only-binary", + str(req), + "--target", + target_dir, + str(req), + ] + ) yield target_dir return bootstrap_pip diff --git a/pex/pip/version.py b/pex/pip/version.py index a723bcab8..45a8d8593 100644 --- a/pex/pip/version.py +++ b/pex/pip/version.py @@ -45,11 +45,12 @@ def overridden(cls): def __init__( self, version, # type: str + setuptools_version, # type: str + wheel_version, # type: str + requires_python, # type: str name=None, # type: Optional[str] requirement=None, # type: Optional[str] - setuptools_version=None, # type: Optional[str] - wheel_version=None, # type: Optional[str] - requires_python=None, # type: Optional[str] + setuptools_requirement=None, # type: Optional[str] hidden=False, # type: bool ): # type: (...) -> None @@ -57,22 +58,26 @@ def __init__( def to_requirement( project_name, # type: str - project_version=None, # type: Optional[str] + project_version, # type: str ): # type: (...) -> Requirement return Requirement.parse( "{project_name}=={project_version}".format( project_name=project_name, project_version=project_version ) - if project_version - else project_name ) self.version = Version(version) self.requirement = ( Requirement.parse(requirement) if requirement else to_requirement("pip", version) ) - self.setuptools_requirement = to_requirement("setuptools", setuptools_version) + self.setuptools_version = setuptools_version + self.setuptools_requirement = ( + Requirement.parse(setuptools_requirement) + if setuptools_requirement + else to_requirement("setuptools", setuptools_version) + ) + self.wheel_version = wheel_version self.wheel_requirement = to_requirement("wheel", wheel_version) self.requires_python = SpecifierSet(requires_python) if requires_python else None self.hidden = hidden @@ -174,6 +179,8 @@ def values(cls): name="20.3.4-patched", version="20.3.4+patched", requirement=vendor.PIP_SPEC.requirement, + setuptools_version="44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863", + setuptools_requirement="setuptools", wheel_version="0.37.1", requires_python="<3.12", ) diff --git a/tox.ini b/tox.ini index 08f680d01..3fd9c43e4 100644 --- a/tox.ini +++ b/tox.ini @@ -156,6 +156,9 @@ deps = pip==20.3.4 # This version should track the version in pex/vendor/__init__.py. setuptools==44.0.0 # This version should track the version in pex/vendor/__init__.py. sphinx + # This is just used as a constraint - the dep is via: + # spinx 5.1.1 -> jinja2>=2.3 -> MarkupSafe>=2.0 + MarkupSafe<3 toml==0.10.2 # This version should track the version in pex/vendor/__init__.py. PyGithub==2.4.0