Add support for Python 3.12.

The crux here is supporting a version of Pip that works in 3.12. There is no such released version yet; so this change adds an unreleased Pip version but goes to some length to hide this version from users and make it only activatable by those in the know / CI. What follows is fixing or adjusting many tests. The result is Pex known to work with Python 3.12 ahead of its release by several months and the spectre of Pex 3 / a Pex branch split, etc., dispelled. It turns out Pex can still ship supporting Python 2.7, 3.5, etc. along side supprting 3.12. The main trick here is to use `python3.12 -mvenv` to spirit up a bootstrap Pip that works at least enough to install the unreleased Pip that truly works with Python 3.12. Previously all Pip version bootstrapping was handled exclusively by the vendored Pip.
jsirois · Jul 6, 2023 · 746d863 · 746d863
1 parent 814ac93
commit 746d863
Show file tree

Hide file tree

Showing 60 changed files with 758 additions and 263 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -51,15 +51,27 @@ jobs:
           - os: macos-12
             python-version: [ 3, 11 ]
             pip-version: 20
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 20
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 22_3_1
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 23_1_2
+            tox-env-python: python
+          - os: macos-12
+            python-version: [ 3, 12, "0-beta.3" ]
+            pip-version: 23_2
+            tox-env-python: python3.11
+          - os: ubuntu-22.04
+            python-version: [ 3, 12, "0-beta.3" ]
+            pip-version: 23_2
+            tox-env-python: python3.11
     steps:
       - name: Calculate Pythons to Expose
         id: calculate-pythons-to-expose
@@ -85,8 +97,9 @@ jobs:
           path: ${{ env._PEX_TEST_PYENV_ROOT }}
           key: ${{ matrix.os }}-${{ runner.arch }}-pyenv-root-v1
       - name: Run Unit Tests
-        uses: pantsbuild/actions/run-tox@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb
+        uses: pantsbuild/actions/run-tox@addae8ed8cce2b0a359f447d1bb2ec69ff18f9ca
         with:
+          python: ${{ matrix.tox-env-python }}
           tox-env: py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-pip${{ matrix.pip-version }}
   cpython-unit-tests-legacy:
     name: (${{ matrix.os }}) Pip ${{ matrix.pip-version }} TOXENV=py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}
@@ -192,15 +205,27 @@ jobs:
           - os: macos-12
             python-version: [ 3, 11 ]
             pip-version: 20
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 20
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 22_3_1
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 23_1_2
+            tox-env-python: python
+          - os: macos-12
+            python-version: [ 3, 12, "0-beta.3" ]
+            pip-version: 23_2
+            tox-env-python: python3.11
+          - os: ubuntu-22.04
+            python-version: [ 3, 12, "0-beta.3" ]
+            pip-version: 23_2
+            tox-env-python: python3.11
     steps:
       - name: Calculate Pythons to Expose
         id: calculate-pythons-to-expose
@@ -236,8 +261,9 @@ jobs:
         with:
           ssh-private-key: ${{ env.SSH_PRIVATE_KEY }}
       - name: Run Integration Tests
-        uses: pantsbuild/actions/run-tox@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb
+        uses: pantsbuild/actions/run-tox@addae8ed8cce2b0a359f447d1bb2ec69ff18f9ca
         with:
+          python: ${{ matrix.tox-env-python }}
           tox-env: py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-pip${{ matrix.pip-version }}-integration
   cpython-integration-tests-legacy:
     name: (${{ matrix.os }}) Pip ${{ matrix.pip-version }} TOXENV=py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-integration

diff --git a/pex/build_system/pep_517.py b/pex/build_system/pep_517.py
@@ -30,27 +30,31 @@
 
 
 def _default_build_system(
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> BuildSystem
     global _DEFAULT_BUILD_SYSTEMS
-    build_system = _DEFAULT_BUILD_SYSTEMS.get(pip_version)
+    selected_pip_version = pip_version or PipVersion.DEFAULT
+    build_system = _DEFAULT_BUILD_SYSTEMS.get(selected_pip_version)
     if build_system is None:
         with TRACER.timed(
             "Building {build_backend} build_backend PEX".format(build_backend=DEFAULT_BUILD_BACKEND)
         ):
             extra_env = {}  # type: Dict[str, str]
-            if pip_version is PipVersion.VENDORED:
+            if selected_pip_version is PipVersion.VENDORED:
                 requires = ["setuptools", "wheel"]
                 resolved = tuple(
                     Distribution.load(dist_location)
                     for dist_location in third_party.expose(requires)
                 )
                 extra_env.update(__PEX_UNVENDORED__="1")
             else:
-                requires = [pip_version.setuptools_requirement, pip_version.wheel_requirement]
+                requires = [
+                    selected_pip_version.setuptools_requirement,
+                    selected_pip_version.wheel_requirement,
+                ]
                 resolved = tuple(
                     installed_distribution.fingerprinted_distribution.distribution
                     for installed_distribution in resolver.resolve_requirements(
@@ -68,24 +72,24 @@ def _default_build_system(
                     **extra_env
                 )
             )
-            _DEFAULT_BUILD_SYSTEMS[pip_version] = build_system
+            _DEFAULT_BUILD_SYSTEMS[selected_pip_version] = build_system
     return build_system
 
 
 def _get_build_system(
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
     project_directory,  # type: str
     extra_requirements=None,  # type: Optional[Iterable[str]]
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> Union[BuildSystem, Error]
     custom_build_system_or_error = load_build_system(
         target, resolver, project_directory, extra_requirements=extra_requirements
     )
     if custom_build_system_or_error:
         return custom_build_system_or_error
-    return _default_build_system(pip_version, target, resolver)
+    return _default_build_system(target, resolver, pip_version=pip_version)
 
 
 # Exit code 75 is EX_TEMPFAIL defined in /usr/include/sysexits.h
@@ -100,7 +104,6 @@ def is_hook_unavailable_error(error):
 
 def _invoke_build_hook(
     project_directory,  # type: str
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
     hook_method,  # type: str
@@ -109,6 +112,7 @@ def _invoke_build_hook(
     hook_kwargs=None,  # type: Optional[Mapping[str, Any]]
     stdout=None,  # type: Optional[int]
     stderr=None,  # type: Optional[int]
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> Union[SpawnedJob[Any], Error]
 
@@ -126,7 +130,11 @@ def _invoke_build_hook(
         )
 
     build_system_or_error = _get_build_system(
-        pip_version, target, resolver, project_directory, extra_requirements=hook_extra_requirements
+        target,
+        resolver,
+        project_directory,
+        extra_requirements=hook_extra_requirements,
+        pip_version=pip_version,
     )
     if isinstance(build_system_or_error, Error):
         return build_system_or_error
@@ -180,19 +188,19 @@ def _invoke_build_hook(
 def build_sdist(
     project_directory,  # type: str
     dist_dir,  # type: str
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> Union[Text, Error]
 
     extra_requirements = []
     spawned_job_or_error = _invoke_build_hook(
         project_directory,
-        pip_version,
         target,
         resolver,
         hook_method="get_requires_for_build_sdist",
+        pip_version=pip_version,
     )
     if isinstance(spawned_job_or_error, Error):
         return spawned_job_or_error
@@ -208,12 +216,12 @@ def build_sdist(
 
     spawned_job_or_error = _invoke_build_hook(
         project_directory,
-        pip_version,
         target,
         resolver,
         hook_method="build_sdist",
         hook_args=[dist_dir],
         hook_extra_requirements=extra_requirements,
+        pip_version=pip_version,
     )
     if isinstance(spawned_job_or_error, Error):
         return spawned_job_or_error
@@ -229,20 +237,20 @@ def build_sdist(
 
 def spawn_prepare_metadata(
     project_directory,  # type: str
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> SpawnedJob[DistMetadata]
 
     extra_requirements = []
     spawned_job = try_(
         _invoke_build_hook(
             project_directory,
-            pip_version,
             target,
             resolver,
             hook_method="get_requires_for_build_wheel",
+            pip_version=pip_version,
         )
     )
     try:
@@ -256,12 +264,12 @@ def spawn_prepare_metadata(
     spawned_job = try_(
         _invoke_build_hook(
             project_directory,
-            pip_version,
             target,
             resolver,
             hook_method="prepare_metadata_for_build_wheel",
             hook_args=[build_dir],
             hook_extra_requirements=extra_requirements,
+            pip_version=pip_version,
         )
     )
     return spawned_job.map(lambda _: DistMetadata.load(build_dir))
diff --git a/pex/build_system/testing.py b/pex/build_system/testing.py
@@ -37,16 +37,16 @@ def assert_expected_dist(dist):
 
     sdist_dir = os.path.join(str(tmpdir), "sdist_dir")
 
-    # This test utility is used by all versions of Python Pex supports; so we need to use the
-    # vendored Pip which is guaranteed to work with all those Python versions.
-    pip_version = PipVersion.VENDORED
+    # This test utility is used by all versions of Python Pex supports; so we need to use a Pip
+    # which is guaranteed to work with the current Python version.
+    pip_version = PipVersion.DEFAULT
 
     location = build_sdist(
         project_dir,
         sdist_dir,
-        pip_version,
         LocalInterpreter.create(),
         ConfiguredResolver(PipConfiguration(version=pip_version)),
+        pip_version=pip_version,
     )
     assert not isinstance(location, Error), location
     assert sdist_dir == os.path.dirname(location)
@@ -56,7 +56,9 @@ def assert_expected_dist(dist):
 
     # Verify the sdist is valid such that we can build a wheel from it.
     wheel_dir = os.path.join(str(tmpdir), "wheel_dir")
-    get_pip().spawn_build_wheels(distributions=[sdist.location], wheel_dir=wheel_dir).wait()
+    get_pip(resolver=ConfiguredResolver(pip_configuration=PipConfiguration())).spawn_build_wheels(
+        distributions=[sdist.location], wheel_dir=wheel_dir
+    ).wait()
     wheels = glob.glob(os.path.join(wheel_dir, "*.whl"))
     assert 1 == len(wheels)
     assert_expected_dist(Distribution.load(wheels[0]))
diff --git a/pex/common.py b/pex/common.py
@@ -630,6 +630,8 @@ def get_parent_dir(path):
 
             def maybe_write_parent_dirs(path):
                 # type: (str) -> None
+                if path == strip_prefix:
+                    return
                 parent_dir = get_parent_dir(path)
                 if parent_dir is None or parent_dir in written_dirs:
                     return

diff --git a/pex/environment.py b/pex/environment.py
@@ -14,12 +14,13 @@
 from pex.dist_metadata import Distribution, Requirement
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.inherit_path import InheritPath
+from pex.interpreter import PythonInterpreter
 from pex.layout import maybe_install
 from pex.orderedset import OrderedSet
 from pex.pep_425 import CompatibilityTags, TagRank
 from pex.pep_503 import ProjectName
 from pex.pex_info import PexInfo
-from pex.targets import Target
+from pex.targets import LocalInterpreter, Target
 from pex.third_party.packaging import specifiers
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
@@ -52,9 +53,12 @@ def _import_pkg_resources():
         from pex import third_party
 
         third_party.install(expose=["setuptools"])
-        import pkg_resources  # vendor:skip
+        try:
+            import pkg_resources  # vendor:skip
 
-        return pkg_resources, True
+            return pkg_resources, True
+        except ImportError:
+            return None, False
 
 
 @attr.s(frozen=True)
@@ -642,7 +646,7 @@ def _declare_namespace_packages(cls, resolved_dists):
         # since we'll only introduce our shaded version when no other standard version is present and
         # even then tear it all down when we hand off from the bootstrap to user code.
         pkg_resources, vendored = _import_pkg_resources()
-        if vendored:
+        if not pkg_resources or vendored:
             dists = "\n".join(
                 "\n{index}. {dist} namespace packages:\n  {ns_packages}".format(
                     index=index + 1,
@@ -651,6 +655,20 @@ def _declare_namespace_packages(cls, resolved_dists):
                 )
                 for index, (dist, ns_packages) in enumerate(namespace_packages_by_dist.items())
             )
+            if not pkg_resources:
+                current_interpreter = PythonInterpreter.get()
+                pex_warnings.warn(
+                    "The legacy `pkg_resources` package cannot be imported by the "
+                    "{interpreter} {version} interpreter at {path}.\n"
+                    "These distributions will fail to work properly:\n{dists}".format(
+                        interpreter=current_interpreter.identity.interpreter,
+                        version=current_interpreter.python,
+                        path=current_interpreter.binary,
+                        dists=dists,
+                    )
+                )
+                return
+
             pex_warnings.warn(
                 "The `pkg_resources` package was loaded from a pex vendored version when "
                 "declaring namespace packages defined by:\n{dists}\n\nThese distributions "

diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py
@@ -316,11 +316,12 @@ def iter_compatible_versions(
     # N.B.: Pex does not support the missing 3.x versions here.
     PythonVersion(Lifecycle.EOL, 3, 5, 10),
     PythonVersion(Lifecycle.EOL, 3, 6, 15),
-    PythonVersion(Lifecycle.STABLE, 3, 7, 15),
-    PythonVersion(Lifecycle.STABLE, 3, 8, 15),
-    PythonVersion(Lifecycle.STABLE, 3, 9, 15),
-    PythonVersion(Lifecycle.STABLE, 3, 10, 8),
-    PythonVersion(Lifecycle.STABLE, 3, 11, 0),
+    # ^-- EOL --^
+    PythonVersion(Lifecycle.STABLE, 3, 7, 17),
+    PythonVersion(Lifecycle.STABLE, 3, 8, 17),
+    PythonVersion(Lifecycle.STABLE, 3, 9, 17),
+    PythonVersion(Lifecycle.STABLE, 3, 10, 12),
+    PythonVersion(Lifecycle.STABLE, 3, 11, 4),
     PythonVersion(Lifecycle.DEV, 3, 12, 0),
 )