From 145a1efe0eca837bc9cf26ec7e2684e9b3657119 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 11 Jul 2023 11:07:40 -0700 Subject: [PATCH] Prepare for Pip 23.2 release. A recent Pip commit has broken `pip --use-deprecated legacy-resolver download ...` which is our backwards-compatible default use case. Whether or not Pip decides to fix, the legacy resolver will be yanked at some point and it seems reasonable to just require any Pex user on Python 3.12 and thus Pip 23.2 to only use the Pip 2020 resolver. See: https://github.com/pypa/pip/issues/12138 --- pex/build_system/testing.py | 15 ++++++++---- pex/pip/tool.py | 4 ++-- pex/pip/version.py | 4 ++-- pex/platforms.py | 6 ++--- pex/resolve/configured_resolver.py | 11 ++++++--- pex/resolve/lockfile/model.py | 7 +++--- pex/resolve/resolver_configuration.py | 24 +++++++++++++++++-- pex/resolve/resolver_options.py | 4 ++-- tests/build_system/test_pep_518.py | 4 +--- .../cli/commands/test_issue_1801.py | 11 ++++++++- .../cli/commands/test_issue_2050.py | 1 - tests/integration/cli/commands/test_lock.py | 1 + tests/integration/resolve/test_issue_1918.py | 1 + tests/integration/test_downloads.py | 2 +- tests/integration/test_integration.py | 1 + tests/test_environment.py | 8 +++---- tests/test_pex.py | 6 ++--- tests/test_resolver.py | 6 ++--- tox.ini | 2 +- 19 files changed, 79 insertions(+), 39 deletions(-) diff --git a/pex/build_system/testing.py b/pex/build_system/testing.py index eca8ec8ad..018810c45 100644 --- a/pex/build_system/testing.py +++ b/pex/build_system/testing.py @@ -13,7 +13,7 @@ from pex.pip.installation import get_pip from pex.pip.version import PipVersion from pex.resolve.configured_resolver import ConfiguredResolver -from pex.resolve.resolver_configuration import PipConfiguration +from pex.resolve.resolver_configuration import PipConfiguration, ResolverVersion from pex.result import Error from pex.targets import LocalInterpreter from pex.typing import TYPE_CHECKING @@ -38,14 +38,19 @@ def assert_expected_dist(dist): sdist_dir = os.path.join(str(tmpdir), "sdist_dir") # This test utility is used by all versions of Python Pex supports; so we need to use a Pip - # which is guaranteed to work with the current Python version. + # setup which is guaranteed to work with the current Python version. pip_version = PipVersion.DEFAULT + resolver_version = ResolverVersion.default(pip_version) + target = LocalInterpreter.create() + resolver = ConfiguredResolver( + PipConfiguration(version=pip_version, resolver_version=resolver_version) + ) location = build_sdist( project_dir, sdist_dir, - LocalInterpreter.create(), - ConfiguredResolver(PipConfiguration(version=pip_version)), + target, + resolver, pip_version=pip_version, ) assert not isinstance(location, Error), location @@ -56,7 +61,7 @@ def assert_expected_dist(dist): # Verify the sdist is valid such that we can build a wheel from it. wheel_dir = os.path.join(str(tmpdir), "wheel_dir") - get_pip(resolver=ConfiguredResolver(pip_configuration=PipConfiguration())).spawn_build_wheels( + get_pip(resolver=resolver).spawn_build_wheels( distributions=[sdist.location], wheel_dir=wheel_dir ).wait() wheels = glob.glob(os.path.join(wheel_dir, "*.whl")) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 08b5613bc..6fe382fed 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -141,7 +141,7 @@ def create( password_entries=(), # type: Iterable[PasswordEntry] ): # type: (...) -> PackageIndexConfiguration - resolver_version = resolver_version or ResolverVersion.PIP_LEGACY + resolver_version = resolver_version or ResolverVersion.default(pip_version) network_configuration = network_configuration or NetworkConfiguration() # We must pass `--client-cert` via PIP_CLIENT_CERT to work around @@ -241,7 +241,7 @@ def _calculate_resolver_version(package_index_configuration=None): return ( package_index_configuration.resolver_version if package_index_configuration - else ResolverVersion.PIP_LEGACY + else ResolverVersion.default() ) @classmethod diff --git a/pex/pip/version.py b/pex/pip/version.py index 1663bf337..9e58df55f 100644 --- a/pex/pip/version.py +++ b/pex/pip/version.py @@ -202,8 +202,8 @@ def values(cls): ) v23_2 = PipVersionValue( - version="23.2.dev0+8a1eea4a", - requirement="pip @ git+https://github.com/pypa/pip@8a1eea4aaedb1fb1c6b4c652cd0c43502f05ff37", + version="23.2.dev0+ea727e4d", + requirement="pip @ git+https://github.com/pypa/pip@ea727e4d6ab598f34f97c50a22350febc1214a97", setuptools_version="67.8.0", wheel_version="0.40.0", requires_python=">=3.7", diff --git a/pex/platforms.py b/pex/platforms.py index 355836083..49e1fbad5 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -240,9 +240,9 @@ def parse_tags(output): job = SpawnedJob.stdout( # TODO(John Sirois): Plumb pip_version and the user-configured resolver: # https://github.com/pantsbuild/pex/issues/1894 - job=get_pip( - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()) - ).spawn_debug(platform=self, manylinux=manylinux), + job=get_pip(resolver=ConfiguredResolver.default()).spawn_debug( + platform=self, manylinux=manylinux + ), result_func=parse_tags, ) return job.await_result() diff --git a/pex/resolve/configured_resolver.py b/pex/resolve/configured_resolver.py index 661ee0343..db8ac319b 100644 --- a/pex/resolve/configured_resolver.py +++ b/pex/resolve/configured_resolver.py @@ -4,10 +4,10 @@ from __future__ import absolute_import from pex import resolver -from pex.pip.version import PipVersionValue +from pex.pip.version import PipVersion, PipVersionValue from pex.resolve import lock_resolver from pex.resolve.lockfile.model import Lockfile -from pex.resolve.resolver_configuration import PipConfiguration, ReposConfiguration +from pex.resolve.resolver_configuration import PipConfiguration, ReposConfiguration, ResolverVersion from pex.resolve.resolvers import Installed, Resolver from pex.result import try_ from pex.targets import Targets @@ -29,7 +29,12 @@ class ConfiguredResolver(Resolver): @classmethod def default(cls): # type: () -> ConfiguredResolver - return cls(PipConfiguration()) + pip_version = PipVersion.DEFAULT + return cls( + PipConfiguration( + version=pip_version, resolver_version=ResolverVersion.default(pip_version) + ) + ) pip_configuration = attr.ib() # type: PipConfiguration diff --git a/pex/resolve/lockfile/model.py b/pex/resolve/lockfile/model.py index 2afd17504..792646dc2 100644 --- a/pex/resolve/lockfile/model.py +++ b/pex/resolve/lockfile/model.py @@ -34,7 +34,6 @@ def create( style, # type: LockStyle.Value requires_python, # type: Iterable[str] target_systems, # type: Iterable[TargetSystem.Value] - resolver_version, # type: ResolverVersion.Value requirements, # type: Iterable[Union[Requirement, ParsedRequirement]] constraints, # type: Iterable[Requirement] allow_prereleases, # type: bool @@ -47,6 +46,7 @@ def create( locked_resolves, # type: Iterable[LockedResolve] source=None, # type: Optional[str] pip_version=None, # type: Optional[PipVersionValue] + resolver_version=None, # type: Optional[ResolverVersion.Value] ): # type: (...) -> Lockfile @@ -88,13 +88,14 @@ def extract_requirement(req): resolve_requirements = OrderedSet(extract_requirement(req) for req in requirements) + pip_ver = pip_version or PipVersion.DEFAULT return cls( pex_version=pex_version, style=style, requires_python=SortedTuple(requires_python), target_systems=SortedTuple(target_systems), - pip_version=pip_version or PipVersion.DEFAULT, - resolver_version=resolver_version, + pip_version=pip_ver, + resolver_version=resolver_version or ResolverVersion.default(pip_ver), requirements=SortedTuple(resolve_requirements, key=str), constraints=SortedTuple(constraints, key=str), allow_prereleases=allow_prereleases, diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index c19bbc28d..582828f09 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -4,12 +4,12 @@ from __future__ import absolute_import import itertools -import os from pex.auth import PasswordEntry from pex.enum import Enum from pex.jobs import DEFAULT_MAX_JOBS from pex.network_configuration import NetworkConfiguration +from pex.pep_440 import Version from pex.pip.version import PipVersion, PipVersionValue from pex.typing import TYPE_CHECKING @@ -31,6 +31,26 @@ class ResolverVersion(Enum["ResolverVersion.Value"]): class Value(Enum.Value): pass + @staticmethod + def _supports_legacy_resolver(pip_version=None): + # type: (Optional[PipVersionValue]) -> bool + pip_ver = pip_version or PipVersion.DEFAULT + return pip_ver.version < Version("23.2") + + @classmethod + def applies( + cls, + resolver_version, # type: ResolverVersion.Value + pip_version=None, + ): + # type: (...) -> bool + return resolver_version is cls.PIP_2020 or cls._supports_legacy_resolver(pip_version) + + @classmethod + def default(cls, pip_version=None): + # type: (Optional[PipVersionValue]) -> ResolverVersion.Value + return cls.PIP_LEGACY if cls._supports_legacy_resolver(pip_version) else cls.PIP_2020 + PIP_LEGACY = Value("pip-legacy-resolver") PIP_2020 = Value("pip-2020-resolver") @@ -63,7 +83,6 @@ def create( @attr.s(frozen=True) class PipConfiguration(object): - resolver_version = attr.ib(default=ResolverVersion.PIP_LEGACY) # type: ResolverVersion.Value repos_configuration = attr.ib(default=ReposConfiguration()) # type: ReposConfiguration network_configuration = attr.ib(default=NetworkConfiguration()) # type: NetworkConfiguration allow_prereleases = attr.ib(default=False) # type: bool @@ -76,6 +95,7 @@ class PipConfiguration(object): max_jobs = attr.ib(default=DEFAULT_MAX_JOBS) # type: int preserve_log = attr.ib(default=False) # type: bool version = attr.ib(default=None) # type: Optional[PipVersionValue] + resolver_version = attr.ib(default=None) # type: Optional[ResolverVersion.Value] allow_version_fallback = attr.ib(default=True) # type: bool diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 2baecf148..08e013434 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -72,7 +72,7 @@ def register( parser.add_argument( "--resolver-version", dest="resolver_version", - default=default_resolver_configuration.resolver_version, + default=ResolverVersion.default(), choices=ResolverVersion.values(), type=ResolverVersion.for_value, help=( @@ -446,7 +446,6 @@ def create_pip_configuration(options): pip_version = PipVersion.for_value(options.pip_version) return PipConfiguration( - resolver_version=options.resolver_version, repos_configuration=repos_configuration, network_configuration=create_network_configuration(options), allow_prereleases=options.allow_prereleases, @@ -459,6 +458,7 @@ def create_pip_configuration(options): max_jobs=get_max_jobs_value(options), preserve_log=options.preserve_pip_download_log, version=pip_version, + resolver_version=options.resolver_version, allow_version_fallback=options.allow_pip_version_fallback, ) diff --git a/tests/build_system/test_pep_518.py b/tests/build_system/test_pep_518.py index bb44cafc2..94daf142d 100644 --- a/tests/build_system/test_pep_518.py +++ b/tests/build_system/test_pep_518.py @@ -10,9 +10,7 @@ from pex.common import touch from pex.environment import PEXEnvironment from pex.pep_503 import ProjectName -from pex.pip.version import PipVersion from pex.resolve.configured_resolver import ConfiguredResolver -from pex.resolve.resolver_configuration import PipConfiguration from pex.result import Error from pex.targets import LocalInterpreter from pex.typing import TYPE_CHECKING @@ -26,7 +24,7 @@ def load_build_system(project_directory): # type: (...) -> Union[Optional[BuildSystem], Error] return pep_518.load_build_system( LocalInterpreter.create(), - ConfiguredResolver(PipConfiguration(version=PipVersion.DEFAULT)), + ConfiguredResolver.default(), project_directory, ) diff --git a/tests/integration/cli/commands/test_issue_1801.py b/tests/integration/cli/commands/test_issue_1801.py index 9d92c9852..a172e22dc 100644 --- a/tests/integration/cli/commands/test_issue_1801.py +++ b/tests/integration/cli/commands/test_issue_1801.py @@ -4,10 +4,12 @@ import os.path import re +import pytest from colors import green from pex.cli.testing import run_pex3 from pex.resolve.lockfile import json_codec +from pex.resolve.resolver_configuration import ResolverVersion from pex.testing import run_pex_command @@ -58,6 +60,13 @@ def test_preserve_pip_download_log(): assert expected_hash == artifact.fingerprint.hash +@pytest.mark.skipif( + ResolverVersion.default() is ResolverVersion.PIP_2020, + reason=( + "The PIP_2020 resolver triggers download analysis in normal resolves but this test is " + "concerned with the case when there is no analysis to be performed." + ), +) def test_preserve_pip_download_log_none(): # type: () -> None @@ -76,4 +85,4 @@ def test_preserve_pip_download_log_none(): assert ( "pex: The `pip download` log is not being utilized, to see more `pip download` details, " "re-run with more Pex verbosity (more `-v`s).\n" - ) in result.error + ) in result.error, result.error diff --git a/tests/integration/cli/commands/test_issue_2050.py b/tests/integration/cli/commands/test_issue_2050.py index 685b5ebe1..b2cbb3760 100644 --- a/tests/integration/cli/commands/test_issue_2050.py +++ b/tests/integration/cli/commands/test_issue_2050.py @@ -15,7 +15,6 @@ from pex.dist_metadata import Requirement from pex.pep_440 import Version from pex.pep_503 import ProjectName -from pex.pip.version import PipVersion from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.locked_resolve import LockedRequirement from pex.resolve.lockfile import json_codec diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py index 95f01a30e..cf725ee5d 100644 --- a/tests/integration/cli/commands/test_lock.py +++ b/tests/integration/cli/commands/test_lock.py @@ -1821,6 +1821,7 @@ def test_excludes_pep517_build_requirements_issue_1565(tmpdir): resolver_version, EXPECTED_LOCKFILES.get(resolver_version), id=resolver_version.value ) for resolver_version in ResolverVersion.values() + if ResolverVersion.applies(resolver_version) ], ) def test_universal_lock( diff --git a/tests/integration/resolve/test_issue_1918.py b/tests/integration/resolve/test_issue_1918.py index 8c0b105ce..6a24468ee 100644 --- a/tests/integration/resolve/test_issue_1918.py +++ b/tests/integration/resolve/test_issue_1918.py @@ -71,6 +71,7 @@ def has_ssh_access(): [ pytest.param(resolver_version, id=resolver_version.value) for resolver_version in ResolverVersion.values() + if ResolverVersion.applies(resolver_version) ], ) def test_redacted_requirement_handling( diff --git a/tests/integration/test_downloads.py b/tests/integration/test_downloads.py index 09cf69006..46c607743 100644 --- a/tests/integration/test_downloads.py +++ b/tests/integration/test_downloads.py @@ -56,7 +56,7 @@ def file_artifact( @pytest.fixture def downloader(): # type: () -> ArtifactDownloader - return ArtifactDownloader(ConfiguredResolver(PipConfiguration())) + return ArtifactDownloader(ConfiguredResolver.default()) def test_issue_1849_download_foreign_artifact( diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index a8d7f8500..4d97e8626 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -1666,6 +1666,7 @@ def test_constraint_file_from_url(tmpdir): # requirements URL contains `requests[security]>=2.20.1` which uses an extra; so we use # older Pip here. "--pip-version=20.3.4-patched", + "--resolver-version=pip-legacy-resolver", "-o", pex_file, ], diff --git a/tests/test_environment.py b/tests/test_environment.py index 86336edab..329018092 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -141,7 +141,7 @@ def add_requirements(builder): for installed_dist in resolver.resolve( targets=Targets(interpreters=(builder.interpreter,)), requirements=requirements, - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + resolver=ConfiguredResolver.default(), ).installed_distributions: builder.add_distribution(installed_dist.distribution) for direct_req in installed_dist.direct_requirements: @@ -274,7 +274,7 @@ def bad_interpreter(): for installed_dist in resolver.resolve( targets=Targets(interpreters=(pb.interpreter,)), requirements=["psutil==5.4.3"], - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + resolver=ConfiguredResolver.default(), ).installed_distributions: pb.add_dist_location(installed_dist.distribution.location) pb.build(pex_file) @@ -342,7 +342,7 @@ def test_activate_extras_issue_615(): for installed_dist in resolver.resolve( targets=Targets(interpreters=(pb.interpreter,)), requirements=["pex[requests]==1.6.3"], - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + resolver=ConfiguredResolver.default(), ).installed_distributions: for direct_req in installed_dist.direct_requirements: pb.add_requirement(direct_req) @@ -369,7 +369,7 @@ def assert_namespace_packages_warning(distribution, version, expected_warning): pb = PEXBuilder() for installed_dist in resolver.resolve( requirements=[requirement], - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + resolver=ConfiguredResolver.default(), ).installed_distributions: pb.add_dist_location(installed_dist.distribution.location) pb.freeze() diff --git a/tests/test_pex.py b/tests/test_pex.py index cd44ce688..53b5a69c4 100644 --- a/tests/test_pex.py +++ b/tests/test_pex.py @@ -839,7 +839,7 @@ def test_pex_run_custom_setuptools_useable( # type: (...) -> None result = resolver.resolve( requirements=[setuptools_requirement], - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + resolver=ConfiguredResolver.default(), ) dists = [installed_dist.distribution for installed_dist in result.installed_distributions] with temporary_dir() as temp_dir: @@ -864,7 +864,7 @@ def test_pex_run_conflicting_custom_setuptools_useable( result = resolver.resolve( requirements=[setuptools_requirement], - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + resolver=ConfiguredResolver.default(), ) dists = [installed_dist.distribution for installed_dist in result.installed_distributions] with temporary_dir() as temp_dir: @@ -891,7 +891,7 @@ def test_pex_run_custom_pex_useable(): old_pex_version = "0.7.0" result = resolver.resolve( requirements=["pex=={}".format(old_pex_version), "setuptools==40.6.3"], - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + resolver=ConfiguredResolver.default(), ) dists = [installed_dist.distribution for installed_dist in result.installed_distributions] with temporary_dir() as temp_dir: diff --git a/tests/test_resolver.py b/tests/test_resolver.py index a22f0effe..b7373b7b5 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -52,7 +52,7 @@ def create_sdist(**kwargs): project_directory=project_dir, dist_dir=dist_dir, target=targets.current(), - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + resolver=ConfiguredResolver.default(), ) dists = os.listdir(dist_dir) @@ -68,7 +68,7 @@ def build_wheel(**kwargs): def resolve(**kwargs): # type: (**Any) -> Installed - kwargs.setdefault("resolver", ConfiguredResolver(pip_configuration=PipConfiguration())) + kwargs.setdefault("resolver", ConfiguredResolver.default()) return resolve_under_test(**kwargs) @@ -500,7 +500,7 @@ def test_download(): result = download( requirements=["{}[foo]".format(project1_sdist)], find_links=[os.path.dirname(project2_wheel)], - resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + resolver=ConfiguredResolver.default(), ) for local_distribution in result.local_distributions: distribution = pkginfo.get_metadata(local_distribution.path) diff --git a/tox.ini b/tox.ini index 799d5bd0f..99e5c2fab 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,7 @@ setenv = pip23_1: _PEX_PIP_VERSION=23.1 pip23_1_1: _PEX_PIP_VERSION=23.1.1 pip23_1_2: _PEX_PIP_VERSION=23.1.2 - pip23_2: _PEX_PIP_VERSION=23.2.dev0+8a1eea4a + pip23_2: _PEX_PIP_VERSION=23.2.dev0+ea727e4d # Python 3 (until a fix here in 3.9: https://bugs.python.org/issue13601) switched from stderr # being unbuffered to stderr being buffered by default. This can lead to tests checking stderr # failing to see what they expect if the stderr buffer block has not been flushed. Force stderr