From ca94e77ef485d84d04b8c0d05d92ba8b8f1492e5 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 19 Sep 2024 11:33:50 -0700 Subject: [PATCH 1/4] Pre-emptively cull unsatisfiable interpreter constraints. When `--interpreter-constraint`s are specified that are unsatisfiable, Pex now either errors if all given interpreter constraints are unsatisfiable or else warns and continues with only the remaining valid interpreter constraints after culling the unsatisfiable ones. Fixes #432 --- CHANGES.md | 12 +++++ pex/interpreter_constraints.py | 63 ++++++++++++++++++++++++--- pex/specifier_sets.py | 27 +++++++++++- pex/version.py | 2 +- tests/test_interpreter_constraints.py | 60 ++++++++++++++++++++++++- tests/test_specifier_sets.py | 28 +++++++++++- 6 files changed, 181 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9183f6c1c..8ca086c25 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,17 @@ # Release Notes +## 2.20.1 + +This release fixes Pex `--interpreter-constraint` handling such that +any supplied interpreter constraints which are in principle +unstaisfiable either raise an error or else cause a warning to be issued +when other viable interpreter constraints have also been specified. For +example, `--interpreter-constraint ==3.11.*,==3.12.*` now errors and +`--interpreter-constraint '>=3.8,<3.8' --interpreter-constraint ==3.9.*` +now warns, culling `>3.8,<3.8` and continuing using only `==3.9.*`. + +* Pre-emptively cull unsatisfiable interpreter constraints. (#2542) + ## 2.20.0 This release adds the `--pip-log` alias for the existing diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index 86b65bf61..afd026869 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -7,16 +7,19 @@ import itertools +from pex import pex_warnings +from pex.common import pluralize from pex.compatibility import indent from pex.dist_metadata import Requirement, RequirementParseError from pex.enum import Enum from pex.interpreter import PythonInterpreter from pex.orderedset import OrderedSet +from pex.specifier_sets import UnsatisfiableSpecifierSet, as_range from pex.third_party.packaging.specifiers import SpecifierSet from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Iterable, Iterator, Optional, Tuple + from typing import Any, Iterable, Iterator, List, Optional, Tuple import attr # vendor:skip @@ -25,6 +28,10 @@ from pex.third_party import attr +class UnsatisfiableError(ValueError): + """Indicates an unsatisfiable interpreter constraint, e.g. `>=3.8,<3.8`.""" + + @attr.s(frozen=True) class InterpreterConstraint(object): @classmethod @@ -75,6 +82,18 @@ def exact_version(cls, interpreter=None): specifier = attr.ib() # type: SpecifierSet name = attr.ib(default=None) # type: Optional[str] + @specifier.validator + def _validate_specifier( + self, + _attribute, # type: Any + value, # type: SpecifierSet + ): + # type: (...) -> None + if isinstance(as_range(value), UnsatisfiableSpecifierSet): + raise UnsatisfiableError( + "The interpreter constraint {constraint} is unsatisfiable.".format(constraint=self) + ) + def iter_matching(self, paths=None): # type: (Optional[Iterable[str]]) -> Iterator[PythonInterpreter] for interp in PythonInterpreter.iter(paths=paths): @@ -103,11 +122,45 @@ class InterpreterConstraints(object): @classmethod def parse(cls, *constraints): # type: (str) -> InterpreterConstraints - return cls( - constraints=tuple( - InterpreterConstraint.parse(constraint) for constraint in OrderedSet(constraints) + + interpreter_constraints = [] # type: List[InterpreterConstraint] + unsatisfiable = [] # type: List[Tuple[str, UnsatisfiableError]] + all_constraints = OrderedSet(constraints) + for constraint in all_constraints: + try: + interpreter_constraints.append(InterpreterConstraint.parse(constraint)) + except UnsatisfiableError as e: + unsatisfiable.append((constraint, e)) + + if unsatisfiable: + if len(unsatisfiable) == 1: + _, err = unsatisfiable[0] + if not interpreter_constraints: + raise err + message = str(err) + else: + message = ( + "Given interpreter constraints are unsatisfiable:\n{unsatisfiable}".format( + unsatisfiable="\n".join(constraint for constraint, _ in unsatisfiable) + ) + ) + + if not interpreter_constraints: + raise UnsatisfiableError(message) + pex_warnings.warn( + "Only {count} interpreter {constraints} {are} valid amongst: {all_constraints}.\n" + "{message}\n" + "Continuing using only {interpreter_constraints}".format( + count=len(interpreter_constraints), + constraints=pluralize(interpreter_constraints, "constraint"), + are="is" if len(interpreter_constraints) == 1 else "are", + all_constraints=" or ".join(all_constraints), + message=message, + interpreter_constraints=" or ".join(map(str, interpreter_constraints)), + ) ) - ) + + return cls(constraints=tuple(interpreter_constraints)) constraints = attr.ib(default=()) # type: Tuple[InterpreterConstraint, ...] diff --git a/pex/specifier_sets.py b/pex/specifier_sets.py index 37263abe9..b4222f396 100644 --- a/pex/specifier_sets.py +++ b/pex/specifier_sets.py @@ -18,6 +18,16 @@ from pex.third_party import attr +def _ensure_specifier_set(specifier_set): + # type: (Union[str, SpecifierSet]) -> SpecifierSet + return specifier_set if isinstance(specifier_set, SpecifierSet) else SpecifierSet(specifier_set) + + +@attr.s(frozen=True) +class UnsatisfiableSpecifierSet(object): + specifier_set = attr.ib(converter=_ensure_specifier_set) # type: SpecifierSet + + @attr.s(frozen=True) class ArbitraryEquality(object): version = attr.ib() # type: str @@ -239,7 +249,7 @@ def _bounds(specifier_set): def as_range(specifier_set): - # type: (Union[str, SpecifierSet]) -> Union[ArbitraryEquality, Range] + # type: (Union[str, SpecifierSet]) -> Union[ArbitraryEquality, Range, UnsatisfiableSpecifierSet] lower_bounds = [] # type: List[LowerBound] upper_bounds = [] # type: List[UpperBound] @@ -281,6 +291,14 @@ def as_range(specifier_set): upper = new_upper excludes.remove(exclude) + # N.B.: Since we went through exclude merging above, there is no need to consider those here + # when checking for unsatisfiable specifier sets. + if lower and upper: + if lower.version > upper.version: + return UnsatisfiableSpecifierSet(specifier_set) + if lower.version == upper.version and (not lower.inclusive or not upper.inclusive): + return UnsatisfiableSpecifierSet(specifier_set) + return Range( lower=lower, upper=upper, @@ -295,9 +313,16 @@ def includes( # type: (...) -> bool included_range = as_range(specifier) + if isinstance(included_range, UnsatisfiableSpecifierSet): + return False + candidate_range = as_range(candidate) + if isinstance(candidate_range, UnsatisfiableSpecifierSet): + return False + if isinstance(included_range, ArbitraryEquality) or isinstance( candidate_range, ArbitraryEquality ): return included_range == candidate_range + return candidate_range in included_range diff --git a/pex/version.py b/pex/version.py index 9c268901c..451441830 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.20.0" +__version__ = "2.20.1" diff --git a/tests/test_interpreter_constraints.py b/tests/test_interpreter_constraints.py index 80cc5f461..e89ece187 100644 --- a/tests/test_interpreter_constraints.py +++ b/tests/test_interpreter_constraints.py @@ -1,12 +1,21 @@ # Copyright 2022 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). - import itertools import sys +from textwrap import dedent + +import pytest from pex import interpreter_constraints from pex.interpreter import PythonInterpreter -from pex.interpreter_constraints import COMPATIBLE_PYTHON_VERSIONS, InterpreterConstraint, Lifecycle +from pex.interpreter_constraints import ( + COMPATIBLE_PYTHON_VERSIONS, + InterpreterConstraint, + InterpreterConstraints, + Lifecycle, + UnsatisfiableError, +) +from pex.pex_warnings import PEXWarning from pex.typing import TYPE_CHECKING from testing import PY38, ensure_python_interpreter @@ -23,6 +32,53 @@ def test_parse(): assert py38 not in InterpreterConstraint.parse("==3.8.*", default_interpreter="PyPy") assert py38 not in InterpreterConstraint.parse("PyPy==3.8.*") + with pytest.raises( + UnsatisfiableError, match="The interpreter constraint ==3.8.*,==3.9.* is unsatisfiable." + ): + InterpreterConstraint.parse("==3.8.*,==3.9.*") + + with pytest.raises( + UnsatisfiableError, match="The interpreter constraint ==3.8.*,==3.9.* is unsatisfiable." + ): + InterpreterConstraints.parse("==3.8.*,==3.9.*") + + with pytest.raises( + UnsatisfiableError, + match=dedent( + """\ + Given interpreter constraints are unsatisfiable: + ==3.8.*,==3.9.* + ==3.9.*,<3.9 + """ + ).strip(), + ): + InterpreterConstraints.parse("==3.8.*,==3.9.*", "==3.9.*,<3.9") + + with pytest.warns( + PEXWarning, + match=dedent( + """\ + Only 2 interpreter constraints are valid amongst: CPython==3.10.*,==3.11.* or CPython==3.10.*,==3.12.* or CPython==3.11.* or CPython==3.11.*,==3.12.* or CPython==3.11.*,==3.9.* or CPython==3.12.* or CPython==3.12.*,==3.9.*. + Given interpreter constraints are unsatisfiable: + CPython==3.10.*,==3.11.* + CPython==3.10.*,==3.12.* + CPython==3.11.*,==3.12.* + CPython==3.11.*,==3.9.* + CPython==3.12.*,==3.9.* + Continuing using only CPython==3.11.* or CPython==3.12.* + """ + ).strip(), + ): + InterpreterConstraints.parse( + "CPython==3.10.*,==3.11.*", + "CPython==3.10.*,==3.12.*", + "CPython==3.11.*", + "CPython==3.11.*,==3.12.*", + "CPython==3.11.*,==3.9.*", + "CPython==3.12.*", + "CPython==3.12.*,==3.9.*", + ) + def iter_compatible_versions(*requires_python): # type: (*str) -> List[Tuple[int, int, int]] diff --git a/tests/test_specifier_sets.py b/tests/test_specifier_sets.py index e199c22a9..fc6e2fd2d 100644 --- a/tests/test_specifier_sets.py +++ b/tests/test_specifier_sets.py @@ -8,9 +8,17 @@ import pytest from pex.pep_440 import Version -from pex.specifier_sets import ExcludedRange, LowerBound, Range, UpperBound, as_range, includes +from pex.specifier_sets import ( + ExcludedRange, + LowerBound, + Range, + UnsatisfiableSpecifierSet, + UpperBound, + as_range, + includes, +) from pex.third_party.packaging.specifiers import InvalidSpecifier -from pex.typing import TYPE_CHECKING +from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: pass @@ -302,3 +310,19 @@ def test_wildcard_version_suffix_handling(): # Local-release specifiers. assert_wildcard_handling("+bob") + + +def test_unsatisfiable(): + # type: () -> None + + assert isinstance(as_range(">3,<2"), UnsatisfiableSpecifierSet) + + assert isinstance(as_range(">=2.7,<2.7"), UnsatisfiableSpecifierSet) + assert isinstance(as_range(">2.7,<=2.7"), UnsatisfiableSpecifierSet) + assert isinstance(as_range(">2.7,<2.7"), UnsatisfiableSpecifierSet) + assert cast(Range, as_range("==2.7")) in cast(Range, as_range(">=2.7,<=2.7")) + + assert isinstance(as_range(">=3.8,!=3.8.*,!=3.9.*,<3.10"), UnsatisfiableSpecifierSet) + + assert not includes(">2,<3", ">=2.7,<2.7") + assert not includes(">=2.7,<2.7", ">2,<3") From 2ed31da41aa1fdba5e6ead398c34bbfe9d668728 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 19 Sep 2024 12:00:03 -0700 Subject: [PATCH 2/4] Good catch, new code. --- tests/resolve/test_target_options.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/resolve/test_target_options.py b/tests/resolve/test_target_options.py index 114fc9993..faf53a60d 100644 --- a/tests/resolve/test_target_options.py +++ b/tests/resolve/test_target_options.py @@ -326,7 +326,6 @@ def assert_interpreter_constraint_not_satisfied(interpreter_constraints): ) assert_interpreter_constraint_not_satisfied(["==3.9.*"]) - assert_interpreter_constraint_not_satisfied(["==3.8.*,!=3.8.*"]) assert_interpreter_constraint_not_satisfied(["==3.9.*", "==2.6.*"]) From c41973f734b1e24391b9d354cbf9597943231746 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 19 Sep 2024 12:06:02 -0700 Subject: [PATCH 3/4] Update target options test. --- tests/resolve/test_target_options.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/resolve/test_target_options.py b/tests/resolve/test_target_options.py index faf53a60d..832cbb7ae 100644 --- a/tests/resolve/test_target_options.py +++ b/tests/resolve/test_target_options.py @@ -16,13 +16,14 @@ from pex.platforms import Platform from pex.resolve import abbreviated_platforms, target_options from pex.resolve.resolver_configuration import PipConfiguration +from pex.resolve.target_configuration import InterpreterConstraintsNotSatisfied from pex.targets import CompletePlatform, Targets from pex.typing import TYPE_CHECKING from pex.variables import ENV from testing import IS_MAC, environment_as if TYPE_CHECKING: - from typing import Any, Dict, Iterable, List, Optional, Tuple + from typing import Any, Dict, Iterable, List, Optional, Tuple, Type def compute_target_configuration( @@ -318,15 +319,25 @@ def assert_interpreter_constraint( assert_interpreter_constraint([">=3.8,<3.9"], [py38], expected_interpreter=py38) assert_interpreter_constraint(["==3.10.*", "==2.7.*"], [py310, py27], expected_interpreter=py27) - def assert_interpreter_constraint_not_satisfied(interpreter_constraints): - # type: (List[str]) -> None - with pytest.raises(pex.resolve.target_configuration.InterpreterConstraintsNotSatisfied): + def assert_interpreter_constraint_not_satisfied( + interpreter_constraints, # type: List[str] + expected_error_type, # type: Type[Exception] + ): + # type: (...) -> None + with pytest.raises(expected_error_type): compute_target_configuration( parser, interpreter_constraint_args(interpreter_constraints) ) - assert_interpreter_constraint_not_satisfied(["==3.9.*"]) - assert_interpreter_constraint_not_satisfied(["==3.9.*", "==2.6.*"]) + assert_interpreter_constraint_not_satisfied( + ["==3.9.*"], expected_error_type=InterpreterConstraintsNotSatisfied + ) + assert_interpreter_constraint_not_satisfied( + ["==3.8.*,!=3.8.*"], expected_error_type=ArgumentTypeError + ) + assert_interpreter_constraint_not_satisfied( + ["==3.9.*", "==2.6.*"], expected_error_type=InterpreterConstraintsNotSatisfied + ) def test_configure_resolve_local_platforms( From 17edb02bef7f716090e483dc834f0bcb032b4e7a Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 19 Sep 2024 14:25:10 -0700 Subject: [PATCH 4/4] Update CHANGES.md Co-authored-by: Benjy Weinberger --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8ca086c25..e63e88a3a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ This release fixes Pex `--interpreter-constraint` handling such that any supplied interpreter constraints which are in principle -unstaisfiable either raise an error or else cause a warning to be issued +unsatisfiable either raise an error or else cause a warning to be issued when other viable interpreter constraints have also been specified. For example, `--interpreter-constraint ==3.11.*,==3.12.*` now errors and `--interpreter-constraint '>=3.8,<3.8' --interpreter-constraint ==3.9.*`