From 96e8b0a3a40566c7cf132e5c85d83352bd83887e Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 8 Aug 2023 06:00:53 -0600 Subject: [PATCH] Handle backtracking due to sdist build errors. (#2213) A narrow range of pip-2020-resolver supporting Pip versions are robust to build failures attempting to gather sdist metdata; namely `>=20.3.2,!=20.3.3,<22`. Fix Pex to handle backtracks triggered by build failures. Fixes #2211 --- pex/resolve/locker.py | 45 ++++++----- .../cli/commands/test_issue_2211.py | 79 +++++++++++++++++++ 2 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 tests/integration/cli/commands/test_issue_2211.py diff --git a/pex/resolve/locker.py b/pex/resolve/locker.py index 9ed27b028..821060991 100644 --- a/pex/resolve/locker.py +++ b/pex/resolve/locker.py @@ -217,12 +217,12 @@ class ArtifactBuildResult(object): @attr.s(frozen=True) class ArtifactBuildObserver(object): - _done_building_pattern = attr.ib() # type: Pattern + _done_building_patterns = attr.ib() # type: Iterable[Pattern] _artifact_url = attr.ib() # type: ArtifactURL def is_done_building(self, line): # type: (str) -> bool - return self._done_building_pattern.search(line) is not None + return any(pattern.search(line) is not None for pattern in self._done_building_patterns) def build_result(self, line): # type: (str) -> Optional[ArtifactBuildResult] @@ -315,16 +315,17 @@ def analyze(self, line): # The log sequence for processing a resolved requirement is as follows (log lines irrelevant # to our purposes omitted): # - # 1.) "... Found link ..." + # 1.) "... Found link ..." # ... - # 1.) "... Found link ..." - # 2.) "... Added to build tracker ..." - # * 3.) Lines related to extracting metadata from if the selected - # distribution is an sdist in any form (VCS, local directory, source archive). - # * 3.5.) "... Source in has version , which satisfies requirement " - # " from ..." - # 4.) "... Removed from ... from build tracker ..." - # 5.) "... Saved / + # 1.) "... Found link ..." + # 2.) "... Added to build tracker ..." + # * 3.) Lines related to extracting metadata from if the selected + # distribution is an sdist in any form (VCS, local directory, source archive). + # * 3.5. ERR) "... WARNING: Discarding . Command errored out with ... + # * 3.5. SUC) "... Source in has version , which satisfies requirement " + # " from ..." + # 4.) "... Removed from ... from build tracker ..." + # 5.) "... Saved / # The lines in section 3 can contain this same pattern of lines if the metadata extraction # proceeds via PEP-517 which recursively uses Pip to resolve build dependencies. We want to @@ -451,10 +452,13 @@ def analyze(self, line): self._selected_path_to_pin[os.path.basename(url.path)] = pin else: self._artifact_build_observer = ArtifactBuildObserver( - done_building_pattern=re.compile( - r"Removed {requirement} from {url} (?:.* )?from build tracker".format( - requirement=re.escape(raw_requirement), url=re.escape(url.raw_url) - ) + done_building_patterns=( + re.compile( + r"Removed {requirement} from {url} (?:.* )?from build tracker".format( + requirement=re.escape(raw_requirement), url=re.escape(url.raw_url) + ) + ), + re.compile(r"WARNING: Discarding {url}".format(url=re.escape(url.raw_url))), ), artifact_url=url, ) @@ -464,10 +468,13 @@ def analyze(self, line): if match: file_url = match.group("file_url") self._artifact_build_observer = ArtifactBuildObserver( - done_building_pattern=re.compile( - r"Removed .+ from {file_url} from build tracker".format( - file_url=re.escape(file_url) - ) + done_building_patterns=( + re.compile( + r"Removed .+ from {file_url} from build tracker".format( + file_url=re.escape(file_url) + ) + ), + re.compile(r"WARNING: Discarding {url}".format(url=re.escape(file_url))), ), artifact_url=ArtifactURL.parse(file_url), ) diff --git a/tests/integration/cli/commands/test_issue_2211.py b/tests/integration/cli/commands/test_issue_2211.py new file mode 100644 index 000000000..2bb449cdc --- /dev/null +++ b/tests/integration/cli/commands/test_issue_2211.py @@ -0,0 +1,79 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import os.path + +import pytest + +from pex.interpreter_constraints import InterpreterConstraint +from pex.pep_440 import Version +from pex.pip.version import PipVersion, PipVersionValue +from pex.typing import TYPE_CHECKING +from testing import run_pex_command +from testing.cli import run_pex3 + +if TYPE_CHECKING: + from typing import Any + + +# N.B.: awscli==1.28.1 Only resolves with the pip-2020-resolver for Pip versions earlier than 22. +# In the end this means only Pip >=20.3.2,<22 can run this test successfully. The underlying issue +# comes from PyYAML and is documented here: https://github.com/yaml/pyyaml/issues/601 +@pytest.mark.parametrize( + "pip_version", + [ + pytest.param(pip_version, id=str(pip_version)) + for pip_version in PipVersion.values() + if Version("20.3.2") <= pip_version.version < Version("22") + ], +) +def test_backtracking( + tmpdir, # type: Any + pip_version, # type: PipVersionValue +): + # type: (...) -> None + + lock = os.path.join(str(tmpdir), "lock.json") + run_pex3( + "lock", + "create", + "-v", + "-o", + lock, + "--indent", + "2", + "--pip-version", + str(pip_version), + "--resolver-version", + "pip-2020-resolver", + "--interpreter-constraint", + "CPython==3.11.*", + "--style", + "universal", + "--target-system", + "linux", + "--target-system", + "mac", + "--manylinux", + "manylinux2014", + "awscli==1.28.1", + ).assert_success() + + try: + python311 = next(InterpreterConstraint.parse("CPython==3.11.*").iter_matching()) + except StopIteration: + pytest.skip("Skipping lock verification since no CPython 3.11 interpreter is available.") + + result = run_pex_command( + args=[ + "--lock", + lock, + "-c", + "aws", + "--", + "--version", + ], + python=python311.binary, + ) + result.assert_success() + assert result.output.startswith("aws-cli/1.28.1 "), result.output