Handle backtracking due to sdist build errors. (pex-tool#2213)

A narrow range of pip-2020-resolver supporting Pip versions are robust
to build failures attempting to gather sdist metdata; namely
`>=20.3.2,!=20.3.3,<22`. Fix Pex to handle backtracks triggered by build
failures.

Fixes pex-tool#2211

pex/resolve/locker.py

@@ -217,12 +217,12 @@ class ArtifactBuildResult(object):
 
 @attr.s(frozen=True)
 class ArtifactBuildObserver(object):
-    _done_building_pattern = attr.ib()  # type: Pattern
+    _done_building_patterns = attr.ib()  # type: Iterable[Pattern]
     _artifact_url = attr.ib()  # type: ArtifactURL
 
     def is_done_building(self, line):
         # type: (str) -> bool
-        return self._done_building_pattern.search(line) is not None
+        return any(pattern.search(line) is not None for pattern in self._done_building_patterns)
 
     def build_result(self, line):
         # type: (str) -> Optional[ArtifactBuildResult]
@@ -315,16 +315,17 @@ def analyze(self, line):
         # The log sequence for processing a resolved requirement is as follows (log lines irrelevant
         # to our purposes omitted):
         #
-        #   1.)   "... Found link <url1> ..."
+        #   1.)       "... Found link <url1> ..."
         #   ...
-        #   1.)   "... Found link <urlN> ..."
-        #   2.)   "... Added <varying info ...> to build tracker ..."
-        # * 3.)   Lines related to extracting metadata from <requirement> if the selected
-        #         distribution is an sdist in any form (VCS, local directory, source archive).
-        # * 3.5.) "... Source in <tmp> has version <version>, which satisfies requirement "
-        #         "<requirement> from <url> ..."
-        #   4.)   "... Removed <requirement> from <url> ... from build tracker ..."
-        #   5.)   "... Saved <download dir>/<artifact file>
+        #   1.)       "... Found link <urlN> ..."
+        #   2.)       "... Added <varying info ...> to build tracker ..."
+        # * 3.)       Lines related to extracting metadata from <requirement> if the selected
+        #             distribution is an sdist in any form (VCS, local directory, source archive).
+        # * 3.5. ERR) "... WARNING: Discarding <url> <varying info...>. Command errored out with ...
+        # * 3.5. SUC) "... Source in <tmp> has version <version>, which satisfies requirement "
+        #             "<requirement> from <url> ..."
+        #   4.)       "... Removed <requirement> from <url> ... from build tracker ..."
+        #   5.)       "... Saved <download dir>/<artifact file>
 
         # The lines in section 3 can contain this same pattern of lines if the metadata extraction
         # proceeds via PEP-517 which recursively uses Pip to resolve build dependencies. We want to
@@ -451,10 +452,13 @@ def analyze(self, line):
                 self._selected_path_to_pin[os.path.basename(url.path)] = pin
             else:
                 self._artifact_build_observer = ArtifactBuildObserver(
-                    done_building_pattern=re.compile(
-                        r"Removed {requirement} from {url} (?:.* )?from build tracker".format(
-                            requirement=re.escape(raw_requirement), url=re.escape(url.raw_url)
-                        )
+                    done_building_patterns=(
+                        re.compile(
+                            r"Removed {requirement} from {url} (?:.* )?from build tracker".format(
+                                requirement=re.escape(raw_requirement), url=re.escape(url.raw_url)
+                            )
+                        ),
+                        re.compile(r"WARNING: Discarding {url}".format(url=re.escape(url.raw_url))),
                     ),
                     artifact_url=url,
                 )
@@ -464,10 +468,13 @@ def analyze(self, line):
         if match:
             file_url = match.group("file_url")
             self._artifact_build_observer = ArtifactBuildObserver(
-                done_building_pattern=re.compile(
-                    r"Removed .+ from {file_url} from build tracker".format(
-                        file_url=re.escape(file_url)
-                    )
+                done_building_patterns=(
+                    re.compile(
+                        r"Removed .+ from {file_url} from build tracker".format(
+                            file_url=re.escape(file_url)
+                        )
+                    ),
+                    re.compile(r"WARNING: Discarding {url}".format(url=re.escape(file_url))),
                 ),
                 artifact_url=ArtifactURL.parse(file_url),
             )

tests/integration/cli/commands/test_issue_2211.py

@@ -0,0 +1,79 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os.path
+
+import pytest
+
+from pex.interpreter_constraints import InterpreterConstraint
+from pex.pep_440 import Version
+from pex.pip.version import PipVersion, PipVersionValue
+from pex.typing import TYPE_CHECKING
+from testing import run_pex_command
+from testing.cli import run_pex3
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+# N.B.: awscli==1.28.1 Only resolves with the pip-2020-resolver for Pip versions earlier than 22.
+# In the end this means only Pip >=20.3.2,<22 can run this test successfully. The underlying issue
+# comes from PyYAML and is documented here: https://github.com/yaml/pyyaml/issues/601
+@pytest.mark.parametrize(
+    "pip_version",
+    [
+        pytest.param(pip_version, id=str(pip_version))
+        for pip_version in PipVersion.values()
+        if Version("20.3.2") <= pip_version.version < Version("22")
+    ],
+)
+def test_backtracking(
+    tmpdir,  # type: Any
+    pip_version,  # type: PipVersionValue
+):
+    # type: (...) -> None
+
+    lock = os.path.join(str(tmpdir), "lock.json")
+    run_pex3(
+        "lock",
+        "create",
+        "-v",
+        "-o",
+        lock,
+        "--indent",
+        "2",
+        "--pip-version",
+        str(pip_version),
+        "--resolver-version",
+        "pip-2020-resolver",
+        "--interpreter-constraint",
+        "CPython==3.11.*",
+        "--style",
+        "universal",
+        "--target-system",
+        "linux",
+        "--target-system",
+        "mac",
+        "--manylinux",
+        "manylinux2014",
+        "awscli==1.28.1",
+    ).assert_success()
+
+    try:
+        python311 = next(InterpreterConstraint.parse("CPython==3.11.*").iter_matching())
+    except StopIteration:
+        pytest.skip("Skipping lock verification since no CPython 3.11 interpreter is available.")
+
+    result = run_pex_command(
+        args=[
+            "--lock",
+            lock,
+            "-c",
+            "aws",
+            "--",
+            "--version",
+        ],
+        python=python311.binary,
+    )
+    result.assert_success()
+    assert result.output.startswith("aws-cli/1.28.1 "), result.output