Handle URL requirements with constraints files.

src/python/pants/backend/python/util_rules/pex_from_targets.py

@@ -244,13 +244,25 @@ async def pex_from_targets(
         # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let
         # packaging canonicalize for us.
         # See: https://www.python.org/dev/peps/pep-0503/#normalized-names
-        exact_req_projects = {
-            canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs
-        }
+
+        url_reqs = set()  # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch'
+        name_reqs = set()  # E.g., foobar>=1.2.3
+        name_req_projects = set()
+
+        for req_str in exact_reqs:
+            req = Requirement.parse(req_str)
+            if req.url:  # type: ignore[attr-defined]
+                url_reqs.add(req)
+            else:
+                name_reqs.add(req)
+                name_req_projects.add(canonicalize_project_name(req.project_name))
+
         constraint_file_projects = {
             canonicalize_project_name(req.project_name) for req in constraints_file_reqs
         }
-        unconstrained_projects = exact_req_projects - constraint_file_projects
+        # Constraints files must only contain name reqs, not URL reqs (those are already
+        # constrained by their very nature). See https://github.com/pypa/pip/issues/8210.
+        unconstrained_projects = name_req_projects - constraint_file_projects
         if unconstrained_projects:
             constraints_descr = (
                 f"constraints file {constraints_file.path}"
@@ -272,15 +284,27 @@ async def pex_from_targets(
                     "file does not cover all requirements."
                 )
             else:
+                # To get a full set of requirements we must add the URL requirements to the
+                # constraints file, since the latter cannot contain URL requirements.
+                # NB: We can only add the URL requirements we know about here, i.e., those that
+                #  are transitive deps of the targets in play. There may be others in the repo.
+                #  So we may end up creating a few different repository pexes, each with identical
+                #  name requirements but different subsets of URL requirements. Fortunately since
+                #  all these repository pexes will have identical pinned versions of everything,
+                #  this is not a correctness issue, only a performance one.
+                # TODO: Address this as part of providing proper lockfile support. However we
+                #  generate lockfiles, they must be able to include URL requirements.
+                all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)}
                 repository_pex = await Get(
                     Pex,
                     PexRequest(
                         description=f"Resolving {python_setup.requirement_constraints}",
                         output_filename="repository.pex",
                         internal_only=request.internal_only,
-                        requirements=PexRequirements(str(req) for req in constraints_file_reqs),
+                        requirements=PexRequirements(all_constraints),
                         interpreter_constraints=interpreter_constraints,
                         platforms=request.platforms,
+                        additional_args=["-vvv"],
                     ),
                 )
     elif (

src/python/pants/backend/python/util_rules/pex_from_targets_test.py

@@ -20,6 +20,7 @@
 from pants.engine.internals.scheduler import ExecutionError
 from pants.python.python_setup import ResolveAllConstraintsOption
 from pants.testutil.rule_runner import QueryRule, RuleRunner
+from pants.util.contextutil import pushd
 
 
 @pytest.fixture
@@ -39,35 +40,50 @@ class Project:
     version: str
 
 
-def create_dists(workdir: Path, project: Project, *projects: Project) -> PurePath:
-    project_dirs = []
-    for proj in (project, *projects):
-        project_dir = workdir / "projects" / proj.name
-        project_dir.mkdir(parents=True)
-        project_dirs.append(project_dir)
-
-        (project_dir / "pyproject.toml").write_text(
-            dedent(
-                """\
-                [build-system]
-                requires = ["setuptools==54.1.2", "wheel==0.36.2"]
-                build-backend = "setuptools.build_meta"
-                """
-            )
+build_deps = ["setuptools==54.1.2", "wheel==0.36.2"]
+
+
+def create_project_dir(workdir: Path, project: Project) -> PurePath:
+    project_dir = workdir / "projects" / project.name
+    project_dir.mkdir(parents=True)
+
+    (project_dir / "pyproject.toml").write_text(
+        dedent(
+            f"""\
+            [build-system]
+            requires = {build_deps}
+            build-backend = "setuptools.build_meta"
+            """
         )
-        (project_dir / "setup.cfg").write_text(
-            dedent(
-                f"""\
+    )
+    (project_dir / "setup.cfg").write_text(
+        dedent(
+            f"""\
                 [metadata]
-                name = {proj.name}
-                version = {proj.version}
+                name = {project.name}
+                version = {project.version}
                 """
-            )
         )
+    )
+    return project_dir
+
+
+def create_dists(workdir: Path, project: Project, *projects: Project) -> PurePath:
+    project_dirs = [create_project_dir(workdir, proj) for proj in (project, *projects)]
 
     pex = workdir / "pex"
     subprocess.run(
-        args=[sys.executable, "-m", "pex", *project_dirs, "--include-tools", "-o", pex], check=True
+        args=[
+            sys.executable,
+            "-m",
+            "pex",
+            *project_dirs,
+            *build_deps,
+            "--include-tools",
+            "-o",
+            pex,
+        ],
+        check=True,
     )
 
     find_links = workdir / "find-links"
@@ -113,15 +129,30 @@ def test_constraints_validation(tmp_path_factory: TempPathFactory, rule_runner:
         Project("QUX", "3.4.5"),
     )
 
+    # Turn the project dir into a git repo, so it can be cloned.
+    foorl_dir = create_project_dir(tmp_path_factory.mktemp("git"), Project("foorl", "9.8.7"))
+    with pushd(str(foorl_dir)):
+        subprocess.check_call(["git", "init"])
+        subprocess.check_call(["git", "config", "user.name", "dummy"])
+        subprocess.check_call(["git", "config", "user.email", "dummy@dummy.com"])
+        subprocess.check_call(["git", "add", "--all"])
+        subprocess.check_call(["git", "commit", "-m", "initial commit"])
+        subprocess.check_call(["git", "branch", "9.8.7"])
+
+    # This string won't parse as a Requirement if it doesn't contain a netloc,
+    # so we explicitly mention localhost.
+    url_req = f"foorl@ git+file://localhost{foorl_dir.as_posix()}@9.8.7"
+
     rule_runner.add_to_build_file(
         "",
         dedent(
-            """
+            f"""
             python_requirement_library(name="foo", requirements=["foo-bar>=0.1.2"])
             python_requirement_library(name="bar", requirements=["bar==5.5.5"])
             python_requirement_library(name="baz", requirements=["baz"])
+            python_requirement_library(name="foorl", requirements=["{url_req}"])
             python_library(name="util", sources=[], dependencies=[":foo", ":bar"])
-            python_library(name="app", sources=[], dependencies=[":util", ":baz"])
+            python_library(name="app", sources=[], dependencies=[":util", ":baz", ":foorl"])
             """
         ),
     )
@@ -135,6 +166,8 @@ def test_constraints_validation(tmp_path_factory: TempPathFactory, rule_runner:
             bar==5.5.5
             baz==2.2.2
             qux==3.4.5
+            # Note that pip does not allow URL requirements in constraints files,
+            # so there is no mention of foorl here.
         """
         ),
     )
@@ -162,27 +195,31 @@ def get_pex_request(
         return rule_runner.request(PexRequest, [request])
 
     pex_req1 = get_pex_request("constraints1.txt", ResolveAllConstraintsOption.NEVER)
-    assert pex_req1.requirements == PexRequirements(["foo-bar>=0.1.2", "bar==5.5.5", "baz"])
+    assert pex_req1.requirements == PexRequirements(
+        ["foo-bar>=0.1.2", "bar==5.5.5", "baz", url_req]
+    )
     assert pex_req1.repository_pex is None
 
     pex_req1_direct = get_pex_request(
         "constraints1.txt", ResolveAllConstraintsOption.NEVER, direct_deps_only=True
     )
-    assert pex_req1_direct.requirements == PexRequirements(["baz"])
+    assert pex_req1_direct.requirements == PexRequirements(["baz", url_req])
     assert pex_req1_direct.repository_pex is None
 
     pex_req2 = get_pex_request("constraints1.txt", ResolveAllConstraintsOption.ALWAYS)
-    assert pex_req2.requirements == PexRequirements(["foo-bar>=0.1.2", "bar==5.5.5", "baz"])
+    assert pex_req2.requirements == PexRequirements(
+        ["foo-bar>=0.1.2", "bar==5.5.5", "baz", url_req]
+    )
     assert pex_req2.repository_pex is not None
     repository_pex = pex_req2.repository_pex
-    assert ["Foo._-BAR==1.0.0", "bar==5.5.5", "baz==2.2.2", "qux==3.4.5"] == requirements(
+    assert ["Foo._-BAR==1.0.0", "bar==5.5.5", "baz==2.2.2", "foorl", "qux==3.4.5"] == requirements(
         rule_runner, repository_pex
     )
 
     pex_req2_direct = get_pex_request(
         "constraints1.txt", ResolveAllConstraintsOption.ALWAYS, direct_deps_only=True
     )
-    assert pex_req2_direct.requirements == PexRequirements(["baz"])
+    assert pex_req2_direct.requirements == PexRequirements(["baz", url_req])
     assert pex_req2_direct.repository_pex == repository_pex
 
     with pytest.raises(ExecutionError) as err: