Un-pin hatchling. (#2408)

Although hatchling fixed a few issues that broke Pex packaging, it has
since more strictly adhered to PEP-621 which breaks the existing project
metadata customization scheme. Update the scheme to conform to PEP-621
strictures. Also update the tox configuration to work with modern
hatchling.

Picks up from #2403 by @ofek

build-backend/pex_build/hatchling/metadata_hook.py

@@ -32,30 +32,71 @@ class AdjustMetadata(MetadataHookInterface):
 
     The following mutations are supported:
     + Specifying alternate requires-python metadata via _PEX_REQUIRES_PYTHON env var.
-    + Expanding format string placeholder (`{name}`) with metadata values via the `expand` mapping of placeholder name
-      to metadata value.
+    + Expanding format string placeholder (`{name}`) with metadata values via the `expand` mapping
+      of placeholder name to metadata value.
     """
 
     PLUGIN_NAME = "pex-adjust-metadata"
 
     def update(self, metadata):
         # type: (Dict[str, Any]) -> None
-        requires_python = os.environ.get("_PEX_REQUIRES_PYTHON")
-        if requires_python:
+
+        dynamic_metadata = self.config.get("project")
+        if not dynamic_metadata:
+            return
+
+        self._update_requires_python(metadata, dynamic_metadata)
+        self._expand_placeholders(metadata, dynamic_metadata)
+
+    def _update_requires_python(
+        self,
+        metadata,  # type: Dict[str, Any]
+        dynamic_metadata,  # type: Dict[str, Any]
+    ):
+        # type: (...) -> None
+
+        dynamic_requires_python = os.environ.get("_PEX_REQUIRES_PYTHON")
+        static_requires_python = dynamic_metadata.get("requires-python")
+
+        if dynamic_requires_python:
+            if not static_requires_python:
+                raise ValueError(
+                    "A dynamic override of requires-python metadata was specified via "
+                    "`_PEX_REQUIRES_PYTHON={dynamic_requires_python}` but there was no "
+                    "corresponding static value defined in "
+                    "`tool.hatch.metadata.hooks.{plugin_name}.project`.".format(
+                        plugin_name=self.PLUGIN_NAME,
+                        dynamic_requires_python=dynamic_requires_python,
+                    )
+                )
+
             print(
                 "pex_build: Dynamically modifying pyproject.toml requires-python of {original} to "
-                "{dynamic}".format(original=metadata["requires-python"], dynamic=requires_python),
+                "{dynamic}".format(
+                    original=static_requires_python, dynamic=dynamic_requires_python
+                ),
                 file=sys.stderr,
             )
-            metadata["requires-python"] = requires_python
+            dynamic_metadata["requires-python"] = dynamic_requires_python
+
+        if dynamic_metadata["requires-python"]:
+            metadata["requires-python"] = dynamic_metadata["requires-python"]
+
+    def _expand_placeholders(
+        self,
+        metadata,  # type: Dict[str, Any]
+        dynamic_metadata,  # type: Dict[str, Any]
+    ):
+        # type: (...) -> None
 
         expand = self.config.get("expand")
-        if expand:
-            metadata.update(
-                (
-                    key,
-                    expand_value(value, **{key: metadata[value] for key, value in expand.items()}),
-                )
-                for key, value in metadata.items()
-                if key != "version"
+        if not expand:
+            return
+
+        metadata.update(
+            (
+                key,
+                expand_value(value, **{key: metadata[value] for key, value in expand.items()}),
             )
+            for key, value in dynamic_metadata.items()
+        )

pyproject.toml

@@ -12,18 +12,27 @@ build-backend = "pex_build.hatchling.build"
 #   runs, which foils our metadata expansion since our README.rst contains inadvertant expansion
 #   tokens.
 # This pin low buys time to work through the issues.
-requires = ["hatchling<1.22.0"]
+requires = ["hatchling"]
 
 [tool.hatch.metadata.hooks.pex-adjust-metadata]
 expand = {"pex_version" = "version"}
 
+[tool.hatch.metadata.hooks.pex-adjust-metadata.project]
+requires-python = ">=2.7,<3.13,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
+
+[tool.hatch.metadata.hooks.pex-adjust-metadata.project.urls]
+Changelog = "https://github.com/pex-tool/pex/blob/v{pex_version}/CHANGES.md"
+Documentation = "https://docs.pex-tool.org/"
+Download = "https://github.com/pex-tool/pex/releases/download/v{pex_version}/pex"
+Homepage = "https://github.com/pex-tool/pex"
+Source = "https://github.com/pex-tool/pex/tree/v{pex_version}"
+
 [tool.hatch.build.targets.wheel.hooks.pex-adjust-build]
 # We need this empty table to enable our hook.
 
 [project]
 name = "pex"
-dynamic = ["version"]
-requires-python = ">=2.7,<3.13,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
+dynamic = ["version", "requires-python", "urls"]
 authors = [
   {name = "The PEX developers", email="developers@pex-tool.org"}
 ]
@@ -71,13 +80,6 @@ pex-tools = "pex.tools.main:main"
 [project.entry-points."distutils.commands"]
 bdist_pex = "pex.distutils.commands.bdist_pex:bdist_pex"
 
-[project.urls]
-Changelog = "https://github.com/pex-tool/pex/blob/v{pex_version}/CHANGES.md"
-Documentation = "https://docs.pex-tool.org/"
-Download = "https://github.com/pex-tool/pex/releases/download/v{pex_version}/pex"
-Homepage = "https://github.com/pex-tool/pex"
-Source = "https://github.com/pex-tool/pex/tree/v{pex_version}"
-
 [tool.hatch.version]
 path = "pex/version.py"
 pattern = '__version__ = "(?P<version>[^"]+)"'

tests/bin/test_sh_boot.py

@@ -42,10 +42,10 @@ def requires_python(pex_project_dir):
     requires_python = os.environ.get("_PEX_REQUIRES_PYTHON")
     if requires_python:
         return requires_python
-    return cast(
-        str,
-        toml.load(os.path.join(pex_project_dir, "pyproject.toml"))["project"]["requires-python"],
-    )
+    pyproject_data = toml.load(os.path.join(pex_project_dir, "pyproject.toml"))
+    metadata_hooks_data = pyproject_data["tool"]["hatch"]["metadata"]["hooks"]
+    pex_adjust_metadata_data = metadata_hooks_data["pex-adjust-metadata"]
+    return cast(str, pex_adjust_metadata_data["project"]["requires-python"])
 
 
 def expected(

tox.ini

@@ -1,5 +1,8 @@
 [tox]
-isolated_build = true
+# N.B.: We handle installing Pex in the envs that need it with a custom testenv install_command
+# below.
+skipsdist = true
+
 skip_missing_interpreters = true
 minversion = 3.25.1
 requires =
@@ -11,9 +14,19 @@ requires =
 # N.B.: We need modern setuptools downloaded out of band by virtualenv to work with Python>=3.12.
 # Trying to upgrade via Pip is too late and Pip blows up.
 download = true
+
+# N.B.: We configure tox to disable its build sdist, then install sdist in venv scheme for all envs
+# with `skip_install = false` (the default). As such, we use a custom `install_command` for all
+# envs that need Pex installed. The install command is tox's default, with the one addition of
+# `{toxinidir}`, which adds `.` to the requirement list handed to Pip to install.
+install_command =
+    docs,check,typecheck,py{py27,py35,py36,py37,py38,py39,py310,27,35,36,37,38,39,310,311,312,313}: \
+        python -m pip install {opts} {toxinidir} {packages}
 commands =
-    python testing/bin/run_tests.py {posargs:-vvs}
+    !integration: python testing/bin/run_tests.py {posargs:-vvs}
+    integration: python testing/bin/run_tests.py --it {posargs:-vvs}
 deps =
+    integration: pytest-xdist==1.34.0
     ansicolors==1.1.8
     coloredlogs==15.0.1
     # The more-itertools project is an indirect requirement of pytest and its broken for
@@ -78,13 +91,6 @@ allowlist_externals =
     bash
     git
 
-[testenv:py{py27-subprocess,py27,py35,py36,py37,py38,py39,py310,27,35,36,37,38,39,310,311,312,313}-{,pip20-,pip22_2-,pip22_3-,pip22_3_1-,pip23_0-,pip23_0_1-,pip23_1-,pip23_1_1-,pip23_1_2-,pip23_2-,pip23_3_1-,pip23_3_2-,pip24_0-,pip24_0_patched-}integration]
-deps =
-    pytest-xdist==1.34.0
-    {[testenv]deps}
-commands =
-    python testing/bin/run_tests.py --it {posargs:-vvs}
-
 [testenv:{format-run,fmt}]
 skip_install = true
 deps =