Improve core metadata generation for compatibility with plugins (#4043)

newsfragments/4043.bugfix.rst

@@ -0,0 +1,4 @@
+Avoid using caching attributes in ``Distribution.metadata`` for requirements.
+This is done for backwards compatibility with customizations that attempt to
+modify ``install_requires`` or ``extras_require`` at a late point (still not
+recommended).

setuptools/_core_metadata.py

@@ -13,7 +13,7 @@
 
 from distutils.util import rfc822_escape
 
-from . import _normalization
+from . import _normalization, _reqs
 from .extern.packaging.markers import Marker
 from .extern.packaging.requirements import Requirement
 from .extern.packaging.version import Version
@@ -211,27 +211,27 @@ def write_field(key, value):
 
 
 def _write_requirements(self, file):
-    for req in self._normalized_install_requires:
+    for req in _reqs.parse(self.install_requires):
         file.write(f"Requires-Dist: {req}\n")
 
     processed_extras = {}
-    for augmented_extra, reqs in self._normalized_extras_require.items():
+    for augmented_extra, reqs in self.extras_require.items():
         # Historically, setuptools allows "augmented extras": `<extra>:<condition>`
         unsafe_extra, _, condition = augmented_extra.partition(":")
         unsafe_extra = unsafe_extra.strip()
         extra = _normalization.safe_extra(unsafe_extra)
 
         if extra:
             _write_provides_extra(file, processed_extras, extra, unsafe_extra)
-        for req in reqs:
+        for req in _reqs.parse_strings(reqs):
             r = _include_extra(req, extra, condition.strip())
             file.write(f"Requires-Dist: {r}\n")
 
     return processed_extras
 
 
 def _include_extra(req: str, extra: str, condition: str) -> Requirement:
-    r = Requirement(req)
+    r = Requirement(req)  # create a fresh object that can be modified
     parts = (
         f"({r.marker})" if r.marker else None,
         f"({condition})" if condition else None,

setuptools/_reqs.py

@@ -1,3 +1,4 @@
+from functools import lru_cache
 from typing import Callable, Iterable, Iterator, TypeVar, Union, overload
 
 import setuptools.extern.jaraco.text as text
@@ -7,6 +8,12 @@
 _StrOrIter = Union[str, Iterable[str]]
 
 
+parse_req: Callable[[str], Requirement] = lru_cache()(Requirement)
+# Setuptools parses the same requirement many times
+# (e.g. first for validation than for normalisation),
+# so it might be worth to cache.
+
+
 def parse_strings(strs: _StrOrIter) -> Iterator[str]:
     """
     Yield requirement strings for each specification in `strs`.
@@ -26,7 +33,7 @@ def parse(strs: _StrOrIter, parser: Callable[[str], _T]) -> Iterator[_T]:
     ...
 
 
-def parse(strs, parser=Requirement):
+def parse(strs, parser=parse_req):
     """
     Replacement for ``pkg_resources.parse_requirements`` that uses ``packaging``.
     """

setuptools/command/_requirestxt.py

@@ -11,6 +11,7 @@
 from itertools import filterfalse
 from typing import Dict, List, Tuple, Mapping, TypeVar
 
+from .. import _reqs
 from ..extern.jaraco.text import yield_lines
 from ..extern.packaging.requirements import Requirement
 
@@ -19,11 +20,11 @@
 _T = TypeVar("_T")
 _Ordered = Dict[_T, None]
 _ordered = dict
+_StrOrIter = _reqs._StrOrIter
 
 
 def _prepare(
-    install_requires: Dict[str, Requirement],
-    extras_require: Mapping[str, Dict[str, Requirement]],
+    install_requires: _StrOrIter, extras_require: Mapping[str, _StrOrIter]
 ) -> Tuple[List[str], Dict[str, List[str]]]:
     """Given values for ``install_requires`` and ``extras_require``
     create modified versions in a way that can be written in ``requires.txt``
@@ -33,7 +34,7 @@ def _prepare(
 
 
 def _convert_extras_requirements(
-    extras_require: Dict[str, Dict[str, Requirement]],
+    extras_require: _StrOrIter,
 ) -> Mapping[str, _Ordered[Requirement]]:
     """
     Convert requirements in `extras_require` of the form
@@ -44,15 +45,14 @@ def _convert_extras_requirements(
     for section, v in extras_require.items():
         # Do not strip empty sections.
         output[section]
-        for r in v.values():
+        for r in _reqs.parse(v):
             output[section + _suffix_for(r)].setdefault(r)
 
     return output
 
 
 def _move_install_requirements_markers(
-    install_requires: Dict[str, Requirement],
-    extras_require: Mapping[str, _Ordered[Requirement]],
+    install_requires: _StrOrIter, extras_require: Mapping[str, _Ordered[Requirement]]
 ) -> Tuple[List[str], Dict[str, List[str]]]:
     """
     The ``requires.txt`` file has an specific format:
@@ -66,7 +66,7 @@ def _move_install_requirements_markers(
     # divide the install_requires into two sets, simple ones still
     # handled by install_requires and more complex ones handled by extras_require.
 
-    inst_reqs = install_requires.values()
+    inst_reqs = list(_reqs.parse(install_requires))
     simple_reqs = filter(_no_marker, inst_reqs)
     complex_reqs = filterfalse(_no_marker, inst_reqs)
     simple_install_requires = list(map(str, simple_reqs))
@@ -90,7 +90,7 @@ def _suffix_for(req):
 
 def _clean_req(req):
     """Given a Requirement, remove environment markers and return it"""
-    r = Requirement(str(req))  # create a copy before modifying.
+    r = Requirement(str(req))  # create a copy before modifying
     r.marker = None
     return r
 
@@ -111,10 +111,9 @@ def append_cr(line):
 
 def write_requirements(cmd, basename, filename):
     dist = cmd.distribution
-    meta = dist.metadata
     data = io.StringIO()
     install_requires, extras_require = _prepare(
-        meta._normalized_install_requires, meta._normalized_extras_require
+        dist.install_requires or (), dist.extras_require or {}
     )
     _write_requirements(data, install_requires)
     for extra in sorted(extras_require):

setuptools/config/_apply_pyprojecttoml.py

@@ -125,7 +125,7 @@ def _set_config(dist: "Distribution", field: str, value: Any):
         setter(value)
     elif hasattr(dist.metadata, field) or field in SETUPTOOLS_PATCHES:
         setattr(dist.metadata, field, value)
-    else:
+    if hasattr(dist, field):
         setattr(dist, field, value)
 
 

setuptools/dist.py

@@ -263,11 +263,8 @@ class Distribution(_Distribution):
         'provides_extras': OrderedSet,
         'license_file': lambda: None,
         'license_files': lambda: None,
-        # Both install_requires and extras_require are needed to write PKG-INFO,
-        # So we take this opportunity to cache parsed requirement objects.
-        # These attributes are not part of the public API and intended for internal use.
-        '_normalized_install_requires': dict,  # Dict[str, Requirement]
-        '_normalized_extras_require': dict,  # Dict[str, Dict[str, Requirement]]
+        'install_requires': list,
+        'extras_require': dict,
     }
 
     _patched_dist = None
@@ -299,14 +296,11 @@ def __init__(self, attrs=None):
         self.setup_requires = attrs.pop('setup_requires', [])
         for ep in metadata.entry_points(group='distutils.setup_keywords'):
             vars(self).setdefault(ep.name, None)
-        _Distribution.__init__(
-            self,
-            {
-                k: v
-                for k, v in attrs.items()
-                if k not in self._DISTUTILS_UNSUPPORTED_METADATA
-            },
-        )
+
+        metadata_only = set(self._DISTUTILS_UNSUPPORTED_METADATA)
+        metadata_only -= {"install_requires", "extras_require"}
+        dist_attrs = {k: v for k, v in attrs.items() if k not in metadata_only}
+        _Distribution.__init__(self, dist_attrs)
 
         # Private API (setuptools-use only, not restricted to Distribution)
         # Stores files that are referenced by the configuration and need to be in the
@@ -394,6 +388,8 @@ def _finalize_requires(self):
             self.metadata.python_requires = self.python_requires
 
         self._normalize_requires()
+        self.metadata.install_requires = self.install_requires
+        self.metadata.extras_require = self.extras_require
 
         if self.extras_require:
             for extra in self.extras_require.keys():
@@ -406,17 +402,9 @@ def _normalize_requires(self):
         """Make sure requirement-related attributes exist and are normalized"""
         install_requires = getattr(self, "install_requires", None) or []
         extras_require = getattr(self, "extras_require", None) or {}
-        meta = self.metadata
-        meta._normalized_install_requires = {
-            str(r): r for r in _reqs.parse(install_requires)
-        }
-        meta._normalized_extras_require = {
-            k: {str(r): r for r in _reqs.parse(v or [])}
-            for k, v in extras_require.items()
-        }
-        self.install_requires = list(meta._normalized_install_requires)
+        self.install_requires = list(map(str, _reqs.parse(install_requires)))
         self.extras_require = {
-            k: list(v) for k, v in meta._normalized_extras_require.items()
+            k: list(map(str, _reqs.parse(v or []))) for k, v in extras_require.items()
         }
 
     def _finalize_license_files(self):