Refactor manylinux

.github/workflows/release.yaml

@@ -16,7 +16,7 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
 
@@ -37,7 +37,7 @@ jobs:
 
     - name: Publish a Python distribution to PyPI
       if: ${{ github.event_name == 'release' }}
-      uses: pypa/gh-action-pypi-publish@v1.8.10
+      uses: pypa/gh-action-pypi-publish@v1.8.11
       with:
         user: __token__
         password: ${{ secrets.PYPI_PASSWORD }}

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
     args: [--fix, --exit-non-zero-on-fix]
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.7.0
+  rev: v1.7.1
   hooks:
   - id: mypy
     additional_dependencies:

CODE_OF_CONDUCT.md

@@ -1,6 +1,6 @@
 # Conda Organization Code of Conduct
 
-> **Note**
+> [!NOTE]
 > Below is the short version of our CoC, see the long version [here](https://github.com/conda-incubator/governance/blob/main/CODE_OF_CONDUCT.md).
 
 # The Short Version

conda_lock/conda_lock.py

@@ -761,6 +761,11 @@ def _solve_for_arch(
             conda_locked={dep.name: dep for dep in conda_deps.values()},
             python_version=conda_deps["python"].version,
             platform=platform,
+            platform_virtual_packages=spec.virtual_package_repo.all_repodata.get(
+                platform, {"packages": None}
+            )["packages"]
+            if spec.virtual_package_repo
+            else None,
             pip_repositories=pip_repositories,
             allow_pypi_requests=spec.allow_pypi_requests,
             strip_auth=strip_auth,

conda_lock/pypi_solver.py

@@ -1,14 +1,16 @@
 import re
 import sys
+import warnings
 
 from pathlib import Path
 from posixpath import expandvars
-from typing import TYPE_CHECKING, Dict, List, Optional
-from urllib.parse import urldefrag, urlparse, urlsplit, urlunsplit
+from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple
+from urllib.parse import urldefrag, urlsplit, urlunsplit
 
 from clikit.api.io.flags import VERY_VERBOSE
 from clikit.io import ConsoleIO, NullIO
 from packaging.tags import compatible_tags, cpython_tags, mac_platforms
+from packaging.version import Version
 
 from conda_lock.interfaces.vendored_poetry import (
     Chooser,
@@ -39,10 +41,11 @@
 if TYPE_CHECKING:
     from packaging.tags import Tag
 
-
 # NB: in principle these depend on the glibc on the machine creating the conda env.
 # We use tags supported by manylinux Docker images, which are likely the most common
 # in practice, see https://github.com/pypa/manylinux/blob/main/README.rst#docker-images.
+
+# NOTE: Keep the max in sync with the default value used in virtual_packages.py
 MANYLINUX_TAGS = ["1", "2010", "2014", "_2_17", "_2_24", "_2_28"]
 # This needs to be updated periodically as new macOS versions are released.
 MACOS_VERSION = (13, 4)
@@ -53,17 +56,30 @@ class PlatformEnv(Env):
     Fake poetry Env to match PyPI distributions to the target conda environment
     """
 
-    def __init__(self, python_version: str, platform: str):
+    _sys_platform: Literal["darwin", "linux", "win32"]
+    _platform_system: Literal["Darwin", "Linux", "Windows"]
+    _os_name: Literal["posix", "nt"]
+    _platforms: List[str]
+    _python_version: Tuple[int, ...]
+
+    def __init__(
+        self,
+        python_version: str,
+        platform: str,
+        platform_virtual_packages: Optional[Dict[str, dict]] = None,
+    ):
         super().__init__(path=Path(sys.prefix))
         system, arch = platform.split("-")
         if arch == "64":
             arch = "x86_64"
 
         if system == "linux":
+            compatible_manylinux_tags = _compute_compatible_manylinux_tags(
+                platform_virtual_packages=platform_virtual_packages
+            )
             self._platforms = [
-                f"manylinux{tag}_{arch}" for tag in reversed(MANYLINUX_TAGS)
-            ]
-            self._platforms.append(f"linux_{arch}")
+                f"manylinux{tag}_{arch}" for tag in compatible_manylinux_tags
+            ] + [f"linux_{arch}"]
         elif system == "osx":
             self._platforms = list(mac_platforms(MACOS_VERSION, arch))
         elif platform == "win-64":
@@ -110,6 +126,115 @@ def get_marker_env(self) -> Dict[str, str]:
         }
 
 
+def _extract_glibc_version_from_virtual_packages(
+    platform_virtual_packages: Dict[str, dict]
+) -> Optional[Version]:
+    """Get the glibc version from the "package" repodata of a chosen platform.
+
+    Note that the glibc version coming from a virtual package is never a legacy
+    manylinux tag (i.e. 1, 2010, or 2014). Those tags predate PEP 600 which
+    introduced manylinux tags containing the glibc version. Currently, all
+    relevant glibc versions look like 2.XX.
+
+    >>> platform_virtual_packages = {
+    ...     "__glibc-2.17-0.tar.bz2": {
+    ...         "name": "__glibc",
+    ...         "version": "2.17",
+    ...     },
+    ... }
+    >>> _extract_glibc_version_from_virtual_packages(platform_virtual_packages)
+    <Version('2.17')>
+    >>> _extract_glibc_version_from_virtual_packages({}) is None
+    True
+    """
+    matches: List[Version] = []
+    for p in platform_virtual_packages.values():
+        if p["name"] == "__glibc":
+            matches.append(Version(p["version"]))
+    if len(matches) == 0:
+        return None
+    elif len(matches) == 1:
+        return matches[0]
+    else:
+        lowest = min(matches)
+        warnings.warn(
+            f"Multiple __glibc virtual package entries found! "
+            f"{matches=} Using the lowest version {lowest}."
+        )
+        return lowest
+
+
+def _glibc_version_from_manylinux_tag(tag: str) -> Version:
+    """
+    Return the glibc version for the given manylinux tag
+
+    >>> _glibc_version_from_manylinux_tag("2010")
+    <Version('2.12')>
+    >>> _glibc_version_from_manylinux_tag("_2_28")
+    <Version('2.28')>
+    """
+    SPECIAL_CASES = {
+        "1": Version("2.5"),
+        "2010": Version("2.12"),
+        "2014": Version("2.17"),
+    }
+    if tag in SPECIAL_CASES:
+        return SPECIAL_CASES[tag]
+    elif tag.startswith("_"):
+        return Version(tag[1:].replace("_", "."))
+    else:
+        raise ValueError(f"Unknown manylinux tag {tag}")
+
+
+def _compute_compatible_manylinux_tags(
+    platform_virtual_packages: Optional[Dict[str, dict]]
+) -> List[str]:
+    """Determine the manylinux tags that are compatible with the given platform.
+
+    If there is no glibc virtual package, then assume that all manylinux tags are
+    compatible.
+
+    The result is sorted in descending order in order to favor the latest.
+
+    >>> platform_virtual_packages = {
+    ...     "__glibc-2.24-0.tar.bz2": {
+    ...         "name": "__glibc",
+    ...         "version": "2.24",
+    ...     },
+    ... }
+    >>> _compute_compatible_manylinux_tags({}) == list(reversed(MANYLINUX_TAGS))
+    True
+    >>> _compute_compatible_manylinux_tags(platform_virtual_packages)
+    ['_2_24', '_2_17', '2014', '2010', '1']
+    """
+    # We use MANYLINUX_TAGS but only go up to the latest supported version
+    # as provided by __glibc if present
+
+    latest_supported_glibc_version: Optional[Version] = None
+    # Try to get the glibc version from the virtual packages if it exists
+    if platform_virtual_packages:
+        latest_supported_glibc_version = _extract_glibc_version_from_virtual_packages(
+            platform_virtual_packages
+        )
+    # Fall back to the latest of MANYLINUX_TAGS
+    if latest_supported_glibc_version is None:
+        latest_supported_glibc_version = _glibc_version_from_manylinux_tag(
+            MANYLINUX_TAGS[-1]
+        )
+
+    # The glibc versions are backwards compatible, so filter the MANYLINUX_TAGS
+    # to those compatible with less than or equal to the latest supported
+    # glibc version.
+    # Note that MANYLINUX_TAGS is sorted in ascending order. The latest tag
+    # is most preferred so we reverse the order.
+    compatible_manylinux_tags = [
+        tag
+        for tag in reversed(MANYLINUX_TAGS)
+        if _glibc_version_from_manylinux_tag(tag) <= latest_supported_glibc_version
+    ]
+    return compatible_manylinux_tags
+
+
 REQUIREMENT_PATTERN = re.compile(
     r"""
     ^
@@ -260,6 +385,7 @@ def solve_pypi(
     conda_locked: Dict[str, LockedDependency],
     python_version: str,
     platform: str,
+    platform_virtual_packages: Optional[Dict[str, dict]] = None,
     pip_repositories: Optional[List[PipRepository]] = None,
     allow_pypi_requests: bool = True,
     verbose: bool = False,
@@ -284,6 +410,8 @@ def solve_pypi(
         Version of Python in conda_locked
     platform :
         Target platform
+    platform_virtual_packages :
+        Virtual packages for the target platform
     allow_pypi_requests :
         Add pypi.org to the list of repositories (pip packages only)
     verbose :
@@ -347,7 +475,7 @@ def solve_pypi(
     to_update = list(
         {spec.name for spec in pip_locked.values()}.intersection(use_latest)
     )
-    env = PlatformEnv(python_version, platform)
+    env = PlatformEnv(python_version, platform, platform_virtual_packages)
     # find platform-specific solution (e.g. dependencies conditioned on markers)
     with s.use_environment(env):
         result = s.solve(use_latest=to_update)

conda_lock/src_parser/environment_yaml.py

@@ -54,29 +54,32 @@ def _parse_environment_file_for_platform(
         dependencies.append(conda_spec_to_versioned_dep(spec, category))
 
     for mapping_spec in mapping_specs:
-        if "pip" in mapping_spec:
-            for spec in mapping_spec["pip"]:
-                if re.match(r"^-e .*$", spec):
-                    print(
-                        (
-                            f"Warning: editable pip dep '{spec}' will not be included in the lock file. "
-                            "You will need to install it separately."
-                        ),
-                        file=sys.stderr,
-                    )
-                    continue
-
-                dependencies.append(
-                    parse_python_requirement(
-                        spec,
-                        manager="pip",
-                        category=category,
-                        normalize_name=False,
-                    )
+        pip = mapping_spec.get("pip")
+        if pip is None:
+            # might not be present OR might be None due to platform selector
+            continue
+        for spec in pip:
+            if re.match(r"^-e .*$", spec):
+                print(
+                    (
+                        f"Warning: editable pip dep '{spec}' will not be included in the lock file. "
+                        "You will need to install it separately."
+                    ),
+                    file=sys.stderr,
                 )
+                continue
+
+            dependencies.append(
+                parse_python_requirement(
+                    spec,
+                    manager="pip",
+                    category=category,
+                    normalize_name=False,
+                )
+            )
 
-            # ensure pip is in target env
-            dependencies.append(parse_python_requirement("pip", manager="conda"))
+        # ensure pip is in target env
+        dependencies.append(parse_python_requirement("pip", manager="conda"))
 
     return dependencies
 

conda_lock/virtual_package.py

@@ -207,7 +207,8 @@ def default_virtual_package_repodata(cuda_version: str = "11.4") -> FakeRepoData
     )
     repodata.add_package(archspec_ppc64le, subdirs=["linux-ppc64le"])
 
-    glibc_virtual = FakePackage(name="__glibc", version="2.17")
+    # NOTE: Keep this in sync with the MANYLINUX_TAGS maximum in pypi_solver.py
+    glibc_virtual = FakePackage(name="__glibc", version="2.28")
     repodata.add_package(
         glibc_virtual, subdirs=["linux-aarch64", "linux-ppc64le", "linux-64"]
     )

pyproject.toml

@@ -109,7 +109,7 @@ exclude = [
 ]
 
 [tool.pytest.ini_options]
-addopts = "-vrsx -n auto"
+addopts = "--doctest-modules -vrsx -n auto"
 flake8-max-line-length = 105
 flake8-ignore = ["docs/* ALL", "conda_lock/_version.py ALL"]
 filterwarnings = "ignore::DeprecationWarning"

tests/test-pip-respects-glibc-version/environment.yml

@@ -0,0 +1,8 @@
+# environment.yml
+channels:
+  - conda-forge
+
+dependencies:
+  - pip
+  - pip:
+    - cryptography

tests/test-pip-respects-glibc-version/virtual-packages.yml

@@ -0,0 +1,4 @@
+subdirs:
+  linux-64:
+    packages:
+      __glibc: "2.17"

tests/test-pip-with-platform-selector/environment.yml

@@ -0,0 +1,10 @@
+name: test-pip-with-platform-selector
+channels:
+  - conda-forge
+dependencies:
+  - tomli
+  - pip:
+    - psutil  # [linux64]
+platforms:
+  - linux-64
+  - osx-arm64