diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index 631db834b..fbe4149c0 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -762,6 +762,11 @@ def _solve_for_arch( conda_locked={dep.name: dep for dep in conda_deps.values()}, python_version=conda_deps["python"].version, platform=platform, + platform_virtual_packages=spec.virtual_package_repo.all_repodata.get( + platform, {"packages": None} + )["packages"] + if spec.virtual_package_repo + else None, pip_repositories=pip_repositories, allow_pypi_requests=spec.allow_pypi_requests, strip_auth=strip_auth, diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index 335758ae5..046efcbce 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -1,14 +1,16 @@ import re import sys +import warnings from pathlib import Path from posixpath import expandvars -from typing import TYPE_CHECKING, Dict, List, Optional -from urllib.parse import urldefrag, urlparse, urlsplit, urlunsplit +from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple +from urllib.parse import urldefrag, urlsplit, urlunsplit from clikit.api.io.flags import VERY_VERBOSE from clikit.io import ConsoleIO, NullIO from packaging.tags import compatible_tags, cpython_tags, mac_platforms +from packaging.version import Version from conda_lock.interfaces.vendored_poetry import ( Chooser, @@ -39,11 +41,12 @@ if TYPE_CHECKING: from packaging.tags import Tag - # NB: in principle these depend on the glibc on the machine creating the conda env. # We use tags supported by manylinux Docker images, which are likely the most common # in practice, see https://github.com/pypa/manylinux/blob/main/README.rst#docker-images. +# NOTE: Keep the max in sync with the default value used in virtual_packages.py MANYLINUX_TAGS = ["1", "2010", "2014", "_2_17", "_2_24", "_2_28"] + # This needs to be updated periodically as new macOS versions are released. MACOS_VERSION = (13, 4) @@ -53,17 +56,32 @@ class PlatformEnv(Env): Fake poetry Env to match PyPI distributions to the target conda environment """ - def __init__(self, python_version: str, platform: str): + _sys_platform: Literal["darwin", "linux", "win32"] + _platform_system: Literal["Darwin", "Linux", "Windows"] + _os_name: Literal["posix", "nt"] + _platforms: List[str] + _python_version: Tuple[int, ...] + + def __init__( + self, + python_version: str, + platform: str, + platform_virtual_packages: Optional[Dict[str, dict]] = None, + ): super().__init__(path=Path(sys.prefix)) system, arch = platform.split("-") if arch == "64": arch = "x86_64" if system == "linux": + # Summary of the manylinux tag story: + # + compatible_manylinux_tags = _compute_compatible_manylinux_tags( + platform_virtual_packages=platform_virtual_packages + ) self._platforms = [ - f"manylinux{tag}_{arch}" for tag in reversed(MANYLINUX_TAGS) - ] - self._platforms.append(f"linux_{arch}") + f"manylinux{tag}_{arch}" for tag in compatible_manylinux_tags + ] + [f"linux_{arch}"] elif system == "osx": self._platforms = list(mac_platforms(MACOS_VERSION, arch)) elif platform == "win-64": @@ -110,6 +128,115 @@ def get_marker_env(self) -> Dict[str, str]: } +def _extract_glibc_version_from_virtual_packages( + platform_virtual_packages: Dict[str, dict] +) -> Optional[Version]: + """Get the glibc version from the "package" repodata of a chosen platform. + + Note that the glibc version coming from a virtual package is never a legacy + manylinux tag (i.e. 1, 2010, or 2014). Those tags predate PEP 600 which + introduced manylinux tags containing the glibc version. Currently, all + relevant glibc versions look like 2.XX. + + >>> platform_virtual_packages = { + ... "__glibc-2.17-0.tar.bz2": { + ... "name": "__glibc", + ... "version": "2.17", + ... }, + ... } + >>> _extract_glibc_version_from_virtual_packages(platform_virtual_packages) + + >>> _extract_glibc_version_from_virtual_packages({}) is None + True + """ + matches: List[Version] = [] + for p in platform_virtual_packages.values(): + if p["name"] == "__glibc": + matches.append(Version(p["version"])) + if len(matches) == 0: + return None + elif len(matches) == 1: + return matches[0] + else: + lowest = min(matches) + warnings.warn( + f"Multiple __glibc virtual package entries found! " + f"{matches=} Using the lowest version {lowest}." + ) + return lowest + + +def _glibc_version_from_manylinux_tag(tag: str) -> Version: + """ + Return the glibc version for the given manylinux tag + + >>> _glibc_version_from_manylinux_tag("2010") + + >>> _glibc_version_from_manylinux_tag("_2_28") + + """ + SPECIAL_CASES = { + "1": Version("2.5"), + "2010": Version("2.12"), + "2014": Version("2.17"), + } + if tag in SPECIAL_CASES: + return SPECIAL_CASES[tag] + elif tag.startswith("_"): + return Version(tag[1:].replace("_", ".")) + else: + raise ValueError(f"Unknown manylinux tag {tag}") + + +def _compute_compatible_manylinux_tags( + platform_virtual_packages: Optional[Dict[str, dict]] +) -> List[str]: + """Determine the manylinux tags that are compatible with the given platform. + + If there is no glibc virtual package, then assume that all manylinux tags are + compatible. + + The result is sorted in descending order in order to favor the latest. + + >>> platform_virtual_packages = { + ... "__glibc-2.24-0.tar.bz2": { + ... "name": "__glibc", + ... "version": "2.24", + ... }, + ... } + >>> _compute_compatible_manylinux_tags({}) == list(reversed(MANYLINUX_TAGS)) + True + >>> _compute_compatible_manylinux_tags(platform_virtual_packages) + ['_2_24', '_2_17', '2014', '2010', '1'] + """ + # We use MANYLINUX_TAGS but only go up to the latest supported version + # as provided by __glibc if present + + latest_supported_glibc_version: Optional[Version] = None + # Try to get the glibc version from the virtual packages if it exists + if platform_virtual_packages: + latest_supported_glibc_version = _extract_glibc_version_from_virtual_packages( + platform_virtual_packages + ) + # Fall back to the latest of MANYLINUX_TAGS + if latest_supported_glibc_version is None: + latest_supported_glibc_version = _glibc_version_from_manylinux_tag( + MANYLINUX_TAGS[-1] + ) + + # The glibc versions are backwards compatible, so filter the MANYLINUX_TAGS + # to those compatible with less than or equal to the latest supported + # glibc version. + # Note that MANYLINUX_TAGS is sorted in ascending order. The latest tag + # is most preferred so we reverse the order. + compatible_manylinux_tags = [ + tag + for tag in reversed(MANYLINUX_TAGS) + if _glibc_version_from_manylinux_tag(tag) <= latest_supported_glibc_version + ] + return compatible_manylinux_tags + + REQUIREMENT_PATTERN = re.compile( r""" ^ @@ -265,6 +392,7 @@ def solve_pypi( conda_locked: Dict[str, LockedDependency], python_version: str, platform: str, + platform_virtual_packages: Optional[Dict[str, dict]] = None, pip_repositories: Optional[List[PipRepository]] = None, allow_pypi_requests: bool = True, verbose: bool = False, @@ -289,6 +417,8 @@ def solve_pypi( Version of Python in conda_locked platform : Target platform + platform_virtual_packages : + Virtual packages for the target platform allow_pypi_requests : Add pypi.org to the list of repositories (pip packages only) verbose : @@ -352,7 +482,7 @@ def solve_pypi( to_update = list( {spec.name for spec in pip_locked.values()}.intersection(use_latest) ) - env = PlatformEnv(python_version, platform) + env = PlatformEnv(python_version, platform, platform_virtual_packages) # find platform-specific solution (e.g. dependencies conditioned on markers) with s.use_environment(env): result = s.solve(use_latest=to_update) diff --git a/conda_lock/virtual_package.py b/conda_lock/virtual_package.py index aa505efaa..b9b9b4d66 100644 --- a/conda_lock/virtual_package.py +++ b/conda_lock/virtual_package.py @@ -207,7 +207,8 @@ def default_virtual_package_repodata(cuda_version: str = "11.4") -> FakeRepoData ) repodata.add_package(archspec_ppc64le, subdirs=["linux-ppc64le"]) - glibc_virtual = FakePackage(name="__glibc", version="2.17") + # NOTE: Keep this in sync with the MANYLINUX_TAGS maximum in pypi_solver.py + glibc_virtual = FakePackage(name="__glibc", version="2.28") repodata.add_package( glibc_virtual, subdirs=["linux-aarch64", "linux-ppc64le", "linux-64"] ) diff --git a/pyproject.toml b/pyproject.toml index b6523232f..a01e81b3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,7 @@ exclude = [ ] [tool.pytest.ini_options] -addopts = "-vrsx -n auto" +addopts = "--doctest-modules -vrsx -n auto" flake8-max-line-length = 105 flake8-ignore = ["docs/* ALL", "conda_lock/_version.py ALL"] filterwarnings = "ignore::DeprecationWarning" diff --git a/tests/test-pip-respects-glibc-version/environment.yml b/tests/test-pip-respects-glibc-version/environment.yml new file mode 100644 index 000000000..7b7855a15 --- /dev/null +++ b/tests/test-pip-respects-glibc-version/environment.yml @@ -0,0 +1,8 @@ +# environment.yml +channels: + - conda-forge + +dependencies: + - pip + - pip: + - cryptography diff --git a/tests/test-pip-respects-glibc-version/virtual-packages.yml b/tests/test-pip-respects-glibc-version/virtual-packages.yml new file mode 100644 index 000000000..7b304967b --- /dev/null +++ b/tests/test-pip-respects-glibc-version/virtual-packages.yml @@ -0,0 +1,4 @@ +subdirs: + linux-64: + packages: + __glibc: "2.17" \ No newline at end of file diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index cb53f9252..e6997a6ba 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -66,7 +66,13 @@ from conda_lock.models.channel import Channel from conda_lock.models.lock_spec import Dependency, VCSDependency, VersionedDependency from conda_lock.models.pip_repository import PipRepository -from conda_lock.pypi_solver import _strip_auth, parse_pip_requirement, solve_pypi +from conda_lock.pypi_solver import ( + MANYLINUX_TAGS, + PlatformEnv, + _strip_auth, + parse_pip_requirement, + solve_pypi, +) from conda_lock.src_parser import ( DEFAULT_PLATFORMS, LockSpecification, @@ -2230,9 +2236,9 @@ def test_default_virtual_package_input_hash_stability(): vpr = default_virtual_package_repodata() expected = { - "linux-64": "dbd71bccc4b3be81038e44b1f14891ccec40a6d70a43cfe02295fc77a2ea9eb5", - "linux-aarch64": "023611fb84c00fb5aaeddde1e0ac62e6ae007aecf6f69ccb5dbfc3cd6d945436", - "linux-ppc64le": "8533a7bd0e950f7b085eeef7686d2c4895e84b8ffdbfba6d62863072ac41090c", + "linux-64": "a949aac83da089258ce729fcd54dc0a3a1724ea325d67680d7a6d7cc9c0f1d1b", + "linux-aarch64": "f68603a3a28dbb03d20a25e1dacda3c42b6acc8a93bd31e13c4956115820cfa6", + "linux-ppc64le": "ababb6bc556ac8c9e27a499bf9b83b5757f6ded385caa0c3d7bf3f360dfe358d", "osx-64": "b7eebe4be0654740f67e3023f2ede298f390119ef225f50ad7e7288ea22d5c93", "osx-arm64": "cc82018d1b1809b9aebacacc5ed05ee6a4318b3eba039607d2a6957571f8bf2b", "win-64": "44239e9f0175404e62e4a80bb8f4be72e38c536280d6d5e484e52fa04b45c9f6", @@ -2447,6 +2453,139 @@ def test_pip_finds_recent_manylinux_wheels( assert manylinux_version > [2, 17] +def test_manylinux_tags(): + from packaging.version import Version + + MANYLINUX_TAGS = pypi_solver.MANYLINUX_TAGS + + # The irregular tags should come at the beginning: + assert MANYLINUX_TAGS[:4] == ["1", "2010", "2014", "_2_17"] + # All other tags should start with "_" + assert all(tag.startswith("_") for tag in MANYLINUX_TAGS[3:]) + + # Now check that the remaining tags parse to versions in increasing order + versions = [Version(tag[1:].replace("_", ".")) for tag in MANYLINUX_TAGS[3:]] + assert versions[0] == Version("2.17") + assert versions == sorted(versions) + + # Verify that the default repodata uses the highest glibc version + default_repodata = default_virtual_package_repodata() + glibc_versions_in_default_repodata: Set[Version] = { + Version(package.version) + for package in default_repodata.packages_by_subdir + if package.name == "__glibc" + } + max_glibc_version_from_manylinux_tags = versions[-1] + assert glibc_versions_in_default_repodata == {max_glibc_version_from_manylinux_tags} + + +def test_pip_respects_glibc_version( + tmp_path: Path, conda_exe: str, monkeypatch: "pytest.MonkeyPatch" +): + """Ensure that we find a manylinux wheel that respects an older glibc constraint. + + This is somewhat the opposite of test_pip_finds_recent_manylinux_wheels + """ + + env_file = clone_test_dir("test-pip-respects-glibc-version", tmp_path).joinpath( + "environment.yml" + ) + monkeypatch.chdir(env_file.parent) + run_lock( + [env_file], + conda_exe=str(conda_exe), + platforms=["linux-64"], + virtual_package_spec=env_file.parent / "virtual-packages.yml", + ) + + lockfile = parse_conda_lock_file(env_file.parent / DEFAULT_LOCKFILE_NAME) + + (cryptography_dep,) = [p for p in lockfile.package if p.name == "cryptography"] + manylinux_pattern = r"manylinux_(\d+)_(\d+).+\.whl" + # Should return the first match so higher version first. + manylinux_match = re.search(manylinux_pattern, cryptography_dep.url) + assert ( + manylinux_match + ), "No match found for manylinux version in {cryptography_dep.url}" + + manylinux_version = [int(each) for each in manylinux_match.groups()] + # Make sure the manylinux wheel was built with glibc <= 2.17 + # since that is what the virtual package spec requires + assert manylinux_version == [2, 17] + + +def test_platformenv_linux_platforms(): + """Check that PlatformEnv correctly handles Linux platforms for wheels""" + # This is the default and maximal list of platforms that we expect + all_expected_platforms = [ + f"manylinux{glibc_ver}_x86_64" for glibc_ver in reversed(MANYLINUX_TAGS) + ] + ["linux_x86_64"] + + # Check that we get the default platforms when no virtual packages are specified + e = PlatformEnv("3.12", "linux-64") + assert e._platforms == all_expected_platforms + + # Check that we get the default platforms when the virtual packages are empty + e = PlatformEnv("3.12", "linux-64", platform_virtual_packages={}) + assert e._platforms == all_expected_platforms + + # Check that we get the default platforms when the virtual packages are nonempty + # but don't include __glibc + platform_virtual_packages = {"x.bz2": {"name": "not_glibc"}} + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == all_expected_platforms + + # Check that we get the expected platforms when using the default repodata. + # (This should include the glibc corresponding to the latest manylinux tag.) + default_repodata = default_virtual_package_repodata() + platform_virtual_packages = default_repodata.all_repodata["linux-64"]["packages"] + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == all_expected_platforms + + # Check that we get the expected platforms after removing glibc from the + # default repodata. + platform_virtual_packages = { + filename: record + for filename, record in platform_virtual_packages.items() + if record["name"] != "__glibc" + } + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == all_expected_platforms + + # Check that we get a restricted list of platforms when specifying a + # lower glibc version. + restricted_platforms = [ + "manylinux_2_17_x86_64", + "manylinux2014_x86_64", + "manylinux2010_x86_64", + "manylinux1_x86_64", + "linux_x86_64", + ] + platform_virtual_packages["__glibc-2.17-0.tar.bz2"] = dict( + name="__glibc", version="2.17" + ) + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == restricted_platforms + + # Check that a warning is raised when there are multiple glibc versions + platform_virtual_packages["__glibc-2.28-0.tar.bz2"] = dict( + name="__glibc", version="2.28" + ) + with pytest.warns(UserWarning): + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == restricted_platforms + + def test_parse_environment_file_with_pip_and_platform_selector(): """See https://github.com/conda/conda-lock/pull/564 for the context.""" env_file = TEST_DIR / "test-pip-with-platform-selector" / "environment.yml"