diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index e7891d8ec..f77f3eaea 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -761,6 +761,11 @@ def _solve_for_arch( conda_locked={dep.name: dep for dep in conda_deps.values()}, python_version=conda_deps["python"].version, platform=platform, + platform_virtual_packages=spec.virtual_package_repo.all_repodata.get( + platform, {"packages": None} + )["packages"] + if spec.virtual_package_repo + else None, pip_repositories=pip_repositories, allow_pypi_requests=spec.allow_pypi_requests, strip_auth=strip_auth, diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index f00828a15..cf0446636 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -43,6 +43,8 @@ # NB: in principle these depend on the glibc on the machine creating the conda env. # We use tags supported by manylinux Docker images, which are likely the most common # in practice, see https://github.com/pypa/manylinux/blob/main/README.rst#docker-images. + +# NOTE: Keep the max in sync with the default value used in virtual_packages.py MANYLINUX_TAGS = ["1", "2010", "2014", "_2_17", "_2_24", "_2_28"] # This needs to be updated periodically as new macOS versions are released. MACOS_VERSION = (13, 4) @@ -53,16 +55,43 @@ class PlatformEnv(Env): Fake poetry Env to match PyPI distributions to the target conda environment """ - def __init__(self, python_version: str, platform: str): + def __init__( + self, + python_version: str, + platform: str, + platform_virtual_packages: Optional[Dict[str, dict]] = None, + ): super().__init__(path=Path(sys.prefix)) system, arch = platform.split("-") if arch == "64": arch = "x86_64" if system == "linux": - self._platforms = [ - f"manylinux{tag}_{arch}" for tag in reversed(MANYLINUX_TAGS) - ] + # We use MANYLINUX_TAGS but only go up to the latest supported version + # as provided by __glibc if present + self._platforms = [] + if platform_virtual_packages: + # By default, look for all tags in MANYLINUX_TAGS + glibc_version = MANYLINUX_TAGS[-1] + for p in platform_virtual_packages.values(): + if p["name"] == "__glibc": + glibc_version = p["version"] + glibc_version_splits = glibc_version.split(".") + for tag in MANYLINUX_TAGS: + if tag[0] == "_": + # Compare to see if glibc_version is greater than this version + if tag[1:].split("_") > glibc_version_splits: + break + self._platforms.append(f"manylinux{tag}_{arch}") + if tag == glibc_version: # Catches 1 and 2010 case + # We go no further than the maximum specific GLIBC version + break + # Latest tag is most preferred so list first + self._platforms.reverse() + else: + self._platforms = [ + f"manylinux{tag}_{arch}" for tag in reversed(MANYLINUX_TAGS) + ] self._platforms.append(f"linux_{arch}") elif system == "osx": self._platforms = list(mac_platforms(MACOS_VERSION, arch)) @@ -260,6 +289,7 @@ def solve_pypi( conda_locked: Dict[str, LockedDependency], python_version: str, platform: str, + platform_virtual_packages: Optional[Dict[str, dict]] = None, pip_repositories: Optional[List[PipRepository]] = None, allow_pypi_requests: bool = True, verbose: bool = False, @@ -284,6 +314,8 @@ def solve_pypi( Version of Python in conda_locked platform : Target platform + platform_virtual_packages : + Virtual packages for the target platform allow_pypi_requests : Add pypi.org to the list of repositories (pip packages only) verbose : @@ -347,7 +379,7 @@ def solve_pypi( to_update = list( {spec.name for spec in pip_locked.values()}.intersection(use_latest) ) - env = PlatformEnv(python_version, platform) + env = PlatformEnv(python_version, platform, platform_virtual_packages) # find platform-specific solution (e.g. dependencies conditioned on markers) with s.use_environment(env): result = s.solve(use_latest=to_update) diff --git a/conda_lock/virtual_package.py b/conda_lock/virtual_package.py index aa505efaa..b9b9b4d66 100644 --- a/conda_lock/virtual_package.py +++ b/conda_lock/virtual_package.py @@ -207,7 +207,8 @@ def default_virtual_package_repodata(cuda_version: str = "11.4") -> FakeRepoData ) repodata.add_package(archspec_ppc64le, subdirs=["linux-ppc64le"]) - glibc_virtual = FakePackage(name="__glibc", version="2.17") + # NOTE: Keep this in sync with the MANYLINUX_TAGS maximum in pypi_solver.py + glibc_virtual = FakePackage(name="__glibc", version="2.28") repodata.add_package( glibc_virtual, subdirs=["linux-aarch64", "linux-ppc64le", "linux-64"] ) diff --git a/tests/test-pip-respects-glibc-version/environment.yml b/tests/test-pip-respects-glibc-version/environment.yml new file mode 100644 index 000000000..7b7855a15 --- /dev/null +++ b/tests/test-pip-respects-glibc-version/environment.yml @@ -0,0 +1,8 @@ +# environment.yml +channels: + - conda-forge + +dependencies: + - pip + - pip: + - cryptography diff --git a/tests/test-pip-respects-glibc-version/virtual-packages.yml b/tests/test-pip-respects-glibc-version/virtual-packages.yml new file mode 100644 index 000000000..7b304967b --- /dev/null +++ b/tests/test-pip-respects-glibc-version/virtual-packages.yml @@ -0,0 +1,4 @@ +subdirs: + linux-64: + packages: + __glibc: "2.17" \ No newline at end of file diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 0d5d00f39..9265053eb 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -2447,6 +2447,41 @@ def test_pip_finds_recent_manylinux_wheels( assert manylinux_version > [2, 17] +def test_pip_respects_glibc_version( + tmp_path: Path, conda_exe: str, monkeypatch: "pytest.MonkeyPatch" +): + """Ensure that we find a manylinux wheel that respects an older glibc constraint. + + This is somewhat the opposite of test_pip_finds_recent_manylinux_wheels + """ + + env_file = clone_test_dir("test-pip-respects-glibc-version", tmp_path).joinpath( + "environment.yml" + ) + monkeypatch.chdir(env_file.parent) + run_lock( + [env_file], + conda_exe=str(conda_exe), + platforms=["linux-64"], + virtual_package_spec=env_file.parent / "virtual-packages.yml", + ) + + lockfile = parse_conda_lock_file(env_file.parent / DEFAULT_LOCKFILE_NAME) + + (cryptography_dep,) = [p for p in lockfile.package if p.name == "cryptography"] + manylinux_pattern = r"manylinux_(\d+)_(\d+).+\.whl" + # Should return the first match so higher version first. + manylinux_match = re.search(manylinux_pattern, cryptography_dep.url) + assert ( + manylinux_match + ), "No match found for manylinux version in {cryptography_dep.url}" + + manylinux_version = [int(each) for each in manylinux_match.groups()] + # Make sure the manylinux wheel was built with glibc <= 2.17 + # since that is what the virtual package spec requires + assert manylinux_version <= [2, 17] + + def test_parse_environment_file_with_pip_and_platform_selector(): """See https://github.com/conda/conda-lock/pull/564 for the context.""" env_file = TEST_DIR / "test-pip-with-platform-selector" / "environment.yml"