From 3b2dae4cdaaf8a09096e433169bbd6003ffcaa64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Fri, 27 Oct 2023 11:08:19 +0200 Subject: [PATCH 1/5] Make recent manylinux wheels findable by pypi_resolver --- conda_lock/pypi_solver.py | 6 ++++-- tests/test_conda_lock.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index 9fe7f742e..f00828a15 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -40,8 +40,10 @@ from packaging.tags import Tag -# NB: in principle these depend on the glibc in the conda env -MANYLINUX_TAGS = ["1", "2010", "2014", "_2_17"] +# NB: in principle these depend on the glibc on the machine creating the conda env. +# We use tags supported by manylinux Docker images, which are likely the most common +# in practice, see https://github.com/pypa/manylinux/blob/main/README.rst#docker-images. +MANYLINUX_TAGS = ["1", "2010", "2014", "_2_17", "_2_24", "_2_28"] # This needs to be updated periodically as new macOS versions are released. MACOS_VERSION = (13, 4) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 440da233b..d527fbb4f 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -11,6 +11,7 @@ import tempfile import typing import uuid +import re from glob import glob from pathlib import Path @@ -299,6 +300,12 @@ def pip_conda_name_confusion(tmp_path: Path): ) +@pytest.fixture +def lightgbm_environment(tmp_path: Path): + return clone_test_dir("test-pip-finds-recent-manylinux-wheels", tmp_path).joinpath( + "environment.yml" + ) + @pytest.fixture def multi_source_env(tmp_path: Path): f = clone_test_dir("test-multi-sources", tmp_path) @@ -2293,3 +2300,18 @@ def test_cli_version(capsys: "pytest.CaptureFixture[str]"): # the part before, in this case just "0.11.3". version_without_dev = __version__.split(".dev")[0] assert version_without_dev in result.stdout + + +def test_pip_finds_recent_manylinux_wheels(monkeypatch: "pytest.MonkeyPatch", lightgbm_environment: Path, conda_exe: str): + monkeypatch.chdir(lightgbm_environment.parent) + run_lock([lightgbm_environment], conda_exe=conda_exe, platforms=["linux-64"]) + lockfile = parse_conda_lock_file(lightgbm_environment.parent / DEFAULT_LOCKFILE_NAME) + + lightgbm_dep, = [p for p in lockfile.package if p.name == 'lightgbm'] + manylinux_pattern = r'manylinux_(\d+)_(\d+).+\.whl' + manylinux_match = re.search(manylinux_pattern, lightgbm_dep.url) + assert manylinux_match, "No match found for manylinux version in {lightgbm_dep.url}" + + manylinux_version = [int(each) for each in manylinux_match.groups()] + assert manylinux_version > [2, 17] + From e6d0b3b0fe241519e19156e98eb28bfb755f83e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Fri, 27 Oct 2023 12:04:04 +0200 Subject: [PATCH 2/5] Add missing test folder --- .../environment.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/test-pip-finds-recent-manylinux-wheels/environment.yml diff --git a/tests/test-pip-finds-recent-manylinux-wheels/environment.yml b/tests/test-pip-finds-recent-manylinux-wheels/environment.yml new file mode 100644 index 000000000..f3b406705 --- /dev/null +++ b/tests/test-pip-finds-recent-manylinux-wheels/environment.yml @@ -0,0 +1,8 @@ +# environment.yml +channels: + - conda-forge + +dependencies: + - pip + - pip: + - lightgbm From e4f42b221009f5c60b18522448b3e9e470c895f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Fri, 27 Oct 2023 12:05:32 +0200 Subject: [PATCH 3/5] lint --- tests/test_conda_lock.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index d527fbb4f..c8cd3890b 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -5,13 +5,13 @@ import os import pathlib import platform +import re import shutil import subprocess import sys import tempfile import typing import uuid -import re from glob import glob from pathlib import Path @@ -306,6 +306,7 @@ def lightgbm_environment(tmp_path: Path): "environment.yml" ) + @pytest.fixture def multi_source_env(tmp_path: Path): f = clone_test_dir("test-multi-sources", tmp_path) @@ -2302,16 +2303,19 @@ def test_cli_version(capsys: "pytest.CaptureFixture[str]"): assert version_without_dev in result.stdout -def test_pip_finds_recent_manylinux_wheels(monkeypatch: "pytest.MonkeyPatch", lightgbm_environment: Path, conda_exe: str): +def test_pip_finds_recent_manylinux_wheels( + monkeypatch: "pytest.MonkeyPatch", lightgbm_environment: Path, conda_exe: str +): monkeypatch.chdir(lightgbm_environment.parent) run_lock([lightgbm_environment], conda_exe=conda_exe, platforms=["linux-64"]) - lockfile = parse_conda_lock_file(lightgbm_environment.parent / DEFAULT_LOCKFILE_NAME) + lockfile = parse_conda_lock_file( + lightgbm_environment.parent / DEFAULT_LOCKFILE_NAME + ) - lightgbm_dep, = [p for p in lockfile.package if p.name == 'lightgbm'] - manylinux_pattern = r'manylinux_(\d+)_(\d+).+\.whl' + (lightgbm_dep,) = [p for p in lockfile.package if p.name == "lightgbm"] + manylinux_pattern = r"manylinux_(\d+)_(\d+).+\.whl" manylinux_match = re.search(manylinux_pattern, lightgbm_dep.url) assert manylinux_match, "No match found for manylinux version in {lightgbm_dep.url}" manylinux_version = [int(each) for each in manylinux_match.groups()] assert manylinux_version > [2, 17] - From b1f21aa2f99cf5602850a36ded97d416ac17ce15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Fri, 27 Oct 2023 13:30:12 +0200 Subject: [PATCH 4/5] Update tests/test_conda_lock.py Co-authored-by: Ben Mares --- tests/test_conda_lock.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index c8cd3890b..48302f195 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -2306,6 +2306,12 @@ def test_cli_version(capsys: "pytest.CaptureFixture[str]"): def test_pip_finds_recent_manylinux_wheels( monkeypatch: "pytest.MonkeyPatch", lightgbm_environment: Path, conda_exe: str ): + """Ensure that we find a pre-built manylinux wheel for lightgbm. + + If not, installation would trigger a build of lightgbm from source. If this test + fails, it likely means that MANYLINUX_TAGS in `conda_lock/pypi_solver.py` is out of + date. + """ monkeypatch.chdir(lightgbm_environment.parent) run_lock([lightgbm_environment], conda_exe=conda_exe, platforms=["linux-64"]) lockfile = parse_conda_lock_file( From 4ad59d64c9829d0d8c1a5dbc098da70f7718011a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Fri, 27 Oct 2023 13:38:51 +0200 Subject: [PATCH 5/5] Tweak comments --- tests/test_conda_lock.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 48302f195..afbc5f402 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -2306,11 +2306,13 @@ def test_cli_version(capsys: "pytest.CaptureFixture[str]"): def test_pip_finds_recent_manylinux_wheels( monkeypatch: "pytest.MonkeyPatch", lightgbm_environment: Path, conda_exe: str ): - """Ensure that we find a pre-built manylinux wheel for lightgbm. + """Ensure that we find a manylinux wheel with glibc > 2.17 for lightgbm. - If not, installation would trigger a build of lightgbm from source. If this test - fails, it likely means that MANYLINUX_TAGS in `conda_lock/pypi_solver.py` is out of - date. + See https://github.com/conda/conda-lock/issues/517 for more context. + + If not found, installation would trigger a build of lightgbm from source. + If this test fails, it likely means that MANYLINUX_TAGS in + `conda_lock/pypi_solver.py` is out of date. """ monkeypatch.chdir(lightgbm_environment.parent) run_lock([lightgbm_environment], conda_exe=conda_exe, platforms=["linux-64"]) @@ -2324,4 +2326,6 @@ def test_pip_finds_recent_manylinux_wheels( assert manylinux_match, "No match found for manylinux version in {lightgbm_dep.url}" manylinux_version = [int(each) for each in manylinux_match.groups()] + # Make sure the manylinux wheel was built with glibc > 2.17 as a + # non-regression test for #517 assert manylinux_version > [2, 17]