Merge pull request #207 from mariusvniekerk/channel-spec

Allow channel::package in package specifications

conda_lock/conda_solver.py

@@ -88,7 +88,10 @@ class DryRunInstall(TypedDict):
 
 
 def _to_match_spec(
-    conda_dep_name: str, conda_version: Optional[str], build: Optional[str]
+    conda_dep_name: str,
+    conda_version: Optional[str],
+    build: Optional[str],
+    conda_channel: Optional[str],
 ) -> str:
     kwargs = dict(name=conda_dep_name)
     if conda_version:
@@ -97,10 +100,17 @@ def _to_match_spec(
         kwargs["build"] = build
         if "version" not in kwargs:
             kwargs["version"] = "*"
+    if conda_channel:
+        kwargs["channel"] = conda_channel
 
     ms = MatchSpec(**kwargs)
     # Since MatchSpec doesn't round trip to the cli well
-    return ms.conda_build_form()
+    if conda_channel:
+        # this will return "channel_name::package_name"
+        return str(ms)
+    else:
+        # this will return only "package_name" even if there's a channel in the kwargs
+        return ms.conda_build_form()
 
 
 def extract_json_object(proc_stdout: str) -> str:
@@ -139,7 +149,7 @@ def solve_conda(
     """
 
     conda_specs = [
-        _to_match_spec(dep.name, dep.version, dep.build)
+        _to_match_spec(dep.name, dep.version, dep.build, dep.conda_channel)
         for dep in specs.values()
         if isinstance(dep, VersionedDependency) and dep.manager == "conda"
     ]

conda_lock/src_parser/__init__.py

@@ -51,7 +51,8 @@ class Dependency(StrictModel):
 
 class VersionedDependency(Dependency):
     version: str
-    build: Optional[str]
+    build: Optional[str] = None
+    conda_channel: Optional[str] = None
 
 
 class URLDependency(Dependency):

conda_lock/src_parser/conda_common.py

@@ -0,0 +1,33 @@
+from typing import Optional
+
+from ..src_parser import VersionedDependency
+from ..vendor.conda.models.channel import Channel
+from ..vendor.conda.models.match_spec import MatchSpec
+
+
+def conda_spec_to_versioned_dep(spec: str, category: str) -> VersionedDependency:
+    """Convert a string form conda spec into a versioned dependency for a given category.
+
+    This is used by the environment.yaml and meta.yaml specification parser
+    """
+
+    try:
+        ms = MatchSpec(spec)
+    except Exception as e:
+        raise RuntimeError(f"Failed to turn `{spec}` into a MatchSpec") from e
+
+    package_channel: Optional[Channel] = ms.get("channel")
+    if package_channel:
+        channel_str = package_channel.canonical_name
+    else:
+        channel_str = None
+    return VersionedDependency(
+        name=ms.name,
+        version=ms.get("version", ""),
+        manager="conda",
+        optional=category != "main",
+        category=category,
+        extras=[],
+        build=ms.get("build"),
+        conda_channel=channel_str,
+    )

conda_lock/src_parser/environment_yaml.py

@@ -6,7 +6,8 @@
 
 import yaml
 
-from conda_lock.src_parser import Dependency, LockSpecification, VersionedDependency
+from conda_lock.src_parser import Dependency, LockSpecification
+from conda_lock.src_parser.conda_common import conda_spec_to_versioned_dep
 from conda_lock.src_parser.selectors import filter_platform_selectors
 
 from .pyproject_toml import parse_python_requirement
@@ -64,24 +65,8 @@ def parse_environment_file(
     specs = [x for x in specs if isinstance(x, str)]
 
     for spec in specs:
-        from ..vendor.conda.models.match_spec import MatchSpec
-
-        try:
-            ms = MatchSpec(spec)
-        except Exception as e:
-            raise RuntimeError(f"Failed to turn `{spec}` into a MatchSpec") from e
-
-        dependencies.append(
-            VersionedDependency(
-                name=ms.name,
-                version=ms.get("version", ""),
-                manager="conda",
-                optional=category != "main",
-                category=category,
-                extras=[],
-                build=ms.get("build"),
-            )
-        )
+        vdep = conda_spec_to_versioned_dep(spec, category)
+        dependencies.append(vdep)
     for mapping_spec in mapping_specs:
         if "pip" in mapping_spec:
             if pip_support:

conda_lock/src_parser/meta_yaml.py

@@ -141,21 +141,9 @@ def add_spec(spec: str, category: str) -> None:
             return
 
         from ..vendor.conda.models.match_spec import MatchSpec
+        from .conda_common import conda_spec_to_versioned_dep
 
-        try:
-            ms = MatchSpec(spec)
-        except Exception as e:
-            raise RuntimeError(f"Failed to turn `{spec}` into a MatchSpec") from e
-
-        dep = VersionedDependency(
-            name=ms.name,
-            version=ms.get("version", ""),
-            manager="conda",
-            optional=category != "main",
-            category=category,
-            extras=[],
-            build=ms.get("build"),
-        )
+        dep = conda_spec_to_versioned_dep(spec, category)
         dep.selectors.platform = [platform]
         dependencies.append(dep)
 

tests/test-channel-inversion/environment.yaml

@@ -0,0 +1,12 @@
+channels:
+  # This example is constructed in this way to mirror a real life packaging
+  # issue that was experienced in 2022-07.
+  # In this case we have the channel order as recommended by rapids ai for
+  # installing cudf, but we want to force getting cuda-python from conda-forge
+  # instead of from the nvidia channel which would normally have higher priority
+  - rapidsai
+  - nvidia
+  - conda-forge
+dependencies:
+  - cudf
+  - conda-forge::cuda-python

tests/test_conda_lock.py

@@ -51,6 +51,7 @@
 from conda_lock.src_parser import (
     HashModel,
     LockedDependency,
+    Lockfile,
     LockSpecification,
     VersionedDependency,
 )
@@ -60,6 +61,7 @@
     parse_pyproject_toml,
     poetry_version_to_conda_version,
 )
+from conda_lock.vendor.conda.models.match_spec import MatchSpec
 
 
 TEST_DIR = pathlib.Path(__file__).parent
@@ -149,6 +151,12 @@ def pdm_pyproject_toml():
     return TEST_DIR.joinpath("test-pdm").joinpath("pyproject.toml")
 
 
+@pytest.fixture
+def channel_inversion():
+    """Path to an environment.yaml that has a hardcoded channel in one of the dependencies"""
+    return TEST_DIR.joinpath("test-channel-inversion").joinpath("environment.yaml")
+
+
 @pytest.fixture(
     scope="function",
     params=[
@@ -620,6 +628,28 @@ def test_poetry_version_parsing_constraints(package, version, url_pattern, capsy
         assert url_pattern in python.url
 
 
+def test_run_with_channel_inversion(monkeypatch, channel_inversion, mamba_exe):
+    """Given that the cuda_python package is available from a few channels
+    and three of those channels listed
+    and with conda-forge listed as the lowest priority channel
+    and with the cuda_python dependency listed as "conda-forge::cuda_python",
+    ensure that the lock file parse picks up conda-forge as the channel and not one of the higher priority channels
+    """
+    with filelock.FileLock(str(channel_inversion.parent / "filelock")):
+        monkeypatch.chdir(channel_inversion.parent)
+        run_lock([channel_inversion], conda_exe=mamba_exe, platforms=["linux-64"])
+        lockfile = parse_conda_lock_file(
+            channel_inversion.parent / DEFAULT_LOCKFILE_NAME
+        )
+        for package in lockfile.package:
+            if package.name == "cuda-python":
+                ms = MatchSpec(package.url)
+                assert ms.get("channel") == "conda-forge"
+                break
+        else:
+            raise ValueError("cuda-python not found!")
+
+
 def _make_spec(name, constraint="*"):
     return VersionedDependency(
         name=name,
@@ -750,6 +780,21 @@ def conda_exe(request):
     raise pytest.skip(f"{request.param} is not installed")
 
 
+@pytest.fixture(scope="session")
+def mamba_exe():
+    """Provides a fixture for tests that require mamba"""
+    kwargs = dict(
+        mamba=True,
+        micromamba=False,
+        conda=False,
+        conda_exe=False,
+    )
+    _conda_exe = _ensureconda(**kwargs)
+    if _conda_exe is not None:
+        return _conda_exe
+    raise pytest.skip("mamba is not installed")
+
+
 def _check_package_installed(package: str, prefix: str):
     import glob
 
@@ -1160,6 +1205,7 @@ def test_fake_conda_env(conda_exe, conda_lock_yaml):
             assert platform == path.parent.name
 
 
+@flaky
 def test_private_lock(quetz_server, tmp_path, monkeypatch, capsys, conda_exe):
     if is_micromamba(conda_exe):
         res = subprocess.run(