Merge pull request #374 from srilman/pre-determine-platforms

Parse Platforms Before Determining Dependencies

conda_lock/src_parser/__init__.py

@@ -3,10 +3,14 @@
 
 from typing import AbstractSet, List, Optional, Sequence
 
+from conda_lock.common import ordered_union
 from conda_lock.models.channel import Channel
 from conda_lock.models.lock_spec import Dependency, LockSpecification
 from conda_lock.src_parser.aggregation import aggregate_lock_specs
-from conda_lock.src_parser.environment_yaml import parse_environment_file
+from conda_lock.src_parser.environment_yaml import (
+    parse_environment_file,
+    parse_platforms_from_env_file,
+)
 from conda_lock.src_parser.meta_yaml import parse_meta_yaml_file
 from conda_lock.src_parser.pyproject_toml import parse_pyproject_toml
 from conda_lock.virtual_package import FakeRepoData
@@ -18,9 +22,30 @@
 logger = logging.getLogger(__name__)
 
 
+def _parse_platforms_from_srcs(src_files: List[pathlib.Path]) -> List[str]:
+    """
+    Parse a sequence of dependency specifications from source files
+
+    Parameters
+    ----------
+    src_files :
+        Files to parse for dependencies
+    """
+    all_file_platforms: List[List[str]] = []
+    for src_file in src_files:
+        if src_file.name == "meta.yaml":
+            continue
+        elif src_file.name == "pyproject.toml":
+            all_file_platforms.append(parse_pyproject_toml(src_file).platforms)
+        else:
+            all_file_platforms.append(parse_platforms_from_env_file(src_file))
+
+    return ordered_union(all_file_platforms)
+
+
 def _parse_source_files(
     src_files: List[pathlib.Path],
-    platform_overrides: Optional[Sequence[str]],
+    platforms: List[str],
 ) -> List[LockSpecification]:
     """
     Parse a sequence of dependency specifications from source files
@@ -29,27 +54,17 @@ def _parse_source_files(
     ----------
     src_files :
         Files to parse for dependencies
-    platform_overrides :
+    platforms :
         Target platforms to render environment.yaml and meta.yaml files for
     """
     desired_envs: List[LockSpecification] = []
     for src_file in src_files:
         if src_file.name == "meta.yaml":
-            desired_envs.append(
-                parse_meta_yaml_file(
-                    src_file, list(platform_overrides or DEFAULT_PLATFORMS)
-                )
-            )
+            desired_envs.append(parse_meta_yaml_file(src_file, platforms))
         elif src_file.name == "pyproject.toml":
             desired_envs.append(parse_pyproject_toml(src_file))
         else:
-            desired_envs.append(
-                parse_environment_file(
-                    src_file,
-                    platform_overrides,
-                    default_platforms=DEFAULT_PLATFORMS,
-                )
-            )
+            desired_envs.append(parse_environment_file(src_file, platforms))
     return desired_envs
 
 
@@ -62,20 +77,22 @@ def make_lock_spec(
     required_categories: Optional[AbstractSet[str]] = None,
 ) -> LockSpecification:
     """Generate the lockfile specs from a set of input src_files.  If required_categories is set filter out specs that do not match those"""
-    lock_specs = _parse_source_files(
-        src_files=src_files, platform_overrides=platform_overrides
-    )
+    platforms = (
+        list(platform_overrides)
+        if platform_overrides
+        else _parse_platforms_from_srcs(src_files)
+    ) or DEFAULT_PLATFORMS
+
+    lock_specs = _parse_source_files(src_files, platforms)
 
     lock_spec = aggregate_lock_specs(lock_specs)
     lock_spec.virtual_package_repo = virtual_package_repo
+    lock_spec.platforms = platforms
     lock_spec.channels = (
         [Channel.from_string(co) for co in channel_overrides]
         if channel_overrides
         else lock_spec.channels
     )
-    lock_spec.platforms = (
-        list(platform_overrides) if platform_overrides else lock_spec.platforms
-    ) or list(DEFAULT_PLATFORMS)
 
     if required_categories is not None:
 

conda_lock/src_parser/environment_yaml.py

@@ -2,7 +2,7 @@
 import re
 import sys
 
-from typing import List, Optional, Sequence, Tuple
+from typing import List, Tuple
 
 import yaml
 
@@ -96,11 +96,23 @@ def _parse_environment_file_for_platform(
     )
 
 
+def parse_platforms_from_env_file(environment_file: pathlib.Path) -> List[str]:
+    """
+    Parse the list of platforms from an environment-yaml file
+    """
+    if not environment_file.exists():
+        raise FileNotFoundError(f"{environment_file} not found")
+
+    with environment_file.open("r") as fo:
+        content = fo.read()
+        env_yaml_data = yaml.safe_load(content)
+
+    return env_yaml_data.get("platforms", [])
+
+
 def parse_environment_file(
     environment_file: pathlib.Path,
-    given_platforms: Optional[Sequence[str]],
-    *,
-    default_platforms: List[str] = [],
+    platforms: List[str],
 ) -> LockSpecification:
     """Parse a simple environment-yaml file for dependencies assuming the target platforms.
 
@@ -114,15 +126,6 @@ def parse_environment_file(
 
     with environment_file.open("r") as fo:
         content = fo.read()
-        env_yaml_data = yaml.safe_load(content)
-
-    # Get list of platforms from the input file
-    yaml_platforms: Optional[List[str]] = env_yaml_data.get("platforms")
-    # Final list of platforms is the following order of priority
-    # 1) List Passed in via the -p flag (if any given)
-    # 2) List From the YAML File (if specified)
-    # 3) Default List of Platforms to Render
-    platforms = list(given_platforms or yaml_platforms or default_platforms)
 
     # Parse with selectors for each target platform
     spec = aggregate_lock_specs(

tests/test-multi-sources/dev.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - pytest

tests/test-multi-sources/main.yml

@@ -0,0 +1,6 @@
+dependencies:
+  - python <3.11
+platforms:
+  - osx-arm64
+  - osx-64
+  - linux-64

tests/test-multi-sources/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "conda-lock-test-poetry"
+version = "0.0.1"
+description = ""
+authors = ["conda-lock"]
+
+[tool.poetry.dependencies]
+pandas = "^1.5.0"
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"
+
+[tool.conda-lock]
+platforms = [
+    "win-64",
+    "osx-arm64"
+]

tests/test_conda_lock.py

@@ -67,10 +67,14 @@
 from conda_lock.src_parser import (
     DEFAULT_PLATFORMS,
     LockSpecification,
+    _parse_platforms_from_srcs,
     parse_meta_yaml_file,
 )
 from conda_lock.src_parser.aggregation import aggregate_lock_specs
-from conda_lock.src_parser.environment_yaml import parse_environment_file
+from conda_lock.src_parser.environment_yaml import (
+    parse_environment_file,
+    parse_platforms_from_env_file,
+)
 from conda_lock.src_parser.pyproject_toml import (
     parse_pyproject_toml,
     poetry_version_to_conda_version,
@@ -212,6 +216,12 @@ def git_metadata_zlib_environment(tmp_path: Path):
     return clone_test_dir("zlib", tmp_path).joinpath("environment.yml")
 
 
+@pytest.fixture
+def multi_source_env(tmp_path: Path):
+    f = clone_test_dir("test-multi-sources", tmp_path)
+    return [f.joinpath("main.yml"), f.joinpath("pyproject.toml"), f.joinpath("dev.yml")]
+
+
 @pytest.fixture(
     scope="function",
     params=[
@@ -360,7 +370,8 @@ def test_parse_environment_file_with_pip(pip_environment: Path):
 
 
 def test_parse_env_file_with_filters_no_args(filter_conda_environment: Path):
-    res = parse_environment_file(filter_conda_environment, None)
+    platforms = parse_platforms_from_env_file(filter_conda_environment)
+    res = parse_environment_file(filter_conda_environment, platforms)
     assert all(x in res.platforms for x in ["osx-arm64", "osx-64", "linux-64"])
     assert res.channels == [Channel.from_string("conda-forge")]
 
@@ -423,6 +434,11 @@ def test_parse_env_file_with_filters_defaults(filter_conda_environment: Path):
     )
 
 
+def test_parse_platforms_from_multi_sources(multi_source_env):
+    platforms = _parse_platforms_from_srcs(multi_source_env)
+    assert platforms == ["osx-arm64", "osx-64", "linux-64", "win-64"]
+
+
 def test_choose_wheel() -> None:
     solution = solve_pypi(
         {