diff --git a/conda_lock/interfaces/vendored_poetry_markers.py b/conda_lock/interfaces/vendored_poetry_markers.py new file mode 100644 index 00000000..b141a09d --- /dev/null +++ b/conda_lock/interfaces/vendored_poetry_markers.py @@ -0,0 +1,20 @@ +from conda_lock._vendor.poetry.core.version.markers import ( + AnyMarker, + BaseMarker, + EmptyMarker, + MarkerUnion, + MultiMarker, + SingleMarker, + parse_marker, +) + + +__all__ = [ + "AnyMarker", + "BaseMarker", + "EmptyMarker", + "MarkerUnion", + "MultiMarker", + "SingleMarker", + "parse_marker", +] diff --git a/conda_lock/models/lock_spec.py b/conda_lock/models/lock_spec.py index db69960d..bcba92c5 100644 --- a/conda_lock/models/lock_spec.py +++ b/conda_lock/models/lock_spec.py @@ -19,6 +19,7 @@ class _BaseDependency(StrictModel): manager: Literal["conda", "pip"] = "conda" category: str = "main" extras: List[str] = [] + markers: Optional[str] = None @validator("extras") def sorted_extras(cls, v: List[str]) -> List[str]: @@ -55,6 +56,7 @@ class PoetryMappedDependencySpec(StrictModel): url: Optional[str] manager: Literal["conda", "pip"] extras: List + markers: Optional[str] poetry_version_spec: Optional[str] diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index 1a67caa8..fedddf95 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -74,13 +74,14 @@ class PlatformEnv(Env): _platform_system: Literal["Darwin", "Linux", "Windows"] _os_name: Literal["posix", "nt"] _platforms: List[str] - _python_version: Tuple[int, ...] + _python_version: Optional[Tuple[int, ...]] def __init__( self, - python_version: str, + *, platform: str, platform_virtual_packages: Optional[Dict[str, dict]] = None, + python_version: Optional[str] = None, ): super().__init__(path=Path(sys.prefix)) system, arch = platform.split("-") @@ -102,7 +103,10 @@ def __init__( self._platforms = ["win_amd64"] else: raise ValueError(f"Unsupported platform '{platform}'") - self._python_version = tuple(map(int, python_version.split("."))) + if python_version is None: + self._python_version = None + else: + self._python_version = tuple(map(int, python_version.split("."))) if system == "osx": self._sys_platform = "darwin" @@ -133,13 +137,19 @@ def get_supported_tags(self) -> List["Tag"]: def get_marker_env(self) -> Dict[str, str]: """Return the subset of info needed to match common markers""" - return { - "python_full_version": ".".join([str(c) for c in self._python_version]), - "python_version": ".".join([str(c) for c in self._python_version[:2]]), + result: Dict[str, str] = { "sys_platform": self._sys_platform, "platform_system": self._platform_system, "os_name": self._os_name, } + if self._python_version is not None: + result["python_full_version"] = ".".join( + [str(c) for c in self._python_version] + ) + result["python_version"] = ".".join( + [str(c) for c in self._python_version[:2]] + ) + return result def _extract_glibc_version_from_virtual_packages( @@ -529,7 +539,11 @@ def solve_pypi( use_latest ) ) - env = PlatformEnv(python_version, platform, platform_virtual_packages) + env = PlatformEnv( + python_version=python_version, + platform=platform, + platform_virtual_packages=platform_virtual_packages, + ) # find platform-specific solution (e.g. dependencies conditioned on markers) with s.use_environment(env): result = s.solve(use_latest=to_update) diff --git a/conda_lock/src_parser/environment_yaml.py b/conda_lock/src_parser/environment_yaml.py index df04b131..b397032f 100644 --- a/conda_lock/src_parser/environment_yaml.py +++ b/conda_lock/src_parser/environment_yaml.py @@ -8,6 +8,7 @@ from conda_lock.models.lock_spec import Dependency, LockSpecification from conda_lock.src_parser.conda_common import conda_spec_to_versioned_dep +from conda_lock.src_parser.markers import evaluate_marker from conda_lock.src_parser.selectors import filter_platform_selectors from .pyproject_toml import parse_python_requirement @@ -69,14 +70,14 @@ def _parse_environment_file_for_platform( ) continue - dependencies.append( - parse_python_requirement( - spec, - manager="pip", - category=category, - normalize_name=False, - ) + dependency = parse_python_requirement( + spec, manager="pip", category=category, normalize_name=False ) + if evaluate_marker(dependency.markers, platform): + # The above condition will skip adding the dependency if a + # marker specifies a platform that doesn't match the target, + # e.g. sys_platform == 'win32' for a linux target. + dependencies.append(dependency) # ensure pip is in target env dependencies.append(parse_python_requirement("pip", manager="conda")) diff --git a/conda_lock/src_parser/markers.py b/conda_lock/src_parser/markers.py new file mode 100644 index 00000000..1f61f821 --- /dev/null +++ b/conda_lock/src_parser/markers.py @@ -0,0 +1,77 @@ +"""Evaluate PEP 508 environment markers. + +Environment markers are expressions such as `sys_platform == 'darwin'` that can +be attached to dependency specifications. + + +""" + +import warnings + +from typing import Set, Union + +from conda_lock.interfaces.vendored_poetry_markers import ( + AnyMarker, + BaseMarker, + EmptyMarker, + MarkerUnion, + MultiMarker, + SingleMarker, + parse_marker, +) +from conda_lock.pypi_solver import PlatformEnv + + +def get_names(marker: Union[BaseMarker, str]) -> Set[str]: + """Extract all environment marker names from a marker expression. + + >>> names = get_names( + ... "python_version < '3.9' and os_name == 'nt' or os_name == 'posix'" + ... ) + >>> sorted(names) + ['os_name', 'python_version'] + """ + if isinstance(marker, str): + marker = parse_marker(marker) + if isinstance(marker, SingleMarker): + return {marker.name} + if isinstance(marker, (MarkerUnion, MultiMarker)): + return set.union(*[get_names(m) for m in marker.markers]) + if isinstance(marker, (AnyMarker, EmptyMarker)): + return set() + raise NotImplementedError(f"Unknown marker type: {marker!r}") + + +def evaluate_marker(marker: Union[BaseMarker, str, None], platform: str) -> bool: + """Evaluate a marker expression for a given platform. + + This is intended to be used for parsing lock specifications, before the Python + version is known, so markers like `python_version` are not supported. + If the marker contains any unsupported names, a warning is issued, and the + corresponding clause will evaluate to `True`. + + >>> evaluate_marker("sys_platform == 'darwin'", "osx-arm64") + True + >>> evaluate_marker("sys_platform == 'darwin'", "linux-64") + False + >>> evaluate_marker(None, "win-64") + True + + # Unsupported names evaluate to True + >>> evaluate_marker("python_version < '0' and implementation_name == 'q'", "win-64") + True + """ + if marker is None: + return True + if isinstance(marker, str): + marker = parse_marker(marker) + env = PlatformEnv(platform=platform) + marker_env = env.get_marker_env() + names = get_names(marker) + supported_names = set(marker_env.keys()) + if not names <= supported_names: + warnings.warn( + f"Marker '{marker}' contains environment markers: " + f"{names - supported_names}. Only {supported_names} are supported." + ) + return marker.validate(marker_env) diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index 7d78fb38..c170958a 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -40,6 +40,7 @@ VersionedDependency, ) from conda_lock.src_parser.conda_common import conda_spec_to_versioned_dep +from conda_lock.src_parser.markers import evaluate_marker POETRY_INVALID_EXTRA_LOC = ( @@ -177,9 +178,12 @@ def handle_mapping( # or is a URL dependency, delegate to the pip section if depattrs.get("source", None) == "pypi" or poetry_version_spec is None: manager = "pip" - # TODO: support additional features such as markers for things like sys_platform, platform_system return PoetryMappedDependencySpec( - url=url, manager=manager, extras=extras, poetry_version_spec=poetry_version_spec + url=url, + manager=manager, + extras=extras, + poetry_version_spec=poetry_version_spec, + markers=depattrs.get("markers", None), ) @@ -237,6 +241,7 @@ def parse_poetry_pyproject_toml( url = None extras: List[Any] = [] in_extra: bool = False + markers: Optional[str] = None # Poetry spec includes Python version in "tool.poetry.dependencies" # Cannot be managed by pip @@ -264,11 +269,12 @@ def parse_poetry_pyproject_toml( manager, poetry_version_spec, ) - url, manager, extras, poetry_version_spec = ( + url, manager, extras, poetry_version_spec, markers = ( pvs.url, pvs.manager, pvs.extras, pvs.poetry_version_spec, + pvs.markers, ) elif isinstance(depattrs, str): @@ -297,6 +303,7 @@ def parse_poetry_pyproject_toml( dependencies.append( VCSDependency( name=name, + markers=markers, source=url, manager=manager, vcs="git", @@ -312,6 +319,7 @@ def parse_poetry_pyproject_toml( dependencies.append( URLDependency( name=name, + markers=markers, url=url, hashes=[hashes], manager=manager, @@ -323,6 +331,7 @@ def parse_poetry_pyproject_toml( dependencies.append( VersionedDependency( name=name, + markers=markers, version=version, manager=manager, category=category, @@ -369,8 +378,13 @@ def specification_with_dependencies( ["tool", "conda-lock", "pip-repositories"], toml_contents, [] ) + platform_specific_dependencies: Dict[str, List[Dependency]] = {} + for platform in platforms: + platform_specific_dependencies[platform] = [ + d for d in dependencies if evaluate_marker(d.markers, platform) + ] return LockSpecification( - dependencies={platform: dependencies for platform in platforms}, + dependencies=platform_specific_dependencies, channels=channels, pip_repositories=pip_repositories, sources=[path], @@ -446,7 +460,44 @@ def parse_python_requirement( category: str = "main", normalize_name: bool = True, ) -> Dependency: - """Parse a requirements.txt like requirement to a conda spec""" + """Parse a requirements.txt like requirement to a conda spec. + + >>> parse_python_requirement("my_package") # doctest: +NORMALIZE_WHITESPACE + VersionedDependency(name='my-package', manager='conda', category='main', extras=[], + markers=None, version='*', build=None, conda_channel=None, hash=None) + + >>> parse_python_requirement( + ... "My_Package[extra]==1.23" + ... ) # doctest: +NORMALIZE_WHITESPACE + VersionedDependency(name='my-package', manager='conda', category='main', + extras=['extra'], markers=None, version='==1.23', build=None, + conda_channel=None, hash=None) + + >>> parse_python_requirement( + ... "conda-lock @ git+https://github.com/conda/conda-lock.git@v2.4.1" + ... ) # doctest: +NORMALIZE_WHITESPACE + VCSDependency(name='conda-lock', manager='conda', category='main', extras=[], + markers=None, source='https://github.com/conda/conda-lock.git', vcs='git', + rev='v2.4.1') + + >>> parse_python_requirement( + ... "some-package @ https://some-repository.org/some-package-1.2.3.tar.gz" + ... ) # doctest: +NORMALIZE_WHITESPACE + URLDependency(name='some-package', manager='conda', category='main', extras=[], + markers=None, url='https://some-repository.org/some-package-1.2.3.tar.gz', + hashes=['']) + + >>> parse_python_requirement( + ... "some-package ; sys_platform == 'darwin'" + ... ) # doctest: +NORMALIZE_WHITESPACE + VersionedDependency(name='some-package', manager='conda', category='main', + extras=[], markers="sys_platform == 'darwin'", version='*', build=None, + conda_channel=None, hash=None) + """ + if ";" in requirement: + requirement, markers = (s.strip() for s in requirement.rsplit(";", 1)) + else: + markers = None parsed_req = parse_requirement_specifier(requirement) name = canonicalize_pypi_name(parsed_req.name) collapsed_version = str(parsed_req.specifier) @@ -466,8 +517,10 @@ def parse_python_requirement( name=conda_dep_name, source=url, manager=manager, + category=category, vcs="git", rev=rev, + markers=markers, ) elif parsed_req.url: assert conda_version in {"", "*", None} @@ -479,6 +532,7 @@ def parse_python_requirement( extras=extras, url=url, hashes=[frag.replace("=", ":")], + markers=markers, ) else: return VersionedDependency( @@ -488,6 +542,7 @@ def parse_python_requirement( category=category, extras=extras, hash=parsed_req.hash, + markers=markers, ) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 4899f2b4..451e54ed 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -2652,18 +2652,22 @@ def test_platformenv_linux_platforms(): ] + ["linux_x86_64"] # Check that we get the default platforms when no virtual packages are specified - e = PlatformEnv("3.12", "linux-64") + e = PlatformEnv(python_version="3.12", platform="linux-64") assert e._platforms == all_expected_platforms # Check that we get the default platforms when the virtual packages are empty - e = PlatformEnv("3.12", "linux-64", platform_virtual_packages={}) + e = PlatformEnv( + python_version="3.12", platform="linux-64", platform_virtual_packages={} + ) assert e._platforms == all_expected_platforms # Check that we get the default platforms when the virtual packages are nonempty # but don't include __glibc platform_virtual_packages = {"x.bz2": {"name": "not_glibc"}} e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == all_expected_platforms @@ -2672,7 +2676,9 @@ def test_platformenv_linux_platforms(): default_repodata = default_virtual_package_repodata() platform_virtual_packages = default_repodata.all_repodata["linux-64"]["packages"] e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == all_expected_platforms @@ -2684,7 +2690,9 @@ def test_platformenv_linux_platforms(): if record["name"] != "__glibc" } e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == all_expected_platforms @@ -2701,7 +2709,9 @@ def test_platformenv_linux_platforms(): name="__glibc", version="2.17" ) e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == restricted_platforms @@ -2711,7 +2721,9 @@ def test_platformenv_linux_platforms(): ) with pytest.warns(UserWarning): e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == restricted_platforms diff --git a/tests/test_markers.py b/tests/test_markers.py new file mode 100644 index 00000000..45c61cf8 --- /dev/null +++ b/tests/test_markers.py @@ -0,0 +1,90 @@ +from pathlib import Path + +import pytest + +from conda_lock.src_parser import make_lock_spec + + +ENVIRONMENT_YAML = """ +channels: + - conda-forge + - nodefaults +dependencies: + - pip + - pip: + - cowsay; sys_platform == 'darwin' +platforms: + - osx-64 + - osx-arm64 + - linux-64 +""" + +POETRY_PYPROJECT = """ +[tool.poetry] +name = "conda-lock-test-poetry" +version = "0.0.1" +description = "" +authors = ["conda-lock"] + +[tool.poetry.dependencies] +python = "^3.9" +cowsay = {version = "*", markers = "sys_platform == 'darwin'"} + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" + +[tool.conda-lock] +platforms = [ + "osx-64", + "osx-arm64", + "linux-64", +] +""" + +HATCH_PYPROJECT = """ +[build-system] +requires = ["hatchling >=1.25.0,<2"] +build-backend = "hatchling.build" + +[project] +name = "conda-lock-test-hatch" +version = "0.0.1" +dependencies = ["cowsay; sys_platform == 'darwin'"] + +[tool.conda-lock] +platforms = [ + "osx-64", + "osx-arm64", + "linux-64", +] +""" + + +@pytest.fixture( + params=[ + (ENVIRONMENT_YAML, "environment.yml"), + (POETRY_PYPROJECT, "pyproject.toml"), + (HATCH_PYPROJECT, "pyproject.toml"), + ], + ids=["environment.yml", "poetry", "hatch"], +) +def cowsay_src_file(request, tmp_path: Path): + contents, filename = request.param + src_file = tmp_path / filename + src_file.write_text(contents) + return src_file + + +def test_sys_platform_marker(cowsay_src_file): + lock_spec = make_lock_spec(src_files=[cowsay_src_file]) + dependencies = lock_spec.dependencies + platform_has_cowsay = { + platform: any(dep.name == "cowsay" for dep in platform_deps) + for platform, platform_deps in dependencies.items() + } + assert platform_has_cowsay == { + "osx-64": True, + "osx-arm64": True, + "linux-64": False, + }