From cfd44ccd1642c0b88c71fd266508dfe2d98d30aa Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 10 Nov 2023 13:56:48 -0500 Subject: [PATCH 01/14] feat: add local pypi-to-conda overrides --- conda_lock/lookup.py | 57 +++++++++++++++++++++++-- conda_lock/src_parser/pyproject_toml.py | 9 +++- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index a56e1be35..ee87a1a28 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -1,5 +1,7 @@ +from collections import ChainMap +from contextlib import suppress from functools import cached_property -from typing import Dict +from typing import Dict, Mapping, Optional, Union import requests import yaml @@ -17,6 +19,7 @@ class MappingEntry(TypedDict): class _LookupLoader: _mapping_url: str = "https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/mappings/pypi/grayskull_pypi_mapping.yaml" + _local_mappings: Optional[Dict[NormalizedName, MappingEntry]] = None @property def mapping_url(self) -> str: @@ -24,12 +27,16 @@ def mapping_url(self) -> str: @mapping_url.setter def mapping_url(self, value: str) -> None: - del self.pypi_lookup - del self.conda_lookup + # these will raise AttributeError if they haven't been cached yet. + with suppress(AttributeError): + del self.remote_mappings + with suppress(AttributeError): + del self.conda_lookup self._mapping_url = value @cached_property - def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]: + def remote_mappings(self) -> Dict[NormalizedName, MappingEntry]: + """PyPI to conda name mapping fetched from `_mapping_url`""" res = requests.get(self._mapping_url) res.raise_for_status() lookup = yaml.safe_load(res.content) @@ -40,6 +47,42 @@ def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]: v["pypi_name"] = canonicalize_name(v["pypi_name"]) return lookup + @property + def local_mappings(self) -> Dict[NormalizedName, MappingEntry]: + """PyPI to conda name mappings set by the user.""" + return self._local_mappings or {} + + @local_mappings.setter + def local_mappings(self, mappings: Mapping[str, Union[str, MappingEntry]]) -> None: + """Value should be a mapping from pypi name to conda name or a mapping entry.""" + lookup: Dict[NormalizedName, MappingEntry] = {} + # normalize to Dict[NormalizedName, MappingEntry] + for k, v in mappings.items(): + key = canonicalize_name(k) + if isinstance(v, Mapping): + if "conda_name" not in v or "pypi_name" not in v: + raise ValueError( + "MappingEntries must have both a 'conda_name' and 'pypi_name'" + ) + entry = dict(v) + entry["pypi_name"] = canonicalize_name(entry["pypi_name"]) + elif isinstance(v, str): + entry = {"conda_name": v, "pypi_name": key} + else: + raise TypeError( + "Each entry in the mapping must be a string or a mapping" + ) + lookup[key] = entry + self._local_mappings = lookup + + @property + def pypi_lookup(self) -> Mapping[NormalizedName, MappingEntry]: + """ChainMap of PyPI to conda name mappings. + + Local mappings take precedence over remote mappings fetched from `_mapping_url`. + """ + return ChainMap(self.local_mappings, self.remote_mappings) + @cached_property def conda_lookup(self) -> Dict[str, MappingEntry]: return {record["conda_name"]: record for record in self.pypi_lookup.values()} @@ -66,6 +109,12 @@ def set_lookup_location(lookup_url: str) -> None: LOOKUP_OBJECT.mapping_url = lookup_url +def set_pypi_lookup_overrides(mappings: Mapping[str, Union[str, MappingEntry]]) -> None: + """Set overrides to the pypi lookup""" + global LOOKUP_OBJECT + LOOKUP_OBJECT.local_mappings = mappings + + def conda_name_to_pypi_name(name: str) -> NormalizedName: """return the pypi name for a conda package""" lookup = get_lookup() diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index fcb23e1ff..476f80548 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -30,7 +30,10 @@ from typing_extensions import Literal from conda_lock.common import get_in -from conda_lock.lookup import get_forward_lookup as get_lookup +from conda_lock.lookup import ( + get_forward_lookup as get_lookup, + set_pypi_lookup_overrides, +) from conda_lock.models.lock_spec import ( Dependency, LockSpecification, @@ -559,6 +562,10 @@ def parse_pyproject_toml( contents = toml_load(fp) build_system = get_in(["build-system", "build-backend"], contents) + pypi_map = get_in(["tool", "conda-lock", "pypi-to-conda-name"], contents, False) + if pypi_map: + set_pypi_lookup_overrides(pypi_map) + if get_in( ["tool", "conda-lock", "skip-non-conda-lock"], contents, From e2ecd3d29d034fd4915c7d026accc1bbf4458ebc Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 10 Nov 2023 14:03:18 -0500 Subject: [PATCH 02/14] lint --- conda_lock/lookup.py | 17 +++++++++-------- conda_lock/src_parser/pyproject_toml.py | 2 ++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index ee87a1a28..a2dc42945 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -1,19 +1,19 @@ from collections import ChainMap from contextlib import suppress from functools import cached_property -from typing import Dict, Mapping, Optional, Union +from typing import Dict, Mapping, Optional, Union, cast import requests import yaml from packaging.utils import NormalizedName, canonicalize_name -from typing_extensions import TypedDict +from typing_extensions import NotRequired, TypedDict class MappingEntry(TypedDict): conda_name: str # legacy field, generally not used by anything anymore - conda_forge: str + conda_forge: NotRequired[str] pypi_name: NormalizedName @@ -64,8 +64,8 @@ def local_mappings(self, mappings: Mapping[str, Union[str, MappingEntry]]) -> No raise ValueError( "MappingEntries must have both a 'conda_name' and 'pypi_name'" ) - entry = dict(v) - entry["pypi_name"] = canonicalize_name(entry["pypi_name"]) + entry = cast("MappingEntry", dict(v)) + entry["pypi_name"] = canonicalize_name(str(entry["pypi_name"])) elif isinstance(v, str): entry = {"conda_name": v, "pypi_name": key} else: @@ -78,7 +78,7 @@ def local_mappings(self, mappings: Mapping[str, Union[str, MappingEntry]]) -> No @property def pypi_lookup(self) -> Mapping[NormalizedName, MappingEntry]: """ChainMap of PyPI to conda name mappings. - + Local mappings take precedence over remote mappings fetched from `_mapping_url`. """ return ChainMap(self.local_mappings, self.remote_mappings) @@ -91,7 +91,7 @@ def conda_lookup(self) -> Dict[str, MappingEntry]: LOOKUP_OBJECT = _LookupLoader() -def get_forward_lookup() -> Dict[NormalizedName, MappingEntry]: +def get_forward_lookup() -> Mapping[NormalizedName, MappingEntry]: global LOOKUP_OBJECT return LOOKUP_OBJECT.pypi_lookup @@ -112,7 +112,8 @@ def set_lookup_location(lookup_url: str) -> None: def set_pypi_lookup_overrides(mappings: Mapping[str, Union[str, MappingEntry]]) -> None: """Set overrides to the pypi lookup""" global LOOKUP_OBJECT - LOOKUP_OBJECT.local_mappings = mappings + # type ignore because the setter will normalize the types + LOOKUP_OBJECT.local_mappings = mappings # type: ignore [assignment] def conda_name_to_pypi_name(name: str) -> NormalizedName: diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index 476f80548..a040fb127 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -32,6 +32,8 @@ from conda_lock.common import get_in from conda_lock.lookup import ( get_forward_lookup as get_lookup, +) +from conda_lock.lookup import ( set_pypi_lookup_overrides, ) from conda_lock.models.lock_spec import ( From fce8271be0df4b6cf2142c205f18a1069b2111d8 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 10 Nov 2023 14:09:09 -0500 Subject: [PATCH 03/14] update readme --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 9501f4918..91e58a865 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,18 @@ platforms = [ ] ``` +#### PyPI name mapping + +If you would like to supplement or override the pypi-to-conda name mappings provided by +([pypi-mapping][mapping]), you can do so by adding a `pypi-to-conda-name`: + +```toml +# pyproject.toml + +[tool.conda-lock.pypi-to-conda-name] +cupy-cuda11x = "cupy" +``` + #### Extras If your pyproject.toml file contains optional dependencies/extras these can be referred to by using the `--extras` flag From e2ca84f06a7c6fc78e8c76aa7f9d28a6fa078a34 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 10 Nov 2023 14:10:53 -0500 Subject: [PATCH 04/14] formatting --- conda_lock/src_parser/pyproject_toml.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index a040fb127..3a388723f 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -30,12 +30,8 @@ from typing_extensions import Literal from conda_lock.common import get_in -from conda_lock.lookup import ( - get_forward_lookup as get_lookup, -) -from conda_lock.lookup import ( - set_pypi_lookup_overrides, -) +from conda_lock.lookup import get_forward_lookup as get_lookup +from conda_lock.lookup import set_pypi_lookup_overrides from conda_lock.models.lock_spec import ( Dependency, LockSpecification, From ad139dfa1c04485b3beeddc56805d036e7145c56 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 10 Nov 2023 14:11:54 -0500 Subject: [PATCH 05/14] remove parens --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 91e58a865..50027abf2 100644 --- a/README.md +++ b/README.md @@ -334,7 +334,7 @@ platforms = [ #### PyPI name mapping If you would like to supplement or override the pypi-to-conda name mappings provided by -([pypi-mapping][mapping]), you can do so by adding a `pypi-to-conda-name`: +[pypi-mapping][mapping], you can do so by adding a `pypi-to-conda-name`: ```toml # pyproject.toml From a71bc82db4df4d27d8f24fd783a6e0fd8fa35c8c Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 10 Nov 2023 17:04:06 -0500 Subject: [PATCH 06/14] cleanup a little --- conda_lock/lookup.py | 83 ++++++++++++++----------- conda_lock/src_parser/pyproject_toml.py | 26 ++------ 2 files changed, 49 insertions(+), 60 deletions(-) diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index a2dc42945..93ff93f7d 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -1,25 +1,42 @@ -from collections import ChainMap +import logging + from contextlib import suppress from functools import cached_property -from typing import Dict, Mapping, Optional, Union, cast +from typing import ClassVar, Dict, Optional, Union, cast import requests import yaml from packaging.utils import NormalizedName, canonicalize_name -from typing_extensions import NotRequired, TypedDict +from typing_extensions import TypedDict + + +DEFAULT_MAPPING_URL = "https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/mappings/pypi/grayskull_pypi_mapping.yaml" class MappingEntry(TypedDict): conda_name: str - # legacy field, generally not used by anything anymore - conda_forge: NotRequired[str] pypi_name: NormalizedName class _LookupLoader: - _mapping_url: str = "https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/mappings/pypi/grayskull_pypi_mapping.yaml" - _local_mappings: Optional[Dict[NormalizedName, MappingEntry]] = None + """Object used to map PyPI package names to conda names.""" + + _SINGLETON: ClassVar[Optional["_LookupLoader"]] = None + + @classmethod + def instance(cls) -> "_LookupLoader": + if cls._SINGLETON is None: + cls._SINGLETON = cls() + return cls._SINGLETON + + def __init__( + self, + pypi_lookup_overrides: Optional[Dict[NormalizedName, MappingEntry]] = None, + mapping_url: str = DEFAULT_MAPPING_URL, + ) -> None: + self._mapping_url = mapping_url + self._local_mappings = pypi_lookup_overrides @property def mapping_url(self) -> str: @@ -30,6 +47,8 @@ def mapping_url(self, value: str) -> None: # these will raise AttributeError if they haven't been cached yet. with suppress(AttributeError): del self.remote_mappings + with suppress(AttributeError): + del self.pypi_lookup with suppress(AttributeError): del self.conda_lookup self._mapping_url = value @@ -53,13 +72,13 @@ def local_mappings(self) -> Dict[NormalizedName, MappingEntry]: return self._local_mappings or {} @local_mappings.setter - def local_mappings(self, mappings: Mapping[str, Union[str, MappingEntry]]) -> None: + def local_mappings(self, mappings: Dict[str, Union[str, MappingEntry]]) -> None: """Value should be a mapping from pypi name to conda name or a mapping entry.""" lookup: Dict[NormalizedName, MappingEntry] = {} # normalize to Dict[NormalizedName, MappingEntry] for k, v in mappings.items(): key = canonicalize_name(k) - if isinstance(v, Mapping): + if isinstance(v, dict): if "conda_name" not in v or "pypi_name" not in v: raise ValueError( "MappingEntries must have both a 'conda_name' and 'pypi_name'" @@ -69,56 +88,40 @@ def local_mappings(self, mappings: Mapping[str, Union[str, MappingEntry]]) -> No elif isinstance(v, str): entry = {"conda_name": v, "pypi_name": key} else: - raise TypeError( - "Each entry in the mapping must be a string or a mapping" - ) + raise TypeError("Each entry in the mapping must be a string or a dict") lookup[key] = entry self._local_mappings = lookup - @property - def pypi_lookup(self) -> Mapping[NormalizedName, MappingEntry]: - """ChainMap of PyPI to conda name mappings. + @cached_property + def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]: + """Dict of PyPI to conda name mappings. Local mappings take precedence over remote mappings fetched from `_mapping_url`. """ - return ChainMap(self.local_mappings, self.remote_mappings) + return {**self.remote_mappings, **self.local_mappings} @cached_property def conda_lookup(self) -> Dict[str, MappingEntry]: return {record["conda_name"]: record for record in self.pypi_lookup.values()} -LOOKUP_OBJECT = _LookupLoader() - - -def get_forward_lookup() -> Mapping[NormalizedName, MappingEntry]: - global LOOKUP_OBJECT - return LOOKUP_OBJECT.pypi_lookup - +def set_lookup_location(lookup_url: str) -> None: + """Set the location of the pypi lookup -def get_lookup() -> Dict[str, MappingEntry]: + Used by the `lock` cli command to override the DEFAULT_MAPPING_URL for the lookup. """ - Reverse grayskull name mapping to map conda names onto PyPI - """ - global LOOKUP_OBJECT - return LOOKUP_OBJECT.conda_lookup - - -def set_lookup_location(lookup_url: str) -> None: - global LOOKUP_OBJECT - LOOKUP_OBJECT.mapping_url = lookup_url + _LookupLoader.instance().mapping_url = lookup_url -def set_pypi_lookup_overrides(mappings: Mapping[str, Union[str, MappingEntry]]) -> None: +def set_pypi_lookup_overrides(mappings: Dict[str, Union[str, MappingEntry]]) -> None: """Set overrides to the pypi lookup""" - global LOOKUP_OBJECT # type ignore because the setter will normalize the types - LOOKUP_OBJECT.local_mappings = mappings # type: ignore [assignment] + _LookupLoader.instance().local_mappings = mappings # type: ignore [assignment] def conda_name_to_pypi_name(name: str) -> NormalizedName: """return the pypi name for a conda package""" - lookup = get_lookup() + lookup = _LookupLoader.instance().conda_lookup cname = canonicalize_name(name) return lookup.get(cname, {"pypi_name": cname})["pypi_name"] @@ -126,4 +129,8 @@ def conda_name_to_pypi_name(name: str) -> NormalizedName: def pypi_name_to_conda_name(name: str) -> str: """return the conda name for a pypi package""" cname = canonicalize_name(name) - return get_forward_lookup().get(cname, {"conda_name": cname})["conda_name"] + forward_lookup = _LookupLoader.instance().pypi_lookup + if cname not in forward_lookup: + logging.warning(f"Could not find conda name for {cname!r}. Assuming identity.") + return cname + return forward_lookup[cname]["conda_name"] diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index 3a388723f..586b55b2c 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -30,8 +30,7 @@ from typing_extensions import Literal from conda_lock.common import get_in -from conda_lock.lookup import get_forward_lookup as get_lookup -from conda_lock.lookup import set_pypi_lookup_overrides +from conda_lock.lookup import pypi_name_to_conda_name, set_pypi_lookup_overrides from conda_lock.models.lock_spec import ( Dependency, LockSpecification, @@ -74,22 +73,6 @@ def join_version_components(pieces: Sequence[Union[str, int]]) -> str: return ".".join(str(p) for p in pieces) -def normalize_pypi_name(name: str) -> str: - cname = canonicalize_pypi_name(name) - if cname in get_lookup(): - lookup = get_lookup()[cname] - res = lookup.get("conda_name") or lookup.get("conda_forge") - if res is not None: - return res - else: - logging.warning( - f"Could not find conda name for {cname}. Assuming identity." - ) - return cname - else: - return cname - - def poetry_version_to_conda_version(version_string: Optional[str]) -> Optional[str]: if version_string is None: return None @@ -276,7 +259,7 @@ def parse_poetry_pyproject_toml( ) if manager == "conda": - name = normalize_pypi_name(depname) + name = pypi_name_to_conda_name(depname) version = poetry_version_to_conda_version(poetry_version_spec) else: name = depname @@ -422,16 +405,15 @@ def parse_python_requirement( ) -> Dependency: """Parse a requirements.txt like requirement to a conda spec""" parsed_req = parse_requirement_specifier(requirement) - name = canonicalize_pypi_name(parsed_req.name) collapsed_version = str(parsed_req.specifier) conda_version = poetry_version_to_conda_version(collapsed_version) if conda_version: conda_version = ",".join(sorted(conda_version.split(","))) if normalize_name: - conda_dep_name = normalize_pypi_name(name) + conda_dep_name = pypi_name_to_conda_name(parsed_req.name) else: - conda_dep_name = name + conda_dep_name = canonicalize_pypi_name(parsed_req.name) extras = list(parsed_req.extras) if parsed_req.url and parsed_req.url.startswith("git+"): From 95f8c931eecbf48ec05152007505e1298d9925dd Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Mon, 13 Nov 2023 13:53:14 -0500 Subject: [PATCH 07/14] test: add test --- tests/test-pep621-pypi-override/pyproject.toml | 10 ++++++++++ tests/test_conda_lock.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 tests/test-pep621-pypi-override/pyproject.toml diff --git a/tests/test-pep621-pypi-override/pyproject.toml b/tests/test-pep621-pypi-override/pyproject.toml new file mode 100644 index 000000000..8ff379caa --- /dev/null +++ b/tests/test-pep621-pypi-override/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "conda-lock-test-pypi-naming" +dependencies = ["some-name-i-want-to-override"] + +[tool.conda-lock.pypi-to-conda-name] +some-name-i-want-to-override = "resolved-name" diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index afbc5f402..3d105b336 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -324,6 +324,13 @@ def include_dev_dependencies(request: Any) -> bool: return request.param +@pytest.fixture +def pep621_pyproject_toml_pypi_override(tmp_path: Path): + return clone_test_dir("test-pep621-pypi-override", tmp_path).joinpath( + "pyproject.toml" + ) + + JSON_FIELDS: Dict[str, str] = {"json_unique_field": "test1", "common_field": "test2"} YAML_FIELDS: Dict[str, str] = {"yaml_unique_field": "test3", "common_field": "test4"} @@ -988,6 +995,16 @@ def test_parse_poetry_invalid_optionals(pyproject_optional_toml: Path): ) +def test_parse_pyproject_pypi_overrides(pep621_pyproject_toml_pypi_override: Path): + res = parse_pyproject_toml(pep621_pyproject_toml_pypi_override, ["linux-64"]) + + specs = {dep.name for dep in res.dependencies["linux-64"]} + + # in the pyproject.toml, the package "resolved-name' is provided as an + # override for the package "some-name-i-want-to-override". + assert "resolved-name" in specs + + def test_run_lock( monkeypatch: "pytest.MonkeyPatch", zlib_environment: Path, conda_exe: str ): From 96e645aef880ffed5ed76f9c2d838029417efe27 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Mon, 13 Nov 2023 16:49:36 -0500 Subject: [PATCH 08/14] remove cache --- conda_lock/lookup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index 93ff93f7d..8b2753d1a 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -47,8 +47,6 @@ def mapping_url(self, value: str) -> None: # these will raise AttributeError if they haven't been cached yet. with suppress(AttributeError): del self.remote_mappings - with suppress(AttributeError): - del self.pypi_lookup with suppress(AttributeError): del self.conda_lookup self._mapping_url = value @@ -92,7 +90,7 @@ def local_mappings(self, mappings: Dict[str, Union[str, MappingEntry]]) -> None: lookup[key] = entry self._local_mappings = lookup - @cached_property + @property def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]: """Dict of PyPI to conda name mappings. From f4cb419a2aee1b1626893167cdfee2198a622c53 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 14 Nov 2023 11:48:42 -0500 Subject: [PATCH 09/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50027abf2..374db9447 100644 --- a/README.md +++ b/README.md @@ -334,7 +334,7 @@ platforms = [ #### PyPI name mapping If you would like to supplement or override the pypi-to-conda name mappings provided by -[pypi-mapping][mapping], you can do so by adding a `pypi-to-conda-name`: +[pypi-mapping][mapping], you can do so by adding a `pypi-to-conda-name` section: ```toml # pyproject.toml From 932253b580c14707c4929ac6cbb7b98e3bdbe85e Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 19 Nov 2023 20:45:32 +0100 Subject: [PATCH 10/14] Eliminate SINGLETON in favor of module-level instance --- conda_lock/lookup.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index 8b2753d1a..13e75bf58 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -2,7 +2,7 @@ from contextlib import suppress from functools import cached_property -from typing import ClassVar, Dict, Optional, Union, cast +from typing import Dict, Optional, Union, cast import requests import yaml @@ -22,13 +22,8 @@ class MappingEntry(TypedDict): class _LookupLoader: """Object used to map PyPI package names to conda names.""" - _SINGLETON: ClassVar[Optional["_LookupLoader"]] = None - - @classmethod - def instance(cls) -> "_LookupLoader": - if cls._SINGLETON is None: - cls._SINGLETON = cls() - return cls._SINGLETON + _mapping_url: str + _local_mappings: Optional[Dict[NormalizedName, MappingEntry]] def __init__( self, @@ -103,23 +98,26 @@ def conda_lookup(self) -> Dict[str, MappingEntry]: return {record["conda_name"]: record for record in self.pypi_lookup.values()} +_lookup_loader = _LookupLoader() + + def set_lookup_location(lookup_url: str) -> None: """Set the location of the pypi lookup Used by the `lock` cli command to override the DEFAULT_MAPPING_URL for the lookup. """ - _LookupLoader.instance().mapping_url = lookup_url + _lookup_loader.mapping_url = lookup_url def set_pypi_lookup_overrides(mappings: Dict[str, Union[str, MappingEntry]]) -> None: """Set overrides to the pypi lookup""" # type ignore because the setter will normalize the types - _LookupLoader.instance().local_mappings = mappings # type: ignore [assignment] + _lookup_loader.local_mappings = mappings # type: ignore [assignment] def conda_name_to_pypi_name(name: str) -> NormalizedName: """return the pypi name for a conda package""" - lookup = _LookupLoader.instance().conda_lookup + lookup = _lookup_loader.conda_lookup cname = canonicalize_name(name) return lookup.get(cname, {"pypi_name": cname})["pypi_name"] @@ -127,7 +125,7 @@ def conda_name_to_pypi_name(name: str) -> NormalizedName: def pypi_name_to_conda_name(name: str) -> str: """return the conda name for a pypi package""" cname = canonicalize_name(name) - forward_lookup = _LookupLoader.instance().pypi_lookup + forward_lookup = _lookup_loader.pypi_lookup if cname not in forward_lookup: logging.warning(f"Could not find conda name for {cname!r}. Assuming identity.") return cname From 5e85791739c56811079c3bcd6f22fd9f57dd9c21 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 19 Nov 2023 20:50:45 +0100 Subject: [PATCH 11/14] Eliminate unused arguments --- conda_lock/lookup.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index 13e75bf58..38479f771 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -25,13 +25,9 @@ class _LookupLoader: _mapping_url: str _local_mappings: Optional[Dict[NormalizedName, MappingEntry]] - def __init__( - self, - pypi_lookup_overrides: Optional[Dict[NormalizedName, MappingEntry]] = None, - mapping_url: str = DEFAULT_MAPPING_URL, - ) -> None: - self._mapping_url = mapping_url - self._local_mappings = pypi_lookup_overrides + def __init__(self) -> None: + self._mapping_url = DEFAULT_MAPPING_URL + self._local_mappings = None @property def mapping_url(self) -> str: From e27d9e0affe4051afaee6460ec466d72764f4702 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 19 Nov 2023 20:57:02 +0100 Subject: [PATCH 12/14] Remove extraneous mapping_url property --- conda_lock/lookup.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index 38479f771..63a4e3ff6 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -22,30 +22,17 @@ class MappingEntry(TypedDict): class _LookupLoader: """Object used to map PyPI package names to conda names.""" - _mapping_url: str + mapping_url: str _local_mappings: Optional[Dict[NormalizedName, MappingEntry]] def __init__(self) -> None: - self._mapping_url = DEFAULT_MAPPING_URL + self.mapping_url = DEFAULT_MAPPING_URL self._local_mappings = None - @property - def mapping_url(self) -> str: - return self._mapping_url - - @mapping_url.setter - def mapping_url(self, value: str) -> None: - # these will raise AttributeError if they haven't been cached yet. - with suppress(AttributeError): - del self.remote_mappings - with suppress(AttributeError): - del self.conda_lookup - self._mapping_url = value - @cached_property def remote_mappings(self) -> Dict[NormalizedName, MappingEntry]: - """PyPI to conda name mapping fetched from `_mapping_url`""" - res = requests.get(self._mapping_url) + """PyPI to conda name mapping fetched from `mapping_url`""" + res = requests.get(self.mapping_url) res.raise_for_status() lookup = yaml.safe_load(res.content) # lowercase and kebabcase the pypi names @@ -85,7 +72,7 @@ def local_mappings(self, mappings: Dict[str, Union[str, MappingEntry]]) -> None: def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]: """Dict of PyPI to conda name mappings. - Local mappings take precedence over remote mappings fetched from `_mapping_url`. + Local mappings take precedence over remote mappings fetched from `mapping_url`. """ return {**self.remote_mappings, **self.local_mappings} @@ -102,6 +89,11 @@ def set_lookup_location(lookup_url: str) -> None: Used by the `lock` cli command to override the DEFAULT_MAPPING_URL for the lookup. """ + # these will raise AttributeError if they haven't been cached yet. + with suppress(AttributeError): + del _lookup_loader.remote_mappings + with suppress(AttributeError): + del _lookup_loader.conda_lookup _lookup_loader.mapping_url = lookup_url From 1327b63be16872215651458cd988f309d062a055 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 19 Nov 2023 21:06:55 +0100 Subject: [PATCH 13/14] Remove extraneous local_mappings property --- conda_lock/lookup.py | 51 +++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index 63a4e3ff6..63ecb2b3b 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -2,7 +2,7 @@ from contextlib import suppress from functools import cached_property -from typing import Dict, Optional, Union, cast +from typing import Dict, Union, cast import requests import yaml @@ -23,11 +23,11 @@ class _LookupLoader: """Object used to map PyPI package names to conda names.""" mapping_url: str - _local_mappings: Optional[Dict[NormalizedName, MappingEntry]] + local_mappings: Dict[NormalizedName, MappingEntry] def __init__(self) -> None: self.mapping_url = DEFAULT_MAPPING_URL - self._local_mappings = None + self.local_mappings = {} @cached_property def remote_mappings(self) -> Dict[NormalizedName, MappingEntry]: @@ -42,32 +42,6 @@ def remote_mappings(self) -> Dict[NormalizedName, MappingEntry]: v["pypi_name"] = canonicalize_name(v["pypi_name"]) return lookup - @property - def local_mappings(self) -> Dict[NormalizedName, MappingEntry]: - """PyPI to conda name mappings set by the user.""" - return self._local_mappings or {} - - @local_mappings.setter - def local_mappings(self, mappings: Dict[str, Union[str, MappingEntry]]) -> None: - """Value should be a mapping from pypi name to conda name or a mapping entry.""" - lookup: Dict[NormalizedName, MappingEntry] = {} - # normalize to Dict[NormalizedName, MappingEntry] - for k, v in mappings.items(): - key = canonicalize_name(k) - if isinstance(v, dict): - if "conda_name" not in v or "pypi_name" not in v: - raise ValueError( - "MappingEntries must have both a 'conda_name' and 'pypi_name'" - ) - entry = cast("MappingEntry", dict(v)) - entry["pypi_name"] = canonicalize_name(str(entry["pypi_name"])) - elif isinstance(v, str): - entry = {"conda_name": v, "pypi_name": key} - else: - raise TypeError("Each entry in the mapping must be a string or a dict") - lookup[key] = entry - self._local_mappings = lookup - @property def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]: """Dict of PyPI to conda name mappings. @@ -99,8 +73,23 @@ def set_lookup_location(lookup_url: str) -> None: def set_pypi_lookup_overrides(mappings: Dict[str, Union[str, MappingEntry]]) -> None: """Set overrides to the pypi lookup""" - # type ignore because the setter will normalize the types - _lookup_loader.local_mappings = mappings # type: ignore [assignment] + lookup: Dict[NormalizedName, MappingEntry] = {} + # normalize to Dict[NormalizedName, MappingEntry] + for k, v in mappings.items(): + key = canonicalize_name(k) + if isinstance(v, dict): + if "conda_name" not in v or "pypi_name" not in v: + raise ValueError( + "MappingEntries must have both a 'conda_name' and 'pypi_name'" + ) + entry = cast("MappingEntry", dict(v)) + entry["pypi_name"] = canonicalize_name(str(entry["pypi_name"])) + elif isinstance(v, str): + entry = {"conda_name": v, "pypi_name": key} + else: + raise TypeError("Each entry in the mapping must be a string or a dict") + lookup[key] = entry + _lookup_loader.local_mappings = lookup def conda_name_to_pypi_name(name: str) -> NormalizedName: From 4d10fc9bd26b2ae544afd9c72276db43a2f5229d Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 23 Feb 2024 11:29:44 -0500 Subject: [PATCH 14/14] fix lint --- conda_lock/lookup.py | 2 +- tests/test_conda_lock.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index ccb70fa7e..04ab53995 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -2,8 +2,8 @@ from contextlib import suppress from functools import cached_property -from typing import Dict, Union, cast from pathlib import Path +from typing import Dict, Union, cast import requests import yaml diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index da788303a..19c42894f 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -1025,6 +1025,7 @@ def test_parse_pyproject_pypi_overrides(pep621_pyproject_toml_pypi_override: Pat # override for the package "some-name-i-want-to-override". assert "resolved-name" in specs + def test_explicit_toposorted() -> None: """Verify that explicit lockfiles are topologically sorted.