Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Proposed fix for #579 #588

Merged
merged 4 commits into from
Feb 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions conda_lock/lookup.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from functools import cached_property
from pathlib import Path
from typing import Dict

import requests
Expand All @@ -24,15 +25,32 @@ def mapping_url(self) -> str:

@mapping_url.setter
def mapping_url(self, value: str) -> None:
del self.pypi_lookup
del self.conda_lookup
self._mapping_url = value
if self._mapping_url != value:
self._mapping_url = value
# Invalidate cache
try:
del self.pypi_lookup
except AttributeError:
pass
try:
del self.conda_lookup
except AttributeError:
pass

@cached_property
def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]:
res = requests.get(self._mapping_url)
res.raise_for_status()
lookup = yaml.safe_load(res.content)
url = self.mapping_url
if url.startswith("http://") or url.startswith("https://"):
res = requests.get(self._mapping_url)
res.raise_for_status()
content = res.content
else:
if url.startswith("file://"):
path = url[len("file://") :]
else:
path = url
content = Path(path).read_bytes()
lookup = yaml.safe_load(content)
# lowercase and kebabcase the pypi names
assert lookup is not None
lookup = {canonicalize_name(k): v for k, v in lookup.items()}
Expand Down
8 changes: 8 additions & 0 deletions tests/test-lookup/emoji-to-python-dateutil-lookup.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Trick conda-lock into thinking that the emoji Conda package provides
# python-dateutil as a pip dependency

python-dateutil:
conda_name: emoji
import_name: emoji
mapping_source: conda-lock-test-suite
pypi_name: python-dateutil
1 change: 1 addition & 0 deletions tests/test-lookup/empty-lookup.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
14 changes: 14 additions & 0 deletions tests/test-lookup/environment.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
channels:
- conda-forge
- nodefaults
platforms:
- linux-64
dependencies:
# The emoji package has no dependencies.
- emoji==2.8.0
- pip
- pip:
# Arrow's dependencies are python-dateutil >=2.7.0 and types-python-dateutil >=2.8.10.
# Note that a version 2.8.0 exists of python-dateutil. This makes it possible
# to add an entry to the lookup table that maps emoji to python-dateutil.
- arrow==1.3.0
85 changes: 85 additions & 0 deletions tests/test_conda_lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
LockedDependency,
MetadataOption,
)
from conda_lock.lookup import _LookupLoader
from conda_lock.models.channel import Channel
from conda_lock.models.lock_spec import Dependency, VCSDependency, VersionedDependency
from conda_lock.models.pip_repository import PipRepository
Expand Down Expand Up @@ -2422,6 +2423,90 @@ def run_install():
run_install()


def test_lookup_sources():
# Test that the lookup can be read from a file:// URL
lookup = (
Path(__file__).parent / "test-lookup" / "emoji-to-python-dateutil-lookup.yml"
)
url = f"file://{lookup.absolute()}"
LOOKUP_OBJECT = _LookupLoader()
LOOKUP_OBJECT.mapping_url = url
assert LOOKUP_OBJECT.conda_lookup["emoji"]["pypi_name"] == "python-dateutil"

# Test that the lookup can be read from a straight filename
url = str(lookup.absolute())
LOOKUP_OBJECT = _LookupLoader()
LOOKUP_OBJECT.mapping_url = url
assert LOOKUP_OBJECT.conda_lookup["emoji"]["pypi_name"] == "python-dateutil"

# Test that the default remote lookup contains expected nontrivial mappings
LOOKUP_OBJECT = _LookupLoader()
assert LOOKUP_OBJECT.conda_lookup["python-build"]["pypi_name"] == "build"


@pytest.fixture
def lookup_environment(tmp_path: Path):
return clone_test_dir("test-lookup", tmp_path).joinpath("environment.yml")


@pytest.mark.parametrize(
"lookup_source", ["emoji-to-python-dateutil-lookup.yml", "empty-lookup.yml"]
)
def test_lookup(
lookup_environment: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
lookup_source: str,
):
"""We test that the lookup table is being used to convert conda package names into
pypi package names. We verify this by comparing the results from using two
different lookup tables.

The pip solver runs after the conda solver. The pip solver needs to know which
packages are already installed by conda. The lookup table is used to convert conda
package names into pypi package names.

We test two cases:
1. The lookup table is empty. In this case, the conda package names are converted
directly into pypi package names. As long as there are no discrepancies between
conda and pypi package names, this gives expected results.
2. The lookup table maps emoji to python-dateutil. Arrow is installed as a pip
package and has python-dateutil as a dependency. Due to this lookup table, the
pip solver should believe that the dependency is already satisfied and not add it.
"""
cwd = lookup_environment.parent
monkeypatch.chdir(cwd)
lookup_filename = str((cwd / lookup_source).absolute())
with capsys.disabled():
from click.testing import CliRunner, Result

runner = CliRunner(mix_stderr=False)
result: Result = runner.invoke(
main,
["lock", "--pypi_to_conda_lookup_file", lookup_filename],
catch_exceptions=False,
)
assert result.exit_code == 0

lockfile = cwd / DEFAULT_LOCKFILE_NAME
assert lockfile.is_file()
lockfile_content = parse_conda_lock_file(lockfile)
installed_packages = {p.name for p in lockfile_content.package}
assert "emoji" in installed_packages
assert "arrow" in installed_packages
assert "types-python-dateutil" in installed_packages
if lookup_source == "empty-lookup.yml":
# If the lookup table is empty, then conda package names are converted
# directly into pypi package names. Arrow depends on python-dateutil, so
# it should be installed.
assert "python-dateutil" in installed_packages
else:
# The nonempty lookup table maps emoji to python-dateutil. Thus the pip
# solver should believe that the dependency is already satisfied and not
# add it as a pip dependency.
assert "python-dateutil" not in installed_packages


def test_extract_json_object():
"""It should remove all the characters after the last }"""
assert extract_json_object(' ^[0m {"key1": true } ^[0m') == '{"key1": true }'
Expand Down