From ded05681c5de6a4f84a0abb3a060fee3d5f8c40b Mon Sep 17 00:00:00 2001 From: Mark Forrer Date: Tue, 6 Sep 2022 17:46:50 -0700 Subject: [PATCH 1/6] Clarify removal for non-installed extras & optional groups (#6229) --- docs/cli.md | 8 +++++++- docs/managing-dependencies.md | 12 +++++++++--- docs/pyproject.md | 6 +++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 3ae708c43a0..cfef79d91f9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -144,7 +144,7 @@ This ensures that everyone using the library will get the same versions of the d If there is no `poetry.lock` file, Poetry will create one after dependency resolution. -If you want to exclude one or more dependency group for the installation, you can use +If you want to exclude one or more dependency groups for the installation, you can use the `--without` option. ```bash @@ -201,6 +201,12 @@ poetry install -E mysql -E pgsql poetry install --all-extras ``` +Extras are not sensitive to `--sync`. Any extras not specified will always be removed. + +```bash +poetry install --extras "A B" # C is removed +``` + By default `poetry` will install your project's package every time you run `install`: ```bash diff --git a/docs/managing-dependencies.md b/docs/managing-dependencies.md index e138cd838e0..995bb823700 100644 --- a/docs/managing-dependencies.md +++ b/docs/managing-dependencies.md @@ -184,10 +184,9 @@ to remove packages from a specific group: poetry remove mkdocs --group docs ``` - ## Synchronizing dependencies -Poetry supports what's called dependency synchronization. What this does is ensuring +Poetry supports what's called dependency synchronization. Dependency synchronization ensures that the locked dependencies in the `poetry.lock` file are the only ones present in the environment, removing anything that's not necessary. @@ -198,7 +197,8 @@ poetry install --sync ``` The `--sync` option can be combined with any [dependency groups]({{< relref "#dependency-groups" >}}) related options -to synchronize the environment with specific groups. +to synchronize the environment with specific groups. Note that extras are separate. Any +extras not selected for install are always removed, regardless of `--sync`. ```bash poetry install --without dev --sync @@ -209,3 +209,9 @@ poetry install --only dev {{% note %}} The `--sync` option replaces the `--remove-untracked` option which is now deprecated. {{% /note %}} + +## Layering optional groups + +When you omit the `--sync` option, you can install any subset of optional groups without removing +those that are already installed. This is very useful, for example, in multi-stage +Docker builds, where you run `poetry install` multiple times in different build stages. diff --git a/docs/pyproject.md b/docs/pyproject.md index defb4636aa3..ba9d785c949 100644 --- a/docs/pyproject.md +++ b/docs/pyproject.md @@ -385,7 +385,11 @@ poetry install --extras "mysql pgsql" poetry install -E mysql -E pgsql ``` -Or, you can install all extras with the `--all-extras` option: +Any extras you don't specify will be removed. Note this behavior is different from +[optional dependency groups]({{< relref "managing-dependencies#optional-groups" >}}) not +selected for install, e.g. those not specified via `install --with`. + +You can install all extras with the `--all-extras` option: ```bash poetry install --all-extras From 06e61fae04320eb24f4132452ee6f20e3e9049a2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 01:11:29 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/managing-dependencies.md | 4 ++-- docs/pyproject.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/managing-dependencies.md b/docs/managing-dependencies.md index 995bb823700..c1fa129674c 100644 --- a/docs/managing-dependencies.md +++ b/docs/managing-dependencies.md @@ -197,7 +197,7 @@ poetry install --sync ``` The `--sync` option can be combined with any [dependency groups]({{< relref "#dependency-groups" >}}) related options -to synchronize the environment with specific groups. Note that extras are separate. Any +to synchronize the environment with specific groups. Note that extras are separate. Any extras not selected for install are always removed, regardless of `--sync`. ```bash @@ -212,6 +212,6 @@ The `--sync` option replaces the `--remove-untracked` option which is now deprec ## Layering optional groups -When you omit the `--sync` option, you can install any subset of optional groups without removing +When you omit the `--sync` option, you can install any subset of optional groups without removing those that are already installed. This is very useful, for example, in multi-stage Docker builds, where you run `poetry install` multiple times in different build stages. diff --git a/docs/pyproject.md b/docs/pyproject.md index ba9d785c949..55004615ccb 100644 --- a/docs/pyproject.md +++ b/docs/pyproject.md @@ -385,8 +385,8 @@ poetry install --extras "mysql pgsql" poetry install -E mysql -E pgsql ``` -Any extras you don't specify will be removed. Note this behavior is different from -[optional dependency groups]({{< relref "managing-dependencies#optional-groups" >}}) not +Any extras you don't specify will be removed. Note this behavior is different from +[optional dependency groups]({{< relref "managing-dependencies#optional-groups" >}}) not selected for install, e.g. those not specified via `install --with`. You can install all extras with the `--all-extras` option: From b4c4fc1caecf1787b62f38ee0abb68e26dc45cb1 Mon Sep 17 00:00:00 2001 From: Charles Durieux Date: Mon, 12 Sep 2022 16:19:32 +0100 Subject: [PATCH 3/6] Rework `_validate_archive_hash()` to support multiple hash algorithms. --- src/poetry/installation/executor.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index da0734ed334..84dc274d4b9 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -658,16 +658,20 @@ def _download_link(self, operation: Install | Update, link: Link) -> Path: @staticmethod def _validate_archive_hash(archive: Path, package: Package) -> str: file_dep = FileDependency(package.name, archive) - archive_hash: str = "sha256:" + file_dep.hash() known_hashes = {f["hash"] for f in package.files} - if archive_hash not in known_hashes: - raise RuntimeError( - f"Hash for {package} from archive {archive.name} not found in" - f" known hashes (was: {archive_hash})" - ) + for known_hash in known_hashes: + hash_name, known_hash_value = known_hash.split(':', 1) + archive_hash_value = file_dep.hash(hash_name=hash_name) + + if known_hash_value == archive_hash_value: + return f"{hash_name}:{archive_hash_value}" + + raise RuntimeError( + f"Hash for {package} from archive {archive.name} not found in" + f" known hashes (was: {archive_hash})" + ) - return archive_hash def _download_archive(self, operation: Install | Update, link: Link) -> Path: response = self._authenticator.request( From d3a6b56648243ffc531b5f2487c67f6d75fffdf4 Mon Sep 17 00:00:00 2001 From: Charles Durieux Date: Mon, 12 Sep 2022 16:27:06 +0100 Subject: [PATCH 4/6] Fix syntax error and remove package hash from exception output --- src/poetry/installation/executor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index 84dc274d4b9..c21e3340c22 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -667,10 +667,10 @@ def _validate_archive_hash(archive: Path, package: Package) -> str: if known_hash_value == archive_hash_value: return f"{hash_name}:{archive_hash_value}" - raise RuntimeError( - f"Hash for {package} from archive {archive.name} not found in" - f" known hashes (was: {archive_hash})" - ) + raise RuntimeError( + f"Hash for {package} from archive {archive.name} not found in" + f" known hashes" + ) def _download_archive(self, operation: Install | Update, link: Link) -> Path: From ae23cf6dcdbc813f993a8d3ce0bb7910cd1d4b87 Mon Sep 17 00:00:00 2001 From: Charles Durieux Date: Mon, 12 Sep 2022 17:04:27 +0100 Subject: [PATCH 5/6] Add test --- tests/installation/test_executor.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py index b2e06de6f99..11f2799d510 100644 --- a/tests/installation/test_executor.py +++ b/tests/installation/test_executor.py @@ -643,6 +643,14 @@ def test_executor_should_write_pep610_url_references_for_git_with_subdirectories ) +@pytest.mark.parametrize( + "file_hash", + [ + ("sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a"), + ("md5:15507846fd4299596661d0197bfb4f90"), + ("sha512:44cfa6b8696e5008c1c4dfe32c52a8b6511c261d8f3a25882366c0718eb42a17da5e4b9196623597661871be65c48e613d8f5c50a5c5c827e0efbf8b612fe5be"), # noqa: E501 + ] +) def test_executor_should_use_cached_link_and_hash( tmp_venv: VirtualEnv, pool: Pool, @@ -650,6 +658,7 @@ def test_executor_should_use_cached_link_and_hash( io: BufferedIO, mocker: MockerFixture, fixture_dir: FixtureDirGetter, + file_hash, ): link_cached = fixture_dir("distributions") / "demo-0.1.0-py2.py3-none-any.whl" @@ -663,7 +672,7 @@ def test_executor_should_use_cached_link_and_hash( package.files = [ { "file": "demo-0.1.0-py2.py3-none-any.whl", - "hash": "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a", # noqa: E501 + "hash": file_hash, } ] From b4758d8a34d4a808c55be8bfb5f125c6ae66c998 Mon Sep 17 00:00:00 2001 From: Charles Durieux Date: Mon, 12 Sep 2022 17:12:13 +0100 Subject: [PATCH 6/6] lint --- src/poetry/installation/executor.py | 6 ++---- tests/installation/test_executor.py | 10 +++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index c21e3340c22..3178ba8740d 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -661,18 +661,16 @@ def _validate_archive_hash(archive: Path, package: Package) -> str: known_hashes = {f["hash"] for f in package.files} for known_hash in known_hashes: - hash_name, known_hash_value = known_hash.split(':', 1) + hash_name, known_hash_value = known_hash.split(":", 1) archive_hash_value = file_dep.hash(hash_name=hash_name) if known_hash_value == archive_hash_value: return f"{hash_name}:{archive_hash_value}" raise RuntimeError( - f"Hash for {package} from archive {archive.name} not found in" - f" known hashes" + f"Hash for {package} from archive {archive.name} not found in known hashes" ) - def _download_archive(self, operation: Install | Update, link: Link) -> Path: response = self._authenticator.request( "get", link.url, stream=True, io=self._sections.get(id(operation), self._io) diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py index 11f2799d510..ef02640b7d8 100644 --- a/tests/installation/test_executor.py +++ b/tests/installation/test_executor.py @@ -646,10 +646,10 @@ def test_executor_should_write_pep610_url_references_for_git_with_subdirectories @pytest.mark.parametrize( "file_hash", [ - ("sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a"), - ("md5:15507846fd4299596661d0197bfb4f90"), - ("sha512:44cfa6b8696e5008c1c4dfe32c52a8b6511c261d8f3a25882366c0718eb42a17da5e4b9196623597661871be65c48e613d8f5c50a5c5c827e0efbf8b612fe5be"), # noqa: E501 - ] + "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a", + "md5:15507846fd4299596661d0197bfb4f90", + "sha512:44cfa6b8696e5008c1c4dfe32c52a8b6511c261d8f3a25882366c0718eb42a17da5e4b9196623597661871be65c48e613d8f5c50a5c5c827e0efbf8b612fe5be", # noqa: E501 + ], ) def test_executor_should_use_cached_link_and_hash( tmp_venv: VirtualEnv, @@ -658,7 +658,7 @@ def test_executor_should_use_cached_link_and_hash( io: BufferedIO, mocker: MockerFixture, fixture_dir: FixtureDirGetter, - file_hash, + file_hash: str, ): link_cached = fixture_dir("distributions") / "demo-0.1.0-py2.py3-none-any.whl"