From 68fc43f52f1862af2cd9cd114a39519e40a2818f Mon Sep 17 00:00:00 2001 From: David Vujic Date: Tue, 5 Aug 2025 10:20:30 +0200 Subject: [PATCH 1/5] fix(poly check): cached 'packages_distributions' from importlib.metadata, always run with same input --- components/polylith/distributions/core.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/components/polylith/distributions/core.py b/components/polylith/distributions/core.py index 05539399..5269eaf4 100644 --- a/components/polylith/distributions/core.py +++ b/components/polylith/distributions/core.py @@ -83,6 +83,14 @@ def get_distributions() -> list: return list(importlib.metadata.distributions()) +@lru_cache +def _get_package_dists() -> dict: + # added in Python 3.10 + fn = getattr(importlib.metadata, "packages_distributions", None) + + return fn() if fn else {} + + def get_packages_distributions(project_dependencies: set) -> set: """Return the mapped top namespace from an import @@ -93,10 +101,7 @@ def get_packages_distributions(project_dependencies: set) -> set: Note: available for Python >= 3.10 """ - # added in Python 3.10 - fn = getattr(importlib.metadata, "packages_distributions", None) - - dists = fn() if fn else {} + dists = _get_package_dists() common = {k for k, v in dists.items() if project_dependencies.intersection(set(v))} From 11d48ef07e6d4b50a7b1e5cac9da788db68f054a Mon Sep 17 00:00:00 2001 From: David Vujic Date: Tue, 5 Aug 2025 10:48:24 +0200 Subject: [PATCH 2/5] fix(poly check): local cache for dist.files lookup when figuring out dependencies and sub-dependencies --- components/polylith/distributions/core.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/components/polylith/distributions/core.py b/components/polylith/distributions/core.py index 5269eaf4..5a331ff3 100644 --- a/components/polylith/distributions/core.py +++ b/components/polylith/distributions/core.py @@ -25,9 +25,13 @@ def map_sub_packages(acc, dist) -> dict: return {**acc, **dist_subpackages(dist)} -def parsed_namespaces_from_files(dist) -> List[str]: - name = dist.metadata["name"] - files = dist.files or [] +_cached_dist_files = {} + + +def parsed_namespaces_from_files(dist, name: str) -> List[str]: + if name not in _cached_dist_files: + files = dist.files or [] + _cached_dist_files[name] = [f for f in files if f.suffix == ".py"] normalized_name = str.replace(name, "-", "_") to_ignore = { @@ -38,7 +42,7 @@ def parsed_namespaces_from_files(dist) -> List[str]: "..", } - filtered = [f for f in files if f.suffix == ".py"] + filtered = _cached_dist_files[name] top_folders = {f.parts[0] for f in filtered if len(f.parts) > 1} namespaces = {t for t in top_folders if t not in to_ignore} @@ -49,17 +53,19 @@ def parsed_top_level_namespace(namespaces: List[str]) -> List[str]: return [str.replace(ns, "/", ".") for ns in namespaces] -def top_level_packages(dist) -> List[str]: +def top_level_packages(dist, name: str) -> List[str]: top_level = dist.read_text("top_level.txt") namespaces = str.split(top_level or "") - return parsed_top_level_namespace(namespaces) or parsed_namespaces_from_files(dist) + return parsed_top_level_namespace(namespaces) or parsed_namespaces_from_files( + dist, name + ) def mapped_packages(dist) -> dict: - packages = top_level_packages(dist) name = dist.metadata["name"] + packages = top_level_packages(dist, name) return {name: packages} if packages else {} From ba16988eb9d003536ea813f00da83649b035e6b4 Mon Sep 17 00:00:00 2001 From: David Vujic Date: Tue, 5 Aug 2025 11:23:51 +0200 Subject: [PATCH 3/5] refactor(distributions): extract custom caching into separate module, also: unit tests resetting cached state before run --- components/polylith/distributions/__init__.py | 2 ++ components/polylith/distributions/caching.py | 17 +++++++++++++++++ components/polylith/distributions/core.py | 17 +++++++++-------- .../polylith/distributions/test_core.py | 10 ++++++++-- 4 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 components/polylith/distributions/caching.py diff --git a/components/polylith/distributions/__init__.py b/components/polylith/distributions/__init__.py index b4b4b607..8204bbea 100644 --- a/components/polylith/distributions/__init__.py +++ b/components/polylith/distributions/__init__.py @@ -1,3 +1,4 @@ +from polylith.distributions import caching from polylith.distributions.collect import known_aliases_and_sub_dependencies from polylith.distributions.core import ( distributions_packages, @@ -6,6 +7,7 @@ ) __all__ = [ + "caching", "distributions_packages", "distributions_sub_packages", "get_distributions", diff --git a/components/polylith/distributions/caching.py b/components/polylith/distributions/caching.py new file mode 100644 index 00000000..8798d6a7 --- /dev/null +++ b/components/polylith/distributions/caching.py @@ -0,0 +1,17 @@ +_cache = {} + + +def add(key: str, value) -> None: + _cache[key] = value + + +def get(key: str): + return _cache[key] + + +def exists(key: str) -> bool: + return key in _cache + + +def clear() -> None: + _cache.clear() diff --git a/components/polylith/distributions/core.py b/components/polylith/distributions/core.py index 5a331ff3..6f57fc74 100644 --- a/components/polylith/distributions/core.py +++ b/components/polylith/distributions/core.py @@ -1,8 +1,11 @@ import importlib.metadata +import pathlib import re from functools import lru_cache, reduce from typing import Dict, List +from polylith.distributions import caching + SUB_DEP_SEPARATORS = r"[\s!=;><\^~]" @@ -25,13 +28,11 @@ def map_sub_packages(acc, dist) -> dict: return {**acc, **dist_subpackages(dist)} -_cached_dist_files = {} - - def parsed_namespaces_from_files(dist, name: str) -> List[str]: - if name not in _cached_dist_files: + if not caching.exists(name): files = dist.files or [] - _cached_dist_files[name] = [f for f in files if f.suffix == ".py"] + python_files = [f for f in files if f.suffix == ".py"] + caching.add(name, python_files) normalized_name = str.replace(name, "-", "_") to_ignore = { @@ -42,7 +43,7 @@ def parsed_namespaces_from_files(dist, name: str) -> List[str]: "..", } - filtered = _cached_dist_files[name] + filtered: List[pathlib.PurePosixPath] = caching.get(name) top_folders = {f.parts[0] for f in filtered if len(f.parts) > 1} namespaces = {t for t in top_folders if t not in to_ignore} @@ -90,7 +91,7 @@ def get_distributions() -> list: @lru_cache -def _get_package_dists() -> dict: +def package_distributions_from_importlib() -> dict: # added in Python 3.10 fn = getattr(importlib.metadata, "packages_distributions", None) @@ -107,7 +108,7 @@ def get_packages_distributions(project_dependencies: set) -> set: Note: available for Python >= 3.10 """ - dists = _get_package_dists() + dists = package_distributions_from_importlib() common = {k for k, v in dists.items() if project_dependencies.intersection(set(v))} diff --git a/test/components/polylith/distributions/test_core.py b/test/components/polylith/distributions/test_core.py index a8301ec9..e7182a51 100644 --- a/test/components/polylith/distributions/test_core.py +++ b/test/components/polylith/distributions/test_core.py @@ -21,6 +21,12 @@ def read_text(self, *args): return self.read_text_data +@pytest.fixture +def setup(): + distributions.caching.clear() + distributions.core.package_distributions_from_importlib.cache_clear() + + def test_distribution_packages(): dists = list(importlib.metadata.distributions()) @@ -61,7 +67,7 @@ def test_distribution_packages_for_missing_metadata_is_handled(): assert res == {} -def test_distribution_packages_with_top_level_ns_information_in_files(): +def test_distribution_packages_with_top_level_ns_information_in_files(setup): files = [ importlib.metadata.PackagePath("some_module.py"), importlib.metadata.PackagePath("hello/world.py"), @@ -104,7 +110,7 @@ def test_distribution_sub_packages(): @pytest.mark.skipif(sys.version_info < (3, 10), reason="requires python3.10 or higher") -def test_package_distributions_returning_top_namespace(monkeypatch): +def test_package_distributions_returning_top_namespace(setup, monkeypatch): fake_dists = { "something": ["something-subnamespace"], "opentelemetry": ["opentelemetry-instrumentation-fastapi"], From df612c852724f9d9d943393595e7340a8b5ac76b Mon Sep 17 00:00:00 2001 From: David Vujic Date: Tue, 5 Aug 2025 11:34:49 +0200 Subject: [PATCH 4/5] bump Poetry plugin to 1.38.2 --- projects/poetry_polylith_plugin/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/poetry_polylith_plugin/pyproject.toml b/projects/poetry_polylith_plugin/pyproject.toml index 39c159b2..b1fa2e19 100644 --- a/projects/poetry_polylith_plugin/pyproject.toml +++ b/projects/poetry_polylith_plugin/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "poetry-polylith-plugin" -version = "1.38.1" +version = "1.38.2" description = "A Poetry plugin that adds tooling support for the Polylith Architecture" authors = ["David Vujic"] homepage = "https://davidvujic.github.io/python-polylith-docs/" From ff4486c857eb1bfb934c3d5e731ff4096253ca30 Mon Sep 17 00:00:00 2001 From: David Vujic Date: Tue, 5 Aug 2025 11:35:07 +0200 Subject: [PATCH 5/5] bump CLI to 1.31.3 --- projects/polylith_cli/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/polylith_cli/pyproject.toml b/projects/polylith_cli/pyproject.toml index aa17983d..6436ea52 100644 --- a/projects/polylith_cli/pyproject.toml +++ b/projects/polylith_cli/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polylith-cli" -version = "1.31.2" +version = "1.31.3" description = "Python tooling support for the Polylith Architecture" authors = ['David Vujic'] homepage = "https://davidvujic.github.io/python-polylith-docs/"