From b112e8fe07e340a89d9036f1e24da3f16da4e2c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Tue, 23 Aug 2022 14:40:22 +0200 Subject: [PATCH 1/9] Fix setuptools>=64 import hooks --- ChangeLog | 3 +++ astroid/interpreter/_import/spec.py | 34 +++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3f4b867a14..d173d2c885 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,9 @@ What's New in astroid 2.15.0? ============================= Release date: TBA +* ``Astroid`` now supports custom import hooks. + + Refs PyCQA/pylint#7306 What's New in astroid 2.14.2? diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index ecf330b09d..d111de79ea 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -14,6 +14,7 @@ import sys import zipimport from collections.abc import Iterator, Sequence +from importlib.abc import MetaPathFinder from pathlib import Path from typing import Any, NamedTuple @@ -43,6 +44,13 @@ class ModuleType(enum.Enum): PY_NAMESPACE = enum.auto() +_MetaPathFinderModuleTypes: dict[str, ModuleType] = { + # Finders created by setuptools editable installs + "_EditableFinder": ModuleType.PY_SOURCE, + "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE, +} + + class ModuleSpec(NamedTuple): """Defines a class similar to PEP 420's ModuleSpec. @@ -349,7 +357,7 @@ def _find_spec_with_path( module_parts: list[str], processed: list[str], submodule_path: Sequence[str] | None, -) -> tuple[Finder, ModuleSpec]: +) -> tuple[Finder | MetaPathFinder, ModuleSpec]: for finder in _SPEC_FINDERS: finder_instance = finder(search_path) spec = finder_instance.find_module( @@ -359,6 +367,28 @@ def _find_spec_with_path( continue return finder_instance, spec + # Support for custom finders + for meta_finder in sys.meta_path: + spec = meta_finder.find_spec(modname, submodule_path) + if spec: + try: + module_type = _MetaPathFinderModuleTypes[ + meta_finder.__name__ # type: ignore[attr-defined] + ] + except KeyError: + # If we don't recognise the finder, we assume it's a regular module + module_type = ModuleType.PY_SOURCE + return ( # type: ignore[return-value] + meta_finder, + ModuleSpec( + spec.name, + module_type, + spec.origin, + spec.origin, + spec.submodule_search_locations, + ), + ) + raise ImportError(f"No module named {'.'.join(module_parts)}") @@ -394,7 +424,7 @@ def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSp _path, modname, module_parts, processed, submodule_path or path ) processed.append(modname) - if modpath: + if modpath and isinstance(finder, Finder): submodule_path = finder.contribute_to_path(spec, processed) if spec.type == ModuleType.PKG_DIRECTORY: From bd8b86989a24e9d4d14c05ef8bdf82aca84d9396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Tue, 23 Aug 2022 14:48:24 +0200 Subject: [PATCH 2/9] Catch AttributeError --- astroid/interpreter/_import/spec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index d111de79ea..93b5465828 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -373,9 +373,9 @@ def _find_spec_with_path( if spec: try: module_type = _MetaPathFinderModuleTypes[ - meta_finder.__name__ # type: ignore[attr-defined] + meta_finder.__name__ # type: ignore[attr-defined], caught by AttributeError ] - except KeyError: + except (KeyError, AttributeError): # If we don't recognise the finder, we assume it's a regular module module_type = ModuleType.PY_SOURCE return ( # type: ignore[return-value] From 4ec016e436d177c12fae062595579b8e819307ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Tue, 23 Aug 2022 14:53:06 +0200 Subject: [PATCH 3/9] Update test --- tests/unittest_modutils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/unittest_modutils.py b/tests/unittest_modutils.py index ab1acaac37..9c058f2a21 100644 --- a/tests/unittest_modutils.py +++ b/tests/unittest_modutils.py @@ -445,11 +445,7 @@ def test_is_module_name_part_of_extension_package_whitelist_success(self) -> Non @pytest.mark.skipif(not HAS_URLLIB3, reason="This test requires urllib3.") def test_file_info_from_modpath__SixMetaPathImporter() -> None: - pytest.raises( - ImportError, - modutils.file_info_from_modpath, - ["urllib3.packages.six.moves.http_client"], - ) + assert modutils.file_info_from_modpath(["urllib3.packages.six.moves.http_client"]) if __name__ == "__main__": From 223cc98c2f6e42ceb32c1437359cdc6b5bc18d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Sat, 4 Feb 2023 16:11:32 +0100 Subject: [PATCH 4/9] Use ``_MetaPathFinder`` ``Protocol`` --- astroid/interpreter/_import/spec.py | 30 +++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index 93b5465828..09bd348553 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -12,9 +12,9 @@ import os import pathlib import sys +import types import zipimport from collections.abc import Iterator, Sequence -from importlib.abc import MetaPathFinder from pathlib import Path from typing import Any, NamedTuple @@ -24,9 +24,21 @@ from . import util if sys.version_info >= (3, 8): - from typing import Literal + from typing import Literal, Protocol else: - from typing_extensions import Literal + from typing_extensions import Literal, Protocol + + +# The MetaPathFinder protocol comes from typeshed, which says: +# Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder` +class _MetaPathFinder(Protocol): + def find_spec( + self, + fullname: str, + path: Sequence[str] | None, + target: types.ModuleType | None = ..., + ) -> importlib.machinery.ModuleSpec | None: + ... class ModuleType(enum.Enum): @@ -130,8 +142,10 @@ def find_module( try: spec = importlib.util.find_spec(modname) if ( - spec and spec.loader is importlib.machinery.FrozenImporter - ): # noqa: E501 # type: ignore[comparison-overlap] + spec + and spec.loader # type: ignore[comparison-overlap] # noqa: E501 + is importlib.machinery.FrozenImporter + ): # No need for BuiltinImporter; builtins handled above return ModuleSpec( name=modname, @@ -357,7 +371,7 @@ def _find_spec_with_path( module_parts: list[str], processed: list[str], submodule_path: Sequence[str] | None, -) -> tuple[Finder | MetaPathFinder, ModuleSpec]: +) -> tuple[Finder | _MetaPathFinder, ModuleSpec]: for finder in _SPEC_FINDERS: finder_instance = finder(search_path) spec = finder_instance.find_module( @@ -373,12 +387,12 @@ def _find_spec_with_path( if spec: try: module_type = _MetaPathFinderModuleTypes[ - meta_finder.__name__ # type: ignore[attr-defined], caught by AttributeError + meta_finder.__name__ # type: ignore[attr-defined] # Caught by AttributeError ] except (KeyError, AttributeError): # If we don't recognise the finder, we assume it's a regular module module_type = ModuleType.PY_SOURCE - return ( # type: ignore[return-value] + return ( meta_finder, ModuleSpec( spec.name, From 539690fbf9be6357625ce2d8e57d34a2f387e777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Sun, 5 Feb 2023 13:44:12 +0100 Subject: [PATCH 5/9] Changes to typing and check order --- astroid/interpreter/_import/spec.py | 31 ++++++++++++++++++----------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index 09bd348553..ca2210c840 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -38,7 +38,7 @@ def find_spec( path: Sequence[str] | None, target: types.ModuleType | None = ..., ) -> importlib.machinery.ModuleSpec | None: - ... + ... # pragma: no cover class ModuleType(enum.Enum): @@ -248,7 +248,6 @@ def __init__(self, path: Sequence[str]) -> None: super().__init__(path) for entry_path in path: if entry_path not in sys.path_importer_cache: - # pylint: disable=no-member try: sys.path_importer_cache[entry_path] = zipimport.zipimporter( # type: ignore[assignment] entry_path @@ -332,7 +331,6 @@ def _is_setuptools_namespace(location: pathlib.Path) -> bool: def _get_zipimporters() -> Iterator[tuple[str, zipimport.zipimporter]]: for filepath, importer in sys.path_importer_cache.items(): - # pylint: disable-next=no-member if isinstance(importer, zipimport.zipimporter): yield filepath, importer @@ -383,15 +381,24 @@ def _find_spec_with_path( # Support for custom finders for meta_finder in sys.meta_path: - spec = meta_finder.find_spec(modname, submodule_path) - if spec: - try: - module_type = _MetaPathFinderModuleTypes[ - meta_finder.__name__ # type: ignore[attr-defined] # Caught by AttributeError - ] - except (KeyError, AttributeError): - # If we don't recognise the finder, we assume it's a regular module - module_type = ModuleType.PY_SOURCE + # See if we support the customer import hook of the meta_finder + try: + meta_finder_name: str = meta_finder.__name__ # type: ignore[attr-defined] + except AttributeError: + continue + if meta_finder_name not in _MetaPathFinderModuleTypes: + continue + + module_type = _MetaPathFinderModuleTypes[meta_finder_name] + + # Meta path finders are supposed to have a find_spec method since + # Python 3.4. However, some third-party finders do not implement it. + # PEP302 does not refer to find_spec as well. + # See: https://github.com/PyCQA/astroid/pull/1752/ + if not hasattr(meta_finder, "find_spec"): + continue + + if spec := meta_finder.find_spec(modname, submodule_path): return ( meta_finder, ModuleSpec( From f889b5d6bb49278d9736b799172f19b3f09631df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Sun, 5 Feb 2023 13:46:11 +0100 Subject: [PATCH 6/9] Don't use the walrus --- astroid/interpreter/_import/spec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index ca2210c840..7a842aad53 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -398,7 +398,8 @@ def _find_spec_with_path( if not hasattr(meta_finder, "find_spec"): continue - if spec := meta_finder.find_spec(modname, submodule_path): + spec = meta_finder.find_spec(modname, submodule_path) + if spec: return ( meta_finder, ModuleSpec( From 7198a97d0d4943a9d3f28d0b03f54bf4efb5ec65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Sun, 5 Feb 2023 15:52:24 +0100 Subject: [PATCH 7/9] Add support for ``_SixMetaPathImporter`` --- astroid/interpreter/_import/spec.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index 7a842aad53..136a91c885 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -60,6 +60,8 @@ class ModuleType(enum.Enum): # Finders created by setuptools editable installs "_EditableFinder": ModuleType.PY_SOURCE, "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE, + # Finders create by six + "_SixMetaPathImporter": ModuleType.PY_SOURCE, } @@ -382,10 +384,7 @@ def _find_spec_with_path( # Support for custom finders for meta_finder in sys.meta_path: # See if we support the customer import hook of the meta_finder - try: - meta_finder_name: str = meta_finder.__name__ # type: ignore[attr-defined] - except AttributeError: - continue + meta_finder_name = meta_finder.__class__.__name__ if meta_finder_name not in _MetaPathFinderModuleTypes: continue From d8c5a48e15dffd40b8f9c9cd8e5bed36e2ffebef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Sun, 5 Feb 2023 16:04:48 +0100 Subject: [PATCH 8/9] Fix support for setuptools --- astroid/interpreter/_import/spec.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index 136a91c885..bd12be7a86 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -386,7 +386,15 @@ def _find_spec_with_path( # See if we support the customer import hook of the meta_finder meta_finder_name = meta_finder.__class__.__name__ if meta_finder_name not in _MetaPathFinderModuleTypes: - continue + # Setuptools>62 creates its EditableFinders dynamically and have + # "type" as their __clas__.__name__. We check __name__ as well + # to see if we can support the finder. + try: + meta_finder_name = meta_finder.__name__ + except AttributeError: + continue + if meta_finder_name not in _MetaPathFinderModuleTypes: + continue module_type = _MetaPathFinderModuleTypes[meta_finder_name] From 6ea1a7d1cfa1a0c7b9e609c7edb49c51423afe18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Sun, 5 Feb 2023 16:28:05 +0100 Subject: [PATCH 9/9] Update astroid/interpreter/_import/spec.py Co-authored-by: Jacob Walls --- astroid/interpreter/_import/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index bd12be7a86..77e71016ac 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -387,7 +387,7 @@ def _find_spec_with_path( meta_finder_name = meta_finder.__class__.__name__ if meta_finder_name not in _MetaPathFinderModuleTypes: # Setuptools>62 creates its EditableFinders dynamically and have - # "type" as their __clas__.__name__. We check __name__ as well + # "type" as their __class__.__name__. We check __name__ as well # to see if we can support the finder. try: meta_finder_name = meta_finder.__name__