From 2abf64e73bdf90b9afa9b5406ba39997570341c8 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 31 Oct 2023 10:46:36 +0000 Subject: [PATCH 1/4] Add try/except import. Remove sys.modules 'shortcut'. This was made buggy by the inclusion of relative imports - potentially incorrect modules could be picked up from sys.modules. --- src/ducktools/lazyimporter.py | 207 +++++++++++++----- tests/example_modules/ex_othermod/__init__.py | 2 + tests/test_basic_imports.py | 19 +- 3 files changed, 171 insertions(+), 57 deletions(-) diff --git a/src/ducktools/lazyimporter.py b/src/ducktools/lazyimporter.py index 1efcb5c..a053f5a 100644 --- a/src/ducktools/lazyimporter.py +++ b/src/ducktools/lazyimporter.py @@ -26,12 +26,13 @@ import abc import sys -__version__ = "v0.1.1" +__version__ = "v0.1.2" __all__ = [ "LazyImporter", "ModuleImport", "FromImport", "MultiFromImport", + "TryExceptImport", "get_importer_state", "get_module_funcs", ] @@ -115,24 +116,21 @@ def __eq__(self, other): return NotImplemented def do_import(self, globs=None): - try: # Already imported - mod = sys.modules[self.module_name_noprefix] - except KeyError: - mod = __import__( - self.module_name_noprefix, - globals=globs, - level=self.import_level, - ) + mod = __import__( + self.module_name_noprefix, + globals=globs, + level=self.import_level, + ) - if self.asname: # return the submodule - submod_used = [self.module_basename] - for submod in self.submodule_names: - submod_used.append(submod) - try: - mod = getattr(mod, submod) - except AttributeError: - invalid_module = ".".join(submod_used) - raise ModuleNotFoundError(f"No module named {invalid_module!r}") + if self.asname: # return the submodule + submod_used = [self.module_basename] + for submod in self.submodule_names: + submod_used.append(submod) + try: + mod = getattr(mod, submod) + except AttributeError: + invalid_module = ".".join(submod_used) + raise ModuleNotFoundError(f"No module named {invalid_module!r}") if self.asname: return {self.asname: mod} @@ -180,17 +178,13 @@ def __eq__(self, other): return NotImplemented def do_import(self, globs=None): - try: - # Module already imported - mod = sys.modules[self.module_name] - except KeyError: - # Perform the import - mod = __import__( - self.module_name_noprefix, - globals=globs, - fromlist=[self.attrib_name], - level=self.import_level, - ) + # Perform the import + mod = __import__( + self.module_name_noprefix, + globals=globs, + fromlist=[self.attrib_name], + level=self.import_level, + ) return {self.asname: getattr(mod, self.attrib_name)} @@ -248,17 +242,13 @@ def asnames(self): def do_import(self, globs=None): from_imports = {} - try: - # Module already imported - mod = sys.modules[self.module_name] - except KeyError: - # Perform the import - mod = __import__( - self.module_name_noprefix, - globals=globs, - fromlist=self.asnames, - level=self.import_level, - ) + # Perform the import + mod = __import__( + self.module_name_noprefix, + globals=globs, + fromlist=self.asnames, + level=self.import_level, + ) for name in self.attrib_names: if isinstance(name, str): @@ -269,6 +259,114 @@ def do_import(self, globs=None): return from_imports +class TryExceptImport(_ImportBase): + module_name: str + except_module: str + asname: str + + def __init__(self, module_name, except_module, asname): + """ + Equivalent to: + + try: + import as + except ImportError: + import as + + Inside a LazyImporter + + :param module_name: Name of the 'try' module + :param except_module: Name of the module to import in the case + that the 'try' module fails + :param asname: Name to use for either on successful import + """ + self.module_name = module_name + self.except_module = except_module + self.asname = asname + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f"module_name={self.module_name!r}, " + f"except_module={self.except_module!r}, " + f"asname={self.asname!r}" + f")" + ) + + def __eq__(self, other): + if self.__class__ is other.__class__: + return (self.module_name, self.except_module, self.asname) == (other.module_name, other.except_module, other.asname) + return NotImplemented + + @property + def except_import_level(self): + level = 0 + for char in self.except_module: + if char != ".": + break + level += 1 + return level + + @property + def except_module_noprefix(self): + """ + Remove any leading '.' characters from the except_module name. + :return: + """ + return self.except_module.lstrip(".") + + @property + def except_module_basename(self): + """ + Get the first part of an except module import name. + eg: 'importlib' from 'importlib.util' + + :return: name of base module + :rtype: str + """ + return self.except_module_noprefix.split(".")[0] + + @property + def except_module_names(self): + """ + Get a list of all except submodule names in order. + eg: ['util'] from 'importlib.util' + :return: List of submodule names. + :rtype: list[str] + """ + return self.except_module_noprefix.split(".")[1:] + + def do_import(self, globs=None): + try: + mod = __import__( + self.module_name_noprefix, + globals=globs, + level=self.import_level, + ) + except ImportError: + mod = __import__( + self.except_module_noprefix, + globals=globs, + level=self.except_import_level + ) + submod_used = [self.except_module_basename] + submodule_names = self.except_module_names + + else: + submod_used = [self.module_basename] + submodule_names = self.submodule_names + + for submod in submodule_names: + submod_used.append(submod) + try: + mod = getattr(mod, submod) + except AttributeError: + invalid_module = ".".join(submod_used) + raise ModuleNotFoundError(f"No module named {invalid_module!r}") + + return {self.asname: mod} + + class _SubmoduleImports(_ImportBase): module_name: str submodules: "set[str]" @@ -304,22 +402,18 @@ def __eq__(self, other): def do_import(self, globs=None): for submod in self.submodules: # Make sure any submodules are in place - try: - _ = sys.modules[submod] - except KeyError: - __import__( - submod, - globals=globs, - level=self.import_level, - ) - try: - mod = sys.modules[self.module_basename] - except KeyError: - mod = __import__( - self.module_basename, + __import__( + submod, globals=globs, level=self.import_level, ) + + mod = __import__( + self.module_basename, + globals=globs, + level=self.import_level, + ) + return {self.module_name: mod} @@ -403,13 +497,14 @@ def group_importers(inst): ) else: raise TypeError( - f"{imp} is not an instance of ModuleImport or FromImport" + f"{imp} is not an instance of " + f"ModuleImport, FromImport, MultiFromImport or TryExceptImport" ) return importers class LazyImporter: - _imports: "list[ModuleImport | FromImport | MultiFromImport]" + _imports: "list[ModuleImport | FromImport | MultiFromImport | TryExceptImport]" _globals: dict _importers = _ImporterGrouper() @@ -422,7 +517,7 @@ def __init__(self, imports, *, globs=None): globals() must be provided to the importer if relative imports are used. :param imports: list of imports - :type imports: list[ModuleImport | FromImport | MultiFromImport] + :type imports: list[ModuleImport | FromImport | MultiFromImport | TryExceptImport] :param globs: globals object for relative imports :type globs: dict[str, typing.Any] """ diff --git a/tests/example_modules/ex_othermod/__init__.py b/tests/example_modules/ex_othermod/__init__.py index c4a71b9..f1ee4eb 100644 --- a/tests/example_modules/ex_othermod/__init__.py +++ b/tests/example_modules/ex_othermod/__init__.py @@ -3,6 +3,8 @@ FromImport, ) +name = "ex_othermod" + laz = LazyImporter( [FromImport("..ex_mod.ex_submod", "name")], globs=globals(), diff --git a/tests/test_basic_imports.py b/tests/test_basic_imports.py index d4f6e7b..3cca3ad 100644 --- a/tests/test_basic_imports.py +++ b/tests/test_basic_imports.py @@ -6,7 +6,8 @@ LazyImporter, ModuleImport, FromImport, - MultiFromImport + MultiFromImport, + TryExceptImport, ) @@ -70,6 +71,22 @@ def test_submod_multifrom(): assert laz.othername == "ex_submod2" +def test_try_except_import(): + # When the first import fails + laz = LazyImporter([ + TryExceptImport("module_does_not_exist", "ex_mod", "ex_mod"), + ]) + + assert laz.ex_mod.name == "ex_mod" + + # When the first import succeeds + laz2 = LazyImporter([ + TryExceptImport("ex_mod", "ex_othermod", "ex_mod"), + ]) + + assert laz2.ex_mod.name == "ex_mod" + + def test_relative_import(): import example_modules.lazy_submod_ex as lse From d9c53399dbe526638f4052eb8c01f7048b6835cf Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 31 Oct 2023 11:31:43 +0000 Subject: [PATCH 2/4] Ignore the htmlcov folder for coverage tests. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e56c54b..f787b64 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ scratch/ build/ _build/ dist/ +htmlcov/ .vscode/ __pycache__/ From bfdf7e1f722e16f9988e2ad3729562ccc6f834a5 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 31 Oct 2023 11:37:40 +0000 Subject: [PATCH 3/4] Black autoformatting --- src/ducktools/lazyimporter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ducktools/lazyimporter.py b/src/ducktools/lazyimporter.py index a053f5a..082c9fc 100644 --- a/src/ducktools/lazyimporter.py +++ b/src/ducktools/lazyimporter.py @@ -295,7 +295,11 @@ def __repr__(self): def __eq__(self, other): if self.__class__ is other.__class__: - return (self.module_name, self.except_module, self.asname) == (other.module_name, other.except_module, other.asname) + return (self.module_name, self.except_module, self.asname) == ( + other.module_name, + other.except_module, + other.asname, + ) return NotImplemented @property @@ -347,7 +351,7 @@ def do_import(self, globs=None): mod = __import__( self.except_module_noprefix, globals=globs, - level=self.except_import_level + level=self.except_import_level, ) submod_used = [self.except_module_basename] submodule_names = self.except_module_names From 40754c01e89993994e5d6c6fb4a9e6806e1a4626 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 31 Oct 2023 11:59:39 +0000 Subject: [PATCH 4/4] Update the readme to document the Import classes. Including the new TryExceptImport class. --- README.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/README.md b/README.md index 09311ea..5fe4223 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,89 @@ laz = LazyImporter( __getattr__, __dir__ = get_module_funcs(laz, __name__) ``` +## The import classes ## + +In all of these instances `modules` is intended as the first argument +to `LazyImporter` and all attributes would be accessed from the +`LazyImporter` instance and not in the global namespace. + +eg: +```python +modules = [ModuleImport("functools")] +laz = LazyImporter(modules) +laz.functools # provides access to the module "functools" +``` + +### ModuleImport ### + +`ModuleImport` is used for your basic module style imports. + +```python +modules = [ + ModuleImport("module"), + ModuleImport("other_module", "other_name"), + ModuleImport("base_module.submodule"), + ModuleImport("base_module.submodule", "short_name"), +] +``` + +is equivalent to + +``` +import module +import other_module as other_name +import base_module.submodule +import base_module.submodule as short_name +``` + +when provided to a LazyImporter. + +### FromImport and MultiFromImport ### + +`FromImport` is used for standard 'from' imports. + +```python +modules = [ + FromImport("dataclasses", "dataclass"), + FromImport("functools", "partial", "partfunc"), + MultiFromImport("collections", ["namedtuple", ("defaultdict", "dd")]), +] +``` + +is equivalent to + +```python +from dataclasses import dataclass +from functools import partial as partfunc +from collections import namedtuple, defaultdict as dd +``` + +when provided to a LazyImporter. + +### TryExceptImport ### + +`TryExceptImport` is used for compatibility where a module may not be available +and so a fallback module providing the same functionality should be used. For +example when a newer version of python has a stdlib module that has replaced +a third party module that was used previously. + +```python +modules = [ + TryExceptImport("tomllib", "tomli", "tomllib"), +] +``` + +is equivalent to + +```python +try: + import tomllib as tomllib +except ImportError: + import tomli as tomllib +``` + +when provided to a LazyImporter. + ## Demonstration of when imports occur ## ```python