Re-adding load_submodules (Project-MONAI#8118)

### Description

This restores load_modules for now to resolve importation issues. Doing
a two-pass loading process seems to allow some references which normally
Python wouldn't permit. The fact that these issues weren't caught in
testing is rather strange, and only showed up when symbols in
`monai.apps` were references in bundles. I'm not sure if this is all
related to circular imports, if parts of MONAI aren't being tested
properly, or the way the symbol resolution works within `monai.bundle`
should be improved.

### Types of changes
<!--- Put an `x` in all the boxes that apply, and remove the not
applicable items -->
- [x] Non-breaking change (fix or new feature that would not break
existing functionality).
- [ ] Breaking change (fix or new feature that would cause existing
functionality to change).
- [ ] New tests added to cover the changes.
- [ ] Integration tests passed locally by running `./runtests.sh -f -u
--net --coverage`.
- [ ] Quick tests passed locally by running `./runtests.sh --quick
--unittests --disttests`.
- [ ] In-line docstrings updated.
- [ ] Documentation updated, tested `make html` command in the `docs/`
folder.

---------

Signed-off-by: Eric Kerfoot <eric.kerfoot@kcl.ac.uk>

monai/__init__.py

@@ -80,25 +80,29 @@ def filter(self, record):
     )
 
 
-from . import (  # noqa: E402
-    apps,
-    auto3dseg,
-    bundle,
-    config,
-    data,
-    engines,
-    fl,
-    handlers,
-    inferers,
-    losses,
-    metrics,
-    networks,
-    optimizers,
-    transforms,
-    utils,
-    visualize,
+from .utils.module import load_submodules  # noqa: E402
+
+# handlers_* have some external decorators the users may not have installed
+# *.so files and folder "_C" may not exist when the cpp extensions are not compiled
+excludes = "|".join(
+    [
+        "(^(monai.handlers))",
+        "(^(monai.bundle))",
+        "(^(monai.fl))",
+        "((\\.so)$)",
+        "(^(monai._C))",
+        "(.*(__main__)$)",
+        "(.*(video_dataset)$)",
+        "(.*(nnunet).*$)",
+    ]
 )
 
+# load directory modules only, skip loading individual files
+load_submodules(sys.modules[__name__], False, exclude_pattern=excludes)
+
+# load all modules, this will trigger all export decorations
+load_submodules(sys.modules[__name__], True, exclude_pattern=excludes)
+
 __all__ = [
     "apps",
     "auto3dseg",

monai/utils/__init__.py

@@ -112,6 +112,7 @@
     get_package_version,
     get_torch_version_tuple,
     instantiate,
+    load_submodules,
     look_up_option,
     min_version,
     optional_import,

monai/utils/module.py

@@ -16,13 +16,15 @@
 import os
 import pdb
 import re
+import sys
 import warnings
 from collections.abc import Callable, Collection, Hashable, Mapping
 from functools import partial, wraps
 from importlib import import_module
+from pkgutil import walk_packages
 from pydoc import locate
 from re import match
-from types import FunctionType
+from types import FunctionType, ModuleType
 from typing import Any, Iterable, cast
 
 import torch
@@ -168,6 +170,38 @@ def damerau_levenshtein_distance(s1: str, s2: str) -> int:
     return d[string_1_length - 1, string_2_length - 1]
 
 
+def load_submodules(
+    basemod: ModuleType, load_all: bool = True, exclude_pattern: str = "(.*[tT]est.*)|(_.*)"
+) -> tuple[list[ModuleType], list[str]]:
+    """
+    Traverse the source of the module structure starting with module `basemod`, loading all packages plus all files if
+    `load_all` is True, excluding anything whose name matches `exclude_pattern`.
+    """
+    submodules = []
+    err_mod: list[str] = []
+    for importer, name, is_pkg in walk_packages(
+        basemod.__path__, prefix=basemod.__name__ + ".", onerror=err_mod.append
+    ):
+        if (is_pkg or load_all) and name not in sys.modules and match(exclude_pattern, name) is None:
+            try:
+                mod = import_module(name)
+                mod_spec = importer.find_spec(name)  # type: ignore
+                if mod_spec and mod_spec.loader:
+                    loader = mod_spec.loader
+                    loader.exec_module(mod)
+                    submodules.append(mod)
+            except OptionalImportError:
+                pass  # could not import the optional deps., they are ignored
+            except ImportError as e:
+                msg = (
+                    "\nMultiple versions of MONAI may have been installed?\n"
+                    "Please see the installation guide: https://docs.monai.io/en/stable/installation.html\n"
+                )  # issue project-monai/monai#5193
+                raise type(e)(f"{e}\n{msg}").with_traceback(e.__traceback__) from e  # raise with modified message
+
+    return submodules, err_mod
+
+
 def instantiate(__path: str, __mode: str, **kwargs: Any) -> Any:
     """
     Create an object instance or call a callable object from a class or function represented by ``_path``.

tests/min_tests.py

@@ -232,6 +232,12 @@ def run_testsuit():
 
 
 if __name__ == "__main__":
+    # testing import submodules
+    from monai.utils.module import load_submodules
+
+    _, err_mod = load_submodules(sys.modules["monai"], True)
+    assert not err_mod, f"err_mod={err_mod} not empty"
+
     # testing all modules
     test_runner = unittest.TextTestRunner(stream=sys.stdout, verbosity=2)
     result = test_runner.run(run_testsuit())