feat: Support dynamically import module from local path if import_mod… (

#23)

Co-authored-by: Dat Nguyen <dat@infinitelambda>

diqu/utils/module.py

@@ -1,7 +1,9 @@
-from importlib import import_module
+import sys
+from importlib import import_module, util
 from types import ModuleType
 
 from diqu.utils import exception
+from diqu.utils.log import logger
 
 
 def load_module(name: str, package: str = "diqu") -> ModuleType:
@@ -16,5 +18,20 @@ def load_module(name: str, package: str = "diqu") -> ModuleType:
     Returns:
         ModuleType: Imported module
     """
-    with exception.handle_module_errors(name, f"{package}.{name}"):
-        return import_module(name=f".{name}", package=package)
+    module_name = f"{package}.{name}"
+    with exception.handle_module_errors(name, module_name):
+        try:
+            mod = import_module(name=f".{name}", package=package)
+        except Exception:
+            mod = None
+            logger.debug(f"Import {module_name} module failed, trying local path...")
+
+        if not mod:
+            spec = util.spec_from_file_location(
+                module_name, f"{package.replace('.', '/')}/{name}.py"
+            )
+            mod = util.module_from_spec(spec)
+            sys.modules[module_name] = mod
+            spec.loader.exec_module(mod)
+
+    return mod

docs/nav/guide/adapters/using_custom_module_script.md

@@ -0,0 +1,64 @@
+<!-- markdownlint-disable no-inline-html -->
+# Using custom module script
+
+In case you're NOT ready for publishing the adapters (`diqu-{new-adapter}`), or would like to keep yours privately, it's also supported here.
+
+This is a basic guideline to let you do that:
+
+## Understand the module paths
+
+Check out the our supported [modules](./../common.md#modules), and here is our skeleton:
+
+```bash
+diqu/
+├── diqu/
+    ├── alerts/     # Alert module
+    ├── packages/   # Package module
+    ├── sources/    # Source module
+```
+
+Let's say that we want to create an alert custom script, named `your_alert_module.py`.
+Now, we must follow the above directory structure and put the script under `(your_repo)/diqu/alerts/` as below:
+
+```bash
+your_repo/
+├── diqu/
+    ├── alerts/
+        ├── your_alert_module.py # custom script here
+├── your_other_dir/
+```
+
+> Follow the same fashion for the `Package` or `Source` module 👍
+
+## Create your module script
+
+Check out the [Build a new adapter](./community_adapter.md#3-build-a-new-adapter) for more details on how to structure your code.
+
+For example: `(your_repo)/diqu/alerts/your_alert_module.py`
+
+```python
+from diqu.utils.log import logger
+from diqu.utils.meta import ResultCode
+
+
+def alert(data) -> ResultCode:
+    # your implementation here
+    # log any necessary messages e.g. logger.info("✅ Done > My Module")
+    return ResultCode.SUCCEEDED # return the ResultCode value
+```
+
+## 3. Run `diqu` with the custom module script
+
+Run `diqu alert -h` to inspect the usage for using custom:
+
+- Alert via `--to` option ([docs](./../config/alerts/))
+- Package via `--package` option ([docs](./../config/packages/))
+- Source via `--profile-name` and `--profiles-dir`  ([docs](./../config/sources/))
+
+For example: `(your_repo)/diqu/alerts/your_alert_module.py`
+
+```bash
+diqu alert --to your_alert_module
+```
+
+**_Happy Customizing 🚀_**

mkdocs.yml

@@ -29,6 +29,7 @@ nav:
     - CLI Reference: nav/guide/cli.md
     - Contribution Guideline: nav/dev/contributing.md
     - Contribute to adapters ❤️: nav/guide/adapters/community_adapter.md
+    - Using custom module script 💡: using_custom_module_script.md
     - Change Log ↗️: https://github.com/infinitelambda/diqu/releases" target="_blank
 
 theme:

pyproject.toml

@@ -32,6 +32,7 @@ pandas = ">=1.0,<2.1"
 snowplow-tracker = "^1.0.1"
 jira = "^3.5.2"
 slack-sdk = "^3.23.0"
+numpy = "<2.0"
 
 [tool.poetry.dev-dependencies]
 snowflake-connector-python = {version = "^3.3.1", extras = ["secure-local-storage", "pandas"]}