add/update docstrings for top-level files and enable API docs

docs/gen_ref_pages.py

@@ -6,10 +6,24 @@
 
 nav = mkdocs_gen_files.Nav()
 
-# print(sorted(Path("merlin").rglob("*.py")))
+IGNORE_PATTERNS = [
+    Path("merlin/examples/workflows"),
+    Path("merlin/examples/dev_workflows"),
+    "*/ascii_art.py",
+]
+
+def should_ignore(path):
+    """Check if the given path matches any ignore patterns."""
+    for pattern in IGNORE_PATTERNS:
+        # if Path(pattern).is_relative_to(path):
+        if path.is_relative_to(Path(pattern)):
+            return True
+        if path.match(pattern):
+            return True
+    return False
 
 for path in sorted(Path("merlin").rglob("*.py")):
-    if "merlin/examples" in str(path):
+    if should_ignore(path):
         continue
     module_path = path.relative_to("merlin").with_suffix("")
     doc_path = path.relative_to("merlin").with_suffix(".md")

merlin/celery.py

@@ -33,7 +33,7 @@
 
 import logging
 import os
-from typing import Dict, Optional, Union
+from typing import Any, Dict, Optional, Union
 
 import billiard
 import celery
@@ -58,7 +58,8 @@ def patch_celery():
     Celery has error callbacks but they do not work properly on chords that
     are nested within chains.
 
-    Credit to this function goes to: https://danidee10.github.io/2019/07/09/celery-chords.html
+    Credit to this function goes to
+    [the following post](https://danidee10.github.io/2019/07/09/celery-chords.html).
     """
 
     def _unpack_chord_result(
@@ -84,9 +85,33 @@ def _unpack_chord_result(
 
 
 # This function has to have specific args/return values for celery so ignore pylint
-def route_for_task(name, args, kwargs, options, task=None, **kw):  # pylint: disable=W0613,R1710
+def route_for_task(
+        name: str,
+        args: List[Any],
+        kwargs: Dict[Any, Any],
+        options: Dict[Any, Any],
+        task: celery.Task = None,
+        **kw: Dict[Any, Any],
+) -> Dict[Any, Any]:  # pylint: disable=W0613,R1710
     """
-    Custom task router for queues
+    Custom task router for Celery queues.
+
+    This function routes tasks to specific queues based on the task name.
+    If the task name contains a colon, it splits the name to determine the queue.
+
+    Args:
+        name: The name of the task being routed.
+        args: The positional arguments passed to the task.
+        kwargs: The keyword arguments passed to the task.
+        options: Additional options for the task.
+        task: The task instance (default is None).
+        **kw: Additional keyword arguments.
+    
+    Returns:
+        A dictionary specifying the queue to route the task to. 
+            If the task name contains a colon, it returns a dictionary with 
+            the key "queue" set to the queue name. Otherwise, it returns 
+            an empty dictionary.
     """
     if ":" in name:
         queue, _ = name.split(":")
@@ -169,11 +194,12 @@ def route_for_task(name, args, kwargs, options, task=None, **kw):  # pylint: dis
 
 # Pylint believes the args are unused, I believe they're used after decoration
 @worker_process_init.connect()
-def setup(**kwargs):  # pylint: disable=W0613
+def setup(**kwargs: Dict[Any, Any]):  # pylint: disable=W0613
     """
-    Set affinity for the worker on startup (works on toss3 nodes)
+    Set affinity for the worker on startup (works on toss3 nodes).
 
-    :param `**kwargs`: keyword arguments
+    Args:
+        **kwargs: Keyword arguments.
     """
     if "CELERY_AFFINITY" in os.environ and int(os.environ["CELERY_AFFINITY"]) > 1:
         # Number of cpus between workers.