DOC: Automatically document settings

docs/build-docs.sh

@@ -10,6 +10,9 @@ python $STEP_DIR/source/examples/gen_examples.py
 echo "Generating pipeline table …"
 python $STEP_DIR/source/features/gen_steps.py
 
+echo "Generating config docs …"
+python $STEP_DIR/source/settings/gen_settings.py
+
 echo "Building the documentation …"
 cd $STEP_DIR
 mkdocs build

docs/mkdocs.yml

@@ -103,6 +103,7 @@ nav:
             - Source space & forward solution: settings/source/forward.md
             - Inverse solution: settings/source/inverse.md
         - Report generation: settings/reports/report_generation.md
+        - Execution: settings/execution.md
     - Examples:
         - Examples Gallery: examples/examples.md
         - examples/ds003392.md

docs/source/settings/execution.md

@@ -0,0 +1,19 @@
+These options control how the pipeline is executed but should not affect
+what outputs get produced.
+
+::: mne_bids_pipeline._config
+    options:
+      members:
+        - n_jobs
+        - parallel_backend
+        - dask_open_dashboard
+        - dask_temp_dir
+        - dask_worker_memory_limit
+        - log_level
+        - mne_log_level
+        - on_error
+        - memory_location
+        - memory_subdir
+        - memory_file_method
+        - memory_verbose
+        - config_validation

docs/source/settings/gen_settings.py

@@ -0,0 +1,144 @@
+"""Generate settings .md files."""
+
+# Any changes to the overall structure need to be reflected in mkdocs.yml nav section.
+
+import re
+from pathlib import Path
+
+from tqdm import tqdm
+
+import mne_bids_pipeline._config
+
+root_file = Path(mne_bids_pipeline._config.__file__)
+settings_dir = Path(__file__).parent
+
+# Mapping between first two lower-case words in the section name and the desired
+# file or folder name
+section_to_file = {  # .md will be added to the files
+    # root file
+    "general settings": "general",
+    # folder
+    "preprocessing": "preprocessing",
+    "break detection": "breaks",
+    "bad channel": "autobads",
+    "maxwell filter": "maxfilter",
+    "filtering": "filter",
+    "resampling": "resample",
+    "epoching": "epochs",
+    "filtering &": None,  # just a header
+    "artifact removal": None,
+    "stimulation artifact": "stim_artifact",
+    "ssp, ica,": "ssp_ica",
+    "amplitude-based artifact": "artifacts",
+    # folder
+    "sensor-level analysis": "sensor",
+    "condition contrasts": "contrasts",
+    "decoding /": "mvpa",
+    "time-frequency analysis": "time_freq",
+    "group-level analysis": "group_level",
+    # folder
+    "source-level analysis": "source",
+    "bem surface": "bem",
+    "source space": "forward",
+    "inverse solution": "inverse",
+    # folder
+    "reports": "reports",
+    "report generation": "report_generation",
+    # root file
+    "execution": "execution",
+}
+
+option_header = """\
+::: mne_bids_pipeline._config
+    options:
+      members:"""
+prefix = """\
+        - """
+
+# We cannot use ast for this because it doesn't preserve comments. We could use
+# something like redbaron, but our code is hopefully simple enough!
+assign_re = re.compile(
+    # Line starts with annotation syntax (name captured by the first group).
+    r"^(\w+): "
+    # Then the annotation can be ...
+    "("
+    # ... a standard assignment ...
+    ".+ = .+"
+    # ... or ...
+    "|"
+    # ... the start of a multiline type annotation like "a: Union["
+    r"(Union|Optional|Literal)\["
+    # To the end of the line.
+    ")$",
+    re.MULTILINE,
+)
+
+
+def main():
+    print(f"Parsing {root_file} to generate settings .md files.")
+    # max file-level depth is 2 even though we have 3 subsection levels
+    levels = [None, None]
+    current_path, current_lines = None, list()
+    text = root_file.read_text("utf-8")
+    lines = text.splitlines()
+    lines += ["# #"]  # add a dummy line to trigger the last write
+    in_header = False
+    for li, line in enumerate(tqdm(lines)):
+        line = line.rstrip()
+        if line.startswith("# #"):  # a new (sub)section / file
+            this_def = line[2:]
+            this_level = this_def.split()[0]
+            assert this_level.count("#") == len(this_level), this_level
+            this_level = this_level.count("#") - 1
+            if this_level == 2:
+                # flatten preprocessing/filtering/filter to preprocessing/filter
+                # for example
+                this_level = 1
+            assert this_level in (0, 1), (this_level, this_def)
+            this_def = this_def[this_level + 2 :]
+            levels[this_level] = this_def
+            # Write current lines and reset
+            if len(current_lines) > 1:  # more than just the header
+                assert current_path is not None, levels
+                if current_lines[0] == "":  # this happens with tags
+                    current_lines = current_lines[1:]
+                current_path.write_text("\n".join(current_lines + [""]), "utf-8")
+            if this_level == 0:
+                this_root = settings_dir
+            else:
+                this_root = settings_dir / f"{section_to_file[levels[0].lower()]}"
+            this_root.mkdir(exist_ok=True)
+            key = " ".join(this_def.split()[:2]).lower()
+            if key == "":
+                assert li == len(lines) - 1, (li, line)
+                continue  # our dummy line
+            fname = section_to_file[key]
+            if fname is None:
+                current_path = None
+            else:
+                current_path = this_root / f"{fname}.md"
+            current_lines = []
+            in_header = True
+            continue
+
+        if in_header:
+            if line == "":
+                in_header = False
+                if current_lines:
+                    current_lines.append("")
+                current_lines.append(option_header)
+            else:
+                assert line == "#" or line.startswith("# "), (li, line)  # a comment
+                current_lines.append(line[2:])
+            continue
+
+        # Could be an option
+        match = assign_re.match(line)
+        if match is not None:
+            name, typ, desc = match.groups()
+            current_lines.append(f"{prefix}{name}")
+            continue
+
+
+if __name__ == "__main__":
+    main()

docs/source/settings/general.md

@@ -27,22 +27,9 @@
         - eeg_reference
         - eeg_template_montage
         - drop_channels
+        - analyze_channels
         - reader_extra_params
         - read_raw_bids_verbose
-        - analyze_channels
         - plot_psd_for_runs
-        - n_jobs
-        - parallel_backend
-        - dask_open_dashboard
-        - dask_temp_dir
-        - dask_worker_memory_limit
         - random_state
         - shortest_event
-        - memory_location
-        - memory_subdir
-        - memory_file_method
-        - memory_verbose
-        - config_validation
-        - log_level
-        - mne_log_level
-        - on_error

docs/source/settings/preprocessing/epochs.md

@@ -13,14 +13,14 @@ tags:
         - rename_events
         - on_rename_missing_events
         - event_repeated
-        - conditions
-        - epochs_tmin
-        - epochs_tmax
-        - baseline
         - epochs_metadata_tmin
         - epochs_metadata_tmax
         - epochs_metadata_keep_first
         - epochs_metadata_keep_last
         - epochs_metadata_query
+        - conditions
+        - epochs_tmin
+        - epochs_tmax
         - rest_epochs_duration
         - rest_epochs_overlap
+        - baseline

docs/source/settings/preprocessing/ssp_ica.md

@@ -17,11 +17,11 @@ tags:
         - min_eog_epochs
         - n_proj_eog
         - n_proj_ecg
-        - ssp_meg
         - ecg_proj_from_average
         - eog_proj_from_average
-        - ssp_reject_eog
+        - ssp_meg
         - ssp_reject_ecg
+        - ssp_reject_eog
         - ssp_ecg_channel
         - ica_reject
         - ica_algorithm

docs/source/settings/source/forward.md

@@ -11,4 +11,3 @@ tags:
         - mri_landmarks_kind
         - spacing
         - mindist
-        - source_info_path_update

docs/source/v1.6.md.inc

@@ -37,3 +37,4 @@
 - Code formatting now uses `ruff format` instead of `black` (#834, #838 by @larsoner)
 - Code caching is now tested using GitHub Actions (#836 by @larsoner)
 - Steps in the documentation are now automatically parsed into flowcharts (#859 by @larsoner)
+- New configuration options are now automatically added to the docs (#863 by @larsoner)