diff --git a/docs/build-docs.sh b/docs/build-docs.sh
index ccb159aae..4a94ab206 100755
--- a/docs/build-docs.sh
+++ b/docs/build-docs.sh
@@ -10,6 +10,9 @@ python $STEP_DIR/source/examples/gen_examples.py
 echo "Generating pipeline table …"
 python $STEP_DIR/source/features/gen_steps.py
 
+echo "Generating config docs …"
+python $STEP_DIR/source/settings/gen_settings.py
+
 echo "Building the documentation …"
 cd $STEP_DIR
 mkdocs build
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 29107ff32..ab4e493e4 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -103,6 +103,7 @@ nav:
             - Source space & forward solution: settings/source/forward.md
             - Inverse solution: settings/source/inverse.md
         - Report generation: settings/reports/report_generation.md
+        - Execution: settings/execution.md
     - Examples:
         - Examples Gallery: examples/examples.md
         - examples/ds003392.md
diff --git a/docs/source/.gitignore b/docs/source/.gitignore
index ce1332a62..b909cc5d3 100644
--- a/docs/source/.gitignore
+++ b/docs/source/.gitignore
@@ -1,2 +1,4 @@
 features/steps.md
 features/overview.md
+settings/**/
+settings/*.md
diff --git a/docs/source/settings/gen_settings.py b/docs/source/settings/gen_settings.py
new file mode 100755
index 000000000..6f7eaf7d3
--- /dev/null
+++ b/docs/source/settings/gen_settings.py
@@ -0,0 +1,182 @@
+"""Generate settings .md files."""
+
+# Any changes to the overall structure need to be reflected in mkdocs.yml nav section.
+
+import re
+from pathlib import Path
+
+from tqdm import tqdm
+
+import mne_bids_pipeline._config
+
+config_path = Path(mne_bids_pipeline._config.__file__)
+settings_dir = Path(__file__).parent
+
+# Mapping between first two lower-case words in the section name and the desired
+# file or folder name
+section_to_file = {  # .md will be added to the files
+    # root file
+    "general settings": "general",
+    # folder
+    "preprocessing": "preprocessing",
+    "break detection": "breaks",
+    "bad channel": "autobads",
+    "maxwell filter": "maxfilter",
+    "filtering": "filter",
+    "resampling": "resample",
+    "epoching": "epochs",
+    "filtering &": None,  # just a header
+    "artifact removal": None,
+    "stimulation artifact": "stim_artifact",
+    "ssp, ica,": "ssp_ica",
+    "amplitude-based artifact": "artifacts",
+    # folder
+    "sensor-level analysis": "sensor",
+    "condition contrasts": "contrasts",
+    "decoding /": "mvpa",
+    "time-frequency analysis": "time_freq",
+    "group-level analysis": "group_level",
+    # folder
+    "source-level analysis": "source",
+    "general source": "general",
+    "bem surface": "bem",
+    "source space": "forward",
+    "inverse solution": "inverse",
+    # folder
+    "reports": "reports",
+    "report generation": "report_generation",
+    # root file
+    "execution": "execution",
+}
+# TODO: Make sure these are consistent, autogenerate some based on section names,
+# and/or autogenerate based on inputs/outputs of actual functions.
+section_tags = {
+    "general settings": (),
+    "preprocessing": (),
+    "filtering &": (),
+    "artifact removal": (),
+    "break detection": ("preprocessing", "artifact-removal", "raw", "events"),
+    "bad channel": ("preprocessing", "raw", "bad-channels"),
+    "maxwell filter": ("preprocessing", "maxwell-filter", "raw"),
+    "filtering": ("preprocessing", "frequency-filter", "raw"),
+    "resampling": ("preprocessing", "resampling", "decimation", "raw", "epochs"),
+    "epoching": ("preprocessing", "epochs", "events", "metadata", "resting-state"),
+    "stimulation artifact": ("preprocessing", "artifact-removal", "raw", "epochs"),
+    "ssp, ica,": ("preprocessing", "artifact-removal", "raw", "epochs", "ssp", "ica"),
+    "amplitude-based artifact": ("preprocessing", "artifact-removal", "epochs"),
+    "sensor-level analysis": (),
+    "condition contrasts": ("epochs", "evoked", "contrast"),
+    "decoding /": ("epochs", "evoked", "contrast", "decoding", "mvpa"),
+    "time-frequency analysis": ("epochs", "evoked", "time-frequency"),
+    "group-level analysis": ("evoked", "group-level"),
+    "source-level analysis": (),
+    "general source": ("inverse-solution",),
+    "bem surface": ("inverse-solution", "bem", "freesurfer"),
+    "source space": ("inverse-solution", "forward-model"),
+    "inverse solution": ("inverse-solution",),
+    "reports": (),
+    "report generation": ("report",),
+    "execution": (),
+}
+
+option_header = """\
+::: mne_bids_pipeline._config
+    options:
+      members:"""
+prefix = """\
+        - """
+
+# We cannot use ast for this because it doesn't preserve comments. We could use
+# something like redbaron, but our code is hopefully simple enough!
+assign_re = re.compile(
+    # Line starts with annotation syntax (name captured by the first group).
+    r"^(\w+): "
+    # Then the annotation can be ...
+    "("
+    # ... a standard assignment ...
+    ".+ = .+"
+    # ... or ...
+    "|"
+    # ... the start of a multiline type annotation like "a: Union["
+    r"(Union|Optional|Literal)\["
+    # To the end of the line.
+    ")$",
+    re.MULTILINE,
+)
+
+
+def main():
+    print(f"Parsing {config_path} to generate settings .md files.")
+    # max file-level depth is 2 even though we have 3 subsection levels
+    levels = [None, None]
+    current_path, current_lines = None, list()
+    text = config_path.read_text("utf-8")
+    lines = text.splitlines()
+    lines += ["# #"]  # add a dummy line to trigger the last write
+    in_header = False
+    have_params = False
+    for li, line in enumerate(tqdm(lines)):
+        line = line.rstrip()
+        if line.startswith("# #"):  # a new (sub)section / file
+            this_def = line[2:]
+            this_level = this_def.split()[0]
+            assert this_level.count("#") == len(this_level), this_level
+            this_level = this_level.count("#") - 1
+            if this_level == 2:
+                # flatten preprocessing/filtering/filter to preprocessing/filter
+                # for example
+                this_level = 1
+            assert this_level in (0, 1), (this_level, this_def)
+            this_def = this_def[this_level + 2 :]
+            levels[this_level] = this_def
+            # Write current lines and reset
+            if have_params:  # more than just the header
+                assert current_path is not None, levels
+                if current_lines[0] == "":  # this happens with tags
+                    current_lines = current_lines[1:]
+                current_path.write_text("\n".join(current_lines + [""]), "utf-8")
+            have_params = False
+            if this_level == 0:
+                this_root = settings_dir
+            else:
+                this_root = settings_dir / f"{section_to_file[levels[0].lower()]}"
+            this_root.mkdir(exist_ok=True)
+            key = " ".join(this_def.split()[:2]).lower()
+            if key == "":
+                assert li == len(lines) - 1, (li, line)
+                continue  # our dummy line
+            fname = section_to_file[key]
+            if fname is None:
+                current_path = None
+            else:
+                current_path = this_root / f"{fname}.md"
+            in_header = True
+            current_lines = list()
+            if len(section_tags[key]):
+                current_lines += ["---", "tags:"]
+                current_lines += [f"  - {tag}" for tag in section_tags[key]]
+                current_lines += ["---"]
+            continue
+
+        if in_header:
+            if line == "":
+                in_header = False
+                if current_lines:
+                    current_lines.append("")
+                current_lines.append(option_header)
+            else:
+                assert line == "#" or line.startswith("# "), (li, line)  # a comment
+                current_lines.append(line[2:])
+            continue
+
+        # Could be an option
+        match = assign_re.match(line)
+        if match is not None:
+            have_params = True
+            name, typ, desc = match.groups()
+            current_lines.append(f"{prefix}{name}")
+            continue
+
+
+if __name__ == "__main__":
+    main()
diff --git a/docs/source/settings/general.md b/docs/source/settings/general.md
deleted file mode 100644
index 2640f5f2b..000000000
--- a/docs/source/settings/general.md
+++ /dev/null
@@ -1,48 +0,0 @@
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - study_name
-        - bids_root
-        - deriv_root
-        - subjects_dir
-        - interactive
-        - sessions
-        - task
-        - task_is_rest
-        - runs
-        - exclude_runs
-        - crop_runs
-        - acq
-        - proc
-        - rec
-        - space
-        - subjects
-        - exclude_subjects
-        - process_empty_room
-        - process_rest
-        - ch_types
-        - data_type
-        - eog_channels
-        - eeg_bipolar_channels
-        - eeg_reference
-        - eeg_template_montage
-        - drop_channels
-        - reader_extra_params
-        - read_raw_bids_verbose
-        - analyze_channels
-        - plot_psd_for_runs
-        - n_jobs
-        - parallel_backend
-        - dask_open_dashboard
-        - dask_temp_dir
-        - dask_worker_memory_limit
-        - random_state
-        - shortest_event
-        - memory_location
-        - memory_subdir
-        - memory_file_method
-        - memory_verbose
-        - config_validation
-        - log_level
-        - mne_log_level
-        - on_error
diff --git a/docs/source/settings/preprocessing/artifacts.md b/docs/source/settings/preprocessing/artifacts.md
deleted file mode 100644
index 88407cd2c..000000000
--- a/docs/source/settings/preprocessing/artifacts.md
+++ /dev/null
@@ -1,20 +0,0 @@
----
-tags:
-  - preprocessing
-  - artifact-removal
-  - epochs
----
-
-???+ info "Good Practice / Advice"
-    Have a look at your raw data and train yourself to detect a blink, a heart
-    beat and an eye movement.
-    You can do a quick average of blink data and check what the amplitude looks
-    like.
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - reject
-        - reject_tmin
-        - reject_tmax
-        - autoreject_n_interpolate
diff --git a/docs/source/settings/preprocessing/autobads.md b/docs/source/settings/preprocessing/autobads.md
deleted file mode 100644
index a118917a1..000000000
--- a/docs/source/settings/preprocessing/autobads.md
+++ /dev/null
@@ -1,27 +0,0 @@
----
-tags:
-  - preprocessing
-  - raw
-  - bad-channels
----
-
-!!! warning
-    This functionality will soon be removed from the pipeline, and
-    will be integrated into MNE-BIDS.
-
-"Bad", i.e. flat and overly noisy channels, can be automatically detected
-using a procedure inspired by the commercial MaxFilter by Elekta. First,
-a copy of the data is low-pass filtered at 40 Hz. Then, channels with
-unusually low variability are flagged as "flat", while channels with
-excessively high variability are flagged as "noisy". Flat and noisy channels
-are marked as "bad" and excluded from subsequent analysis. See
-:func:`mne.preprocssessing.find_bad_channels_maxwell` for more information
-on this procedure. The list of bad channels detected through this procedure
-will be merged with the list of bad channels already present in the dataset,
-if any.
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - find_flat_channels_meg
-        - find_noisy_channels_meg
diff --git a/docs/source/settings/preprocessing/breaks.md b/docs/source/settings/preprocessing/breaks.md
deleted file mode 100644
index 01e3159eb..000000000
--- a/docs/source/settings/preprocessing/breaks.md
+++ /dev/null
@@ -1,15 +0,0 @@
----
-tags:
-  - preprocessing
-  - artifact-removal
-  - raw
-  - events
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - find_breaks
-        - min_break_duration
-        - t_break_annot_start_after_previous_event
-        - t_break_annot_stop_before_next_event
diff --git a/docs/source/settings/preprocessing/epochs.md b/docs/source/settings/preprocessing/epochs.md
deleted file mode 100644
index 02dd1f71d..000000000
--- a/docs/source/settings/preprocessing/epochs.md
+++ /dev/null
@@ -1,26 +0,0 @@
----
-tags:
-  - preprocessing
-  - epochs
-  - events
-  - metadata
-  - resting-state
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - rename_events
-        - on_rename_missing_events
-        - event_repeated
-        - conditions
-        - epochs_tmin
-        - epochs_tmax
-        - baseline
-        - epochs_metadata_tmin
-        - epochs_metadata_tmax
-        - epochs_metadata_keep_first
-        - epochs_metadata_keep_last
-        - epochs_metadata_query
-        - rest_epochs_duration
-        - rest_epochs_overlap
diff --git a/docs/source/settings/preprocessing/filter.md b/docs/source/settings/preprocessing/filter.md
deleted file mode 100644
index 9d1301412..000000000
--- a/docs/source/settings/preprocessing/filter.md
+++ /dev/null
@@ -1,37 +0,0 @@
----
-tags:
-  - preprocessing
-  - frequency-filter
-  - raw
----
-
-It is typically better to set your filtering properties on the raw data so
-as to avoid what we call border (or edge) effects.
-
-If you use this pipeline for evoked responses, you could consider
-a low-pass filter cut-off of h_freq = 40 Hz
-and possibly a high-pass filter cut-off of l_freq = 1 Hz
-so you would preserve only the power in the 1Hz to 40 Hz band.
-Note that highpass filtering is not necessarily recommended as it can
-distort waveforms of evoked components, or simply wash out any low
-frequency that can may contain brain signal. It can also act as
-a replacement for baseline correction in Epochs. See below.
-
-If you use this pipeline for time-frequency analysis, a default filtering
-could be a high-pass filter cut-off of l_freq = 1 Hz
-a low-pass filter cut-off of h_freq = 120 Hz
-so you would preserve only the power in the 1Hz to 120 Hz band.
-
-If you need more fancy analysis, you are already likely past this kind
-of tips! 😇
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - l_freq
-        - h_freq
-        - l_trans_bandwidth
-        - h_trans_bandwidth
-        - notch_freq
-        - notch_trans_bandwidth
-        - notch_widths
diff --git a/docs/source/settings/preprocessing/maxfilter.md b/docs/source/settings/preprocessing/maxfilter.md
deleted file mode 100644
index 3cd32d9d7..000000000
--- a/docs/source/settings/preprocessing/maxfilter.md
+++ /dev/null
@@ -1,29 +0,0 @@
----
-tags:
-  - preprocessing
-  - maxwell-filter
-  - raw
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - use_maxwell_filter
-        - mf_st_duration
-        - mf_st_correlation
-        - mf_head_origin
-        - mf_destination
-        - mf_int_order
-        - mf_reference_run
-        - mf_cal_fname
-        - mf_ctc_fname
-        - mf_esss
-        - mf_esss_reject
-        - mf_mc
-        - mf_mc_t_step_min
-        - mf_mc_t_window
-        - mf_mc_gof_limit
-        - mf_mc_dist_limit
-        - mf_mc_rotation_velocity_limit
-        - mf_mc_translation_velocity_limit
-        - mf_filter_chpi
diff --git a/docs/source/settings/preprocessing/resample.md b/docs/source/settings/preprocessing/resample.md
deleted file mode 100644
index 6aa824a6c..000000000
--- a/docs/source/settings/preprocessing/resample.md
+++ /dev/null
@@ -1,21 +0,0 @@
----
-tags:
-  - preprocessing
-  - resampling
-  - decimation
-  - raw
-  - epochs
----
-
-If you have acquired data with a very high sampling frequency (e.g. 2 kHz)
-you will likely want to downsample to lighten up the size of the files you
-are working with (pragmatics)
-If you are interested in typical analysis (up to 120 Hz) you can typically
-resample your data down to 500 Hz without preventing reliable time-frequency
-exploration of your data.
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - raw_resample_sfreq
-        - epochs_decim
diff --git a/docs/source/settings/preprocessing/ssp_ica.md b/docs/source/settings/preprocessing/ssp_ica.md
deleted file mode 100644
index f25110729..000000000
--- a/docs/source/settings/preprocessing/ssp_ica.md
+++ /dev/null
@@ -1,33 +0,0 @@
----
-tags:
-  - preprocessing
-  - artifact-removal
-  - raw
-  - epochs
-  - ssp
-  - ica
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - regress_artifact
-        - spatial_filter
-        - min_ecg_epochs
-        - min_eog_epochs
-        - n_proj_eog
-        - n_proj_ecg
-        - ssp_meg
-        - ecg_proj_from_average
-        - eog_proj_from_average
-        - ssp_reject_eog
-        - ssp_reject_ecg
-        - ssp_ecg_channel
-        - ica_reject
-        - ica_algorithm
-        - ica_l_freq
-        - ica_max_iterations
-        - ica_n_components
-        - ica_decim
-        - ica_ctps_ecg_threshold
-        - ica_eog_threshold
diff --git a/docs/source/settings/preprocessing/stim_artifact.md b/docs/source/settings/preprocessing/stim_artifact.md
deleted file mode 100644
index cbc142550..000000000
--- a/docs/source/settings/preprocessing/stim_artifact.md
+++ /dev/null
@@ -1,19 +0,0 @@
----
-tags:
-  - preprocessing
-  - artifact-removal
-  - raw
-  - epochs
----
-
-When using electric stimulation systems, e.g. for median nerve or index
-stimulation, it is frequent to have a stimulation artifact. This option
-allows to fix it by linear interpolation early in the pipeline on the raw
-data.
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - fix_stim_artifact
-        - stim_artifact_tmin
-        - stim_artifact_tmax
diff --git a/docs/source/settings/reports/report_generation.md b/docs/source/settings/reports/report_generation.md
deleted file mode 100644
index 2ccf520ed..000000000
--- a/docs/source/settings/reports/report_generation.md
+++ /dev/null
@@ -1,10 +0,0 @@
----
-tags:
-  - report
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - report_evoked_n_time_points
-        - report_stc_n_time_points
diff --git a/docs/source/settings/sensor/contrasts.md b/docs/source/settings/sensor/contrasts.md
deleted file mode 100644
index 576e45ee3..000000000
--- a/docs/source/settings/sensor/contrasts.md
+++ /dev/null
@@ -1,11 +0,0 @@
----
-tags:
-  - epochs
-  - evoked
-  - contrast
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - contrasts
diff --git a/docs/source/settings/sensor/group_level.md b/docs/source/settings/sensor/group_level.md
deleted file mode 100644
index a330a9703..000000000
--- a/docs/source/settings/sensor/group_level.md
+++ /dev/null
@@ -1,10 +0,0 @@
----
-tags:
-  - evoked
-  - group-level
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - interpolate_bads_grand_average
diff --git a/docs/source/settings/sensor/mvpa.md b/docs/source/settings/sensor/mvpa.md
deleted file mode 100644
index 3a56d22d7..000000000
--- a/docs/source/settings/sensor/mvpa.md
+++ /dev/null
@@ -1,27 +0,0 @@
----
-tags:
-  - epochs
-  - evoked
-  - contrast
-  - decoding
-  - mvpa
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - decode
-        - decoding_which_epochs
-        - decoding_epochs_tmin
-        - decoding_epochs_tmax
-        - decoding_metric
-        - decoding_n_splits
-        - decoding_time_generalization
-        - decoding_time_generalization_decim
-        - decoding_csp
-        - decoding_csp_times
-        - decoding_csp_freqs
-        - n_boot
-        - cluster_forming_t_threshold
-        - cluster_n_permutations
-        - cluster_permutation_p_threshold
diff --git a/docs/source/settings/sensor/time_freq.md b/docs/source/settings/sensor/time_freq.md
deleted file mode 100644
index 492296dc0..000000000
--- a/docs/source/settings/sensor/time_freq.md
+++ /dev/null
@@ -1,18 +0,0 @@
----
-tags:
-  - epochs
-  - evoked
-  - time-frequency
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - time_frequency_conditions
-        - time_frequency_freq_min
-        - time_frequency_freq_max
-        - time_frequency_cycles
-        - time_frequency_subtract_evoked
-        - time_frequency_baseline
-        - time_frequency_baseline_mode
-        - time_frequency_crop
diff --git a/docs/source/settings/source/bem.md b/docs/source/settings/source/bem.md
deleted file mode 100644
index f55972baf..000000000
--- a/docs/source/settings/source/bem.md
+++ /dev/null
@@ -1,16 +0,0 @@
----
-tags:
-  - inverse-solution
-  - bem
-  - freesurfer
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - use_template_mri
-        - adjust_coreg
-        - bem_mri_images
-        - recreate_bem
-        - recreate_scalp_surface
-        - freesurfer_verbose
diff --git a/docs/source/settings/source/forward.md b/docs/source/settings/source/forward.md
deleted file mode 100644
index 8ce5c87ad..000000000
--- a/docs/source/settings/source/forward.md
+++ /dev/null
@@ -1,14 +0,0 @@
----
-tags:
-  - inverse-solution
-  - forward-model
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - mri_t1_path_generator
-        - mri_landmarks_kind
-        - spacing
-        - mindist
-        - source_info_path_update
diff --git a/docs/source/settings/source/general.md b/docs/source/settings/source/general.md
deleted file mode 100644
index 09eac741f..000000000
--- a/docs/source/settings/source/general.md
+++ /dev/null
@@ -1,9 +0,0 @@
----
-tags:
-  - inverse-solution
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - run_source_estimation
diff --git a/docs/source/settings/source/inverse.md b/docs/source/settings/source/inverse.md
deleted file mode 100644
index 367275071..000000000
--- a/docs/source/settings/source/inverse.md
+++ /dev/null
@@ -1,15 +0,0 @@
----
-tags:
-  - inverse-solution
----
-
-::: mne_bids_pipeline._config
-    options:
-      members:
-        - loose
-        - depth
-        - inverse_method
-        - noise_cov
-        - noise_cov_method
-        - source_info_path_update
-        - inverse_targets
diff --git a/docs/source/v1.6.md.inc b/docs/source/v1.6.md.inc
index aba57baea..0c0e204c0 100644
--- a/docs/source/v1.6.md.inc
+++ b/docs/source/v1.6.md.inc
@@ -37,3 +37,4 @@
 - Code formatting now uses `ruff format` instead of `black` (#834, #838 by @larsoner)
 - Code caching is now tested using GitHub Actions (#836 by @larsoner)
 - Steps in the documentation are now automatically parsed into flowcharts (#859 by @larsoner)
+- New configuration options are now automatically added to the docs (#863 by @larsoner)
diff --git a/mne_bids_pipeline/_config.py b/mne_bids_pipeline/_config.py
index b45753966..06f468cb7 100644
--- a/mne_bids_pipeline/_config.py
+++ b/mne_bids_pipeline/_config.py
@@ -13,9 +13,8 @@
     PathLike,
 )
 
-###############################################################################
-# Config parameters
-# -----------------
+# %%
+# # General settings
 
 study_name: str = ""
 """
@@ -95,6 +94,11 @@
 The task to process.
 """
 
+task_is_rest: bool = False
+"""
+Whether the task should be treated as resting-state data.
+"""
+
 runs: Union[Sequence, Literal["all"]] = "all"
 """
 The runs to process. If `'all'`, will process all runs found in the
@@ -144,14 +148,6 @@
 The BIDS `space` entity.
 """
 
-plot_psd_for_runs: Union[Literal["all"], Sequence[str]] = "all"
-"""
-For which runs to add a power spectral density (PSD) plot to the generated
-report. This can take a considerable amount of time if you have many long
-runs. In this case, specify the runs, or pass an empty list to disable raw PSD
-plotting.
-"""
-
 subjects: Union[Sequence[str], Literal["all"]] = "all"
 """
 Subjects to analyze. If `'all'`, include all subjects. To only
@@ -426,9 +422,32 @@
 `'error'` to suppress warnings emitted by read_raw_bids.
 """
 
-###############################################################################
-# BREAK DETECTION
-# ---------------
+plot_psd_for_runs: Union[Literal["all"], Sequence[str]] = "all"
+"""
+For which runs to add a power spectral density (PSD) plot to the generated
+report. This can take a considerable amount of time if you have many long
+runs. In this case, specify the runs, or pass an empty list to disable raw PSD
+plotting.
+"""
+
+random_state: Optional[int] = 42
+"""
+You can specify the seed of the random number generator (RNG).
+This setting is passed to the ICA algorithm and to the decoding function,
+ensuring reproducible results. Set to `None` to avoid setting the RNG
+to a defined state.
+"""
+
+shortest_event: int = 1
+"""
+Minimum number of samples an event must last. If the
+duration is less than this, an exception will be raised.
+"""
+
+# %%
+# # Preprocessing
+
+# ## Break detection
 
 find_breaks: bool = False
 """
@@ -527,10 +546,23 @@
     ```
 """
 
-###############################################################################
-# MAXWELL FILTER PARAMETERS
-# -------------------------
-# done in 01-import_and_maxfilter.py
+# %%
+# ## Bad channel detection
+#
+# !!! warning
+#     This functionality will soon be removed from the pipeline, and
+#     will be integrated into MNE-BIDS.
+#
+# "Bad", i.e. flat and overly noisy channels, can be automatically detected
+# using a procedure inspired by the commercial MaxFilter by Elekta. First,
+# a copy of the data is low-pass filtered at 40 Hz. Then, channels with
+# unusually low variability are flagged as "flat", while channels with
+# excessively high variability are flagged as "noisy". Flat and noisy channels
+# are marked as "bad" and excluded from subsequent analysis. See
+# :func:`mne.preprocssessing.find_bad_channels_maxwell` for more information
+# on this procedure. The list of bad channels detected through this procedure
+# will be merged with the list of bad channels already present in the dataset,
+# if any.
 
 find_flat_channels_meg: bool = False
 """
@@ -543,6 +575,9 @@
 Auto-detect "noisy" channels and mark them as bad.
 """
 
+# %%
+# ## Maxwell filter
+
 use_maxwell_filter: bool = False
 """
 Whether or not to use Maxwell filtering to preprocess the data.
@@ -738,45 +773,29 @@
 Only used when [`use_maxwell_filter=True`][mne_bids_pipeline._config.use_maxwell_filter]
 """  # noqa: E501
 
-###############################################################################
-# STIMULATION ARTIFACT
-# --------------------
-# used in 01-import_and_maxfilter.py
+# ## Filtering & resampling
 
-fix_stim_artifact: bool = False
-"""
-Apply interpolation to fix stimulation artifact.
-
-???+ example "Example"
-    ```python
-    fix_stim_artifact = False
-    ```
-"""
-
-stim_artifact_tmin: float = 0.0
-"""
-Start time of the interpolation window in seconds.
-
-???+ example "Example"
-    ```python
-    stim_artifact_tmin = 0.  # on stim onset
-    ```
-"""
-
-stim_artifact_tmax: float = 0.01
-"""
-End time of the interpolation window in seconds.
-
-???+ example "Example"
-    ```python
-    stim_artifact_tmax = 0.01  # up to 10ms post-stimulation
-    ```
-"""
-
-###############################################################################
-# FREQUENCY FILTERING & RESAMPLING
-# --------------------------------
-# done in 02-frequency_filter.py
+# ### Filtering
+#
+# It is typically better to set your filtering properties on the raw data so
+# as to avoid what we call border (or edge) effects.
+#
+# If you use this pipeline for evoked responses, you could consider
+# a low-pass filter cut-off of h_freq = 40 Hz
+# and possibly a high-pass filter cut-off of l_freq = 1 Hz
+# so you would preserve only the power in the 1Hz to 40 Hz band.
+# Note that highpass filtering is not necessarily recommended as it can
+# distort waveforms of evoked components, or simply wash out any low
+# frequency that can may contain brain signal. It can also act as
+# a replacement for baseline correction in Epochs. See below.
+#
+# If you use this pipeline for time-frequency analysis, a default filtering
+# could be a high-pass filter cut-off of l_freq = 1 Hz
+# a low-pass filter cut-off of h_freq = 120 Hz
+# so you would preserve only the power in the 1Hz to 120 Hz band.
+#
+# If you need more fancy analysis, you are already likely past this kind
+# of tips! 😇
 
 l_freq: Optional[float] = None
 """
@@ -790,6 +809,20 @@
 Keep it `None` if no lowpass filtering should be applied.
 """
 
+l_trans_bandwidth: Union[float, Literal["auto"]] = "auto"
+"""
+Specifies the transition bandwidth of the
+highpass filter. By default it's `'auto'` and uses default MNE
+parameters.
+"""
+
+h_trans_bandwidth: Union[float, Literal["auto"]] = "auto"
+"""
+Specifies the transition bandwidth of the
+lowpass filter. By default it's `'auto'` and uses default MNE
+parameters.
+"""
+
 notch_freq: Optional[Union[float, Sequence[float]]] = None
 """
 Notch filter frequency. More than one frequency can be supplied, e.g. to remove
@@ -809,20 +842,6 @@
     ```
 """
 
-l_trans_bandwidth: Union[float, Literal["auto"]] = "auto"
-"""
-Specifies the transition bandwidth of the
-highpass filter. By default it's `'auto'` and uses default MNE
-parameters.
-"""
-
-h_trans_bandwidth: Union[float, Literal["auto"]] = "auto"
-"""
-Specifies the transition bandwidth of the
-lowpass filter. By default it's `'auto'` and uses default MNE
-parameters.
-"""
-
 notch_trans_bandwidth: float = 1.0
 """
 Specifies the transition bandwidth of the notch filter. The default is `1.`.
@@ -833,6 +852,15 @@
 Specifies the width of each stop band. `None` uses the MNE default.
 """
 
+# ### Resampling
+#
+# If you have acquired data with a very high sampling frequency (e.g. 2 kHz)
+# you will likely want to downsample to lighten up the size of the files you
+# are working with (pragmatics)
+# If you are interested in typical analysis (up to 120 Hz) you can typically
+# resample your data down to 500 Hz without preventing reliable time-frequency
+# exploration of your data.
+
 raw_resample_sfreq: Optional[float] = None
 """
 Specifies at which sampling frequency the data should be resampled.
@@ -845,10 +873,6 @@
     ```
 """
 
-###############################################################################
-# DECIMATION
-# ----------
-
 epochs_decim: int = 1
 """
 Says how much to decimate data at the epochs level.
@@ -867,9 +891,7 @@
 """
 
 
-###############################################################################
-# RENAME EXPERIMENTAL EVENTS
-# --------------------------
+# ## Epoching
 
 rename_events: dict = dict()
 """
@@ -895,10 +917,6 @@
 to only get a warning instead, or `'ignore'` to ignore it completely.
 """
 
-###############################################################################
-# HANDLING OF REPEATED EVENTS
-# ---------------------------
-
 event_repeated: Literal["error", "drop", "merge"] = "error"
 """
 How to handle repeated events. We call events "repeated" if more than one event
@@ -914,10 +932,6 @@
     April 1st, 2021.
 """
 
-###############################################################################
-# EPOCHING
-# --------
-
 epochs_metadata_tmin: Optional[float] = None
 """
 The beginning of the time window for metadata generation, in seconds,
@@ -1032,11 +1046,6 @@
     ```
 """
 
-task_is_rest: bool = False
-"""
-Whether the task should be treated as resting-state data.
-"""
-
 rest_epochs_duration: Optional[float] = None
 """
 Duration of epochs in seconds.
@@ -1059,72 +1068,46 @@
     ```
 """
 
-contrasts: Sequence[Union[tuple[str, str], ArbitraryContrast]] = []
-"""
-The conditions to contrast via a subtraction of ERPs / ERFs. The list elements
-can either be tuples or dictionaries (or a mix of both). Each element in the
-list corresponds to a single contrast.
-
-A tuple specifies a one-vs-one contrast, where the second condition is
-subtracted from the first.
-
-If a dictionary, must contain the following keys:
+# ## Artifact removal
 
-- `name`: a custom name of the contrast
-- `conditions`: the conditions to contrast
-- `weights`: the weights associated with each condition.
-
-Pass an empty list to avoid calculation of any contrasts.
+# ### Stimulation artifact
+#
+# When using electric stimulation systems, e.g. for median nerve or index
+# stimulation, it is frequent to have a stimulation artifact. This option
+# allows to fix it by linear interpolation early in the pipeline on the raw
+# data.
 
-For the contrasts to be computed, the appropriate conditions must have been
-epoched, and therefore the conditions should either match or be subsets of
-`conditions` above.
+fix_stim_artifact: bool = False
+"""
+Apply interpolation to fix stimulation artifact.
 
 ???+ example "Example"
-    Contrast the "left" and the "right" conditions by calculating
-    `left - right` at every time point of the evoked responses:
     ```python
-    contrasts = [('left', 'right')]  # Note we pass a tuple inside the list!
+    fix_stim_artifact = False
     ```
+"""
 
-    Contrast the "left" and the "right" conditions within the "auditory" and
-    the "visual" modality, and "auditory" vs "visual" regardless of side:
+stim_artifact_tmin: float = 0.0
+"""
+Start time of the interpolation window in seconds.
+
+???+ example "Example"
     ```python
-    contrasts = [('auditory/left', 'auditory/right'),
-                 ('visual/left', 'visual/right'),
-                 ('auditory', 'visual')]
+    stim_artifact_tmin = 0.  # on stim onset
     ```
+"""
 
-    Contrast the "left" and the "right" regardless of side, and compute an
-    arbitrary contrast with a gradient of weights:
+stim_artifact_tmax: float = 0.01
+"""
+End time of the interpolation window in seconds.
+
+???+ example "Example"
     ```python
-    contrasts = [
-        ('auditory/left', 'auditory/right'),
-        {
-            'name': 'gradedContrast',
-            'conditions': [
-                'auditory/left',
-                'auditory/right',
-                'visual/left',
-                'visual/right'
-            ],
-            'weights': [-1.5, -.5, .5, 1.5]
-        }
-    ]
+    stim_artifact_tmax = 0.01  # up to 10ms post-stimulation
     ```
 """
 
-###############################################################################
-# ARTIFACT REMOVAL
-# ----------------
-#
-# You can choose between ICA and SSP to remove eye and heart artifacts.
-# SSP: https://mne-tools.github.io/stable/auto_tutorials/plot_artifacts_correction_ssp.html?highlight=ssp # noqa
-# ICA: https://mne-tools.github.io/stable/auto_tutorials/plot_artifacts_correction_ica.html?highlight=ica # noqa
-# if you choose ICA, run steps 5a and 6a
-# if you choose SSP, run steps 5b and 6b
-#
-# Currently you cannot use both.
+# ### SSP, ICA, and artifact regression
 
 regress_artifact: Optional[dict[str, Any]] = None
 """
@@ -1171,9 +1154,6 @@
 Minimal number of EOG epochs needed to compute SSP projectors.
 """
 
-
-# Rejection based on SSP
-# ~~~~~~~~~~~~~~~~~~~~~~
 n_proj_eog: dict[str, float] = dict(n_mag=1, n_grad=1, n_eeg=1)
 """
 Number of SSP vectors to create for EOG artifacts for each channel type.
@@ -1249,8 +1229,6 @@
 is not reliable.
 """
 
-# Rejection based on ICA
-# ~~~~~~~~~~~~~~~~~~~~~~
 ica_reject: Optional[Union[dict[str, float], Literal["autoreject_local"]]] = None
 """
 Peak-to-peak amplitude limits to exclude epochs from ICA fitting. This allows you to
@@ -1388,8 +1366,13 @@
 false-alarm rate increases dramatically.
 """
 
-# Rejection based on peak-to-peak amplitude
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# ### Amplitude-based artifact rejection
+#
+# ???+ info "Good Practice / Advice"
+#     Have a look at your raw data and train yourself to detect a blink, a heart
+#     beat and an eye movement.
+#     You can do a quick average of blink data and check what the amplitude looks
+#     like.
 
 reject: Optional[
     Union[dict[str, float], Literal["autoreject_global", "autoreject_local"]]
@@ -1471,9 +1454,67 @@
     be considered (i.e., will remain marked as bad and not analyzed by autoreject).
 """
 
-###############################################################################
-# DECODING
-# --------
+# %%
+# # Sensor-level analysis
+
+# ## Condition contrasts
+
+contrasts: Sequence[Union[tuple[str, str], ArbitraryContrast]] = []
+"""
+The conditions to contrast via a subtraction of ERPs / ERFs. The list elements
+can either be tuples or dictionaries (or a mix of both). Each element in the
+list corresponds to a single contrast.
+
+A tuple specifies a one-vs-one contrast, where the second condition is
+subtracted from the first.
+
+If a dictionary, must contain the following keys:
+
+- `name`: a custom name of the contrast
+- `conditions`: the conditions to contrast
+- `weights`: the weights associated with each condition.
+
+Pass an empty list to avoid calculation of any contrasts.
+
+For the contrasts to be computed, the appropriate conditions must have been
+epoched, and therefore the conditions should either match or be subsets of
+`conditions` above.
+
+???+ example "Example"
+    Contrast the "left" and the "right" conditions by calculating
+    `left - right` at every time point of the evoked responses:
+    ```python
+    contrasts = [('left', 'right')]  # Note we pass a tuple inside the list!
+    ```
+
+    Contrast the "left" and the "right" conditions within the "auditory" and
+    the "visual" modality, and "auditory" vs "visual" regardless of side:
+    ```python
+    contrasts = [('auditory/left', 'auditory/right'),
+                 ('visual/left', 'visual/right'),
+                 ('auditory', 'visual')]
+    ```
+
+    Contrast the "left" and the "right" regardless of side, and compute an
+    arbitrary contrast with a gradient of weights:
+    ```python
+    contrasts = [
+        ('auditory/left', 'auditory/right'),
+        {
+            'name': 'gradedContrast',
+            'conditions': [
+                'auditory/left',
+                'auditory/right',
+                'visual/left',
+                'visual/right'
+            ],
+            'weights': [-1.5, -.5, .5, 1.5]
+        }
+    ]
+    ```
+"""
+
+# ## Decoding / MVPA
 
 decode: bool = True
 """
@@ -1572,6 +1613,78 @@
 resolution in the resulting matrix.
 """
 
+decoding_csp: bool = False
+"""
+Whether to run decoding via Common Spatial Patterns (CSP) analysis on the
+data. CSP takes as input data covariances that are estimated on different
+time and frequency ranges. This allows to obtain decoding scores defined over
+time and frequency.
+"""
+
+decoding_csp_times: Optional[FloatArrayLike] = None
+"""
+The edges of the time bins to use for CSP decoding.
+Must contain at least two elements. By default, 5 equally-spaced bins are
+created across the non-negative time range of the epochs.
+All specified time points must be contained in the epochs interval.
+If `None`, do not perform **time-frequency** analysis, and only run CSP on
+**frequency** data.
+
+???+ example "Example"
+    Create 3 equidistant time bins (0–0.2, 0.2–0.4, 0.4–0.6 sec):
+    ```python
+    decoding_csp_times = np.linspace(start=0, stop=0.6, num=4)
+    ```
+    Create 2 time bins of different durations (0–0.4, 0.4–0.6 sec):
+    ```python
+    decoding_csp_times = [0, 0.4, 0.6]
+    ```
+"""
+
+decoding_csp_freqs: Optional[dict[str, FloatArrayLike]] = None
+"""
+The edges of the frequency bins to use for CSP decoding.
+
+This parameter must be a dictionary with:
+- keys specifying the unique identifier or "name" to use for the frequency
+  range to be treated jointly during statistical testing (such as "alpha" or
+  "beta"), and
+- values must be list-like objects containing at least two scalar values,
+  specifying the edges of the respective frequency bin(s), e.g., `[8, 12]`.
+
+Defaults to two frequency bins, one from
+[`time_frequency_freq_min`][mne_bids_pipeline._config.time_frequency_freq_min]
+to the midpoint between this value and
+[`time_frequency_freq_max`][mne_bids_pipeline._config.time_frequency_freq_max];
+and the other from that midpoint to `time_frequency_freq_max`.
+???+ example "Example"
+    Create two frequency bins, one for 4–8 Hz, and another for 8–14 Hz, which
+    will be clustered together during statistical testing (in the
+    time-frequency plane):
+    ```python
+    decoding_csp_freqs = {
+        'custom_range': [4, 8, 14]
+    }
+    ```
+    Create the same two frequency bins, but treat them separately during
+    statistical testing (i.e., temporal clustering only):
+    ```python
+    decoding_csp_freqs = {
+        'theta': [4, 8],
+        'alpha': [8, 14]
+    }
+    ```
+    Create 5 equidistant frequency bins from 4 to 14 Hz:
+    ```python
+    decoding_csp_freqs = {
+        'custom_range': np.linspace(
+            start=4,
+            stop=14,
+            num=5+1  # We need one more to account for the endpoint!
+        )
+    }
+"""
+
 n_boot: int = 5000
 """
 The number of bootstrap resamples when estimating the standard error and
@@ -1608,26 +1721,7 @@
     [`cluster_forming_t_threshold`][mne_bids_pipeline._config.cluster_forming_t_threshold].
 """
 
-###############################################################################
-# GROUP AVERAGE SENSORS
-# ---------------------
-
-interpolate_bads_grand_average: bool = True
-"""
-Interpolate bad sensors in each dataset before calculating the grand
-average. This parameter is passed to the `mne.grand_average` function via
-the keyword argument `interpolate_bads`. It requires to have channel
-locations set.
-
-???+ example "Example"
-    ```python
-    interpolate_bads_grand_average = True
-    ```
-"""
-
-###############################################################################
-# TIME-FREQUENCY
-# --------------
+# ## Time-frequency analysis
 
 time_frequency_conditions: Sequence[str] = []
 """
@@ -1676,82 +1770,6 @@
      This also applies to CSP analysis.
 """
 
-###############################################################################
-# TIME-FREQUENCY CSP
-# ------------------
-
-decoding_csp: bool = False
-"""
-Whether to run decoding via Common Spatial Patterns (CSP) analysis on the
-data. CSP takes as input data covariances that are estimated on different
-time and frequency ranges. This allows to obtain decoding scores defined over
-time and frequency.
-"""
-
-decoding_csp_times: Optional[FloatArrayLike] = None
-"""
-The edges of the time bins to use for CSP decoding.
-Must contain at least two elements. By default, 5 equally-spaced bins are
-created across the non-negative time range of the epochs.
-All specified time points must be contained in the epochs interval.
-If `None`, do not perform **time-frequency** analysis, and only run CSP on
-**frequency** data.
-
-???+ example "Example"
-    Create 3 equidistant time bins (0–0.2, 0.2–0.4, 0.4–0.6 sec):
-    ```python
-    decoding_csp_times = np.linspace(start=0, stop=0.6, num=4)
-    ```
-    Create 2 time bins of different durations (0–0.4, 0.4–0.6 sec):
-    ```python
-    decoding_csp_times = [0, 0.4, 0.6]
-    ```
-"""
-
-decoding_csp_freqs: Optional[dict[str, FloatArrayLike]] = None
-"""
-The edges of the frequency bins to use for CSP decoding.
-
-This parameter must be a dictionary with:
-- keys specifying the unique identifier or "name" to use for the frequency
-  range to be treated jointly during statistical testing (such as "alpha" or
-  "beta"), and
-- values must be list-like objects containing at least two scalar values,
-  specifying the edges of the respective frequency bin(s), e.g., `[8, 12]`.
-
-Defaults to two frequency bins, one from
-[`time_frequency_freq_min`][mne_bids_pipeline._config.time_frequency_freq_min]
-to the midpoint between this value and
-[`time_frequency_freq_max`][mne_bids_pipeline._config.time_frequency_freq_max];
-and the other from that midpoint to `time_frequency_freq_max`.
-???+ example "Example"
-    Create two frequency bins, one for 4–8 Hz, and another for 8–14 Hz, which
-    will be clustered together during statistical testing (in the
-    time-frequency plane):
-    ```python
-    decoding_csp_freqs = {
-        'custom_range': [4, 8, 14]
-    }
-    ```
-    Create the same two frequency bins, but treat them separately during
-    statistical testing (i.e., temporal clustering only):
-    ```python
-    decoding_csp_freqs = {
-        'theta': [4, 8],
-        'alpha': [8, 14]
-    }
-    ```
-    Create 5 equidistant frequency bins from 4 to 14 Hz:
-    ```python
-    decoding_csp_freqs = {
-        'custom_range': np.linspace(
-            start=4,
-            stop=14,
-            num=5+1  # We need one more to account for the endpoint!
-        )
-    }
-"""
-
 time_frequency_baseline: Optional[tuple[float, float]] = None
 """
 Baseline period to use for the time-frequency analysis. If `None`, no baseline.
@@ -1782,16 +1800,33 @@
     ```
 """
 
-###############################################################################
-# SOURCE ESTIMATION PARAMETERS
-# ----------------------------
-#
+# ## Group-level analysis
+
+interpolate_bads_grand_average: bool = True
+"""
+Interpolate bad sensors in each dataset before calculating the grand
+average. This parameter is passed to the `mne.grand_average` function via
+the keyword argument `interpolate_bads`. It requires to have channel
+locations set.
+
+???+ example "Example"
+    ```python
+    interpolate_bads_grand_average = True
+    ```
+"""
+
+# %%
+# # Source-level analysis
+
+# ## General source analysis settings
 
 run_source_estimation: bool = True
 """
 Whether to run source estimation processing steps if not explicitly requested.
 """
 
+# ## BEM surface
+
 use_template_mri: Optional[str] = None
 """
 Whether to use a template MRI subject such as FreeSurfer's `fsaverage` subject.
@@ -1864,6 +1899,8 @@
 Whether to print the complete output of FreeSurfer commands. Note that if
 `False`, no FreeSurfer output might be displayed at all!"""
 
+# ## Source space & forward solution
+
 mri_t1_path_generator: Optional[Callable[[BIDSPath], BIDSPath]] = None
 """
 To perform source-level analyses, the Pipeline needs to generate a
@@ -1956,6 +1993,8 @@ def mri_landmarks_kind(bids_path):
 Exclude points closer than this distance (mm) to the bounding surface.
 """
 
+# ## Inverse solution
+
 loose: Union[float, Literal["auto"]] = 0.2
 """
 Value that weights the source variances of the dipole components
@@ -2103,9 +2142,10 @@ def noise_cov(bids_path):
     ```
 """
 
-###############################################################################
-# Report generation
-# -----------------
+# %%
+# # Reports
+
+# ## Report generation
 
 report_evoked_n_time_points: Optional[int] = None
 """
@@ -2131,9 +2171,11 @@ def noise_cov(bids_path):
     ```
 """
 
-###############################################################################
-# Execution
-# ---------
+# %%
+# # Execution
+#
+# These options control how the pipeline is executed but should not affect
+# what outputs get produced.
 
 n_jobs: int = 1
 """
@@ -2173,20 +2215,6 @@ def noise_cov(bids_path):
 The maximum amount of RAM per Dask worker.
 """
 
-random_state: Optional[int] = 42
-"""
-You can specify the seed of the random number generator (RNG).
-This setting is passed to the ICA algorithm and to the decoding function,
-ensuring reproducible results. Set to `None` to avoid setting the RNG
-to a defined state.
-"""
-
-shortest_event: int = 1
-"""
-Minimum number of samples an event must last. If the
-duration is less than this, an exception will be raised.
-"""
-
 log_level: Literal["info", "error"] = "info"
 """
 Set the pipeline logging verbosity.
diff --git a/mne_bids_pipeline/tests/test_documented.py b/mne_bids_pipeline/tests/test_documented.py
index dd90f7ad5..906bba3f6 100644
--- a/mne_bids_pipeline/tests/test_documented.py
+++ b/mne_bids_pipeline/tests/test_documented.py
@@ -2,6 +2,7 @@
 import ast
 import os
 import re
+import sys
 from pathlib import Path
 
 import yaml
@@ -29,31 +30,40 @@ def test_options_documented():
     config_names = set(d for d in dir(config) if not d.startswith("_"))
     assert in_config == config_names
     settings_path = root_path.parent / "docs" / "source" / "settings"
+    sys.path.append(str(settings_path))
+    try:
+        from gen_settings import main
+    finally:
+        sys.path.pop()
+    main()
     assert settings_path.is_dir()
-    in_doc = set()
+    in_doc = dict()
     key = "        - "
-    allowed_duplicates = set(
-        [
-            "source_info_path_update",
-        ]
-    )
     for dirpath, _, fnames in os.walk(settings_path):
         for fname in fnames:
             if not fname.endswith(".md"):
                 continue
             # This is a .md file
-            with open(Path(dirpath) / fname) as fid:
+            # convert to relative path
+            fname = os.path.join(os.path.relpath(dirpath, settings_path), fname)
+            assert fname not in in_doc
+            in_doc[fname] = set()
+            with open(settings_path / fname) as fid:
                 for line in fid:
                     if not line.startswith(key):
                         continue
                     # The line starts with our magic key
                     val = line[len(key) :].strip()
-                    if val not in allowed_duplicates:
-                        assert val not in in_doc, "Duplicate documentation"
-                    in_doc.add(val)
+                    for other in in_doc:
+                        why = f"Duplicate docs in {fname} and {other} for {val}"
+                        assert val not in in_doc[other], why
+                    in_doc[fname].add(val)
     what = "docs/source/settings doc"
-    assert in_doc.difference(in_config) == set(), f"Extra values in {what}"
-    assert in_config.difference(in_doc) == set(), f"Values missing from {what}"
+    in_doc_all = set()
+    for vals in in_doc.values():
+        in_doc_all.update(vals)
+    assert in_doc_all.difference(in_config) == set(), f"Extra values in {what}"
+    assert in_config.difference(in_doc_all) == set(), f"Values missing from {what}"
 
 
 def test_datasets_in_doc():