From dbce9405dbfc4447c96d2fb1bc9b0de690026561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Tue, 23 Apr 2024 14:53:38 +0200 Subject: [PATCH 1/6] Fix report section on original raw data (didn't contain bad channels and subject or experimenter name) Fixes #930 --- .../steps/preprocessing/_01_data_quality.py | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py index 2f5f98bb7..0bb822768 100644 --- a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py +++ b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py @@ -101,7 +101,7 @@ def assess_data_quality( cfg=cfg, data_is_rest=data_is_rest, ) - preexisting_bads = set(raw.info["bads"]) + preexisting_bads = sorted(raw.info["bads"]) if _do_mf_autobad(cfg=cfg): ( @@ -118,13 +118,13 @@ def assess_data_quality( task=task, ) bads = sorted(set(raw.info["bads"] + auto_noisy_chs + auto_flat_chs)) - msg = f"Found {len(bads)} channel{_pl(bads)} as bad." + msg = f"Found {len(bads)} bad channel{_pl(bads)}." raw.info["bads"] = bads del bads logger.info(**gen_log_kwargs(message=msg)) else: auto_scores = auto_noisy_chs = auto_flat_chs = None - del key, raw + del key # Always output the scores and bads TSV out_files["auto_scores"] = bids_path_in.copy().update( @@ -149,27 +149,40 @@ def assess_data_quality( reasons = [] if auto_flat_chs: - bads_for_tsv.extend(auto_flat_chs) - reasons.extend(["auto-flat"] * len(auto_flat_chs)) - preexisting_bads -= set(auto_flat_chs) + for ch in auto_flat_chs: + reason = ( + "pre-existing (before MNE-BIDS-pipeline was run) & auto-flat" + if ch in preexisting_bads + else "auto-flat" + ) + bads_for_tsv.append(ch) + reasons.append(reason) - if auto_noisy_chs is not None: - bads_for_tsv.extend(auto_noisy_chs) - reasons.extend(["auto-noisy"] * len(auto_noisy_chs)) - preexisting_bads -= set(auto_noisy_chs) + if auto_noisy_chs: + for ch in auto_noisy_chs: + reason = ( + "pre-existing (before MNE-BIDS-pipeline was run) & auto-noisy" + if ch in preexisting_bads + else "auto-noisy" + ) + bads_for_tsv.append(ch) + reasons.append(reason) - preexisting_bads = sorted(preexisting_bads) if preexisting_bads: - bads_for_tsv.extend(preexisting_bads) - reasons.extend( - ["pre-existing (before MNE-BIDS-pipeline was run)"] * len(preexisting_bads) - ) + for ch in preexisting_bads: + if ch in bads_for_tsv: + continue + bads_for_tsv.append(ch) + reasons.append(["pre-existing (before MNE-BIDS-pipeline was run)"]) tsv_data = pd.DataFrame(dict(name=bads_for_tsv, reason=reasons)) tsv_data = tsv_data.sort_values(by="name") tsv_data.to_csv(out_files["bads_tsv"], sep="\t", index=False) # Report + # Restore bads to their original state + raw.info["bads"] = preexisting_bads + with _open_report( cfg=cfg, exec_params=exec_params, @@ -186,6 +199,7 @@ def assess_data_quality( cfg=cfg, report=report, bids_path_in=bids_path_in, + raw=raw, title=f"Raw ({kind})", tags=("data-quality",), ) From 17f457a255469a2ebc4c3f4e344d952fb989d2cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Tue, 23 Apr 2024 14:56:34 +0200 Subject: [PATCH 2/6] Fix --- mne_bids_pipeline/steps/preprocessing/_01_data_quality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py index 0bb822768..fdd95a607 100644 --- a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py +++ b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py @@ -173,7 +173,7 @@ def assess_data_quality( if ch in bads_for_tsv: continue bads_for_tsv.append(ch) - reasons.append(["pre-existing (before MNE-BIDS-pipeline was run)"]) + reasons.append("pre-existing (before MNE-BIDS-pipeline was run)") tsv_data = pd.DataFrame(dict(name=bads_for_tsv, reason=reasons)) tsv_data = tsv_data.sort_values(by="name") From 7a0a6cdb5ea119262b9fb5902b54f9f5769152f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Tue, 23 Apr 2024 14:56:59 +0200 Subject: [PATCH 3/6] Update mne_bids_pipeline/steps/preprocessing/_01_data_quality.py --- mne_bids_pipeline/steps/preprocessing/_01_data_quality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py index fdd95a607..c36a0a159 100644 --- a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py +++ b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py @@ -180,7 +180,7 @@ def assess_data_quality( tsv_data.to_csv(out_files["bads_tsv"], sep="\t", index=False) # Report - # Restore bads to their original state + # Restore bads to their original state before generating the report raw.info["bads"] = preexisting_bads with _open_report( From 7f2048b84ae2717e7ed220fea947333d17f149d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Tue, 23 Apr 2024 15:03:56 +0200 Subject: [PATCH 4/6] Cleaner --- mne_bids_pipeline/_report.py | 4 ++-- mne_bids_pipeline/steps/preprocessing/_01_data_quality.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mne_bids_pipeline/_report.py b/mne_bids_pipeline/_report.py index fe77d7951..a9313c3e6 100644 --- a/mne_bids_pipeline/_report.py +++ b/mne_bids_pipeline/_report.py @@ -848,9 +848,9 @@ def _add_raw( cfg: SimpleNamespace, report: mne.report.Report, bids_path_in: BIDSPath, + raw: BaseRaw, title: str, tags: tuple = (), - raw: BaseRaw | None = None, extra_html: str | None = None, ): if bids_path_in.run is not None: @@ -865,7 +865,7 @@ def _add_raw( tags = ("raw", f"run-{bids_path_in.run}") + tags with mne.use_log_level("error"): report.add_raw( - raw=raw or bids_path_in, + raw=raw, title=title, butterfly=5, psd=plot_raw_psd, diff --git a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py index fdd95a607..5a988530c 100644 --- a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py +++ b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py @@ -232,7 +232,7 @@ def _find_bads_maxwell( *, cfg: SimpleNamespace, exec_params: SimpleNamespace, - raw: mne.io.Raw, + raw: mne.io.BaseRaw, subject: str, session: str | None, run: str | None, From f9c10da272df5a8ef14154dc44810afe8b77dd45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Tue, 23 Apr 2024 15:05:31 +0200 Subject: [PATCH 5/6] Update mne_bids_pipeline/steps/preprocessing/_01_data_quality.py --- mne_bids_pipeline/steps/preprocessing/_01_data_quality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py index 4e760da32..86a2c6b6e 100644 --- a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py +++ b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py @@ -180,7 +180,7 @@ def assess_data_quality( tsv_data.to_csv(out_files["bads_tsv"], sep="\t", index=False) # Report - # Restore bads to their original state before generating the report + # Restore bads to their original state so they will show up in the report raw.info["bads"] = preexisting_bads with _open_report( From a319e772b06138460b81d17aeed2b2ccdc7e769c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Tue, 23 Apr 2024 16:54:57 +0200 Subject: [PATCH 6/6] Update changelog --- docs/source/v1.9.md.inc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/source/v1.9.md.inc b/docs/source/v1.9.md.inc index a88a4848d..0581747af 100644 --- a/docs/source/v1.9.md.inc +++ b/docs/source/v1.9.md.inc @@ -13,12 +13,15 @@ ### :warning: Behavior changes - All ICA HTML reports have been consolidated in the standard subject `*_report.html` - file instead of producing separate files (#899 by @larsoner). + file instead of producing separate files. (#899 by @larsoner) - Changed default for `source_info_path_update` to `None`. In `_04_make_forward.py` and `_05_make_inverse.py`, we retrieve the info from the file from which - the `noise_cov` is computed (#919 by @SophieHerbst) + the `noise_cov` is computed. (#919 by @SophieHerbst) - The [`depth`][mne_bids_pipeline._config.depth] parameter doesn't accept `None` anymore. Please use `0` instead. (#915 by @hoechenberger) +- When using automated bad channel detection, now indicate the generated `*_bads.tsv` files whether a channel + had previously already been marked as bad in the dataset. Resulting entries in the TSV file may now look like: + `"pre-existing (before MNE-BIDS-pipeline was run) & auto-noisy"` (previously: only `"auto-noisy"`). (#930 by @hoechenberger) ### :package: Requirements @@ -38,7 +41,9 @@ bad channels from the first pipeline run. Now, we ensure that the original bad channels would be used and the related section is removed from the report in this case. (#902 by @larsoner) - Fixed group-average decoding statistics were not updated in some cases, even if relevant configuration options had been changed. (#902 by @larsoner) -- Fixed a compatibility bug with joblib 1.4.0 +- Fixed a compatibility bug with joblib 1.4.0. (#899 by @larsoner) +- Fixed how "original" raw data is included in the report. Previously, bad channels, subject, and experimenter name would not + be displayed correctly. (#930 by @hoechenberger) ### :medical_symbol: Code health and infrastructure