FIX: Paths and pyarrow (#834)

.pre-commit-config.yaml

@@ -5,17 +5,11 @@ files: ^(.*\.(py|yaml))$
 # for example
 exclude: ^(\.[^/]*cache/.*|.*/freesurfer/contrib/.*)$
 repos:
-  - repo: https://github.com/psf/black
-    rev: 23.12.1
-    hooks:
-      - id: black
-        args:
-          - --safe
-          - --quiet
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.13
+    rev: v0.1.14
     hooks:
       - id: ruff
+      - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:

docs/source/v1.6.md.inc

@@ -14,10 +14,11 @@
 
 - MNE-BIDS-Pipeline now requires Python 3.9 or newer. (#825 by @hoechenberger)
 
-[//]: # (### :bug: Bug fixes)
+### :bug: Bug fixes
 
-[//]: # (- Whatever (#000 by @whoever))
+- Fix minor issues with path handling for cross-talk and calibration files (#834 by @larsoner)
 
 ### :medical_symbol: Code health
 
 - The package build backend has been switched from `setuptools` to `hatchling`. (#825 by @hoechenberger)
+- Code formatting now uses `ruff format` instead of `black` (#834 by @larsoner)

mne_bids_pipeline/_config_utils.py

@@ -353,15 +353,19 @@ def get_mf_cal_fname(
     *, config: SimpleNamespace, subject: str, session: str
 ) -> pathlib.Path:
     if config.mf_cal_fname is None:
-        mf_cal_fpath = BIDSPath(
+        bids_path = BIDSPath(
             subject=subject,
             session=session,
             suffix="meg",
             datatype="meg",
             root=config.bids_root,
-        ).meg_calibration_fpath
+        ).match()[0]
+        mf_cal_fpath = bids_path.meg_calibration_fpath
         if mf_cal_fpath is None:
-            raise ValueError("Could not find Maxwell Filter Calibration file.")
+            raise ValueError(
+                "Could not determine Maxwell Filter Calibration file from BIDS "
+                f"definition for file {bids_path}."
+            )
     else:
         mf_cal_fpath = pathlib.Path(config.mf_cal_fname).expanduser().absolute()
         if not mf_cal_fpath.exists():

mne_bids_pipeline/_main.py

@@ -37,7 +37,7 @@ def main():
         metavar="FILE",
         help="Create a template configuration file with the specified name. "
         "If specified, all other parameters will be ignored.",
-    ),
+    )
     parser.add_argument(
         "--steps",
         dest="steps",
@@ -70,7 +70,7 @@ def main():
         If unspecified, this will be derivatives/mne-bids-pipeline
         inside the BIDS root."""
         ),
-    ),
+    )
     parser.add_argument(
         "--subject", dest="subject", default=None, help="The subject to process."
     )

mne_bids_pipeline/steps/preprocessing/_04_frequency_filter.py

@@ -173,7 +173,7 @@ def filter_data(
         raw = import_er_data(
             cfg=cfg,
             bids_path_er_in=bids_path_in,
-            bids_path_ref_in=in_files.pop("raw_ref_run"),
+            bids_path_ref_in=in_files.pop("raw_ref_run", None),
             bids_path_er_bads_in=bids_path_bads_in,
             # take bads from this run (0)
             bids_path_ref_bads_in=in_files.pop("raw_ref_run-bads", None),
@@ -196,6 +196,7 @@ def filter_data(
         split=None,
         task=task,
         run=run,
+        check=False,
     )
 
     raw.load_data()
@@ -232,6 +233,9 @@ def filter_data(
         run_type=run_type,
     )
 
+    # For example, might need to create
+    # derivatives/mne-bids-pipeline/sub-emptyroom/ses-20230412/meg
+    out_files[in_key].fpath.parent.mkdir(exist_ok=True, parents=True)
     raw.save(
         out_files[in_key],
         overwrite=True,

mne_bids_pipeline/steps/sensor/_99_group_average.py

@@ -814,7 +814,8 @@ def average_csp_decoding(
         import scipy.stats
 
         cluster_forming_t_threshold = scipy.stats.t.ppf(
-            1 - 0.05, len(cfg.subjects) - 1  # one-sided test
+            1 - 0.05,
+            len(cfg.subjects) - 1,  # one-sided test
         )
     else:
         cluster_forming_t_threshold = cfg.cluster_forming_t_threshold

mne_bids_pipeline/tests/configs/config_ds003392.py

@@ -21,7 +21,7 @@
 # Artifact correction.
 spatial_filter = "ica"
 ica_algorithm = "picard-extended_infomax"
-ica_max_iterations = 500
+ica_max_iterations = 1000
 ica_l_freq = 1.0
 ica_n_components = 0.99
 ica_reject_components = "auto"

mne_bids_pipeline/tests/datasets.py

@@ -85,6 +85,8 @@ class DATASET_OPTIONS_T(TypedDict, total=False):
     "ds003775": {
         "openneuro": "ds003775",
         "include": ["sub-010"],
+        # See https://github.com/OpenNeuroOrg/openneuro/issues/2976
+        "exclude": ["sub-010/ses-t1/sub-010_ses-t1_scans.tsv"],
     },
     "ds001810": {
         "openneuro": "ds001810",

mne_bids_pipeline/tests/sub-010_ses-t1_scans.tsv

@@ -0,0 +1,2 @@
+filename	acq_time
+eeg/sub-010_ses-t1_task-resteyesc_eeg.edf	2017-05-09T12:11:44

mne_bids_pipeline/tests/test_run.py

@@ -169,6 +169,15 @@ def test_run(dataset, monkeypatch, dataset_test, capsys, tmp_path):
             src=fix_path / "ds001971_participants.tsv",
             dst=DATA_DIR / "ds001971" / "participants.tsv",
         )
+    elif dataset == "ds003775":
+        shutil.copy(
+            src=fix_path / "sub-010_ses-t1_scans.tsv",
+            dst=DATA_DIR
+            / "ds003775"
+            / "sub-010"
+            / "ses-t1"
+            / "sub-010_ses-t1_scans.tsv",
+        )
 
     # Run the tests.
     steps = test_options.get("steps", ("preprocessing", "sensor"))

pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
   "jupyter-server-proxy", # to have dask and jupyter working together
   "scikit-learn",
   "pandas",
+  "pyarrow",  # from pandas
   "seaborn",
   "json_tricks",
   "pydantic >= 2.0.0",