eyelink sync

.circleci/remove_examples.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -eo pipefail
+
+VER=$1
+if [ -z "$VER" ]; then
+  echo "Usage: $0 <version>"
+  exit 1
+fi
+ROOT="$PWD/$VER/examples/"
+if [ ! -d ${ROOT} ]; then
+  echo "Version directory does not exist or appears incorrect:"
+  echo
+  echo "$ROOT"
+  echo
+  echo "Are you on the gh-pages branch and is the ds000117 directory present?"
+  exit 1
+fi
+if [ ! -d ${ROOT}ds000117 ]; then
+  echo "Directory does not exist:"
+  echo
+  echo "${ROOT}ds000117"
+  echo
+  echo "Assuming already pruned and exiting."
+  exit 0
+fi
+echo "Pruning examples in ${ROOT} ..."
+
+find $ROOT -type d -name "*" | tail -n +2 | xargs rm -Rf
+find $ROOT -name "*.html" -exec sed -i /^\<h2\ id=\"generated-output\"\>Generated/,/^\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ $/{d} {} \;
+find $ROOT -name "*.html" -exec sed -i '/^  <a href="#generated-output"/,/^  <\/a>$/{d}' {} \;
+
+echo "Done"

.circleci/run_dataset_and_copy_files.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -eo pipefail
+
+COPY_FILES="true"
+RERUN_TEST="true"
+while getopts "cr" option; do
+   echo $option
+   case $option in
+      c)
+        COPY_FILES="false";;
+      r)
+        RERUN_TEST="false";;
+   esac
+done
+shift "$(($OPTIND -1))"
+
+DS_RUN=$1
+if [[ -z $1 ]]; then
+  echo "Missing dataset argument"
+  exit 1
+fi
+if [[ "$DS_RUN" == "ERP_CORE_"* ]]; then
+  DS="ERP_CORE"
+else
+  DS="$1"
+fi
+
+SECONDS=0
+pytest mne_bids_pipeline --junit-xml=test-results/junit-results.xml -k ${DS_RUN}
+echo "Runtime: ${SECONDS} seconds"
+
+# rerun test (check caching)!
+SECONDS=0
+RERUN_LIMIT=30
+if [[ "$RERUN_TEST" == "false" ]]; then
+  echo "Skipping rerun test"
+  RUN_TIME=0
+else
+  pytest mne_bids_pipeline --cov-append -k $DS_RUN
+  RUN_TIME=$SECONDS
+  echo "Runtime: ${RUN_TIME} seconds (should be <= $RERUN_LIMIT)"
+fi
+test $RUN_TIME -le $RERUN_LIMIT
+
+if [[ "$COPY_FILES" == "false" ]]; then
+  echo "Not copying files"
+  exit 0
+fi
+mkdir -p ~/reports/${DS}
+# these should always exist
+cp -av ~/mne_data/derivatives/mne-bids-pipeline/${DS}/*/**/*.html ~/reports/${DS}/
+cp -av ~/mne_data/derivatives/mne-bids-pipeline/${DS}/*.xlsx ~/reports/${DS}/
+# these are allowed to be optional
+cp -av ~/mne_data/derivatives/mne-bids-pipeline/${DS}/*/**/*.json ~/reports/${DS}/ || :
+cp -av ~/mne_data/derivatives/mne-bids-pipeline/${DS}/*/**/*.tsv ~/reports/${DS}/ || :
+ls -al test-results/*.xml

.circleci/setup_bash.sh

@@ -33,15 +33,13 @@ fi
 
 # Set up image
 sudo ln -s /usr/lib/x86_64-linux-gnu/libxcb-util.so.0 /usr/lib/x86_64-linux-gnu/libxcb-util.so.1
-wget -q -O- http://neuro.debian.net/lists/focal.us-tn.libre | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
-sudo apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9
-echo "export RUN_TESTS=\"pytest mne_bids_pipeline --junit-xml=test-results/junit-results.xml -k\"" >> "$BASH_ENV"
-echo "export DOWNLOAD_DATA=\"python -m mne_bids_pipeline._download\"" >> "$BASH_ENV"
+echo "export RUN_TESTS=\".circleci/run_dataset_and_copy_files.sh\"" >> "$BASH_ENV"
+echo "export DOWNLOAD_DATA=\"coverage run -m mne_bids_pipeline._download\"" >> "$BASH_ENV"
 
 # Similar CircleCI setup to mne-python (Xvfb, venv, minimal commands, env vars)
 wget -q https://raw.githubusercontent.com/mne-tools/mne-python/main/tools/setup_xvfb.sh
 bash setup_xvfb.sh
-sudo apt install -qq tcsh git-annex-standalone python3.10-venv python3-venv libxft2
+sudo apt install -qq tcsh python3.10-venv python3-venv libxft2
 python3.10 -m venv ~/python_env
 wget -q https://raw.githubusercontent.com/mne-tools/mne-python/main/tools/get_minimal_commands.sh
 source get_minimal_commands.sh
@@ -57,6 +55,7 @@ echo 'export MPLBACKEND=Agg' >> "$BASH_ENV"
 echo "source ~/python_env/bin/activate" >> "$BASH_ENV"
 echo "export MNE_3D_OPTION_MULTI_SAMPLES=1" >> "$BASH_ENV"
 echo "export MNE_BIDS_PIPELINE_FORCE_TERMINAL=true" >> "$BASH_ENV"
+echo "export FORCE_COLOR=1" >> "$BASH_ENV"  # for rich to use color in logs
 mkdir -p ~/.local/bin
 if [[ ! -f ~/.local/bin/python ]]; then
     ln -s ~/python_env/bin/python ~/.local/bin/python

.git_archival.txt

@@ -0,0 +1,3 @@
+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$

.gitattributes

@@ -0,0 +1 @@
+.git_archival.txt  export-subst

.github/dependabot.yml

@@ -0,0 +1,15 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "dependabot"
+    commit-message:
+      prefix: "[dependabot]"

.github/workflows/circleci-redirector.yml

@@ -10,5 +10,6 @@ jobs:
         uses: larsoner/circleci-artifacts-redirector-action@master
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+          api-token: ${{ secrets.CIRCLECI_TOKEN }}
           artifact-path: 0/site/index.html
           circleci-jobs: build_docs

.github/workflows/release.yml

@@ -18,9 +18,9 @@ jobs:
   package:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: Install dependencies
@@ -34,7 +34,7 @@ jobs:
     - name: Check env vars
       run: |
         echo "Triggered by: ${{ github.event_name }}"
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: dist
         path: dist
@@ -45,7 +45,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'release'
     steps:
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: dist
         path: dist

.github/workflows/run-tests.yml

@@ -6,32 +6,70 @@ concurrency:
 on: [push, pull_request]
 
 jobs:
-  check-style:
-    name: Style
-    runs-on: "ubuntu-latest"
-    defaults:
-      run:
-        shell: bash -l {0}
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-    - name: Install ruff and codespell
-      run: pip install ruff codespell tomli
-    - run: make ruff
-    - run: make codespell-error
-    - uses: psf/black@stable
   check-doc:
-    name: Doc consistency
+    name: Doc consistency and codespell
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-    - run: pip install --upgrade setuptools[toml] pip
-    - run: pip install --no-build-isolation -ve .[tests]
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+    - run: pip install --upgrade pip
+    - run: pip install -ve .[tests] codespell tomli
+    - run: make codespell-error
     - run: pytest mne_bids_pipeline -m "not dataset_test"
-    - uses: codecov/codecov-action@v3
+    - uses: codecov/codecov-action@v4
       if: success()
       name: 'Upload coverage to CodeCov'
+  caching:
+    name: 'Caching on ${{ matrix.os }}'
+    timeout-minutes: 30
+    continue-on-error: true
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+          - os: macos-latest
+          - os: windows-latest
+    env:
+      MNE_BIDS_PIPELINE_LEGACY_WINDOWS: "false"
+      PYTHONIOENCODING: 'utf8'  # for Windows
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pyvista/setup-headless-display-action@main
+        with:
+          qt: true
+          pyvista: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"  # no "multidict" wheels on 3.12 yet
+      - run: pip install -ve .[tests]
+      - uses: actions/cache@v4
+        with:
+          key: ds001971
+          path: ~/mne_data/ds001971
+        id: ds001971-cache
+      - run: python -m mne_bids_pipeline._download ds001971
+        if: steps.ds001971-cache.outputs.cache-hit != 'true'
+      - run: pytest --cov-append -k ds001971 mne_bids_pipeline/
+      - run: pytest --cov-append -k ds001971 mne_bids_pipeline/  # uses "hash" method
+        timeout-minutes: 1
+      - uses: actions/cache@v4
+        with:
+          key: ds003392
+          path: ~/mne_data/ds003392
+        id: ds003392-cache
+      - run: python -m mne_bids_pipeline._download ds003392
+        if: steps.ds003392-cache.outputs.cache-hit != 'true'
+      - run: pytest --cov-append -k ds003392 mne_bids_pipeline/
+      - run: pytest --cov-append -k ds003392 mne_bids_pipeline/  # uses "mtime" method
+        timeout-minutes: 1
+      - uses: codecov/codecov-action@v4
+        if: success()

.pre-commit-config.yaml

@@ -1,22 +1,24 @@
 ---
-# Eventually we should use yamllint, too
 files: ^(.*\.(py|yaml))$
-exclude: ^(\.[^/]*cache/.*)$
+# We need to match the exclude list in pyproject.toml because pre-commit.ci
+# passes filenames and these do not get passed through the tool.black filter
+# for example
+exclude: ^(\.[^/]*cache/.*|.*/freesurfer/contrib/.*)$
 repos:
-  - repo: https://github.com/psf/black
-    rev: 22.10.0
-    hooks:
-      - id: black
-        args:
-          - --safe
-          - --quiet
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.178
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.4
     hooks:
       - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
+    rev: v2.2.6
     hooks:
       - id: codespell
         additional_dependencies:
           - tomli
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.35.1
+    hooks:
+      - id: yamllint
+        args: [--strict]

CONTRIBUTING.md

@@ -1,27 +1,21 @@
-# Overview
+# Contributing to MNE-BIDS-Pipeline
 
 Contributors to MNE-BIDS-Pipeline are expected to follow our
 [Code of Conduct](https://github.com/mne-tools/.github/blob/main/CODE_OF_CONDUCT.md).
 
-# Installation
+## Installation
 
 First, you need to make sure you have MNE-Python installed and working on your
 system. See the [installation instructions](http://martinos.org/mne/stable/install_mne_python.html).
 Once this is done, you should be able to run this in a terminal:
 
 `$ python -c "import mne; mne.sys_info()"`
 
-You can then install the following additional packages via `pip`. Note that
+You can then install the following additional package via `pip`. Note that
 the URL points to the bleeding edge version of `mne_bids`:
 
-`$ pip install datalad`
 `$ pip install https://github.com/mne-tools/mne-bids/zipball/main`
 
-To get the test data, you need to install `git-annex` on your system. If you
-installed MNE-Python via `conda`, you can simply call:
-
-`conda install -c conda-forge git-annex`
-
 Now, get the pipeline through git:
 
 `$ git clone https://github.com/mne-tools/mne-bids-pipeline.git`
@@ -32,9 +26,9 @@ If you do not know how to use git, download the pipeline as a zip file
 Finally, for source analysis you'll also need `FreeSurfer`, follow the
 instructions on [their website](https://surfer.nmr.mgh.harvard.edu/).
 
-# Testing
+## Testing
 
-## Running the tests, and continuous integration
+### Running the tests, and continuous integration
 
 The tests are run using `pytest`. You can run them by calling
 `pytest mne_bids_pipeline` to run
@@ -48,17 +42,17 @@ For every pull request or merge into the `main` branch of the
 [CircleCI](https://circleci.com/gh/brainthemind/CogBrainDyn_MEG_Pipeline)
 will run tests as defined in `./circleci/config.yml`.
 
-## Debugging
+### Debugging
 
 To run the test in debugging mode, just pass `--pdb` to the `pytest` call
 as usual. This will place you in debugging mode on failure.
 See the
 [pdb help](https://docs.python.org/3/library/pdb.html#debugger-commands)
 for more commands.
 
-## Config files
+### Config files
 
-Nested in the `/tests` directory is a `/configs` directory, which contains
+Nested in the `tests` directory is a `configs` directory, which contains
 config files for specific test datasets. For example, the `config_ds001810.py`
 file specifies parameters only for the `ds001810` data, which should overwrite
 the more general parameters in the main `_config.py` file.

Makefile

@@ -23,8 +23,6 @@ doc:
 
 check:
 	which python
-	git-annex version
-	datalad --version
 	openneuro-py --version
 	mri_convert --version
 	mne_bids --version
@@ -33,10 +31,6 @@ check:
 trailing-spaces:
 	find . -name "*.py" | xargs perl -pi -e 's/[ \t]*$$//'
 
-ruff:
-	ruff .
-	@echo "ruff passed"
-
 codespell:  # running manually; auto-fix spelling mistakes
 	@codespell --write-changes $(CODESPELL_DIRS)