From 2d748719b656160ea10fe94f29ff5c613e35f957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Sun, 27 Mar 2022 20:09:12 +0200 Subject: [PATCH] Revert PR #167: "ci: fix broken CI" (#169) --- .github/workflows/docs.yml | 10 +- .github/workflows/test.yml | 62 ++++-- Makefile | 32 --- doc/conf.py | 4 +- eegnb/analysis/utils.py | 55 ++--- eegnb/analysis/utils_old.py | 294 ++++++++++++++++++++++++++ eegnb/cli/__main__.py | 8 +- eegnb/datasets/datasets.py | 23 +- examples/visual_n170/01r__n170_viz.py | 2 +- requirements.txt | 34 +-- 10 files changed, 391 insertions(+), 133 deletions(-) delete mode 100644 Makefile create mode 100755 eegnb/analysis/utils_old.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e0bf5414..d208acd8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - name: Set up Python @@ -17,14 +17,16 @@ jobs: python-version: 3.8 - name: Install dependencies run: | - make install-deps-apt python -m pip install --upgrade pip wheel - make install-deps-wxpython + # Install wxPython wheels since they are distribution-specific and therefore not on PyPI + # See: https://wxpython.org/pages/downloads/index.html + pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython + pip install . - name: Build docs run: | - make docs + cd doc && make html - name: Deploy Docs uses: peaceiris/actions-gh-pages@v3 if: github.ref == 'refs/heads/master' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f808173..185caf2d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,14 +8,16 @@ on: jobs: test: - name: test (${{ matrix.os }}, py-${{ matrix.python_version }}) + name: ${{ matrix.os }}, py-${{ matrix.python_version }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - python_version: [3.8] - #include: + python_version: [3.7] + include: + - os: ubuntu-latest + python_version: 3.8 # Experimental: Python 3.9 # Works fine, commented out because mostly covered (at least installing/building deps) by the typecheck job # See issue: https://github.com/NeuroTechX/eeg-notebooks/issues/50 @@ -38,22 +40,33 @@ jobs: - name: Install APT dependencies if: "startsWith(runner.os, 'Linux')" run: | - make install-deps-apt - - name: Upgrade pip - run: | - python -m pip install --upgrade pip wheel + # update archive links + sudo apt-get update + # xvfb is a dependency to create a virtual display + # libgtk-3-dev is a requirement for wxPython + # freeglut3-dev is a requirement for a wxPython dependency + sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev - name: Install Linux dependencies if: "startsWith(runner.os, 'Linux')" run: | - make install-deps-wxpython - - name: Install dependencies + python -m pip install --upgrade pip wheel + + # Install wxPython wheels since they are distribution-specific and therefore not on PyPI + # See: https://wxpython.org/pages/downloads/index.html + pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython + + pip install . + - name: Install MacOS/Windows dependencies run: | - make build + python -m pip install --upgrade pip wheel + pip install . - name: Run eegnb install test shell: bash + pip install -U psychtoolbox # JG_ADD run: | if [ "$RUNNER_OS" == "Linux" ]; then Xvfb :0 -screen 0 1024x768x24 -ac +extension GLX +render -noreset &> xvfb.log & + / export DISPLAY=:0 fi eegnb --help @@ -65,10 +78,9 @@ jobs: Xvfb :0 -screen 0 1024x768x24 -ac +extension GLX +render -noreset &> xvfb.log & export DISPLAY=:0 fi - make test + pytest typecheck: - name: typecheck (${{ matrix.os }}, py-${{ matrix.python_version }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -85,17 +97,27 @@ jobs: - name: Install APT dependencies if: "startsWith(runner.os, 'Linux')" run: | - make install-deps-apt - - name: Upgrade pip - run: | - python -m pip install --upgrade pip wheel + # update archive links + sudo apt-get update + # xvfb is a dependency to create a virtual display + # libgtk-3-dev is a requirement for wxPython + # freeglut3-dev is a requirement for a wxPython dependency + sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev - name: Install Linux dependencies if: "startsWith(runner.os, 'Linux')" run: | - make install-deps-wxpython - - name: Install dependencies + python -m pip install --upgrade pip wheel + + # Install wxPython wheels since they are distribution-specific and therefore not on PyPI + # See: https://wxpython.org/pages/downloads/index.html + pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython + + pip install . + - name: Install MacOS/Windows dependencies run: | - make build + python -m pip install --upgrade pip wheel + pip install . - name: Typecheck run: | - make typecheck + # Exclude visual_cueing due to errors + python -m mypy --exclude 'examples/visual_cueing' diff --git a/Makefile b/Makefile deleted file mode 100644 index 4175cb02..00000000 --- a/Makefile +++ /dev/null @@ -1,32 +0,0 @@ -build: - pip install . - -test: - pytest - -typecheck: - # Exclude visual_cueing due to errors - python -m mypy --exclude 'examples/visual_cueing' - -docs: - cd doc && make html - -clean: - cd doc && make clean - -install-deps-apt: - sudo apt-get update # update archive links - - # xvfb is a dependency to create a virtual display - # libgtk-3-dev is a requirement for wxPython - # freeglut3-dev is a requirement for a wxPython dependency - # portaudio19-dev *might* be required to import psychopy on Ubuntu - # pulseaudio *might* be required to actually run the tests (on PsychoPy import) - # libpulse-dev required to build pocketsphinx (speech recognition dependency of psychopy) - # libsdl2-dev required by psychopy - sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev pulseaudio libsdl2-dev - -install-deps-wxpython: - # Install wxPython wheels since they are distribution-specific and therefore not on PyPI - # See: https://wxpython.org/pages/downloads/index.html - pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython diff --git a/doc/conf.py b/doc/conf.py index d4503b43..cae9a5dc 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -263,7 +263,7 @@ def setup(app): 'backreferences_dir': 'generated', # Where to drop linking files between examples & API 'doc_module': ('eeg-notebooks'), 'reference_url': {'eeg-notebooks': None}, - 'remove_config_comments': True} + 'remove_conffig_comments': True} """ sphinx_gallery_conf = { @@ -284,7 +284,7 @@ def setup(app): 'backreferences_dir': 'generated', # Where to drop linking files between examples & API 'doc_module': ('eeg-notebooks',), 'reference_url': {'eeg-notebooksS': None}, - 'remove_config_comments': True, + 'remove_conffig_comments': True, } """ diff --git a/eegnb/analysis/utils.py b/eegnb/analysis/utils.py index 603ed14c..bd67baa9 100644 --- a/eegnb/analysis/utils.py +++ b/eegnb/analysis/utils.py @@ -2,11 +2,11 @@ from copy import deepcopy import math import logging -import sys from collections import OrderedDict from glob import glob -from typing import Union, List +from typing import Union, List, Dict from time import sleep, time +from numpy.core.fromnumeric import std import pandas as pd import numpy as np @@ -16,13 +16,13 @@ from mne.channels import make_standard_montage from mne.filter import create_filter from matplotlib import pyplot as plt -from scipy import stats from scipy.signal import lfilter, lfilter_zi from eegnb import _get_recording_dir from eegnb.devices.eeg import EEG from eegnb.devices.utils import EEG_INDICES, SAMPLE_FREQS + # this should probably not be done here sns.set_context("talk") @@ -32,33 +32,6 @@ logger = logging.getLogger(__name__) -def _bootstrap(data, n_boot: int, ci: float): - """From: https://stackoverflow.com/a/47582329/965332""" - boot_dist = [] - for i in range(int(n_boot)): - resampler = np.random.randint(0, data.shape[0], data.shape[0]) - sample = data.take(resampler, axis=0) - boot_dist.append(np.mean(sample, axis=0)) - b = np.array(boot_dist) - s1 = np.apply_along_axis(stats.scoreatpercentile, 0, b, 50 - ci / 2) - s2 = np.apply_along_axis(stats.scoreatpercentile, 0, b, 50 + ci / 2) - return (s1, s2) - - -def _tsplotboot(ax, data, time: list, n_boot: int, ci: float, color): - """From: https://stackoverflow.com/a/47582329/965332""" - # Time forms the xaxis of the plot - if time is None: - x = np.arange(data.shape[1]) - else: - x = np.asarray(time) - est = np.mean(data, axis=0) - cis = _bootstrap(data, n_boot, ci) - ax.fill_between(x, cis[0], cis[1], alpha=0.2, color=color) - ax.plot(x, est, color=color) - ax.margins(x=0) - - def load_csv_as_raw( fnames: List[str], sfreq: float, @@ -179,9 +152,7 @@ def load_data( site = "*" data_path = ( - _get_recording_dir( - device_name, experiment, subject_str, session_str, site, data_dir - ) + _get_recording_dir(device_name, experiment, subject_str, session_str, site, data_dir) / "*.csv" ) fnames = glob(str(data_path)) @@ -222,8 +193,7 @@ def plot_conditions( ylim=(-6, 6), diff_waveform=(1, 2), channel_count=4, - channel_order=None, -): + channel_order=None): """Plot ERP conditions. Args: epochs (mne.epochs): EEG epochs @@ -249,9 +219,10 @@ def plot_conditions( """ if channel_order: - channel_order = np.array(channel_order) + channel_order = np.array(channel_order) else: - channel_order = np.array(range(channel_count)) + channel_order = np.array(range(channel_count)) + if isinstance(conditions, dict): conditions = OrderedDict(conditions) @@ -261,7 +232,7 @@ def plot_conditions( X = epochs.get_data() * 1e6 - X = X[:, channel_order] + X = X[:,channel_order] times = epochs.times y = pd.Series(epochs.events[:, -1]) @@ -278,15 +249,13 @@ def plot_conditions( for ch in range(channel_count): for cond, color in zip(conditions.values(), palette): - y_cond = y.isin(cond) - X_cond = X[y_cond, ch] - _tsplotboot( - ax=axes[ch], - data=X_cond, + sns.tsplot( + X[y.isin(cond), ch], time=times, color=color, n_boot=n_boot, ci=ci, + ax=axes[ch], ) if diff_waveform: diff --git a/eegnb/analysis/utils_old.py b/eegnb/analysis/utils_old.py new file mode 100755 index 00000000..16f01ffe --- /dev/null +++ b/eegnb/analysis/utils_old.py @@ -0,0 +1,294 @@ +# -*- coding: utf-8 -*- + +from glob import glob +import os +from collections import OrderedDict + +from mne import create_info, concatenate_raws +from mne.io import RawArray +from mne.channels import make_standard_montage +import pandas as pd +import numpy as np +import seaborn as sns +from matplotlib import pyplot as plt + + +sns.set_context("talk") +sns.set_style("white") + + +def load_muse_csv_as_raw( + filename, + sfreq=256.0, + ch_ind=[0, 1, 2, 3], + stim_ind=5, + replace_ch_names=None, + verbose=1, +): + """Load CSV files into a Raw object. + + Args: + filename (str or list): path or paths to CSV files to load + + Keyword Args: + subject_nb (int or str): subject number. If 'all', load all + subjects. + session_nb (int or str): session number. If 'all', load all + sessions. + sfreq (float): EEG sampling frequency + ch_ind (list): indices of the EEG channels to keep + stim_ind (int): index of the stim channel + replace_ch_names (dict or None): dictionary containing a mapping to + rename channels. Useful when an external electrode was used. + + Returns: + (mne.io.array.array.RawArray): loaded EEG + """ + n_channel = len(ch_ind) + + raw = [] + print(filename) + for fname in filename: + print(fname) + # read the file + data = pd.read_csv(fname, index_col=0) + + # name of each channels + ch_names = list(data.columns)[0:n_channel] + ["Stim"] + + if replace_ch_names is not None: + ch_names = [ + c if c not in replace_ch_names.keys() else replace_ch_names[c] + for c in ch_names + ] + + # type of each channels + ch_types = ["eeg"] * n_channel + ["stim"] + montage = make_standard_montage("standard_1005") + + # get data and exclude Aux channel + data = data.values[:, ch_ind + [stim_ind]].T + + # convert in Volts (from uVolts) + data[:-1] *= 1e-6 + + # create MNE object + info = create_info( + ch_names=ch_names, + ch_types=ch_types, + sfreq=sfreq, + montage=montage, + verbose=verbose, + ) + raw.append(RawArray(data=data, info=info, verbose=verbose)) + + # concatenate all raw objects + print("raw is") + print(raw) + raws = concatenate_raws(raw, verbose=verbose) + + return raws + + +def load_data( + data_dir, + site="eegnb_examples", + experiment="visual_n170", + device="muse2016", + subject_nb=1, + session_nb=1, + sfreq=256.0, + ch_ind=[0, 1, 2, 3], + stim_ind=5, + replace_ch_names=None, + verbose=1, +): + + """Load CSV files from the /data directory into a Raw object. + + Args: + data_dir (str): directory inside /data that contains the + CSV files to load, e.g., 'auditory/P300' + + Keyword Args: + subject_nb (int or str): subject number. If 'all', load all + subjects. + session_nb (int or str): session number. If 'all', load all + sessions. + sfreq (float): EEG sampling frequency + ch_ind (list): indices of the EEG channels to keep + stim_ind (int): index of the stim channel + replace_ch_names (dict or None): dictionary containing a mapping to + rename channels. Useful when an external electrode was used. + + Returns: + (mne.io.array.array.RawArray): loaded EEG + """ + if subject_nb == "all": + subject_nb = "*" + if session_nb == "all": + session_nb = "*" + + subject_nb_str = "%04.f" % subject_nb + session_nb_str = "%03.f" % session_nb + subsess = "subject{}/session{}/*.csv".format(subject_nb_str, session_nb_str) + data_path = os.path.join(data_dir, experiment, site, device, subsess) + fnames = glob(data_path) + + return load_muse_csv_as_raw( + fnames, + sfreq=sfreq, + ch_ind=ch_ind, + stim_ind=stim_ind, + replace_ch_names=replace_ch_names, + verbose=verbose, + ) + + +def plot_conditions( + epochs, + conditions=OrderedDict(), + ci=97.5, + n_boot=1000, + title="", + palette=None, + ylim=(-6, 6), + diff_waveform=(1, 2), +): + """Plot ERP conditions. + + Args: + epochs (mne.epochs): EEG epochs + + Keyword Args: + conditions (OrderedDict): dictionary that contains the names of the + conditions to plot as keys, and the list of corresponding marker + numbers as value. E.g., + + conditions = {'Non-target': [0, 1], + 'Target': [2, 3, 4]} + + ci (float): confidence interval in range [0, 100] + n_boot (int): number of bootstrap samples + title (str): title of the figure + palette (list): color palette to use for conditions + ylim (tuple): (ymin, ymax) + diff_waveform (tuple or None): tuple of ints indicating which + conditions to subtract for producing the difference waveform. + If None, do not plot a difference waveform + + Returns: + (matplotlib.figure.Figure): figure object + (list of matplotlib.axes._subplots.AxesSubplot): list of axes + """ + if isinstance(conditions, dict): + conditions = OrderedDict(conditions) + + if palette is None: + palette = sns.color_palette("hls", len(conditions) + 1) + + X = epochs.get_data() * 1e6 + times = epochs.times + y = pd.Series(epochs.events[:, -1]) + + fig, axes = plt.subplots(2, 2, figsize=[12, 6], sharex=True, sharey=True) + axes = [axes[1, 0], axes[0, 0], axes[0, 1], axes[1, 1]] + + for ch in range(4): + for cond, color in zip(conditions.values(), palette): + sns.tsplot( + X[y.isin(cond), ch], + time=times, + color=color, + n_boot=n_boot, + ci=ci, + ax=axes[ch], + ) + + if diff_waveform: + diff = np.nanmean(X[y == diff_waveform[1], ch], axis=0) - np.nanmean( + X[y == diff_waveform[0], ch], axis=0 + ) + axes[ch].plot(times, diff, color="k", lw=1) + + axes[ch].set_title(epochs.ch_names[ch]) + axes[ch].set_ylim(ylim) + axes[ch].axvline( + x=0, ymin=ylim[0], ymax=ylim[1], color="k", lw=1, label="_nolegend_" + ) + + axes[0].set_xlabel("Time (s)") + axes[0].set_ylabel("Amplitude (uV)") + axes[-1].set_xlabel("Time (s)") + axes[1].set_ylabel("Amplitude (uV)") + + if diff_waveform: + legend = ["{} - {}".format(diff_waveform[1], diff_waveform[0])] + list( + conditions.keys() + ) + else: + legend = conditions.keys() + axes[-1].legend(legend, loc="lower right") + sns.despine() + plt.tight_layout() + + if title: + fig.suptitle(title, fontsize=20) + + return fig, axes + + +def plot_highlight_regions( + x, y, hue, hue_thresh=0, xlabel="", ylabel="", legend_str=() +): + """Plot a line with highlighted regions based on additional value. + + Plot a line and highlight ranges of x for which an additional value + is lower than a threshold. For example, the additional value might be + pvalues, and the threshold might be 0.05. + + Args: + x (array_like): x coordinates + y (array_like): y values of same shape as `x` + + Keyword Args: + hue (array_like): values to be plotted as hue based on `hue_thresh`. + Must be of the same shape as `x` and `y`. + hue_thresh (float): threshold to be applied to `hue`. Regions for which + `hue` is lower than `hue_thresh` will be highlighted. + xlabel (str): x-axis label + ylabel (str): y-axis label + legend_str (tuple): legend for the line and the highlighted regions + + Returns: + (matplotlib.figure.Figure): figure object + (list of matplotlib.axes._subplots.AxesSubplot): list of axes + """ + fig, axes = plt.subplots(1, 1, figsize=(10, 5), sharey=True) + + axes.plot(x, y, lw=2, c="k") + plt.xlabel(xlabel) + plt.ylabel(ylabel) + + kk = 0 + a = [] + while kk < len(hue): + if hue[kk] < hue_thresh: + b = kk + kk += 1 + while kk < len(hue): + if hue[kk] > hue_thresh: + break + else: + kk += 1 + a.append([b, kk - 1]) + else: + kk += 1 + + st = (x[1] - x[0]) / 2.0 + for p in a: + axes.axvspan(x[p[0]] - st, x[p[1]] + st, facecolor="g", alpha=0.5) + plt.legend(legend_str) + sns.despine() + + return fig, axes diff --git a/eegnb/cli/__main__.py b/eegnb/cli/__main__.py index e19aca4c..6e57fbce 100644 --- a/eegnb/cli/__main__.py +++ b/eegnb/cli/__main__.py @@ -1,7 +1,9 @@ +from eegnb import DATA_DIR import click +from time import sleep +from os import path import os - -from eegnb import DATA_DIR +import shutil from eegnb.datasets.datasets import zip_data_folders from .introprompt import intro_prompt @@ -32,7 +34,7 @@ def runexp( recdur: float = None, outfname: str = None, prompt: bool = False, - dosigqualcheck=True, + dosigqualcheck = True, ): """ Run experiment. diff --git a/eegnb/datasets/datasets.py b/eegnb/datasets/datasets.py index 21add031..3fc7ae87 100644 --- a/eegnb/datasets/datasets.py +++ b/eegnb/datasets/datasets.py @@ -1,17 +1,11 @@ -import os -import glob -import shutil -import zipfile - -import requests -import gdown - +import os,sys,glob,shutil,numpy as np, pandas as pd +import requests, zipfile,gdown from datetime import datetime from eegnb import DATA_DIR # eegnb example data sites. do not select these when zipping recordings -eegnb_sites = ["eegnb_examples", "grifflab_dev", "jadinlab_home"] +eegnb_sites = ['eegnb_examples', 'grifflab_dev', 'jadinlab_home'] def fetch_dataset( @@ -70,7 +64,7 @@ def fetch_dataset( } # If no non-default top-level data path specified, use default - if data_dir is None: + if data_dir == None: data_dir = DATA_DIR # check parameter entries @@ -91,9 +85,12 @@ def fetch_dataset( destination = os.path.join(data_dir, "downloaded_data.zip") if download_method == "gdown": + URL = "https://drive.google.com/uc?id=" + gdrive_locs[experiment] gdown.download(URL, destination, quiet=False) + elif download_method == "requests": + URL = "https://docs.google.com/uc?export=download" session = requests.Session() @@ -117,8 +114,6 @@ def fetch_dataset( for chunk in response.iter_content(CHUNK_SIZE): if chunk: f.write(chunk) - else: - raise ValueError("download_method not supported") # unzip the file with zipfile.ZipFile(destination, "r") as zip_ref: @@ -165,7 +160,9 @@ def fetch_dataset( return fnames -def zip_data_folders(experiment: str, site: str = "local"): + +def zip_data_folders(experiment: str, + site: str="local"): """ Run data zipping diff --git a/examples/visual_n170/01r__n170_viz.py b/examples/visual_n170/01r__n170_viz.py index 53ba630e..9aa48f7d 100644 --- a/examples/visual_n170/01r__n170_viz.py +++ b/examples/visual_n170/01r__n170_viz.py @@ -59,7 +59,7 @@ subject = 1 session = 1 raw = load_data(subject,session, - experiment='visual-N170', site='eegnb_examples', device_name='muse2016', + experiment='visual-N170', site='eegnb_examples', device_name='muse2016_bfn', data_dir = eegnb_data_path) ################################################################################################### diff --git a/requirements.txt b/requirements.txt index 2436971c..ea8be01c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,19 @@ # Main repo requirements -psychopy==2022.1.1 +psychopy==2020.2.3 +psychtoolbox scikit-learn>=0.23.2 pandas>=1.1.4 numpy>=1.19.4 mne>=0.20.8 -seaborn>=0.11.0 +seaborn==0.9.0 pyriemann>=0.2.7 jupyter muselsl>=2.0.2 brainflow>=4.8.2 gdown matplotlib>=3.3.3 -pysocks>=1.7.1 -pyserial>=3.5 +pysocks==1.7.1 +pyserial==3.5 h5py>=3.1.0 pytest-shutil pyo>=1.0.3; platform_system == "Linux" @@ -22,6 +23,9 @@ pyo>=1.0.3; platform_system == "Linux" wxPython>=4.0 ; platform_system == "Linux" click +# can be removed once minimum version is Python 3.7 +dataclasses; python_version == '3.6' + # pywinhook needs some special treatment since there are only wheels on PyPI for Python 3.7-3.8, and building requires special tools (swig, VS C++ tools) # See issue: https://github.com/NeuroTechX/eeg-notebooks/issues/29 pywinhook>=1.6.0 ; platform_system == "Windows" and (python_version == "3.7" or python_version == "3.8") @@ -41,14 +45,14 @@ nbval types-requests # Docs requirements -sphinx -sphinx-gallery -sphinx_rtd_theme -sphinx-tabs -sphinx-copybutton -sphinxcontrib-httpdomain -numpydoc -recommonmark -versioneer -rst2pdf -docutils +sphinx==3.1.1 +sphinx-gallery==0.8.1 +sphinx_rtd_theme==0.5.0 +sphinx-tabs==1.3.0 +sphinx-copybutton==0.3.1 +sphinxcontrib-httpdomain==1.7.0 +numpydoc==1.1.0 +recommonmark==0.6.0 +versioneer==0.19 +rst2pdf==0.98 +docutils==0.17