From 8e901d023eb00d396f431601911172c47615dea5 Mon Sep 17 00:00:00 2001 From: luiz Date: Mon, 9 Dec 2024 15:58:27 +0100 Subject: [PATCH 1/5] updates envs --- environments/environment-Linux.yml | 12 +++++++----- environments/environment-MAC-apple-silicon.yml | 17 +++++++++-------- environments/environment-MAC-intel.yml | 12 +++++++----- environments/environment-Windows.yml | 12 +++++++----- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/environments/environment-Linux.yml b/environments/environment-Linux.yml index 00d3773d1..59f82527a 100644 --- a/environments/environment-Linux.yml +++ b/environments/environment-Linux.yml @@ -7,7 +7,7 @@ dependencies: - nodejs = 18.16.1 - numcodecs = 0.11.0 # install these from conda-forge so that dependent packages get included in the distributable - - jsonschema = 4.18.0 # installs jsonschema-specifications + - jsonschema = 4.18.0 # installs jsonschema-specifications - pip - pip: - chardet == 5.1.0 @@ -15,11 +15,13 @@ dependencies: - flask == 2.3.2 - flask-cors == 4.0.0 - flask_restx == 1.1.0 - - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. + - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text] # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation - spikeinterface == 0.100.5 - - scikit-learn == 1.4.0 # Tutorial data generation - - tqdm_publisher >= 0.0.1 # Progress bars - - tzlocal >= 5.2 # Frontend timezone handling + - scikit-learn == 1.4.0 # Tutorial data generation + - tqdm_publisher >= 0.0.1 # Progress bars + - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - tables diff --git a/environments/environment-MAC-apple-silicon.yml b/environments/environment-MAC-apple-silicon.yml index fa40a3b4f..f415885ab 100644 --- a/environments/environment-MAC-apple-silicon.yml +++ b/environments/environment-MAC-apple-silicon.yml @@ -7,11 +7,11 @@ dependencies: - nodejs = 18.16.1 # Install these from conda-forge so that dependent packages get included in the distributable - numcodecs = 0.11.0 - - lxml = 4.9.3 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge - - pyedflib = 0.1.32 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge - - numpy # May have x64/arm64 mismatch issues so install from conda-forge + - lxml = 4.9.3 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge + - pyedflib = 0.1.32 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge + - numpy # May have x64/arm64 mismatch issues so install from conda-forge - pytables = 3.8 # PyPI build fails on arm64 so install from conda-forge (used by neuroconv deps) - - jsonschema = 4.18.0 # Also installs jsonschema-specifications + - jsonschema = 4.18.0 # Also installs jsonschema-specifications - pip - pip: - setuptools==70.0.0 @@ -21,13 +21,14 @@ dependencies: - flask == 2.3.2 - flask-cors == 4.0.0 - flask_restx == 1.1.0 - - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. + - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. # NOTE: the NeuroConv wheel on PyPI includes sonpy which is not compatible with arm64, so build and install # NeuroConv from GitHub, which will remove the sonpy dependency when building from Mac arm64 # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text] # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation - spikeinterface == 0.100.5 - - scikit-learn == 1.4.0 # Tutorial data generation - - tqdm_publisher >= 0.0.1 # Progress bars - - tzlocal >= 5.2 # Frontend timezone handling + - scikit-learn == 1.4.0 # Tutorial data generation + - tqdm_publisher >= 0.0.1 # Progress bars + - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 diff --git a/environments/environment-MAC-intel.yml b/environments/environment-MAC-intel.yml index 41de8e5ac..b9a36e00c 100644 --- a/environments/environment-MAC-intel.yml +++ b/environments/environment-MAC-intel.yml @@ -7,7 +7,7 @@ dependencies: - nodejs = 18.16.1 - numcodecs = 0.11.0 # install these from conda-forge so that dependent packages get included in the distributable - - jsonschema = 4.18.0 # installs jsonschema-specifications + - jsonschema = 4.18.0 # installs jsonschema-specifications - pip - pip: - setuptools==70.0.0 @@ -18,11 +18,13 @@ dependencies: - flask == 2.3.2 - flask-cors == 4.0.0 - flask_restx == 1.1.0 - - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. + - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text] # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation - spikeinterface == 0.100.5 - - scikit-learn == 1.4.0 # Tutorial data generation - - tqdm_publisher >= 0.0.1 # Progress bars - - tzlocal >= 5.2 # Frontend timezone handling + - scikit-learn == 1.4.0 # Tutorial data generation + - tqdm_publisher >= 0.0.1 # Progress bars + - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - tables diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 7b04f568e..366360139 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -12,17 +12,19 @@ dependencies: - pip: - setuptools==70.0.0 - PyInstaller==6.7.0 - - pyinstaller-hooks-contrib == 2024.6 # Fix needed for pydantic v2; otherwise imports pydantic.compiled + - pyinstaller-hooks-contrib == 2024.6 # Fix needed for pydantic v2; otherwise imports pydantic.compiled - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - flask-cors === 3.0.10 - flask_restx == 1.1.0 - - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. + - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text] # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation - spikeinterface == 0.100.5 - - scikit-learn == 1.4.0 # Tutorial data generation - - tqdm_publisher >= 0.0.1 # Progress bars - - tzlocal >= 5.2 # Frontend timezone handling + - scikit-learn == 1.4.0 # Tutorial data generation + - tqdm_publisher >= 0.0.1 # Progress bars + - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - tables From 49dd9952069b1437b6ca7a5f2d79e94de48f29c9 Mon Sep 17 00:00:00 2001 From: luiz Date: Mon, 9 Dec 2024 16:44:10 +0100 Subject: [PATCH 2/5] nwbinspector on linux --- environments/environment-Linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/environment-Linux.yml b/environments/environment-Linux.yml index 59f82527a..3d37525a5 100644 --- a/environments/environment-Linux.yml +++ b/environments/environment-Linux.yml @@ -24,4 +24,5 @@ dependencies: - tqdm_publisher >= 0.0.1 # Progress bars - tzlocal >= 5.2 # Frontend timezone handling - ndx-pose == 0.1.1 + - nwbinspector==0.6.1 - tables From b94ce18015eeed35924252b9969dcff3fa1fbebd Mon Sep 17 00:00:00 2001 From: luiz Date: Mon, 9 Dec 2024 16:52:30 +0100 Subject: [PATCH 3/5] nwbinspector to macos and win --- environments/environment-MAC-apple-silicon.yml | 1 + environments/environment-MAC-intel.yml | 1 + environments/environment-Windows.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/environments/environment-MAC-apple-silicon.yml b/environments/environment-MAC-apple-silicon.yml index f415885ab..d193ce791 100644 --- a/environments/environment-MAC-apple-silicon.yml +++ b/environments/environment-MAC-apple-silicon.yml @@ -32,3 +32,4 @@ dependencies: - tqdm_publisher >= 0.0.1 # Progress bars - tzlocal >= 5.2 # Frontend timezone handling - ndx-pose == 0.1.1 + - nwbinspector==0.6.1 diff --git a/environments/environment-MAC-intel.yml b/environments/environment-MAC-intel.yml index b9a36e00c..61d38320d 100644 --- a/environments/environment-MAC-intel.yml +++ b/environments/environment-MAC-intel.yml @@ -27,4 +27,5 @@ dependencies: - tqdm_publisher >= 0.0.1 # Progress bars - tzlocal >= 5.2 # Frontend timezone handling - ndx-pose == 0.1.1 + - nwbinspector==0.6.1 - tables diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 366360139..b7d650661 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -27,4 +27,5 @@ dependencies: - tqdm_publisher >= 0.0.1 # Progress bars - tzlocal >= 5.2 # Frontend timezone handling - ndx-pose == 0.1.1 + - nwbinspector==0.6.1 - tables From 535135e85d0631e6366c9443f9e174ca2dbbda43 Mon Sep 17 00:00:00 2001 From: luiz Date: Mon, 9 Dec 2024 17:51:29 +0100 Subject: [PATCH 4/5] temporarily comment DLC dataset --- .github/workflows/testing_pipelines.yml | 5 ++ src/example_pipelines.yml | 68 ++++++++----------------- 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/.github/workflows/testing_pipelines.yml b/.github/workflows/testing_pipelines.yml index 8892eacd0..400bc20b2 100644 --- a/.github/workflows/testing_pipelines.yml +++ b/.github/workflows/testing_pipelines.yml @@ -74,24 +74,29 @@ jobs: - name: Get ephy_testing_data current head hash id: ephys run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" + - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }} uses: actions/cache@v4 id: cache-ephys-datasets with: path: ./ephy_testing_data key: ephys-datasets-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} uses: actions/cache@v4 id: cache-ophys-datasets with: path: ./ophys_testing_data key: ophys-datasets-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + - name: Get behavior_testing_data current head hash id: behavior run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)" + - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} uses: actions/cache@v4 id: cache-behavior-datasets diff --git a/src/example_pipelines.yml b/src/example_pipelines.yml index cd5803376..353bae86d 100644 --- a/src/example_pipelines.yml +++ b/src/example_pipelines.yml @@ -26,80 +26,66 @@ SpikeGLX-Phy: PhySortingInterface: folder_path: ephy_testing_data/phy/phy_example_0 - SpikeGLX_v1_SingleProbe_AP: SpikeGLXRecordingInterface: file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin - SpikeGLXConverter-Phy: SpikeGLXConverterPipe: folder_path: ephy_testing_data/spikeglx/Noise4Sam_g0 PhySortingInterface: folder_path: ephy_testing_data/phy/phy_example_0 - Suite2P_SinglePlane_SingleChannel: metadata: NWBFile: session_start_time: 2024-05-09T00:00 Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: Suite2pSegmentationInterface: folder_path: ophys_testing_data/segmentation_datasets/suite2p - Caiman_SinglePlane_SingleChannel: metadata: NWBFile: session_start_time: 2024-05-09T00:00 Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: CaimanSegmentationInterface: file_path: ophys_testing_data/segmentation_datasets/caiman/caiman_analysis.hdf5 - ScanImage_SinglePlane_SingleChannel: - metadata: Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: ScanImageImagingInterface: file_path: ophys_testing_data/imaging_datasets/ScanImage/sample_scanimage_version_3_8.tiff - TDT: metadata: - NWBFile: - session_start_time: 2024-05-09T00:00 + NWBFile: + session_start_time: 2024-05-09T00:00 interfaces: TdtRecordingInterface: folder_path: ephy_testing_data/tdt/aep_05 gain: 1 - CellExplorer: metadata: NWBFile: @@ -109,7 +95,6 @@ CellExplorer: CellExplorerRecordingInterface: folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed - CellExplorerUnits: metadata: NWBFile: @@ -121,7 +106,6 @@ CellExplorerUnits: CellExplorerSortingInterface: file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat - CellExplorerSortingOnly: metadata: NWBFile: @@ -131,24 +115,19 @@ CellExplorerSortingOnly: CellExplorerSortingInterface: file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat - IntanSingleFileRecording: metadata: NWBFile: session_start_time: 2024-05-09T00:00 Ecephys: ElectrodeGroup: - - - location: Test location - - - location: Test location - - - location: Test location + - location: Test location + - location: Test location + - location: Test location interfaces: IntanRecordingInterface: file_path: ephy_testing_data/intan/intan_rhd_test_1.rhd - MEArec: metadata: NWBFile: @@ -158,32 +137,27 @@ MEArec: MEArecRecordingInterface: file_path: ephy_testing_data/mearec/mearec_test_10s.h5 - BrukerTiffMultiChannelSinglePlane: metadata: Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: BrukerTiffSinglePlaneConverter: folder_path: ophys_testing_data/imaging_datasets/BrukerTif/NCCR32_2023_02_20_Into_the_void_t_series_baseline-000 +# DeepLabCut: +# metadata: +# NWBFile: +# session_start_time: 2024-05-09T00:00 -DeepLabCut: - metadata: - NWBFile: - session_start_time: 2024-05-09T00:00 - - interfaces: - DeepLabCutInterface: - file_path: behavior_testing_data/DLC/m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5 - config_file_path: behavior_testing_data/DLC/config.yaml - +# interfaces: +# DeepLabCutInterface: +# file_path: behavior_testing_data/DLC/m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5 +# config_file_path: behavior_testing_data/DLC/config.yaml SLEAP: metadata: From b1b45f7efd024ba9971ba5bf8eb82bb99fe671d1 Mon Sep 17 00:00:00 2001 From: luiz Date: Mon, 9 Dec 2024 18:17:23 +0100 Subject: [PATCH 5/5] try with different DLC example data --- src/example_pipelines.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/example_pipelines.yml b/src/example_pipelines.yml index 353bae86d..fe8d11541 100644 --- a/src/example_pipelines.yml +++ b/src/example_pipelines.yml @@ -149,15 +149,15 @@ BrukerTiffMultiChannelSinglePlane: BrukerTiffSinglePlaneConverter: folder_path: ophys_testing_data/imaging_datasets/BrukerTif/NCCR32_2023_02_20_Into_the_void_t_series_baseline-000 -# DeepLabCut: -# metadata: -# NWBFile: -# session_start_time: 2024-05-09T00:00 - -# interfaces: -# DeepLabCutInterface: -# file_path: behavior_testing_data/DLC/m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5 -# config_file_path: behavior_testing_data/DLC/config.yaml +DeepLabCut: + metadata: + NWBFile: + session_start_time: 2024-05-09T00:00 + + interfaces: + DeepLabCutInterface: + file_path: behavior_testing_data/DLC/open_field_with_video/m3v1mp4DLC_resnet50_openfieldOct30shuffle1_1000.h5 + config_file_path: behavior_testing_data/DLC/open_field_with_video/config.yaml SLEAP: metadata: