From 4b68a77751b604c2e3ca489dae72925a426120f1 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 9 Mar 2023 16:36:01 +0100 Subject: [PATCH 01/20] rename --- .../{another_conversion => fiber_photometry}/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/tye_lab_to_nwb/{another_conversion => fiber_photometry}/__init__.py (100%) diff --git a/src/tye_lab_to_nwb/another_conversion/__init__.py b/src/tye_lab_to_nwb/fiber_photometry/__init__.py similarity index 100% rename from src/tye_lab_to_nwb/another_conversion/__init__.py rename to src/tye_lab_to_nwb/fiber_photometry/__init__.py From 1407c389c8ffdef8d8ff9909494f89a0aff7144a Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 9 Mar 2023 16:37:06 +0100 Subject: [PATCH 02/20] add ndx-photometry --- src/tye_lab_to_nwb/fiber_photometry/requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/tye_lab_to_nwb/fiber_photometry/requirements.txt diff --git a/src/tye_lab_to_nwb/fiber_photometry/requirements.txt b/src/tye_lab_to_nwb/fiber_photometry/requirements.txt new file mode 100644 index 0000000..faa38d8 --- /dev/null +++ b/src/tye_lab_to_nwb/fiber_photometry/requirements.txt @@ -0,0 +1 @@ +ndx-photometry>=0.1.0 From dcd77ab3464e8960248a41cbf912e0ae68f60508 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 9 Mar 2023 17:58:18 +0100 Subject: [PATCH 03/20] add metadata --- .../metadata/fiber_photometry_metadata.yaml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml diff --git a/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml new file mode 100644 index 0000000..e62e808 --- /dev/null +++ b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml @@ -0,0 +1,28 @@ +ExcitationSourcesTable: + - name: LED415 + peak_wavelength: 415.0 # in nanometers + source_type: LED + - name: LED470 + peak_wavelength: 470.0 # in nanometers + source_type: LED + - name: LED560 + peak_wavelength: 560.0 # in nanometers + source_type: LED +PhotodetectorsTable: + description: The metadata for the photodetector. + type: CMOS camera +FluorophoresTable: + description: The neurotensin-fluorescent sensor was injected into the BLA. The coordinates are in unit meters relative to Bregma (AP, ML, DV). + label: neurotensin-fluorescent sensor (green) + location: BLA + coordinates: + - -0.0016 # AP (in meters) + - 0.0035 # ML (in meters) + - -0.005 # DV (in meters relative to Bregma) +FibersTable: + description: The metadata for the optical fiber. + location: BLA + notes: The optical fiber was implanted above the BLA. +RoiResponseSeries: + description: The photometry intensity values measured in arbitrary units. + unit: a.u. From 901318f65bcad1b12b59488d1817caf8b33832b2 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 9 Mar 2023 17:58:55 +0100 Subject: [PATCH 04/20] create utility function for adding photometry to nwb --- .../fiber_photometry/tools/photometry.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py diff --git a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py new file mode 100644 index 0000000..8a45909 --- /dev/null +++ b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py @@ -0,0 +1,72 @@ +from typing import Optional + +import pandas as pd +from hdmf.common import DynamicTableRegion +from ndx_photometry import FibersTable, FiberPhotometry, ExcitationSourcesTable, PhotodetectorsTable, FluorophoresTable +from pynwb import NWBFile +from pynwb.ophys import RoiResponseSeries + + +def add_photometry(photometry_dataframe: pd.DataFrame, nwbfile: NWBFile, metadata: Optional[dict]): + # Create the ExcitationSourcesTable that holds metadata for the LED sources + excitation_sources_table = ExcitationSourcesTable(description="The metadata for the excitation sources.") + for source_metadata in metadata["ExcitationSourcesTable"]: + excitation_sources_table.add_row( + peak_wavelength=source_metadata["peak_wavelength"], + source_type=source_metadata["source_type"], + ) + + # Create the PhotodetectorsTable that holds metadata for the photodetector. + photodetectors_table = PhotodetectorsTable(description=metadata["PhotodetectorsTable"]["description"]) + photodetectors_table.add_row(type=metadata["PhotodetectorsTable"]["type"]) + + # Create the FluorophoresTable that holds metadata for the fluorophores. + fluorophores_table = FluorophoresTable(description=metadata["FluorophoresTable"]["description"]) + + fluorophores_table.add_row( + label=metadata["FluorophoresTable"]["label"], + location=metadata["FluorophoresTable"]["location"], + coordinates=metadata["FluorophoresTable"]["coordinates"], + ) + + # Create the FibersTable that holds metadata for fibers + fibers_table = FibersTable(description=metadata["FibersTable"]["description"]) + fiber_photometry = FiberPhotometry( + fibers=fibers_table, + excitation_sources=excitation_sources_table, + photodetectors=photodetectors_table, + fluorophores=fluorophores_table, + ) + + # Add the metadata tables to the metadata section + nwbfile.add_lab_meta_data(fiber_photometry) + + # Add row for each fiber defined in metadata + fibers_table.add_fiber( + excitation_source=0, + photodetector=0, + fluorophores=[0], + location=metadata["FibersTable"]["location"], + notes=metadata["FibersTable"]["notes"], + # from article, confirm? + ) + + # Create reference for fibers + rois = DynamicTableRegion( + name="rois", + data=[0], + description="source fibers", + table=fibers_table, + ) + # Create the RoiResponseSeries that holds the intensity values + roi_response_series_name = "RoiResponseSeries" + roi_response_series = RoiResponseSeries( + name=roi_response_series_name, + description=metadata[roi_response_series_name]["description"], + data=photometry_dataframe["Region1R"].values, + unit=metadata[roi_response_series_name]["unit"], + timestamps=photometry_dataframe["Timestamp"].values, + rois=rois, + ) + + nwbfile.add_acquisition(roi_response_series) From 3a2d8967f2d2abab25db57a2dd43e1b84f7a5938 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 9 Mar 2023 17:59:14 +0100 Subject: [PATCH 05/20] create interface --- .../fiber_photometry/__init__.py | 1 + .../fiber_photometry/convert_session.py | 42 +++++++ .../fiberphotometrydatainterface.py | 105 ++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 src/tye_lab_to_nwb/fiber_photometry/convert_session.py create mode 100644 src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py diff --git a/src/tye_lab_to_nwb/fiber_photometry/__init__.py b/src/tye_lab_to_nwb/fiber_photometry/__init__.py index e69de29..2e4ff6f 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/__init__.py +++ b/src/tye_lab_to_nwb/fiber_photometry/__init__.py @@ -0,0 +1 @@ +from .fiberphotometrydatainterface import FiberPhotometryInterface diff --git a/src/tye_lab_to_nwb/fiber_photometry/convert_session.py b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py new file mode 100644 index 0000000..0abdff1 --- /dev/null +++ b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py @@ -0,0 +1,42 @@ +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +from neuroconv.utils import FilePathType + +from tye_lab_to_nwb.fiber_photometry import FiberPhotometryInterface + + +def session_to_nwb( + file_path: FilePathType, + output_dir_path: FilePathType, + stub_test: bool = False, +): + output_dir_path = Path(output_dir_path) + if stub_test: + output_dir_path = output_dir_path / "nwb_stub" + output_dir_path.mkdir(parents=True, exist_ok=True) + + interface = FiberPhotometryInterface(file_path=str(file_path)) + metadata = interface.get_metadata() + + nwbfile_path = output_dir_path / "fiber_photometry.nwb" + + # Add datetime to conversion + date = datetime(year=2020, month=1, day=1, tzinfo=ZoneInfo("US/Eastern")) # TO-DO: Get this from author + metadata["NWBFile"]["session_start_time"] = date + + interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, stub_test=stub_test) + + +if __name__ == "__main__": + # Parameters for conversion + photometry_file_path = Path("/Volumes/t7-ssd/Hao_NWB/recording/Photometry_data0.csv") + output_dir_path = Path("/Volumes/t7-ssd/Hao_NWB/nwbfiles") + stub_test = False + + session_to_nwb( + file_path=photometry_file_path, + output_dir_path=output_dir_path, + stub_test=stub_test, + ) diff --git a/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py b/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py new file mode 100644 index 0000000..9f517f8 --- /dev/null +++ b/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py @@ -0,0 +1,105 @@ +"""Primary class for converting fiber photometry data.""" +from pathlib import Path +from typing import Optional + +import numpy as np +import pandas as pd +from neuroconv.basedatainterface import BaseDataInterface +from neuroconv.tools.nwb_helpers import make_or_load_nwbfile +from neuroconv.utils import FilePathType, load_dict_from_file, OptionalFilePathType +from pynwb import NWBFile + +from tye_lab_to_nwb.fiber_photometry.tools.photometry import add_photometry + + +class FiberPhotometryInterface(BaseDataInterface): + """Primary interface for converting fiber photometry data in custom CSV format.""" + + def __init__( + self, + file_path: FilePathType, + verbose: bool = True, + ): + """ + Interface for writing fiber photometry data from CSV to NWB. + + Parameters + ---------- + file_path : FilePathType + path to the CSV file that contains the intensity values. + verbose: bool, default: True + controls verbosity. + """ + self.verbose = verbose + super().__init__(file_path=file_path) + + def get_metadata(self) -> dict: + metadata = super().get_metadata() + + photometry_metadata = load_dict_from_file( + file_path=Path(__file__).parent / "metadata" / "fiber_photometry_metadata.yaml" + ) + metadata.update(photometry_metadata) + + return metadata + + def _load_source_data(self) -> pd.DataFrame: + return pd.read_csv(self.source_data["file_path"], header=0) + + def get_original_timestamps(self) -> np.ndarray: + """ + Retrieve the original unaltered timestamps for the data in this interface. + + This function should retrieve the data on-demand by re-initializing the IO. + + Returns + ------- + timestamps: numpy.ndarray + The timestamps for the data stream. + """ + raise NotImplementedError( + "Unable to retrieve the original unaltered timestamps for this interface! " + "Define the `get_original_timestamps` method for this interface." + ) + + def get_timestamps(self) -> np.ndarray: + """ + Retrieve the timestamps for the data in this interface. + + Returns + ------- + timestamps: numpy.ndarray + The timestamps for the data stream. + """ + raise NotImplementedError( + "Unable to retrieve timestamps for this interface! Define the `get_timestamps` method for this interface." + ) + + def align_timestamps(self, aligned_timestamps: np.ndarray): + """ + Replace all timestamps for this interface with those aligned to the common session start time. + + Must be in units seconds relative to the common 'session_start_time'. + + Parameters + ---------- + aligned_timestamps : numpy.ndarray + The synchronized timestamps for data in this interface. + """ + raise NotImplementedError( + "The protocol for synchronizing the timestamps of this interface has not been specified!" + ) + + def run_conversion( + self, + nwbfile_path: OptionalFilePathType = None, + nwbfile: Optional[NWBFile] = None, + metadata: Optional[dict] = None, + stub_test: bool = False, + overwrite: bool = False, + ): + with make_or_load_nwbfile( + nwbfile_path=nwbfile_path, nwbfile=nwbfile, metadata=metadata, overwrite=overwrite, verbose=self.verbose + ) as nwbfile_out: + photometry_data = self._load_source_data() + add_photometry(photometry_dataframe=photometry_data, nwbfile=nwbfile_out, metadata=metadata) From 03dfd94895fc28f4b3df0b220208787e0a88a629 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 13 Mar 2023 14:56:50 +0100 Subject: [PATCH 06/20] add events --- .../fiberphotometrydatainterface.py | 6 +++++- .../metadata/fiber_photometry_metadata.yaml | 16 ++++++++++++++++ .../fiber_photometry/requirements.txt | 1 + .../fiber_photometry/tools/photometry.py | 16 ++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py b/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py index 9f517f8..e5c58a2 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py +++ b/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py @@ -9,7 +9,10 @@ from neuroconv.utils import FilePathType, load_dict_from_file, OptionalFilePathType from pynwb import NWBFile -from tye_lab_to_nwb.fiber_photometry.tools.photometry import add_photometry +from tye_lab_to_nwb.fiber_photometry.tools import ( + add_photometry, + add_events_from_photometry, +) class FiberPhotometryInterface(BaseDataInterface): @@ -102,4 +105,5 @@ def run_conversion( nwbfile_path=nwbfile_path, nwbfile=nwbfile, metadata=metadata, overwrite=overwrite, verbose=self.verbose ) as nwbfile_out: photometry_data = self._load_source_data() + add_events_from_photometry(photometry_dataframe=photometry_data, nwbfile=nwbfile_out, metadata=metadata) add_photometry(photometry_dataframe=photometry_data, nwbfile=nwbfile_out, metadata=metadata) diff --git a/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml index e62e808..947f64a 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml +++ b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml @@ -26,3 +26,19 @@ FibersTable: RoiResponseSeries: description: The photometry intensity values measured in arbitrary units. unit: a.u. +Events: + name: LabeledEvents + description: The LED events during photometry. + labels: + 17: LED415 + 18: LED470 + 20: LED560 + 273: Stimulus presentation while LED415 + 274: Stimulus presentation while LED470 + 276: Stimulus presentation while LED560 + 529: Lick while LED415 + 530: Lick while LED470 + 532: Lick while LED560 + 785: TODO while LED415 # missing description + 786: TODO while LED470 # missing description + 788: TODO while LED560 # missing description diff --git a/src/tye_lab_to_nwb/fiber_photometry/requirements.txt b/src/tye_lab_to_nwb/fiber_photometry/requirements.txt index faa38d8..b19ad26 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/requirements.txt +++ b/src/tye_lab_to_nwb/fiber_photometry/requirements.txt @@ -1 +1,2 @@ ndx-photometry>=0.1.0 +ndx-events>=0.2.0 diff --git a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py index 8a45909..ba81586 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py +++ b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py @@ -2,6 +2,7 @@ import pandas as pd from hdmf.common import DynamicTableRegion +from ndx_events import LabeledEvents, AnnotatedEventsTable from ndx_photometry import FibersTable, FiberPhotometry, ExcitationSourcesTable, PhotodetectorsTable, FluorophoresTable from pynwb import NWBFile from pynwb.ophys import RoiResponseSeries @@ -70,3 +71,18 @@ def add_photometry(photometry_dataframe: pd.DataFrame, nwbfile: NWBFile, metadat ) nwbfile.add_acquisition(roi_response_series) + + +def add_events_from_photometry(photometry_dataframe: pd.DataFrame, nwbfile: NWBFile, metadata: Optional[dict]): + annotated_events = AnnotatedEventsTable( + name=metadata["Events"]["name"], + description=metadata["Events"]["description"], + ) + for event_num, event_label in metadata["Events"]["labels"].items(): + annotated_events.add_event_type( + label=event_label, + event_description=f"The times when the {event_label} was on.", + event_times=photometry_dataframe.loc[photometry_dataframe["Flags"] == event_num, "Timestamp"].values, + ) + + nwbfile.add_acquisition(annotated_events) From 9b61f60b17159a688467f183c8f6c3acfc4d41e1 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 13 Mar 2023 14:59:57 +0100 Subject: [PATCH 07/20] remove unused import --- src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py index ba81586..3333599 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py +++ b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py @@ -2,7 +2,7 @@ import pandas as pd from hdmf.common import DynamicTableRegion -from ndx_events import LabeledEvents, AnnotatedEventsTable +from ndx_events import AnnotatedEventsTable from ndx_photometry import FibersTable, FiberPhotometry, ExcitationSourcesTable, PhotodetectorsTable, FluorophoresTable from pynwb import NWBFile from pynwb.ophys import RoiResponseSeries From 20e9620e911c4f84cad176ed63b997bf1bd9a8f3 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 13 Mar 2023 15:02:30 +0100 Subject: [PATCH 08/20] remove unused stub_test --- src/tye_lab_to_nwb/fiber_photometry/convert_session.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/convert_session.py b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py index 0abdff1..4775ef9 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/convert_session.py +++ b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py @@ -10,11 +10,8 @@ def session_to_nwb( file_path: FilePathType, output_dir_path: FilePathType, - stub_test: bool = False, ): output_dir_path = Path(output_dir_path) - if stub_test: - output_dir_path = output_dir_path / "nwb_stub" output_dir_path.mkdir(parents=True, exist_ok=True) interface = FiberPhotometryInterface(file_path=str(file_path)) @@ -26,17 +23,15 @@ def session_to_nwb( date = datetime(year=2020, month=1, day=1, tzinfo=ZoneInfo("US/Eastern")) # TO-DO: Get this from author metadata["NWBFile"]["session_start_time"] = date - interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, stub_test=stub_test) + interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) if __name__ == "__main__": # Parameters for conversion photometry_file_path = Path("/Volumes/t7-ssd/Hao_NWB/recording/Photometry_data0.csv") output_dir_path = Path("/Volumes/t7-ssd/Hao_NWB/nwbfiles") - stub_test = False session_to_nwb( file_path=photometry_file_path, output_dir_path=output_dir_path, - stub_test=stub_test, ) From 5ebcc8d35c47f02f7b0379e21b7e0eab20cc611a Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 13 Mar 2023 15:18:03 +0100 Subject: [PATCH 09/20] remove unused stub_test --- src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py index 3333599..8b58bc4 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py +++ b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py @@ -49,7 +49,6 @@ def add_photometry(photometry_dataframe: pd.DataFrame, nwbfile: NWBFile, metadat fluorophores=[0], location=metadata["FibersTable"]["location"], notes=metadata["FibersTable"]["notes"], - # from article, confirm? ) # Create reference for fibers From 8136b936e7f83f020236d81862f78b3884e2a4a7 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 13 Mar 2023 15:50:43 +0100 Subject: [PATCH 10/20] add init --- src/tye_lab_to_nwb/fiber_photometry/tools/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/tye_lab_to_nwb/fiber_photometry/tools/__init__.py diff --git a/src/tye_lab_to_nwb/fiber_photometry/tools/__init__.py b/src/tye_lab_to_nwb/fiber_photometry/tools/__init__.py new file mode 100644 index 0000000..dcdd050 --- /dev/null +++ b/src/tye_lab_to_nwb/fiber_photometry/tools/__init__.py @@ -0,0 +1 @@ +from .photometry import add_photometry, add_events_from_photometry From c33f5d712976df16ec0c8c146e000df4639a2f45 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 20 Mar 2023 10:32:03 +0100 Subject: [PATCH 11/20] edit metadata --- .../metadata/fiber_photometry_metadata.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml index 947f64a..c85921f 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml +++ b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml @@ -33,12 +33,12 @@ Events: 17: LED415 18: LED470 20: LED560 - 273: Stimulus presentation while LED415 - 274: Stimulus presentation while LED470 - 276: Stimulus presentation while LED560 + 273: Cued stimulus while LED415 + 274: Cued stimulus while LED470 + 276: Cued stimulus while LED560 529: Lick while LED415 530: Lick while LED470 532: Lick while LED560 - 785: TODO while LED415 # missing description - 786: TODO while LED470 # missing description - 788: TODO while LED560 # missing description + 785: Both cued stimulus and lick while LED415 + 786: Both cued stimulus and lick while LED470 + 788: Both cued stimulus and lick while LED560 From 8261763ae353ca899e619874fb8e70a0e560f114 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 20 Mar 2023 10:48:18 +0100 Subject: [PATCH 12/20] fill timestamps methods --- .../fiberphotometrydatainterface.py | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py b/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py index e5c58a2..a681b38 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py +++ b/src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py @@ -35,6 +35,7 @@ def __init__( """ self.verbose = verbose super().__init__(file_path=file_path) + self.photometry_dataframe = self._read_file() def get_metadata(self) -> dict: metadata = super().get_metadata() @@ -46,10 +47,10 @@ def get_metadata(self) -> dict: return metadata - def _load_source_data(self) -> pd.DataFrame: + def _read_file(self) -> pd.DataFrame: return pd.read_csv(self.source_data["file_path"], header=0) - def get_original_timestamps(self) -> np.ndarray: + def get_original_timestamps(self, column: str = "Timestamp") -> np.ndarray: """ Retrieve the original unaltered timestamps for the data in this interface. @@ -60,12 +61,9 @@ def get_original_timestamps(self) -> np.ndarray: timestamps: numpy.ndarray The timestamps for the data stream. """ - raise NotImplementedError( - "Unable to retrieve the original unaltered timestamps for this interface! " - "Define the `get_original_timestamps` method for this interface." - ) + return self._read_file()[column].values - def get_timestamps(self) -> np.ndarray: + def get_timestamps(self, column: str = "Timestamp") -> np.ndarray: """ Retrieve the timestamps for the data in this interface. @@ -74,11 +72,9 @@ def get_timestamps(self) -> np.ndarray: timestamps: numpy.ndarray The timestamps for the data stream. """ - raise NotImplementedError( - "Unable to retrieve timestamps for this interface! Define the `get_timestamps` method for this interface." - ) + return self.photometry_dataframe[column].values - def align_timestamps(self, aligned_timestamps: np.ndarray): + def align_timestamps(self, aligned_timestamps: np.ndarray, column: str = "Timestamp"): """ Replace all timestamps for this interface with those aligned to the common session start time. @@ -89,9 +85,7 @@ def align_timestamps(self, aligned_timestamps: np.ndarray): aligned_timestamps : numpy.ndarray The synchronized timestamps for data in this interface. """ - raise NotImplementedError( - "The protocol for synchronizing the timestamps of this interface has not been specified!" - ) + self.photometry_dataframe[column] = aligned_timestamps def run_conversion( self, @@ -104,6 +98,7 @@ def run_conversion( with make_or_load_nwbfile( nwbfile_path=nwbfile_path, nwbfile=nwbfile, metadata=metadata, overwrite=overwrite, verbose=self.verbose ) as nwbfile_out: - photometry_data = self._load_source_data() - add_events_from_photometry(photometry_dataframe=photometry_data, nwbfile=nwbfile_out, metadata=metadata) - add_photometry(photometry_dataframe=photometry_data, nwbfile=nwbfile_out, metadata=metadata) + add_events_from_photometry( + photometry_dataframe=self.photometry_dataframe, nwbfile=nwbfile_out, metadata=metadata + ) + add_photometry(photometry_dataframe=self.photometry_dataframe, nwbfile=nwbfile_out, metadata=metadata) From ccbea14c5eeb35ad441db20cd885641cebcc3174 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 20 Mar 2023 10:48:28 +0100 Subject: [PATCH 13/20] edit metadata --- .../metadata/fiber_photometry_metadata.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml index c85921f..3c4abe2 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml +++ b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml @@ -39,6 +39,6 @@ Events: 529: Lick while LED415 530: Lick while LED470 532: Lick while LED560 - 785: Both cued stimulus and lick while LED415 - 786: Both cued stimulus and lick while LED470 - 788: Both cued stimulus and lick while LED560 + 785: Cued stimulus and lick while LED415 + 786: Cued stimulus and lick while LED470 + 788: Cued stimulus and lick while LED560 From 49c2c1f172266d26f8d634d52ca79106f083785a Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 22 Mar 2023 11:45:46 +0100 Subject: [PATCH 14/20] exclude LED560 events --- .../metadata/fiber_photometry_metadata.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml index 3c4abe2..8fe4455 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml +++ b/src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml @@ -5,9 +5,6 @@ ExcitationSourcesTable: - name: LED470 peak_wavelength: 470.0 # in nanometers source_type: LED - - name: LED560 - peak_wavelength: 560.0 # in nanometers - source_type: LED PhotodetectorsTable: description: The metadata for the photodetector. type: CMOS camera @@ -24,7 +21,9 @@ FibersTable: location: BLA notes: The optical fiber was implanted above the BLA. RoiResponseSeries: - description: The photometry intensity values measured in arbitrary units. + - region: Region0G + name: RoiResponseSeriesRegion0G + description: The photometry intensity values measured in arbitrary units and recorded from BLA. unit: a.u. Events: name: LabeledEvents @@ -32,13 +31,9 @@ Events: labels: 17: LED415 18: LED470 - 20: LED560 273: Cued stimulus while LED415 274: Cued stimulus while LED470 - 276: Cued stimulus while LED560 529: Lick while LED415 530: Lick while LED470 - 532: Lick while LED560 785: Cued stimulus and lick while LED415 786: Cued stimulus and lick while LED470 - 788: Cued stimulus and lick while LED560 From 066aa1a43257303ac9f8521abad45840edab2c3d Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 22 Mar 2023 11:47:00 +0100 Subject: [PATCH 15/20] adjust RoiResponseSeries metadata --- .../fiber_photometry/tools/photometry.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py index 8b58bc4..a3f3963 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py +++ b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py @@ -59,17 +59,19 @@ def add_photometry(photometry_dataframe: pd.DataFrame, nwbfile: NWBFile, metadat table=fibers_table, ) # Create the RoiResponseSeries that holds the intensity values - roi_response_series_name = "RoiResponseSeries" - roi_response_series = RoiResponseSeries( - name=roi_response_series_name, - description=metadata[roi_response_series_name]["description"], - data=photometry_dataframe["Region1R"].values, - unit=metadata[roi_response_series_name]["unit"], - timestamps=photometry_dataframe["Timestamp"].values, - rois=rois, - ) + for photometry_metadata in metadata["RoiResponseSeries"]: + column = photometry_metadata["region"] + roi_response_series_name = photometry_metadata["name"] + roi_response_series = RoiResponseSeries( + name=roi_response_series_name, + description=photometry_metadata["description"], + data=photometry_dataframe[column].values, + unit=photometry_metadata["unit"], + timestamps=photometry_dataframe["Timestamp"].values, + rois=rois, + ) - nwbfile.add_acquisition(roi_response_series) + nwbfile.add_acquisition(roi_response_series) def add_events_from_photometry(photometry_dataframe: pd.DataFrame, nwbfile: NWBFile, metadata: Optional[dict]): From e9749eee0a36a379f7f7648a9f57d14245633045 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 22 Mar 2023 12:08:08 +0100 Subject: [PATCH 16/20] add general metadata --- .../fiber_photometry/convert_session.py | 14 ++++++++++---- .../metadata/general_metadata.yaml | 9 +++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 src/tye_lab_to_nwb/fiber_photometry/metadata/general_metadata.yaml diff --git a/src/tye_lab_to_nwb/fiber_photometry/convert_session.py b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py index 4775ef9..448c5ac 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/convert_session.py +++ b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py @@ -2,7 +2,7 @@ from pathlib import Path from zoneinfo import ZoneInfo -from neuroconv.utils import FilePathType +from neuroconv.utils import FilePathType, load_dict_from_file, dict_deep_update from tye_lab_to_nwb.fiber_photometry import FiberPhotometryInterface @@ -17,11 +17,17 @@ def session_to_nwb( interface = FiberPhotometryInterface(file_path=str(file_path)) metadata = interface.get_metadata() - nwbfile_path = output_dir_path / "fiber_photometry.nwb" + # Update default metadata with the editable in the corresponding yaml file + editable_metadata_path = Path(__file__).parent / "metadata" / "general_metadata.yaml" + editable_metadata = load_dict_from_file(editable_metadata_path) + metadata = dict_deep_update(metadata, editable_metadata) + + nwbfile_path = output_dir_path / f"{photometry_file_path.stem}.nwb" # Add datetime to conversion - date = datetime(year=2020, month=1, day=1, tzinfo=ZoneInfo("US/Eastern")) # TO-DO: Get this from author - metadata["NWBFile"]["session_start_time"] = date + if "session_start_time" not in metadata["NWBFile"]: + date = datetime(year=2020, month=1, day=1, tzinfo=ZoneInfo("US/Eastern")) # TO-DO: Get this from author + metadata["NWBFile"].update(session_start_time=date) interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) diff --git a/src/tye_lab_to_nwb/fiber_photometry/metadata/general_metadata.yaml b/src/tye_lab_to_nwb/fiber_photometry/metadata/general_metadata.yaml new file mode 100644 index 0000000..fbedba5 --- /dev/null +++ b/src/tye_lab_to_nwb/fiber_photometry/metadata/general_metadata.yaml @@ -0,0 +1,9 @@ +NWBFile: + session_description: Contains photometry intensity values obtained from a single fiber implanted in BLA. + institution: Salk Institute for Biological Studies + lab: Tye + experimenter: + - Li, Hao + related_publications: https://doi.org/10.1038/s41586-022-04964-y +Subject: + species: Mus musculus From 51a65298002de107e07d0e651919865d6b866253 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 22 Mar 2023 12:11:19 +0100 Subject: [PATCH 17/20] add compression --- src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py index a3f3963..d07fe2c 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py +++ b/src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py @@ -1,6 +1,7 @@ from typing import Optional import pandas as pd +from hdmf.backends.hdf5 import H5DataIO from hdmf.common import DynamicTableRegion from ndx_events import AnnotatedEventsTable from ndx_photometry import FibersTable, FiberPhotometry, ExcitationSourcesTable, PhotodetectorsTable, FluorophoresTable @@ -65,9 +66,9 @@ def add_photometry(photometry_dataframe: pd.DataFrame, nwbfile: NWBFile, metadat roi_response_series = RoiResponseSeries( name=roi_response_series_name, description=photometry_metadata["description"], - data=photometry_dataframe[column].values, + data=H5DataIO(photometry_dataframe[column].values, compression=True), unit=photometry_metadata["unit"], - timestamps=photometry_dataframe["Timestamp"].values, + timestamps=H5DataIO(photometry_dataframe["Timestamp"].values, compression=True), rois=rois, ) From bfce1f3130801403dc7e36dd2964874f750b9bea Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 22 Mar 2023 12:39:52 +0100 Subject: [PATCH 18/20] update metadata --- .../fiber_photometry/convert_session.py | 24 ++++++++++++++----- .../metadata/general_metadata.yaml | 3 +++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/convert_session.py b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py index 448c5ac..1622af7 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/convert_session.py +++ b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py @@ -1,5 +1,6 @@ from datetime import datetime from pathlib import Path +from uuid import uuid4 from zoneinfo import ZoneInfo from neuroconv.utils import FilePathType, load_dict_from_file, dict_deep_update @@ -11,10 +12,9 @@ def session_to_nwb( file_path: FilePathType, output_dir_path: FilePathType, ): - output_dir_path = Path(output_dir_path) - output_dir_path.mkdir(parents=True, exist_ok=True) - + # Initalize interface with photometry source data interface = FiberPhotometryInterface(file_path=str(file_path)) + # Update metadata from interface metadata = interface.get_metadata() # Update default metadata with the editable in the corresponding yaml file @@ -22,12 +22,22 @@ def session_to_nwb( editable_metadata = load_dict_from_file(editable_metadata_path) metadata = dict_deep_update(metadata, editable_metadata) - nwbfile_path = output_dir_path / f"{photometry_file_path.stem}.nwb" - # Add datetime to conversion if "session_start_time" not in metadata["NWBFile"]: date = datetime(year=2020, month=1, day=1, tzinfo=ZoneInfo("US/Eastern")) # TO-DO: Get this from author metadata["NWBFile"].update(session_start_time=date) + # Generate subject identifier if missing from metadata + subject_id = "1" + if "subject_id" not in metadata["Subject"]: + metadata["Subject"].update(subject_id=subject_id) + session_id = str(uuid4()) + if "session_id" not in metadata["NWBFile"]: + metadata["NWBFile"].update(session_id=session_id) + + output_dir_path = Path(output_dir_path) / f"sub-{metadata['Subject']['subject_id']}" + output_dir_path.mkdir(parents=True, exist_ok=True) + nwbfile_name = f"sub-{subject_id}_ses-{session_id}.nwb" + nwbfile_path = output_dir_path / nwbfile_name interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) @@ -35,7 +45,9 @@ def session_to_nwb( if __name__ == "__main__": # Parameters for conversion photometry_file_path = Path("/Volumes/t7-ssd/Hao_NWB/recording/Photometry_data0.csv") - output_dir_path = Path("/Volumes/t7-ssd/Hao_NWB/nwbfiles") + output_dir_path = Path( + "/Users/weian/Library/Mobile Documents/com~apple~CloudDocs/catalystneuro/tye-lab-to-nwb/204134" + ) session_to_nwb( file_path=photometry_file_path, diff --git a/src/tye_lab_to_nwb/fiber_photometry/metadata/general_metadata.yaml b/src/tye_lab_to_nwb/fiber_photometry/metadata/general_metadata.yaml index fbedba5..1738da6 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/metadata/general_metadata.yaml +++ b/src/tye_lab_to_nwb/fiber_photometry/metadata/general_metadata.yaml @@ -7,3 +7,6 @@ NWBFile: related_publications: https://doi.org/10.1038/s41586-022-04964-y Subject: species: Mus musculus + age: P8W/P20W # male and female mice between the ages of 8-20 weeks were used for all experiments + sex: U # "M" is for male, "F" is for female (for this session it is unknown) + #strain: C57BL/6J # the strain of the mouse for this session From b14cefd2c40950c64538b597e7f717908878dbba Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 22 Mar 2023 12:55:04 +0100 Subject: [PATCH 19/20] rollback output_dir_path --- src/tye_lab_to_nwb/fiber_photometry/convert_session.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/convert_session.py b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py index 1622af7..48c23b9 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/convert_session.py +++ b/src/tye_lab_to_nwb/fiber_photometry/convert_session.py @@ -45,9 +45,7 @@ def session_to_nwb( if __name__ == "__main__": # Parameters for conversion photometry_file_path = Path("/Volumes/t7-ssd/Hao_NWB/recording/Photometry_data0.csv") - output_dir_path = Path( - "/Users/weian/Library/Mobile Documents/com~apple~CloudDocs/catalystneuro/tye-lab-to-nwb/204134" - ) + output_dir_path = Path("/Volumes/t7-ssd/Hao_NWB/nwbfiles") session_to_nwb( file_path=photometry_file_path, From 0b2250c26c3212597377d06b9138687dbc657cce Mon Sep 17 00:00:00 2001 From: Szonja Weigl Date: Wed, 22 Mar 2023 15:56:39 +0100 Subject: [PATCH 20/20] Update src/tye_lab_to_nwb/fiber_photometry/requirements.txt --- src/tye_lab_to_nwb/fiber_photometry/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tye_lab_to_nwb/fiber_photometry/requirements.txt b/src/tye_lab_to_nwb/fiber_photometry/requirements.txt index b19ad26..ede7304 100644 --- a/src/tye_lab_to_nwb/fiber_photometry/requirements.txt +++ b/src/tye_lab_to_nwb/fiber_photometry/requirements.txt @@ -1,2 +1,2 @@ -ndx-photometry>=0.1.0 +git+https://github.com/catalystneuro/ndx-photometry ndx-events>=0.2.0