Merge pull request #12 from catalystneuro/add_photometry

Add photometry

src/tye_lab_to_nwb/fiber_photometry/__init__.py

@@ -0,0 +1 @@
+from .fiberphotometrydatainterface import FiberPhotometryInterface

src/tye_lab_to_nwb/fiber_photometry/convert_session.py

@@ -0,0 +1,53 @@
+from datetime import datetime
+from pathlib import Path
+from uuid import uuid4
+from zoneinfo import ZoneInfo
+
+from neuroconv.utils import FilePathType, load_dict_from_file, dict_deep_update
+
+from tye_lab_to_nwb.fiber_photometry import FiberPhotometryInterface
+
+
+def session_to_nwb(
+    file_path: FilePathType,
+    output_dir_path: FilePathType,
+):
+    # Initalize interface with photometry source data
+    interface = FiberPhotometryInterface(file_path=str(file_path))
+    # Update metadata from interface
+    metadata = interface.get_metadata()
+
+    # Update default metadata with the editable in the corresponding yaml file
+    editable_metadata_path = Path(__file__).parent / "metadata" / "general_metadata.yaml"
+    editable_metadata = load_dict_from_file(editable_metadata_path)
+    metadata = dict_deep_update(metadata, editable_metadata)
+
+    # Add datetime to conversion
+    if "session_start_time" not in metadata["NWBFile"]:
+        date = datetime(year=2020, month=1, day=1, tzinfo=ZoneInfo("US/Eastern"))  # TO-DO: Get this from author
+        metadata["NWBFile"].update(session_start_time=date)
+    # Generate subject identifier if missing from metadata
+    subject_id = "1"
+    if "subject_id" not in metadata["Subject"]:
+        metadata["Subject"].update(subject_id=subject_id)
+    session_id = str(uuid4())
+    if "session_id" not in metadata["NWBFile"]:
+        metadata["NWBFile"].update(session_id=session_id)
+
+    output_dir_path = Path(output_dir_path) / f"sub-{metadata['Subject']['subject_id']}"
+    output_dir_path.mkdir(parents=True, exist_ok=True)
+    nwbfile_name = f"sub-{subject_id}_ses-{session_id}.nwb"
+    nwbfile_path = output_dir_path / nwbfile_name
+
+    interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata)
+
+
+if __name__ == "__main__":
+    # Parameters for conversion
+    photometry_file_path = Path("/Volumes/t7-ssd/Hao_NWB/recording/Photometry_data0.csv")
+    output_dir_path = Path("/Volumes/t7-ssd/Hao_NWB/nwbfiles")
+
+    session_to_nwb(
+        file_path=photometry_file_path,
+        output_dir_path=output_dir_path,
+    )

src/tye_lab_to_nwb/fiber_photometry/fiberphotometrydatainterface.py

@@ -0,0 +1,104 @@
+"""Primary class for converting fiber photometry data."""
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+from neuroconv.basedatainterface import BaseDataInterface
+from neuroconv.tools.nwb_helpers import make_or_load_nwbfile
+from neuroconv.utils import FilePathType, load_dict_from_file, OptionalFilePathType
+from pynwb import NWBFile
+
+from tye_lab_to_nwb.fiber_photometry.tools import (
+    add_photometry,
+    add_events_from_photometry,
+)
+
+
+class FiberPhotometryInterface(BaseDataInterface):
+    """Primary interface for converting fiber photometry data in custom CSV format."""
+
+    def __init__(
+        self,
+        file_path: FilePathType,
+        verbose: bool = True,
+    ):
+        """
+        Interface for writing fiber photometry data from CSV to NWB.
+
+        Parameters
+        ----------
+        file_path : FilePathType
+            path to the CSV file that contains the intensity values.
+        verbose: bool, default: True
+            controls verbosity.
+        """
+        self.verbose = verbose
+        super().__init__(file_path=file_path)
+        self.photometry_dataframe = self._read_file()
+
+    def get_metadata(self) -> dict:
+        metadata = super().get_metadata()
+
+        photometry_metadata = load_dict_from_file(
+            file_path=Path(__file__).parent / "metadata" / "fiber_photometry_metadata.yaml"
+        )
+        metadata.update(photometry_metadata)
+
+        return metadata
+
+    def _read_file(self) -> pd.DataFrame:
+        return pd.read_csv(self.source_data["file_path"], header=0)
+
+    def get_original_timestamps(self, column: str = "Timestamp") -> np.ndarray:
+        """
+        Retrieve the original unaltered timestamps for the data in this interface.
+
+        This function should retrieve the data on-demand by re-initializing the IO.
+
+        Returns
+        -------
+        timestamps: numpy.ndarray
+            The timestamps for the data stream.
+        """
+        return self._read_file()[column].values
+
+    def get_timestamps(self, column: str = "Timestamp") -> np.ndarray:
+        """
+        Retrieve the timestamps for the data in this interface.
+
+        Returns
+        -------
+        timestamps: numpy.ndarray
+            The timestamps for the data stream.
+        """
+        return self.photometry_dataframe[column].values
+
+    def align_timestamps(self, aligned_timestamps: np.ndarray, column: str = "Timestamp"):
+        """
+        Replace all timestamps for this interface with those aligned to the common session start time.
+
+        Must be in units seconds relative to the common 'session_start_time'.
+
+        Parameters
+        ----------
+        aligned_timestamps : numpy.ndarray
+            The synchronized timestamps for data in this interface.
+        """
+        self.photometry_dataframe[column] = aligned_timestamps
+
+    def run_conversion(
+        self,
+        nwbfile_path: OptionalFilePathType = None,
+        nwbfile: Optional[NWBFile] = None,
+        metadata: Optional[dict] = None,
+        stub_test: bool = False,
+        overwrite: bool = False,
+    ):
+        with make_or_load_nwbfile(
+            nwbfile_path=nwbfile_path, nwbfile=nwbfile, metadata=metadata, overwrite=overwrite, verbose=self.verbose
+        ) as nwbfile_out:
+            add_events_from_photometry(
+                photometry_dataframe=self.photometry_dataframe, nwbfile=nwbfile_out, metadata=metadata
+            )
+            add_photometry(photometry_dataframe=self.photometry_dataframe, nwbfile=nwbfile_out, metadata=metadata)

src/tye_lab_to_nwb/fiber_photometry/metadata/fiber_photometry_metadata.yaml

@@ -0,0 +1,39 @@
+ExcitationSourcesTable:
+  - name: LED415
+    peak_wavelength: 415.0 # in nanometers
+    source_type: LED
+  - name: LED470
+    peak_wavelength: 470.0 # in nanometers
+    source_type: LED
+PhotodetectorsTable:
+    description: The metadata for the photodetector.
+    type: CMOS camera
+FluorophoresTable:
+    description: The neurotensin-fluorescent sensor was injected into the BLA. The coordinates are in unit meters relative to Bregma (AP, ML, DV).
+    label: neurotensin-fluorescent sensor (green)
+    location: BLA
+    coordinates:
+      - -0.0016 # AP (in meters)
+      - 0.0035 # ML (in meters)
+      - -0.005 # DV (in meters relative to Bregma)
+FibersTable:
+    description: The metadata for the optical fiber.
+    location: BLA
+    notes: The optical fiber was implanted above the BLA.
+RoiResponseSeries:
+  - region: Region0G
+    name: RoiResponseSeriesRegion0G
+    description: The photometry intensity values measured in arbitrary units and recorded from BLA.
+    unit: a.u.
+Events:
+    name: LabeledEvents
+    description: The LED events during photometry.
+    labels:
+      17: LED415
+      18: LED470
+      273: Cued stimulus while LED415
+      274: Cued stimulus while LED470
+      529: Lick while LED415
+      530: Lick while LED470
+      785: Cued stimulus and lick while LED415
+      786: Cued stimulus and lick while LED470

src/tye_lab_to_nwb/fiber_photometry/metadata/general_metadata.yaml

@@ -0,0 +1,12 @@
+NWBFile:
+  session_description: Contains photometry intensity values obtained from a single fiber implanted in BLA.
+  institution: Salk Institute for Biological Studies
+  lab: Tye
+  experimenter:
+    - Li, Hao
+  related_publications: https://doi.org/10.1038/s41586-022-04964-y
+Subject:
+  species: Mus musculus
+  age: P8W/P20W # male and female mice between the ages of 8-20 weeks were used for all experiments
+  sex: U # "M" is for male, "F" is for female (for this session it is unknown)
+  #strain: C57BL/6J # the strain of the mouse for this session

src/tye_lab_to_nwb/fiber_photometry/requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/catalystneuro/ndx-photometry
+ndx-events>=0.2.0

src/tye_lab_to_nwb/fiber_photometry/tools/__init__.py

@@ -0,0 +1 @@
+from .photometry import add_photometry, add_events_from_photometry

src/tye_lab_to_nwb/fiber_photometry/tools/photometry.py

@@ -0,0 +1,90 @@
+from typing import Optional
+
+import pandas as pd
+from hdmf.backends.hdf5 import H5DataIO
+from hdmf.common import DynamicTableRegion
+from ndx_events import AnnotatedEventsTable
+from ndx_photometry import FibersTable, FiberPhotometry, ExcitationSourcesTable, PhotodetectorsTable, FluorophoresTable
+from pynwb import NWBFile
+from pynwb.ophys import RoiResponseSeries
+
+
+def add_photometry(photometry_dataframe: pd.DataFrame, nwbfile: NWBFile, metadata: Optional[dict]):
+    # Create the ExcitationSourcesTable that holds metadata for the LED sources
+    excitation_sources_table = ExcitationSourcesTable(description="The metadata for the excitation sources.")
+    for source_metadata in metadata["ExcitationSourcesTable"]:
+        excitation_sources_table.add_row(
+            peak_wavelength=source_metadata["peak_wavelength"],
+            source_type=source_metadata["source_type"],
+        )
+
+    # Create the PhotodetectorsTable that holds metadata for the photodetector.
+    photodetectors_table = PhotodetectorsTable(description=metadata["PhotodetectorsTable"]["description"])
+    photodetectors_table.add_row(type=metadata["PhotodetectorsTable"]["type"])
+
+    # Create the FluorophoresTable that holds metadata for the fluorophores.
+    fluorophores_table = FluorophoresTable(description=metadata["FluorophoresTable"]["description"])
+
+    fluorophores_table.add_row(
+        label=metadata["FluorophoresTable"]["label"],
+        location=metadata["FluorophoresTable"]["location"],
+        coordinates=metadata["FluorophoresTable"]["coordinates"],
+    )
+
+    # Create the FibersTable that holds metadata for fibers
+    fibers_table = FibersTable(description=metadata["FibersTable"]["description"])
+    fiber_photometry = FiberPhotometry(
+        fibers=fibers_table,
+        excitation_sources=excitation_sources_table,
+        photodetectors=photodetectors_table,
+        fluorophores=fluorophores_table,
+    )
+
+    # Add the metadata tables to the metadata section
+    nwbfile.add_lab_meta_data(fiber_photometry)
+
+    # Add row for each fiber defined in metadata
+    fibers_table.add_fiber(
+        excitation_source=0,
+        photodetector=0,
+        fluorophores=[0],
+        location=metadata["FibersTable"]["location"],
+        notes=metadata["FibersTable"]["notes"],
+    )
+
+    # Create reference for fibers
+    rois = DynamicTableRegion(
+        name="rois",
+        data=[0],
+        description="source fibers",
+        table=fibers_table,
+    )
+    # Create the RoiResponseSeries that holds the intensity values
+    for photometry_metadata in metadata["RoiResponseSeries"]:
+        column = photometry_metadata["region"]
+        roi_response_series_name = photometry_metadata["name"]
+        roi_response_series = RoiResponseSeries(
+            name=roi_response_series_name,
+            description=photometry_metadata["description"],
+            data=H5DataIO(photometry_dataframe[column].values, compression=True),
+            unit=photometry_metadata["unit"],
+            timestamps=H5DataIO(photometry_dataframe["Timestamp"].values, compression=True),
+            rois=rois,
+        )
+
+        nwbfile.add_acquisition(roi_response_series)
+
+
+def add_events_from_photometry(photometry_dataframe: pd.DataFrame, nwbfile: NWBFile, metadata: Optional[dict]):
+    annotated_events = AnnotatedEventsTable(
+        name=metadata["Events"]["name"],
+        description=metadata["Events"]["description"],
+    )
+    for event_num, event_label in metadata["Events"]["labels"].items():
+        annotated_events.add_event_type(
+            label=event_label,
+            event_description=f"The times when the {event_label} was on.",
+            event_times=photometry_dataframe.loc[photometry_dataframe["Flags"] == event_num, "Timestamp"].values,
+        )
+
+    nwbfile.add_acquisition(annotated_events)