Merge pull request #277 from MetaCell/release/0.6.2

Release/0.6.2

README.md

@@ -53,7 +53,7 @@ git clone -b development https://github.com/MetaCell/nwb-explorer
 If you have Docker installed, you can run NWB explorer with one single command:
 
 ```bash
-docker run -it -p8888:8888 gcr.io/metacellllc/nwb-explorer:0.6.0
+docker run -it -p8888:8888 gcr.io/metacellllc/nwb-explorer:0.6.2
 ```
 
 #### Build Docker image

nwb_explorer/nwb_data_manager.py

@@ -27,8 +27,9 @@ def get_file_path(file_name_or_url):
     elif not os.path.exists(file_name_or_url):
         raise NWBFileNotFound("NWB file not found", file_name_or_url)
     else:
-        file_name = get_cache_path(file_name_or_url)
+        file_name = file_name_or_url
         if not os.path.exists(file_name):
+            file_name = get_cache_path(file_name_or_url)
             if not os.path.exists(os.path.dirname(file_name)):
                 os.makedirs(os.path.dirname(file_name))
                 shutil.copyfile(file_name_or_url, file_name)

nwb_explorer/nwb_model_interpreter/nwb_geppetto_mappers.py

@@ -18,12 +18,11 @@
 from pygeppetto.model.values import Image, Text, ImportValue, StringArray
 from pygeppetto.model.variables import Variable, TypeToValueMap
 
-
 nwb_geppetto_mappers = []
 
 
 def is_metadata(value):
-    return isinstance(value, (str, int, float, bool, np.number))
+    return isinstance(value, (str, int, float, bool, bytes, np.number))
 
 
 def is_collection(value):
@@ -34,6 +33,13 @@ def is_array(value):
     return isinstance(value, (np.ndarray, Dataset))
 
 
+def is_multidimensional_data(value):
+    """ Use this function to decide whether to split up the data into multiple rows or not """
+    image_types = ('ImageSeries', 'OpticalSeries', 'TwoPhotonSeries', 'ImageMaskSeries', 'VectorData')
+    return hasattr(value, 'data') and  hasattr(value, 'neurodata_type') and value.data is not None and len(value.data.shape) > 1 and not(
+            value.neurodata_type in image_types)
+
+
 class MapperType(type):
     """Metaclass for mappers: control instances and classes"""
     instances = {}
@@ -139,7 +145,7 @@ def generate_obj_id(self, pynwb_obj):
         return self.single_id(obj_id)
 
     def create_type(self, pynwb_obj, type_id=None, type_name=None):
-        if id(pynwb_obj) in self.created_types:
+        if id(pynwb_obj) in self.created_types and not is_multidimensional_data(pynwb_obj):
             return self.created_types[id(pynwb_obj)]
         obj_type = CompositeType(id=type_id, name=type_name, abstract=False)
 
@@ -188,9 +194,18 @@ def modify_type(self, pynwb_obj, obj_type):
                     logging.debug(f'No mappers are defined for: {value}')
                 continue
 
-            variable = supportingmapper.create_variable(self.sanitize(key), value, pynwb_obj)
-            self.created_variables[id(value)] = variable
-            obj_type.variables.append(variable)
+            if is_multidimensional_data(value):
+                for index in range(value.data.shape[1]):  # loop through data rows to make separate variables
+                    name = f"{self.sanitize(key)}_row{index:02}"  # pad row name so ordered correctly in list view
+                    variable = supportingmapper.create_variable(name, value, pynwb_obj)
+                    self.created_variables[id(value.data[:, index])] = variable
+                    obj_type.variables.append(variable)
+            elif is_multidimensional_data(pynwb_obj) and key == 'data':
+                continue  # create data variable in TimeSeriesMapper.modify_type instead
+            else:
+                variable = supportingmapper.create_variable(self.sanitize(key), value, pynwb_obj)
+                self.created_variables[id(value)] = variable
+                obj_type.variables.append(variable)
 
     def get_object_items(self, pynwb_obj):
         obj_dict = pynwb_obj.fields if hasattr(pynwb_obj, 'fields') else pynwb_obj
@@ -264,11 +279,14 @@ def creates(cls, value):
         return is_collection(value) and value and is_metadata(next(iter(value)))
 
     def create_variable(self, name, pynwb_obj, parent_obj):
-        value = StringArray(tuple(str(v) for v in pynwb_obj))
+        if any(hasattr(pynwb_obj[k],'decode') for k in range(len(pynwb_obj))):
+            obj = tuple(pynwb_obj[k].decode() for k in range(len(pynwb_obj)) if hasattr(pynwb_obj[k],'decode'))
+        else:
+            obj = pynwb_obj
+        value = StringArray(tuple(str(v) for v in obj))
         array_variable = self.model_factory.create_simple_array_variable(name, value)
         return array_variable
 
-
 class VectorDataMapper(NWBGeppettoMapper):
 
     def creates(self, pynwb_obj):
@@ -284,6 +302,8 @@ def create_variable_base(self, name, pynwb_obj):
 
     def get_value(self, v):
         if is_metadata(v):
+            if hasattr(v, 'decode'):
+                return Text(str(v.decode()))
             return Text(str(v))
         if id(v) in self.created_variables:
             return Pointer(path=self.created_variables[id(v)].getPath())
@@ -316,6 +336,16 @@ def modify_type(self, pynwb_obj, geppetto_composite_type):
                                                                text=next(iter(pynwb_obj.timestamp_link)).name)
             geppetto_composite_type.variables.append(variable)
 
+        if is_multidimensional_data(pynwb_obj):  # if multidimensional data, select data from relevant row
+            index = self.type_ids[pynwb_obj.name]
+            timeseries_dict = {**pynwb_obj.fields, 'data': pynwb_obj.data[:, index]}
+            import_val = ImportValueMapper.create_import_value(timeseries_dict)
+            variable = self.model_factory.create_state_variable(id="data", initialValue=import_val)
+            geppetto_composite_type.variables.append(variable)
+
+            message = f"contains values from row {index} of {pynwb_obj.name}"
+            UnsupportedMapper.handle_unsupported(geppetto_composite_type, self.model_factory, message)
+
     def get_object_items(self, pynwb_obj):
         return ((key, value) for key, value in super().get_object_items(pynwb_obj) if key != 'timestamp_link')
 

nwb_explorer/nwb_model_interpreter/nwb_model_interpreter.py

@@ -11,9 +11,9 @@
 from pygeppetto.services.model_interpreter import ModelInterpreter
 
 from nwb_explorer.nwb_model_interpreter.nwb_geppetto_mappers import *
-from .nwb_reader import NWBReader
-from .settings import *
-from ..utils import guess_units
+from nwb_explorer.nwb_model_interpreter.nwb_reader import NWBReader
+from nwb_explorer.nwb_model_interpreter.settings import *
+from nwb_explorer.utils import guess_units
 
 from jupyter_geppetto import settings, PathService
 
@@ -28,8 +28,10 @@ class NWBModelInterpreter(ModelInterpreter):
     def __init__(self, nwb_file_or_filename, source_url=None):
         if source_url == None:
             source_url = nwb_file_or_filename
-        logging.info(f'Creating a Model Interpreter for {nwb_file_or_filename}')
-        self.nwb_file_name = nwb_file_or_filename if isinstance(nwb_file_or_filename, str) else 'in-memory file'
+        logging.info(
+            f'Creating a Model Interpreter for {nwb_file_or_filename}')
+        self.nwb_file_name = nwb_file_or_filename if isinstance(
+            nwb_file_or_filename, str) else 'in-memory file'
         self.source_url = source_url
         self.nwb_reader = NWBReader(nwb_file_or_filename)
         self.library = GeppettoLibrary(name='nwbfile', id='nwbfile')
@@ -48,19 +50,23 @@ def create_model(self):
 
         geppetto_model.libraries.append(self.library)
 
-        obj_type = ImportType(autoresolve=True, url=self.nwb_file_name, id="nwbfile", name='nwbfile')
+        obj_type = ImportType(
+            autoresolve=True, url=self.nwb_file_name, id="nwbfile", name='nwbfile')
         self.library.types.append(obj_type)
-        obj_variable = Variable(id='nwbfile', name='nwbfile', types=(obj_type,))
+        obj_variable = Variable(
+            id='nwbfile', name='nwbfile', types=(obj_type,))
         geppetto_model.variables.append(obj_variable)
 
         return geppetto_model
 
     def importType(self, url, type_name, library, geppetto_model_access: GeppettoModelAccess):
         logging.info(f"Importing type {type_name}, url: {url}")
-        model_factory = GeppettoModelFactory(geppetto_model_access.geppetto_common_library)
+        model_factory = GeppettoModelFactory(
+            geppetto_model_access.geppetto_common_library)
         mapper = GenericCompositeMapper(model_factory, library)
         # build compositeTypes for pynwb objects
-        root_type = mapper.create_type(self.get_nwbfile(), type_name=type_name, type_id=type_name)
+        root_type = mapper.create_type(
+            self.get_nwbfile(), type_name=type_name, type_id=type_name)
         if isinstance(self.nwb_file_name, str) and type_name == 'nwbfile':
 
             root_type.variables.append(
@@ -72,7 +78,8 @@ def importType(self, url, type_name, library, geppetto_model_access: GeppettoMod
         return root_type
 
     def importValue(self, import_value: ImportValue):
-        logging.info(f"Importing value {import_value.eContainer().eContainer().getPath()}")
+        logging.info(
+            f"Importing value {import_value.eContainer().eContainer().getPath()}")
         nwb_obj = ImportValueMapper.import_values[import_value]
         var_to_extract = import_value.eContainer().eContainer().id
 
@@ -82,21 +89,36 @@ def importValue(self, import_value: ImportValue):
                 timestamps = NWBReader.get_timeseries_timestamps(time_series)
                 if time_series.rate is not None:
                     for index, item in enumerate(timestamps):
-                        timestamps[index] = ( timestamps[index] / time_series.rate / time_series.rate ) + time_series.starting_time
+                        timestamps[index] = (
+                            timestamps[index] / time_series.rate / time_series.rate) + time_series.starting_time
                 timestamps_unit = guess_units(time_series.timestamps_unit) if hasattr(time_series,
                                                                                       'timestamps_unit') and time_series.timestamps_unit else 's'
                 return GeppettoModelFactory.create_time_series(timestamps, timestamps_unit)
             else:
 
-                plottable_timeseries = NWBReader.get_plottable_timeseries(time_series)
+                plottable_timeseries = NWBReader.get_plottable_timeseries(
+                    time_series)
 
                 unit = guess_units(time_series.unit)
-                time_series_value = GeppettoModelFactory.create_time_series(plottable_timeseries[0], unit)
+                time_series_value = GeppettoModelFactory.create_time_series(
+                    plottable_timeseries and plottable_timeseries[0], unit)
                 if time_series.conversion is not None:
                     for index, item in enumerate(time_series_value.value):
-                        time_series_value.value[index] = time_series_value.value[index] * time_series.conversion
+                        time_series_value.value[index] = time_series_value.value[index] * \
+                            time_series.conversion
                 stringV = str(time_series_value)
                 return time_series_value
+        elif isinstance(nwb_obj, dict):
+            plottable_timeseries = NWBReader.get_mono_dimensional_timeseries_aux(
+                nwb_obj['data'])
+            unit = guess_units(nwb_obj['unit'])
+            time_series_value = GeppettoModelFactory.create_time_series(
+                plottable_timeseries and plottable_timeseries[0], unit)
+            if nwb_obj['conversion'] is not None:
+                for index, item in enumerate(time_series_value.value):
+                    time_series_value.value[index] = time_series_value.value[index] * \
+                        nwb_obj['conversion']
+            return time_series_value
         else:
             # TODO handle other possible ImportValue(s)
             pass

nwb_explorer/nwb_model_interpreter/nwb_reader.py

@@ -110,7 +110,7 @@ def __init__(self, nwbfile_or_path):
             try:
 
 
-                io = NWBHDF5IO(nwbfile_or_path, 'r')
+                io = NWBHDF5IO(nwbfile_or_path, mode='r', load_namespaces=True)
                 nwbfile = io.read()
             except Exception  as e:
                 raise ValueError('Error reading the NWB file.', e.args)

requirements.txt

@@ -1,10 +1,10 @@
 imageio==2.5.0
 jupyter_geppetto==1.1.1
-nwbwidgets==0.4.0
+nwbwidgets==0.8.0
 pyecore==0.11.6
 pygeppetto==0.8.0
-pynwb==1.3.0
-Pillow==7.0.0
+pynwb==2.0.0
+Pillow==7.1.2
 quantities==0.12.3
 nose==1.3.7
 redis==2.10.6

setup.py

@@ -16,7 +16,7 @@
 
 setuptools.setup(
     name="nwb_explorer",
-    version="0.6.0",
+    version="0.6.2",
     url="https://github.com/tarelli/nwb-explorer",
     author="MetaCell",
     author_email="info@metacell.us",
@@ -49,9 +49,9 @@
         'redis>=2.10.6',
         'seaborn>=0.8.1',
         'uuid>=1.30',
-        'pynwb>=1.2.1',
+        'pynwb>=2.0.0',
         'imageio>=2.5.0',
         'quantities>=0.12.3',
-        'nwbwidgets>=0.2.0'
+        'nwbwidgets>=0.8.0'
     ],
 )

test/test_nwb_model_interpreter.py

@@ -137,7 +137,6 @@ def test_importType(nwbfile):
     assert imported_type.id
     assert imported_type.name
     assert 'acquisition' in var_names
-    assert 'stimulus' in var_names
 
     assert len(imported_type.variables[0].types) == 1
 

test/utils.py

@@ -84,6 +84,7 @@ def create_image(name, nwbfile, external_storage):
     else:
         return ImageSeries(name=name,
                                data=np.array([imageio.imread(image_uri) for image_uri in images_uri]),
+                               unit="x",
                                timestamps=timestamps,
                                starting_frame=[0], 
                                format='png,tiff,png', 

webapp/components/Dialog.js

@@ -44,7 +44,7 @@ const AboutContent = withStyles(styles)(({ classes }) => (
     </Grid>
 
     <Box m={1}>
-      <Typography variant="h5" color={fontColor}>NWB Explorer v0.6.0</Typography>
+      <Typography variant="h5" color={fontColor}>NWB Explorer v0.6.2</Typography>
     </Box>
 
     <Box m={1} >

webapp/components/FileUrlSelector.js

@@ -35,10 +35,9 @@ export default class FileUrlSelector extends React.Component {
         <Box display="flex" alignItems="flex-end" justifyContent="space-between" className="input-with-button">
           <TextField
             id="nwb-url-input"
-            placeholder="Paste a URL pointing to an NWB v2 file"
             // helperText="Insert a public url or local absolute path of an NWB file"
             className = 'input-form-control'
-            placeholder="Paste an URL pointing to an NWB v2 file"
+            placeholder="Paste a URL pointing to an NWB v2 file"
             margin="0"
             InputLabelProps={
               { shrink: true }

webapp/components/Metadata.js

@@ -66,8 +66,7 @@ export default class Metadata extends React.Component {
       } else if (variableType.getChildren && variableType.getChildren()) {
         metadata = variable.getType().getChildren().filter(v => v.getType().getName() == 'Text').map(v => this.formatField(prettyLabel(v.getId()), this.prettyContent(v.getInitialValue().value.text)));
       } else if (variableType.getName() == 'Simple Array') {
-        metadata = variable.getInitialValue().value.elements.join(',');
-        console.log('Array:', metadata);
+        metadata = variable.getInitialValue().value.elements.map(v => v['text'] || v).join(', ');
       } else {
         console.debug('Unsupported variable', variable)
       }

webapp/components/appBar/AppBar.js

@@ -11,6 +11,7 @@ export default class Appbar extends Component {
     this.showList = this.props.showList ? this.props.showList : () => console.debug('showList not defined in ' + typeof this);
     this.showAcquisition = this.props.showAcquisition ? this.props.showAcquisition : () => console.debug('showAcquisition not defined in ' + typeof this);
     this.showStimulus = this.props.showStimulus ? this.props.showStimulus : () => console.debug('showStimulus not defined in ' + typeof this);
+    this.showProcessing = this.props.showProcessing ? this.props.showProcessing : () => console.debug('showProcessing not defined in ' + typeof this);
   }
 
   handleClickBack () {
@@ -20,6 +21,7 @@ export default class Appbar extends Component {
   handleShowLists () {
     this.showAcquisition();
     this.showStimulus();
+    this.showProcessing();
   }
 
   handleShowAll () {

webapp/components/configuration/listViewerConfiguration.js

@@ -16,8 +16,8 @@ const conf = [
       {
         id: "showPlot",
         customComponent: CustomIconComponent,
-        visible: entity =>
-          Instances.getInstance(entity.path + ".data")
+        visible: entity => entity.type !== "ImageSeries"
+          && Instances.getInstance(entity.path + ".data")
           && Instances.getInstance(entity.path + ".timestamps"),
 
         source: entity => entity,
@@ -49,7 +49,7 @@ const conf = [
         customComponent: CustomIconComponent,
         visible: entity =>
           !((Instances.getInstance(entity.path + ".data")
-        && Instances.getInstance(entity.path + ".timestamps")) || entity.type === "ImageSeries"),
+            && Instances.getInstance(entity.path + ".timestamps")) || entity.type === "ImageSeries"),
         source: entity => entity,
         configuration: {
           action: "clickShowDetails",
@@ -66,7 +66,7 @@ const conf = [
         source: entity => entity,
         visible: entity =>
           (Instances.getInstance(entity.path + ".data")
-          && Instances.getInstance(entity.path + ".timestamps")) || entity.type === "ImageSeries",
+            && Instances.getInstance(entity.path + ".timestamps")) || entity.type === "ImageSeries",
         configuration: {
           actions: "clickShowDetails",
           label: "Show details",
@@ -79,10 +79,10 @@ const conf = [
   {
     id: "path",
     title: "Path",
-    customComponent:  ({ action }) => ({ value }) => <span onClick={() => action(value)} style={{ cursor: 'pointer' }}>{value}</span>,
+    customComponent: ({ action }) => ({ value }) => <span onClick={() => action(value)} style={{ cursor: 'pointer' }}>{value}</span>,
     source: ({ path }) => path.slice(FILEVARIABLE_LENGTH),
     configuration: { action: "clickTitleDetails", }
-    
+
   },
   {
     id: "type",