Merge pull request #154 from MetaCell/release/0.4.0

Release/0.4.0

.travis.yml

@@ -1,6 +1,6 @@
 language: node_js
 node_js:
-  - "9"
+  - "13"
 cache: 
   - pip
   - npm

Dockerfile

@@ -5,6 +5,9 @@ ENV FOLDER=nwb-explorer
 USER root
 RUN apt-get update -qq &&\
     apt-get install python3-tk vim nano unzip -qq
+RUN npm i -g npm@6
+
+RUN chown -R 1000:100 $HOME/.npm
 USER $NB_UID
 
 RUN jupyter labextension disable @jupyterlab/hub-extension

README.md

@@ -39,19 +39,37 @@ conda create -n nwb-explorer python=3.7
 conda activate nwb-explorer
 ```
 
-### Local installation
+### Clone "nwb-explorer" Repository
 
-Instructions to get a development environment running.
+Clone repository using the development branch:
 
 ```bash
 git clone -b development https://github.com/MetaCell/nwb-explorer
+```
+
+### Run via Docker
+
+There is a [Dockerfile](./Dockerfile) ready to build a container for NWB-Explorer:
+
+```bash
+cd nwb-explorer
+docker build -t nwb-explorer .
+docker run -it -p8888:8888 nwb-explorer
+```
+Then, after the Docker contained has started, the application is ready at http://localhost:8888
+
+### Local Installation without Docker
+
+Instructions to get a development environment running.
+
+```bash
 cd nwb-explorer
 python utilities/install.py
 ```
 
 ## How to run NWB Explorer
 
-After the installation is complete, run the script:
+After the local installation is complete, run the script:
 
 ```bash
 cd nwb-explorer
@@ -81,7 +99,7 @@ pip install -e .
 
 ### Javascript code from sources
 
-JS/HTML code can be found inside `static/org.geppetto.frontend/src/main/webapp/`. The code needs to be rebuilt with webpack everytime there is a change. To avoid having to do so you can use the Webpack development server running in `/static/org.geppetto.frontend/src/main/webapp/` this command:
+JS/HTML code can be found inside `webapp/`. The code needs to be rebuilt with webpack everytime there is a change. To avoid having to do so you can use the Webpack development server running in `webapp/` this command:
 
 ```bash
 npm run build-dev-noTest:watch
@@ -91,17 +109,6 @@ This will spawn a process that while left running will watch for any changes on
 
 To check if a dependency is installed in development mode, run `pip list`.
 
-### Run via Docker
-
-There is a dockerfile ready to build a container for NWB-Explorer:
-
-```bash
-cd nwb-explorer/k8s
-docker build -t nwb-explorer .
-docker run -it -p8888:8888 nwb-explorer
-```
-Then, after the docker contained has started, the application is ready http://localhost:8888/geppetto
-
 ## Built With
 
 * [Geppetto](http://www.geppetto.org/) - Used to build a web-based application to interpret and visualize the NWB:N 2 files.
@@ -111,11 +118,11 @@ Then, after the docker contained has started, the application is ready http://lo
 
 ## Background
 
-The NWB Explorer was initially created by MetaCell to showcase the features of the Geppetto platform to share
-neurophysiological data in Neurodata Without Borders format. It was further developed as part of a
-Google Summer of Code project with the OpenWorm project. It is currently being extended as part of the Open Source Brain
+The NWB Explorer was initially created by [MetaCell](http://metacell.us) to showcase the features of the [Geppetto](http://www.geppetto.org/) platform to share
+neurophysiological data in [Neurodata Without Borders](https://www.nwb.org/) format. It was further developed as part of a
+[Google Summer of Code](https://summerofcode.withgoogle.com/) project for the [OpenWorm project](http://openworm.org/). It is currently being extended as part of the [Open Source Brain](http://www.opensourcebrain.org/)
 project to provide both a standalone and online application for visualising and analysing the contents of NWB:N 2 files.
-This work is currently funded by the Wellcome Trust.
+This work is currently funded by the [Wellcome Trust](https://wellcome.ac.uk/).
 
 ## Authors
 

k8s/auth.py

@@ -36,7 +36,7 @@ def get(self):
         redirection = self.get_next_url(user)
         user.spawners[server_name].environment["NWBFILE"] = ''
 
-        if 'hub/nwbfile=' in self.request.uri:
+        if 'nwbfile=' in self.request.uri:
             server_name = str(uuid.uuid4()).split('-').pop()
             redirection = f'/hub/spawn/{user.name}/{server_name}'
             url = self.request.uri.split('=').pop()

nwb-extensions/README.md

@@ -0,0 +1,3 @@
+# Custom pynwb extensions
+
+Place your extensions here

nwb-extensions/silverlab/init.py

@@ -0,0 +1 @@
+pynwb.get_class('ZplanePockelsDataset', 'silverlab_extended_schema')

nwb-extensions/silverlab/silverlab.metadata.yaml

@@ -0,0 +1,9 @@
+groups:
+- neurodata_type_def: SilverLabMetaData
+  neurodata_type_inc: LabMetaData
+  doc: A place to store Silver lab specific metadata
+  attributes:
+  - name: silverlab_api_version
+    dtype: text
+    doc: For potential future backwards compatibility, store the 'version' of this
+      API that created the file.

nwb-extensions/silverlab/silverlab.namespace.yaml

@@ -0,0 +1,11 @@
+namespaces:
+- doc: Extensions for acousto-optic lens data
+  full_name: Silver lab data extension to NWB format for acousto-optic lens experiments
+  name: silverlab_extended_schema
+  schema:
+  - namespace: core
+    neurodata_types:
+    - LabMetaData
+  - source: silverlab.ophys.yaml
+  - source: silverlab.metadata.yaml
+  version: '0.2'

nwb-extensions/silverlab/silverlab.ophys.yaml

@@ -0,0 +1,36 @@
+groups:
+- neurodata_type_def: SilverLabOptophysiology
+  neurodata_type_inc: LabMetaData
+  doc: A place to store Silver lab specific optophysiology data
+  attributes:
+  - name: cycle_time
+    dtype: float
+    doc: time in seconds for the microscope to acquire all ROIs once and return to
+      its starting position
+  - name: cycles_per_trial
+    dtype: int
+    doc: how many microscope cycles occur in each experimental trial
+  - name: frame_size
+    dtype: int
+    shape:
+    - 2
+    doc: the 2d imaging frame size in voxels
+  - name: imaging_mode
+    dtype: text
+    doc: the acquisition mode for the experiment; pointing = single-voxel ROIs, miniscan
+      = 2d rectangular ROIs, volume = 3d cuboid ROIs
+  datasets:
+  - neurodata_type_def: ZplanePockelsDataset
+    name: pockels
+    shape:
+    - null
+    - 4
+    doc: "pockels data set, recording calibration data for focusing at different z-planes\
+      \ in four columns: Z offset from focal plane (micrometres), normalised Z, 'Pockels'\
+      \ i.e. laser power in %, and z offset for drive motors"
+    attributes:
+    - name: columns
+      dtype: text
+      shape:
+      - 4
+      doc: column names for the zplane pockels dataset

nwb_explorer/__init__.py

@@ -1,4 +1,8 @@
 import logging
+import glob
+import os
+import pynwb
+from pynwb import load_namespaces
 
 from jupyter_geppetto.webapi import RouteManager
 from pygeppetto.services import model_interpreter, data_manager
@@ -7,6 +11,8 @@
 from nwb_explorer.nwb_data_manager import NWBDataManager
 from nwb_explorer.nwb_model_interpreter import NWBModelInterpreter
 
+EXTENSION_PATH = 'nwb-extensions'
+
 # Add REST API
 RouteManager.add_controller(api.NWBController)
 logging.info("Adding NWBModelInterpreter")
@@ -16,3 +22,24 @@
 logging.info("Adding NWBDataManager")
 # Replace data manager
 data_manager.set_data_manager(NWBDataManager())
+
+
+# This should be temporary. Ideally namespaces should be cached in the NWB files
+# See https://github.com/SilverLabUCL/PySilverLabNWB/issues/26
+def init_extensions():
+
+    for namespace_file in glob.glob(EXTENSION_PATH + '/**/*.namespace.*ml'):
+        extension_path = os.path.dirname(namespace_file)
+        extension_name = os.path.basename(extension_path)
+        logging.info('Initializing extension ' + extension_name)
+        load_namespaces(namespace_file)
+
+        init_file = os.path.join(extension_path, 'init.py')
+        if os.path.exists(init_file):
+            try:
+                with open(init_file) as f:
+                    for line in f:
+                        eval(line)
+            except Exception as e:
+                logging.error("Error evaluating extension init file " + init_file, exc_info=True)
+init_extensions()

nwb_explorer/api.py

@@ -2,15 +2,10 @@
 
 from jupyter_geppetto.webapi import get
 from notebook.base.handlers import IPythonHandler
-from pygeppetto.model.model_serializer import GeppettoModelSerializer
 
 from nwb_explorer.nwb_model_interpreter import NWBModelInterpreter
-from nwb_explorer.plots_manager import PlotManager
-from . import nwb_data_manager
 
 from pygeppetto.managers import GeppettoManager
-from pygeppetto.managers.geppetto_manager import RuntimeProject
-from traitlets import Instance
 cache_model = False
 
 from pygeppetto.services.data_manager import DataManagerHelper
@@ -29,6 +24,8 @@ def createNotebook(filename):
     nb0 = new_notebook(cells=[nbf.v4.new_markdown_cell("""Welcome to the NWB Explorer!
 --
 
+This interface allows you to interact with the data in your NWB file both graphically (click on the icons under the 'Controls' column on the list above) and programmatically.
+
 With this Python console you can programmatically access the loaded data using the [PyNWB Python API](https://pynwb.readthedocs.io/en/stable/).
 
 The loaded NWB:N 2 file can be accessed from the variable `nwbfile`. 

nwb_explorer/nwb_data_manager.py

@@ -61,7 +61,7 @@ def get_project_from_url(self, nwbfile):
         except Exception as e:
             raise Exception("Error retrieving file" + nwbfile) from e
         try:
-            model_interpreter = NWBModelInterpreter(nwbfilename)
+            model_interpreter = NWBModelInterpreter(nwbfilename, source_url=nwbfile)
             add_model_interpreter(model_interpreter.library.id, model_interpreter)
 
             geppetto_model = model_interpreter.create_model()

nwb_explorer/nwb_model_interpreter/nwb_geppetto_mappers.py

@@ -5,7 +5,7 @@
 from pygeppetto.model import Pointer, GenericArray, PointerElement
 
 from pynwb import TimeSeries
-from pynwb.core import NWBBaseType, ElementIdentifiers, VectorData
+from hdmf.common import ElementIdentifiers, VectorData
 from pynwb.core import LabelledDict
 from pynwb.file import Subject
 from pynwb.core import DynamicTable
@@ -18,6 +18,7 @@
 from pygeppetto.model.values import Image, Text, ImportValue, StringArray
 from pygeppetto.model.variables import Variable, TypeToValueMap
 
+
 nwb_geppetto_mappers = []
 
 
@@ -82,7 +83,6 @@ def creates(self, value):
     def create_variable(self, name, pynwb_obj, parent_obj):
         return self.model_factory.create_text_variable(id=name, text=str(pynwb_obj))
 
-
 class ImportValueMapper(NWBGeppettoMapper):
 
     def creates(self, value):
@@ -197,7 +197,7 @@ def get_object_items(self, pynwb_obj):
         if hasattr(obj_dict, 'items'):
             items = obj_dict.items()
         elif is_collection(obj_dict):
-            items = ((obj_dict[k].name, obj_dict[k]) for k in range(len(obj_dict)))
+            items = tuple((obj_dict[k].name, obj_dict[k]) for k in range(len(obj_dict)) if hasattr(obj_dict[k], "name"))
         else:
             items = ()
         return items
@@ -264,7 +264,7 @@ def creates(cls, value):
         return is_collection(value) and value and is_metadata(next(iter(value)))
 
     def create_variable(self, name, pynwb_obj, parent_obj):
-        value = StringArray(str(v) for v in pynwb_obj)
+        value = StringArray(tuple(str(v) for v in pynwb_obj))
         array_variable = self.model_factory.create_simple_array_variable(name, value)
         return array_variable
 

nwb_explorer/nwb_model_interpreter/nwb_model_interpreter.py

@@ -25,11 +25,13 @@ def assign_name_to_type(pynwb_obj):
 
 class NWBModelInterpreter(ModelInterpreter):
 
-    def __init__(self, nwb_file_name):
-
-        logging.info(f'Creating a Model Interpreter for {nwb_file_name}')
-        self.nwb_file_name = nwb_file_name if isinstance(nwb_file_name, str) else 'in-memory file'
-        self.nwb_reader = NWBReader(nwb_file_name)
+    def __init__(self, nwb_file_or_filename, source_url=None):
+        if source_url == None:
+            source_url = nwb_file_or_filename
+        logging.info(f'Creating a Model Interpreter for {nwb_file_or_filename}')
+        self.nwb_file_name = nwb_file_or_filename if isinstance(nwb_file_or_filename, str) else 'in-memory file'
+        self.source_url = source_url
+        self.nwb_reader = NWBReader(nwb_file_or_filename)
         self.library = GeppettoLibrary(name='nwbfile', id='nwbfile')
 
     @staticmethod
@@ -60,11 +62,11 @@ def importType(self, url, type_name, library, geppetto_model_access: GeppettoMod
         # build compositeTypes for pynwb objects
         root_type = mapper.create_type(self.get_nwbfile(), type_name=type_name, type_id=type_name)
         if isinstance(self.nwb_file_name, str) and type_name == 'nwbfile':
-            from nwb_explorer.nwb_data_manager import CACHE_DIRNAME
+
             root_type.variables.append(
                 model_factory.create_url_variable(
                     id='source file',
-                    url=f"{settings.home_page}/{CACHE_DIRNAME}/{os.path.basename(self.nwb_file_name)}"
+                    url=self.source_url if 'http' in self.source_url else 'file://' + self.source_url
                 )
             )
         return root_type