Merge branch 'pipeline2' into pipeline2-sync-00f2

.github/workflows/main.yml

@@ -5,7 +5,7 @@ name: CI
 # Controls when the action will run. Triggers the workflow on push only. ToDo: handle pull requests for testing only
 on:
   push:
-    branches: '*'
+    branches: '**'
     tags: 'v*'
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
@@ -26,27 +26,28 @@ jobs:
       # Decide based on branch which servers to use
       - name: Pass branch
         id: branch
-        run: echo "::set-output name=value::${GITHUB_REF##*/}";
+        run: echo "::set-output name=value::${GITHUB_REF#refs/heads/}";
+            echo "::set-output name=clean::$(echo ${GITHUB_REF#refs/heads/} | sed 's@[/\\]@-@g')";
       - name: Setup local servers
         id: local-servers
         shell: bash
-        run: if [ "${GITHUB_REF##*/}" == master ] ; then
+        run: if [ "${GITHUB_REF#refs/heads/}" == master ] ; then
             echo "::debug::Set to master setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb:7474";
             echo "::set-output name=VFB_OWL_SERVER::http://owl:8080/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://ocpu:80/ocpu/library/vfbr/R/vfb_nblast";
-            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.v4.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
-          elif [ "${GITHUB_REF##*/}" == debug ] || [ "${GITHUB_REF##*/}" == pipeline2 ] || [ "${GITHUB_REF##*/}" == vfb_geppetto_application ] || [ "${GITHUB_REF##*/}" == development ] ; then
+          elif [ "${GITHUB_REF#refs/heads/}" == debug ] || [ "${GITHUB_REF#refs/heads/}" == pipeline2 ] || [ "${GITHUB_REF#refs/heads/}" == vfb_geppetto_application ] || [ "${GITHUB_REF#refs/heads/}" == development ] ; then
             echo "::debug::Set to dev setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb:7474";
             echo "::set-output name=VFB_OWL_SERVER::http://owl:8080/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://ocpu:80/ocpu/library/vfbr/R/vfb_nblast";
             echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb-dev.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr-dev.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::development";
-          elif [ "${GITHUB_REF##*/}" == alpha ] ; then
+          elif [ "${GITHUB_REF#refs/heads/}" == alpha ] ; then
             echo "::debug::Set to alpha setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb:7474";
             echo "::set-output name=VFB_OWL_SERVER::http://owl:8080/kbs/vfb/";
@@ -59,30 +60,30 @@ jobs:
             echo "::set-output name=VFB_PDB_SERVER::http://pdb:7474";
             echo "::set-output name=VFB_OWL_SERVER::http://owl:8080/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://ocpu:80/ocpu/library/vfbr/R/vfb_nblast";
-            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.v4.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
           fi
       - name: Setup remote servers
         id: remote-servers
         shell: bash
-        run: if [ "${GITHUB_REF##*/}" == master ] ; then
+        run: if [ "${GITHUB_REF#refs/heads/}" == master ] ; then
             echo "::debug::Set to master setup";
-            echo "::set-output name=VFB_PDB_SERVER::http://pdb.virtualflybrain.org";
-            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_PDB_SERVER::http://pdb.v4.virtualflybrain.org";
+            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.v4.virtualflybrain.org";
             echo "::set-output name=VFB_OWL_SERVER::http://owl.virtualflybrain.org/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://r.virtualflybrain.org/ocpu/library/vfbr/R/vfb_nblast";
             echo "::set-output name=SOLR_SERVER::https://solr.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
-          elif [ "${GITHUB_REF##*/}" == debug ] || [ "${GITHUB_REF##*/}" == pipeline2 ] || [ "${GITHUB_REF##*/}" == vfb_geppetto_application ] || [ "${GITHUB_REF##*/}" == development ] ; then
+          elif [ "${GITHUB_REF#refs/heads/}" == debug ] || [ "${GITHUB_REF#refs/heads/}" == pipeline2 ] || [ "${GITHUB_REF#refs/heads/}" == vfb_geppetto_application ] || [ "${GITHUB_REF#refs/heads/}" == development ] ; then
             echo "::debug::Set to dev setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb-dev.virtualflybrain.org";
             echo "::set-output name=VFB_OWL_SERVER::http://owl-dev.virtualflybrain.org/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://r.virtualflybrain.org/ocpu/library/vfbr/R/vfb_nblast";
             echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb-dev.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr-dev.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
-          elif [ "${GITHUB_REF##*/}" == alpha ] ; then
+          elif [ "${GITHUB_REF#refs/heads/}" == alpha ] ; then
             echo "::debug::Set to alpha setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb-alpha.virtualflybrain.org";
             echo "::set-output name=VFB_OWL_SERVER::http://owl-alpha.virtualflybrain.org/kbs/vfb/";
@@ -92,14 +93,14 @@ jobs:
             echo "::set-output name=BUILD_TYPE::release";
           else 
             echo "::debug::Set to default setup";
-            echo "::set-output name=VFB_PDB_SERVER::http://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_PDB_SERVER::http://pdb.v4.virtualflybrain.org";
             echo "::set-output name=VFB_OWL_SERVER::http://owl.virtualflybrain.org/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://r.virtualflybrain.org/ocpu/library/vfbr/R/vfb_nblast";
-            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.v4.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
           fi
-        
+
       # Output the chosen servers
       - name: Used remote servers
         run: |
@@ -127,7 +128,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          tags: "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.value }}-local.wss"
+          tags: "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.clean }}-local.wss"
           build-args: |
             VFB_TREE_PDB_SERVER_ARG=${{ steps.local-servers.outputs.VFB_TREE_PDB_SERVER }}
             SOLR_SERVER_ARG=${{ steps.local-servers.outputs.SOLR_SERVER }}
@@ -146,7 +147,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          tags: "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.value }}-remote"
+          tags: "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.clean }}-remote"
           build-args: |
             VFB_TREE_PDB_SERVER_ARG=${{ steps.remote-servers.outputs.VFB_TREE_PDB_SERVER }}
             SOLR_SERVER_ARG=${{ steps.remote-servers.outputs.SOLR_SERVER }}
@@ -163,11 +164,14 @@ jobs:
         run: |
           echo "local:${{ steps.docker_build_local.outputs.digest }}"
           echo "remote:${{ steps.docker_build_remote.outputs.digest }}"
-                
+      
       - name: Install Puppeteer
         run: npm install jest@24.8.0 jest-image-snapshot@4.1.0 puppeteer@1.17.0 jest-puppeteer@4.3.0 @babel/preset-env@7.4.5 url-join@4.0.0 @babel/core@7.4.5
       - name: Start VFB server
-        run: docker run -t -dit --name=testServer -p 8080:8080 "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.value }}-remote";
+        run: docker run -t -dit --name=testServer -p 8080:8080 "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.clean}}-remote";
+            sleep 100;
+            docker logs testServer;
+            if [ $(docker logs testServer | grep "fixable with the " | wc -l) -gt 0 ]; then echo "Lint Error!"; exit 1; fi;
       - name: Wait for VFB server to spin up
         run: |  
           export LANDING_PAGE="http://localhost:8080/org.geppetto.frontend/geppetto"  
@@ -182,7 +186,7 @@ jobs:
       - name: Test under review  
         continue-on-error: true
         run: |
-          if [ "${GITHUB_REF##*/}" == debug ] || [ "${GITHUB_REF##*/}" == pipeline2 ] || [ "${GITHUB_REF##*/}" == vfb_geppetto_application ] || [ "${GITHUB_REF##*/}" == development ] || [[ "${GITHUB_REF##*/}" =~ ^(fix|feature).* ]] ; then
+          if [ "${GITHUB_REF#refs/heads/}" == debug ] || [ "${GITHUB_REF#refs/heads/}" == pipeline2 ] || [ "${GITHUB_REF#refs/heads/}" == vfb_geppetto_application ] || [ "${GITHUB_REF#refs/heads/}" == development ] || [[ "${GITHUB_REF#refs/heads/}" =~ ^(fix|feature).* ]] ; then
             npm test -- --verbose --colors --forceExit --testPathPattern='geppetto-vfb/tests/jest/vfb/review/.*js';
           fi
       - name: Stop VFB server   

components/VFBMain.js

@@ -37,7 +37,7 @@ class VFBMain extends React.Component {
       canvasVisible: true,
       listViewerVisible: true,
       graphVisible : true,
-      circuitBrowserVisible : false,
+      circuitBrowserVisible : true,
       htmlFromToolbar: undefined,
       idSelected: undefined,
       instanceOnFocus: undefined,
@@ -1022,6 +1022,10 @@ class VFBMain extends React.Component {
     } else if (component === "vfbCircuitBrowser") {
       let circuitBrowserVisibility = node.isVisible();
       node.setEventListener("close", () => {
+        self.setState({
+          UIUpdated: true,
+          circuitBrowserVisible: false
+        });
         self.props.vfbCircuitBrowser(ACTIONS.UPDATE_CIRCUIT_QUERY,null,false);
       });
 

components/configuration/VFBCircuitBrowser/circuitBrowserConfiguration.js

@@ -1,11 +1,27 @@
-var locationCypherQuery = ( instances, hops ) => ({
+var locationCypherQuery = ( instances, paths, weight ) => ({
   "statements": [
     {
-      "statement" : "WITH [" + instances + "] AS neurons" 
-      + " MATCH p=(x:Class)-[:synapsed_to*.." + hops.toString() + "]->(y:Class)"
-      + " WHERE x.short_form in neurons and y.short_form in neurons" 
-      + " RETURN p, neurons",
-      "resultDataContents": ["graph"]
+      "statement" : "WITH [" + instances + "] AS neurons"
+      + " WITH neurons[0] as a, neurons[1] AS b"
+      + " MATCH (source:has_neuron_connectivity {short_form: a}), (target:Neuron {short_form: b})"
+      + " CALL gds.beta.shortestPath.yens.stream({"
+      + "  nodeQuery: 'MATCH (n:has_neuron_connectivity) RETURN id(n) AS id',"
+      + "  relationshipQuery: 'MATCH (a:has_neuron_connectivity)-[r:synapsed_to]->(b:has_neuron_connectivity) WHERE exists(r.weight) AND r.weight[0] >= "
+      + weight.toString() + " RETURN id(a) AS source, id(b) AS target, type(r) as type, 5000-r.weight[0] as weight_p',"
+      + "  sourceNode: id(source),"
+      + "  targetNode: id(target),"
+     + "  k: " + paths.toString() + ","
+      + "  relationshipWeightProperty: 'weight_p',"
+      + "  relationshipTypes: ['*'],"
+      + "  path: true"
+      + "})"
+      + " YIELD index, sourceNode, targetNode, nodeIds, path"
+      + " WITH * ORDER BY index DESC"
+     + " OPTIONAL MATCH fp=(source)-[r:synapsed_to*..]->(target) WHERE ALL(n in nodes(fp) WHERE id(n) IN nodeIds)"
+      + " UNWIND r as sr WITH *, collect(id(sr)) as ids, toString(id(sr))+':'+toString(index) as relY OPTIONAL MATCH cp=(source)-[r:synapsed_to*..]-(target)"
+      + " WHERE ALL(n in nodes(cp) WHERE id(n) IN nodeIds) UNWIND ids as id"
+      + " RETURN distinct a as root, collect(distinct fp) as pp, collect(distinct cp) as p, collect(distinct id) as fr, sourceNode as source, targetNode as target, max(length(fp)) as maxHops, collect(distinct relY) as relationshipY ",
+      "resultDataContents": ["row", "graph"]
     }
   ]
 });
@@ -20,26 +36,27 @@ var configuration = {
     },
     "link" : {
       "label" : "label",
+      "weight" : "weight",
       "visible" : true,
       "tooltip" : "label"
     }
   },
-  // Minimum amount of hops allowed
+  // Minimum amount of paths allowed
   minHops : 1,
-  // Maximum amount of hops allowed
+  // Maximum amount of paths allowed
   maxHops : 6,
   // Minimum amount of neurons allowed
   minNeurons : 2,
   // Maximum amount of neurons allowed
-  maxNeurons : 5
+  maxNeurons : 2
 }
 
 var styling = {
   // Background color for canvas
   canvasColor : "black",
   // Color for links between nodes
   defaultLinkColor : "white",
-  // Color apply to links while hovering over them 
+  // Color apply to links while hovering over them
   defaultLinkHoverColor : "#11bffe",
   // Color apply to target and source nodes when hovering over a link or a node.
   defaultNeighborNodesHoverColor : "orange",
@@ -50,39 +67,39 @@ var styling = {
   // Node border color
   defaultBorderColor : "black",
   // When hovering over a node, the node's border color changes to create a halo effect
-  defaultNodeHoverBoderColor : "red", 
+  defaultNodeHoverBoderColor : "red",
   // Title bar (in node) background color
   defaultNodeTitleBackgroundColor : "#11bffe",
   // Description area (in node) background color
   defaultNodeDescriptionBackgroundColor : "white",
   nodeColorsByLabel : {
     "Template" : "#ff6cc8",
-    "Ganglion" : "#d6007d",
-    "Neuromere" : "#d6007d",
     "GABAergic" : "#9551ff",
-    "Dopaminergic" : "#9551ff",
-    "Cholinergic" : "#9551ff",
-    "Glutamatergic" : "#9551ff",
-    "Octopaminergic" : "#9551ff",
-    "Serotonergic" : "#9551ff",
-    "Motor_neuron" : "#ff6a3a",
-    "Sensory_neuron" : "#ff6a3a",
-    "Peptidergic_neuron" : "#ff6a3a",
-    "Glial_cell" : "#ff6a3a",
-    "Cell" : "#ff6a3a",
+    "Dopaminergic" : "#3551ff",
+    "Cholinergic" : "#95515f",
+    "Glutamatergic" : "#95f1ff",
+    "Octopaminergic" : "#f3511f",
+    "Serotonergic" : "#9501f0",
+    "Motor_neuron" : "#fffa30",
+    "Sensory_neuron" : "#ff3a3a",
+    "Peptidergic_neuron" : "#5f6a3a",
+    "Glial_cell" : "#ff3a6a",
     "Clone" : "#d6007d",
     "Synaptic_neuropil" : "#00a2aa",
     "License" : "#0164d8",
     "Person" : "#023f00",
     "Neuron" : "#7f2100",
-    "Neuron_projection_bundle" : "#d6007d",
+    "Neuron_projection_bundle" : "#d6327d",
     "Resource" : "#005f1d",
     "Site" : "#005f1d",
     "Expression_pattern" : "#534700",
     "Split" : "#e012e3",
     "DataSet" : "#b700b5",
-    "Anatomy" : "#00a2aa",
+    "Ganglion" : "#d6007d",
+    "Neuromere" : "#d6507d",
+    "Cell" : "#ff6a3a",
     "Property" : "#005f1d",
+    "Anatomy" : "#00a2aa",
     "_Class" : "#0164d8"
   },
   controlIcons : {
@@ -93,7 +110,7 @@ var styling = {
 }
 
 var restPostConfig = {
-  url: "https://pdb.virtualflybrain.org/db/data/transaction/commit",
+  url: "https://pdb.virtualflybrain.org/db/neo4j/tx/commit",
   contentType: "application/json"
 };
 

components/configuration/VFBGraph/graphConfiguration.js

@@ -12,7 +12,7 @@ var locationCypherQuery = instance => ({
 var whatCypherQuery = instance => ({
   "statements": [
     {
-      "statement": "MATCH (n:Entity {short_form:'" + instance + "'}) OPTIONAL MATCH p=(n)-[:INSTANCEOF|:SUBCLASSOF*..]->(x) "
+      "statement": "MATCH (n:Entity {short_form:'" + instance + "'}) OPTIONAL MATCH p=(n)-[:INSTANCEOF|SUBCLASSOF*..]->(x) "
       + "WHERE ('Anatomy' IN  labels(x)) OR (('Cell' IN  labels(x)) OR ('synaptic neuropil' IN labels(x))) "
       + " OR (('Ganglion' IN  labels(x)) OR ('Neuron_projection_bundle' IN labels(x))) "
       + "RETURN  n,p, n.short_form as root",
@@ -84,7 +84,7 @@ var styling = {
 }
 
 var restPostConfig = {
-  url: "https://pdb.virtualflybrain.org/db/data/transaction/commit",
+  url: "https://pdb-dev.virtualflybrain.org/db/neo4j/tx/commit",
   contentType: "application/json"
 };
 

components/configuration/VFBMain/layoutModel.js

@@ -71,6 +71,11 @@ var modelJson = {
             "type": "tab",
             "name": "Layers",
             "component": "vfbListViewer"
+          },
+          {
+            "type": "tab",
+            "name": "Circuit Browser",
+            "component": "vfbCircuitBrowser"
           }
         ]
       }

components/configuration/VFBMain/searchConfiguration.js

@@ -148,6 +148,11 @@ var searchConfiguration = {
           "filter_name": "Image",
           "enabled": "disabled",
         },
+        {
+          "key": "has_neuron_connectivity",
+          "filter_name": "Neurons with Connectivity",
+          "enabled": "disabled",
+        },
         {
           "key": "Synaptic_neuropil_domain",
           "filter_name": "Synaptic Neuropil",

components/configuration/VFBTermInfo/VFBTermInfoConfiguration.js

@@ -142,9 +142,9 @@ const linksConfiguration = {
   },
   // CircuitBrowser Links configuration, name of key "CircuitBrowser" must not be changed
   "CircuitBrowser": {
-    "title": "Circuit browser For",
+    "title": "Add Neuron to Circuit Browser",
     "visibility": true,
-    "superType": "hasSynapses"
+    "superType": "has_neuron_connectivity"
   }
 }
 

components/configuration/VFBToolbar/vfbtoolbarMenuConfiguration.js

@@ -245,6 +245,14 @@ var toolbarMenu = {
             parameters: ["graphVisible"]
           }
         },
+        {
+          label: "Circuit Browser",
+          icon: "fa fa-cogs",
+          action: {
+            handlerAction: "UIElementHandler",
+            parameters: ["circuitBrowserVisible"]
+          }
+        },
         {
           label: "NBLAST",
           icon: "",

components/configuration/VFBTree/VFBTreeConfiguration.js

@@ -1,5 +1,5 @@
 var restPostConfig = {
-  url: "https://pdb.virtualflybrain.org/db/data/transaction/commit",
+  url: "https://pdb.virtualflybrain.org/db/neo4j/tx/commit",
   contentType: "application/json"
 };
 
@@ -10,8 +10,10 @@ var treeCypherQuery = instance => ({
       + "<-[:depicts]-(tc:Template)<-[ie:in_register_with]-(c:Individual)-[:depicts]->(image:"
       + "Individual)-[r:INSTANCEOF]->(anat:Class:Nervous_system) WHERE exists(ie.index) WITH root, anat,r,image"
       + " MATCH p=allshortestpaths((root)<-[:SUBCLASSOF|part_of*..]-(anat)) "
+      + "UNWIND nodes(p) as n UNWIND nodes(p) as m WITH * WHERE id(n) < id(m) "
+      + "MATCH path = allShortestPaths( (n)-[:SUBCLASSOF|part_of*..1]-(m) ) "
       + "RETURN collect(distinct { node_id: id(anat), short_form: anat.short_form, image: image.short_form })"
-      + " AS image_nodes, id(root) AS root, collect(p)",
+      + " AS image_nodes, id(root) AS root, collect(path)",
       "resultDataContents": ["row", "graph"]
     }
   ]