Merge pull request #1146 from VirtualFlyBrain/debug-sync-1e8d

sync: debug  with debug-sync-1e8d

.github/workflows/main.yml

@@ -5,7 +5,7 @@ name: CI
 # Controls when the action will run. Triggers the workflow on push only. ToDo: handle pull requests for testing only
 on:
   push:
-    branches: '*'
+    branches: '**'
     tags: 'v*'
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
@@ -26,27 +26,28 @@ jobs:
       # Decide based on branch which servers to use
       - name: Pass branch
         id: branch
-        run: echo "::set-output name=value::${GITHUB_REF##*/}";
+        run: echo "::set-output name=value::${GITHUB_REF#refs/heads/}";
+            echo "::set-output name=clean::$(echo ${GITHUB_REF#refs/heads/} | sed 's@[/\\]@-@g')";
       - name: Setup local servers
         id: local-servers
         shell: bash
-        run: if [ "${GITHUB_REF##*/}" == master ] ; then
+        run: if [ "${GITHUB_REF#refs/heads/}" == master ] ; then
             echo "::debug::Set to master setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb:7474";
             echo "::set-output name=VFB_OWL_SERVER::http://owl:8080/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://ocpu:80/ocpu/library/vfbr/R/vfb_nblast";
-            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.v4.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
-          elif [ "${GITHUB_REF##*/}" == debug ] || [ "${GITHUB_REF##*/}" == pipeline2 ] || [ "${GITHUB_REF##*/}" == vfb_geppetto_application ] || [ "${GITHUB_REF##*/}" == development ] ; then
+          elif [ "${GITHUB_REF#refs/heads/}" == debug ] || [ "${GITHUB_REF#refs/heads/}" == pipeline2 ] || [ "${GITHUB_REF#refs/heads/}" == vfb_geppetto_application ] || [ "${GITHUB_REF#refs/heads/}" == development ] ; then
             echo "::debug::Set to dev setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb:7474";
             echo "::set-output name=VFB_OWL_SERVER::http://owl:8080/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://ocpu:80/ocpu/library/vfbr/R/vfb_nblast";
             echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb-dev.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr-dev.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::development";
-          elif [ "${GITHUB_REF##*/}" == alpha ] ; then
+          elif [ "${GITHUB_REF#refs/heads/}" == alpha ] ; then
             echo "::debug::Set to alpha setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb:7474";
             echo "::set-output name=VFB_OWL_SERVER::http://owl:8080/kbs/vfb/";
@@ -59,30 +60,30 @@ jobs:
             echo "::set-output name=VFB_PDB_SERVER::http://pdb:7474";
             echo "::set-output name=VFB_OWL_SERVER::http://owl:8080/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://ocpu:80/ocpu/library/vfbr/R/vfb_nblast";
-            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.v4.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
           fi
       - name: Setup remote servers
         id: remote-servers
         shell: bash
-        run: if [ "${GITHUB_REF##*/}" == master ] ; then
+        run: if [ "${GITHUB_REF#refs/heads/}" == master ] ; then
             echo "::debug::Set to master setup";
-            echo "::set-output name=VFB_PDB_SERVER::http://pdb.virtualflybrain.org";
-            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_PDB_SERVER::http://pdb.v4.virtualflybrain.org";
+            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.v4.virtualflybrain.org";
             echo "::set-output name=VFB_OWL_SERVER::http://owl.virtualflybrain.org/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://r.virtualflybrain.org/ocpu/library/vfbr/R/vfb_nblast";
             echo "::set-output name=SOLR_SERVER::https://solr.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
-          elif [ "${GITHUB_REF##*/}" == debug ] || [ "${GITHUB_REF##*/}" == pipeline2 ] || [ "${GITHUB_REF##*/}" == vfb_geppetto_application ] || [ "${GITHUB_REF##*/}" == development ] ; then
+          elif [ "${GITHUB_REF#refs/heads/}" == debug ] || [ "${GITHUB_REF#refs/heads/}" == pipeline2 ] || [ "${GITHUB_REF#refs/heads/}" == vfb_geppetto_application ] || [ "${GITHUB_REF#refs/heads/}" == development ] ; then
             echo "::debug::Set to dev setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb-dev.virtualflybrain.org";
             echo "::set-output name=VFB_OWL_SERVER::http://owl-dev.virtualflybrain.org/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://r.virtualflybrain.org/ocpu/library/vfbr/R/vfb_nblast";
             echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb-dev.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr-dev.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
-          elif [ "${GITHUB_REF##*/}" == alpha ] ; then
+          elif [ "${GITHUB_REF#refs/heads/}" == alpha ] ; then
             echo "::debug::Set to alpha setup";
             echo "::set-output name=VFB_PDB_SERVER::http://pdb-alpha.virtualflybrain.org";
             echo "::set-output name=VFB_OWL_SERVER::http://owl-alpha.virtualflybrain.org/kbs/vfb/";
@@ -92,14 +93,14 @@ jobs:
             echo "::set-output name=BUILD_TYPE::release";
           else 
             echo "::debug::Set to default setup";
-            echo "::set-output name=VFB_PDB_SERVER::http://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_PDB_SERVER::http://pdb.v4.virtualflybrain.org";
             echo "::set-output name=VFB_OWL_SERVER::http://owl.virtualflybrain.org/kbs/vfb/";
             echo "::set-output name=VFB_R_SERVER::http://r.virtualflybrain.org/ocpu/library/vfbr/R/vfb_nblast";
-            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.virtualflybrain.org";
+            echo "::set-output name=VFB_TREE_PDB_SERVER::https://pdb.v4.virtualflybrain.org";
             echo "::set-output name=SOLR_SERVER::https://solr.virtualflybrain.org/solr/ontology/select";
             echo "::set-output name=BUILD_TYPE::release";
           fi
-        
+
       # Output the chosen servers
       - name: Used remote servers
         run: |
@@ -127,7 +128,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          tags: "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.value }}-local.wss"
+          tags: "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.clean }}-local.wss"
           build-args: |
             VFB_TREE_PDB_SERVER_ARG=${{ steps.local-servers.outputs.VFB_TREE_PDB_SERVER }}
             SOLR_SERVER_ARG=${{ steps.local-servers.outputs.SOLR_SERVER }}
@@ -146,7 +147,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          tags: "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.value }}-remote"
+          tags: "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.clean }}-remote"
           build-args: |
             VFB_TREE_PDB_SERVER_ARG=${{ steps.remote-servers.outputs.VFB_TREE_PDB_SERVER }}
             SOLR_SERVER_ARG=${{ steps.remote-servers.outputs.SOLR_SERVER }}
@@ -163,11 +164,14 @@ jobs:
         run: |
           echo "local:${{ steps.docker_build_local.outputs.digest }}"
           echo "remote:${{ steps.docker_build_remote.outputs.digest }}"
-                
+      
       - name: Install Puppeteer
         run: npm install jest@24.8.0 jest-image-snapshot@4.1.0 puppeteer@1.17.0 jest-puppeteer@4.3.0 @babel/preset-env@7.4.5 url-join@4.0.0 @babel/core@7.4.5
       - name: Start VFB server
-        run: docker run -t -dit --name=testServer -p 8080:8080 "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.value }}-remote";
+        run: docker run -t -dit --name=testServer -p 8080:8080 "virtualflybrain/geppetto-vfb:${{ steps.branch.outputs.clean}}-remote";
+            sleep 100;
+            docker logs testServer;
+            if [ $(docker logs testServer | grep "fixable with the " | wc -l) -gt 0 ]; then echo "Lint Error!"; exit 1; fi;
       - name: Wait for VFB server to spin up
         run: |  
           export LANDING_PAGE="http://localhost:8080/org.geppetto.frontend/geppetto"  
@@ -182,7 +186,7 @@ jobs:
       - name: Test under review  
         continue-on-error: true
         run: |
-          if [ "${GITHUB_REF##*/}" == debug ] || [ "${GITHUB_REF##*/}" == pipeline2 ] || [ "${GITHUB_REF##*/}" == vfb_geppetto_application ] || [ "${GITHUB_REF##*/}" == development ] || [[ "${GITHUB_REF##*/}" =~ ^(fix|feature).* ]] ; then
+          if [ "${GITHUB_REF#refs/heads/}" == debug ] || [ "${GITHUB_REF#refs/heads/}" == pipeline2 ] || [ "${GITHUB_REF#refs/heads/}" == vfb_geppetto_application ] || [ "${GITHUB_REF#refs/heads/}" == development ] || [[ "${GITHUB_REF#refs/heads/}" =~ ^(fix|feature).* ]] ; then
             npm test -- --verbose --colors --forceExit --testPathPattern='geppetto-vfb/tests/jest/vfb/review/.*js';
           fi
       - name: Stop VFB server   

Dockerfile

@@ -21,8 +21,8 @@ ARG ukAcVfbGeppettoRelease=pipeline2
 
 ARG mvnOpt="-Dhttps.protocols=TLSv1.2 -DskipTests --quiet -Pmaster"
 
-ARG VFB_PDB_SERVER_ARG=http://pdb.virtualflybrain.org
-ARG VFB_TREE_PDB_SERVER_ARG=https://pdb.virtualflybrain.org
+ARG VFB_PDB_SERVER_ARG=http://pdb.v4.virtualflybrain.org
+ARG VFB_TREE_PDB_SERVER_ARG=https://pdb.v4.virtualflybrain.org
 ARG VFB_OWL_SERVER_ARG=http://owl.virtualflybrain.org/kbs/vfb/
 ARG VFB_R_SERVER_ARG=http://r.virtualflybrain.org/ocpu/library/vfbr/R/vfb_nblast
 ARG SOLR_SERVER_ARG=https://solr.virtualflybrain.org/solr/ontology/select

actions/generals.js

@@ -11,6 +11,7 @@ export const INSTANCE_SELECTED = 'INSTANCE_SELECTION';
 export const INSTANCE_DELETED = 'INSTANCE_DELETED';
 export const INSTANCE_VISIBILITY_CHANGED = 'INSTANCE_VISIBILITY_CHANGED';
 export const SHOW_LIST_VIEWER = 'SHOW_LIST_VIEWER';
+export const INVALID_ID = 'INVALID_ID';
 
 export const vfbError = errorMessage => ({
   type: VFB_ERROR,
@@ -85,3 +86,8 @@ export const showListViewer = () => ({
   type: SHOW_LIST_VIEWER,
   data : {}
 });
+
+export const invalidIdLoaded = id => ({
+  type: INVALID_ID,
+  data : { id: id }
+});

components/VFBMain.js

@@ -37,7 +37,7 @@ class VFBMain extends React.Component {
       canvasVisible: true,
       listViewerVisible: true,
       graphVisible : true,
-      circuitBrowserVisible : false,
+      circuitBrowserVisible : true,
       htmlFromToolbar: undefined,
       idSelected: undefined,
       instanceOnFocus: undefined,
@@ -230,7 +230,8 @@ class VFBMain extends React.Component {
         meta = Instances.getInstance(variableIds[singleId] + '.' + variableIds[singleId] + '_meta');
       } catch (e) {
         console.log('Instance for ' + variableIds[singleId] + '.' + variableIds[singleId] + '_meta' + ' does not exist in the current model');
-        this.vfbLoadBuffer.splice($.inArray(variableIds[singleId], window.vfbLoadBuffer), 1);
+        this.props.invalidIdLoaded(variableIds[singleId])
+        // this.vfbLoadBuffer.splice($.inArray(variableIds[singleId], window.vfbLoadBuffer), 1);
         continue;
       }
       if (this.hasVisualType(variableIds[singleId])) {
@@ -1022,6 +1023,10 @@ class VFBMain extends React.Component {
     } else if (component === "vfbCircuitBrowser") {
       let circuitBrowserVisibility = node.isVisible();
       node.setEventListener("close", () => {
+        self.setState({
+          UIUpdated: true,
+          circuitBrowserVisible: false
+        });
         self.props.vfbCircuitBrowser(ACTIONS.UPDATE_CIRCUIT_QUERY,null,false);
       });
 

components/configuration/VFBCircuitBrowser/circuitBrowserConfiguration.js

@@ -1,11 +1,26 @@
-var locationCypherQuery = ( instances, hops ) => ({
+var locationCypherQuery = ( instances, paths, weight ) => ({
   "statements": [
     {
-      "statement" : "WITH [" + instances + "] AS neurons" 
-      + " MATCH p=(x:Class)-[:synapsed_to*.." + hops.toString() + "]->(y:Class)"
-      + " WHERE x.short_form in neurons and y.short_form in neurons" 
-      + " RETURN p, neurons",
-      "resultDataContents": ["graph"]
+      "statement" : "WITH [" + instances + "] AS neurons"
+      + " WITH neurons[0] as a, neurons[1] AS b"
+      + " MATCH (source:has_neuron_connectivity {short_form: a}), (target:Neuron {short_form: b})"
+      + " CALL gds.beta.shortestPath.yens.stream({"
+      + "  nodeQuery: 'MATCH (n:Neuron) RETURN id(n) AS id',"
+      + "  relationshipQuery: 'MATCH (a:Neuron:has_neuron_connectivity)-[r:synapsed_to]->(b:Neuron) WHERE exists(r.weight) AND r.weight[0] >= "
+      + weight?.toString() + " RETURN id(a) AS source, id(b) AS target, type(r) as type, 5000-r.weight[0] as weight_p',"
+      + "  sourceNode: id(source),"
+      + "  targetNode: id(target),"
+      + "  k: " + paths?.toString() + ","
+      + "  relationshipWeightProperty: 'weight_p',"
+      + "  relationshipTypes: ['*'],"
+      + "  path: true"
+      + "})"
+      + " YIELD index, sourceNode, targetNode, nodeIds, path"
+      + " WITH * ORDER BY index DESC"
+      + " UNWIND relationships(path) as sr"
+      + " OPTIONAL MATCH cp=(x)-[:synapsed_to]-(y) WHERE x=apoc.rel.startNode(sr) AND y=apoc.rel.endNode(sr) OPTIONAL MATCH fp=(x)-[r:synapsed_to]->(y)"
+      + " RETURN distinct a as root, collect(distinct fp) as pp, collect(distinct cp) as p, collect(distinct id(r)) as fr, sourceNode as source, targetNode as target, max(length(path)) as maxHops, collect(distinct toString(id(r))+':'+toString(index)) as relationshipY ",
+      "resultDataContents": ["row", "graph"]
     }
   ]
 });
@@ -20,26 +35,29 @@ var configuration = {
     },
     "link" : {
       "label" : "label",
+      "weight" : "weight",
       "visible" : true,
       "tooltip" : "label"
     }
   },
-  // Minimum amount of hops allowed
-  minHops : 1,
-  // Maximum amount of hops allowed
-  maxHops : 6,
+  // Minimum amount of paths allowed
+  minPaths : 1,
+  // Maximum amount of paths allowed
+  maxPaths : 6,
   // Minimum amount of neurons allowed
   minNeurons : 2,
   // Maximum amount of neurons allowed
-  maxNeurons : 5
+  maxNeurons : 2,
+  // Curvature of lines, 0 is a straight line
+  linkCurvature : 0
 }
 
 var styling = {
   // Background color for canvas
   canvasColor : "black",
   // Color for links between nodes
   defaultLinkColor : "white",
-  // Color apply to links while hovering over them 
+  // Color apply to links while hovering over them
   defaultLinkHoverColor : "#11bffe",
   // Color apply to target and source nodes when hovering over a link or a node.
   defaultNeighborNodesHoverColor : "orange",
@@ -50,39 +68,39 @@ var styling = {
   // Node border color
   defaultBorderColor : "black",
   // When hovering over a node, the node's border color changes to create a halo effect
-  defaultNodeHoverBoderColor : "red", 
+  defaultNodeHoverBoderColor : "red",
   // Title bar (in node) background color
   defaultNodeTitleBackgroundColor : "#11bffe",
   // Description area (in node) background color
   defaultNodeDescriptionBackgroundColor : "white",
   nodeColorsByLabel : {
     "Template" : "#ff6cc8",
-    "Ganglion" : "#d6007d",
-    "Neuromere" : "#d6007d",
     "GABAergic" : "#9551ff",
-    "Dopaminergic" : "#9551ff",
-    "Cholinergic" : "#9551ff",
-    "Glutamatergic" : "#9551ff",
-    "Octopaminergic" : "#9551ff",
-    "Serotonergic" : "#9551ff",
-    "Motor_neuron" : "#ff6a3a",
-    "Sensory_neuron" : "#ff6a3a",
-    "Peptidergic_neuron" : "#ff6a3a",
-    "Glial_cell" : "#ff6a3a",
-    "Cell" : "#ff6a3a",
+    "Dopaminergic" : "#3551ff",
+    "Cholinergic" : "#95515f",
+    "Glutamatergic" : "#95f1ff",
+    "Octopaminergic" : "#f3511f",
+    "Serotonergic" : "#9501f0",
+    "Motor_neuron" : "#fffa30",
+    "Sensory_neuron" : "#ff3a3a",
+    "Peptidergic_neuron" : "#5f6a3a",
+    "Glial_cell" : "#ff3a6a",
     "Clone" : "#d6007d",
     "Synaptic_neuropil" : "#00a2aa",
     "License" : "#0164d8",
     "Person" : "#023f00",
     "Neuron" : "#7f2100",
-    "Neuron_projection_bundle" : "#d6007d",
+    "Neuron_projection_bundle" : "#d6327d",
     "Resource" : "#005f1d",
     "Site" : "#005f1d",
     "Expression_pattern" : "#534700",
     "Split" : "#e012e3",
     "DataSet" : "#b700b5",
-    "Anatomy" : "#00a2aa",
+    "Ganglion" : "#d6007d",
+    "Neuromere" : "#d6507d",
+    "Cell" : "#ff6a3a",
     "Property" : "#005f1d",
+    "Anatomy" : "#00a2aa",
     "_Class" : "#0164d8"
   },
   controlIcons : {
@@ -93,7 +111,7 @@ var styling = {
 }
 
 var restPostConfig = {
-  url: "https://pdb.virtualflybrain.org/db/neo4j/tx/commit",
+  url: "https://pdb-dev.virtualflybrain.org/db/neo4j/tx/commit",
   contentType: "application/json"
 };
 

components/configuration/VFBGraph/graphConfiguration.js

@@ -12,7 +12,7 @@ var locationCypherQuery = instance => ({
 var whatCypherQuery = instance => ({
   "statements": [
     {
-      "statement": "MATCH (n:Entity {short_form:'" + instance + "'}) OPTIONAL MATCH p=(n)-[:INSTANCEOF|:SUBCLASSOF*..]->(x) "
+      "statement": "MATCH (n:Entity {short_form:'" + instance + "'}) OPTIONAL MATCH p=(n)-[:INSTANCEOF|SUBCLASSOF*..]->(x) "
       + "WHERE ('Anatomy' IN  labels(x)) OR (('Cell' IN  labels(x)) OR ('synaptic neuropil' IN labels(x))) "
       + " OR (('Ganglion' IN  labels(x)) OR ('Neuron_projection_bundle' IN labels(x))) "
       + "RETURN  n,p, n.short_form as root",
@@ -84,7 +84,7 @@ var styling = {
 }
 
 var restPostConfig = {
-  url: "https://pdb.virtualflybrain.org/db/neo4j/tx/commit",
+  url: "https://pdb-dev.virtualflybrain.org/db/neo4j/tx/commit",
   contentType: "application/json"
 };
 

components/configuration/VFBMain/layoutModel.js

@@ -71,6 +71,11 @@ var modelJson = {
             "type": "tab",
             "name": "Layers",
             "component": "vfbListViewer"
+          },
+          {
+            "type": "tab",
+            "name": "Circuit Browser",
+            "component": "vfbCircuitBrowser"
           }
         ]
       }

components/configuration/VFBMain/searchConfiguration.js

@@ -148,6 +148,11 @@ var searchConfiguration = {
           "filter_name": "Image",
           "enabled": "disabled",
         },
+        {
+          "key": "has_neuron_connectivity",
+          "filter_name": "Neurons with Connectivity",
+          "enabled": "disabled",
+        },
         {
           "key": "Synaptic_neuropil_domain",
           "filter_name": "Synaptic Neuropil",