Neo4j graph based on io-context document, using neomodel

files/neomodel/context-graph-model.py

@@ -0,0 +1,124 @@
+import pika
+import pprint
+import json
+import os
+from neomodel import config
+from neo4j import GraphDatabase
+
+from neomodel import (config, StructuredNode, StringProperty, IntegerProperty,
+    UniqueIdProperty, RelationshipTo, One)
+
+rabbitmq_url = os.getenv("RABBITMQ_URL")
+neo4j_url = os.getenv("NEO4J_URL")
+neo4j_username = os.getenv("NEO4J_USERNAME")
+neo4j_password = os.getenv("NEO4J_PASSWORD")
+
+#config.DRIVER = GraphDatabase().driver("bolt://192.168.49.2:31237", auth=('neo4j', 'password'))
+config.DRIVER = GraphDatabase().driver(neo4j_url, auth=(neo4j_username, neo4j_password))
+
+connection = pika.BlockingConnection(pika.URLParameters(rabbitmq_url))
+rabbitmq_channel = connection.channel()
+
+exchange_name = os.getenv('EXCHANGE_NAME')
+exchange_name = "io-context"
+result = rabbitmq_channel.queue_declare(queue="", exclusive=True)
+queue_name = result.method.queue
+
+rabbitmq_channel.queue_bind(exchange=exchange_name, queue=queue_name)
+
+class IoDocument(StructuredNode):
+    input = StringProperty(required = True)
+    output = StringProperty(required = True)
+    io_context = RelationshipTo('IoContext', 'HAS_IO_CONTEXT')
+
+class IoContext(StructuredNode):
+    name = StringProperty(unique_index=True, required = True)
+    io_document = RelationshipTo(IoDocument, 'HAS_IO_DOCUMENT')
+    io_context = RelationshipTo('IoContext', 'HAS_IO_CONTEXT')
+
+    def getLastIoDocument(self):
+        # XXX refact
+        nodes = self.io_document.filter()
+        if (len(nodes)):
+            return nodes[0]
+
+
+
+class TmuxPane(StructuredNode):
+    pane_id = IntegerProperty(required = True)
+    io_context = RelationshipTo(IoContext, 'HAS_IO_CONTEXT')
+    active_io_context = RelationshipTo(IoContext, 'ACTIVE_IO_CONTEXT')
+
+    def switchActiveIoContext(self, active_context):
+        self.active_io_context.disconnect_all()
+        self.active_io_context.connect(active_context)
+
+    def getActiveIoContext(self):
+        nodes = self.active_io_context.filter()
+        if (len(nodes)):
+            return nodes[0]
+
+
+class TmuxSession(StructuredNode):
+    name = StringProperty(unique_index=True, required = True)
+    pane = RelationshipTo(TmuxPane, 'HAS_TMUX_PANE')
+
+class TmuxHost(StructuredNode):
+    name = StringProperty(unique_index=True, required = True)
+    session = RelationshipTo(TmuxSession, 'HAS_TMUX_SESSION')
+
+
+def callback(ch, method, properties, body):
+    data = json.loads(body.decode())
+    pprint.pprint(data)
+
+
+    host = TmuxHost.get_or_create(
+        {
+            "name": data['host']
+        }
+    )[0]
+
+    tmux_session = host.session.get_or_none(name = data['metadata']['tmux']['session_name'])
+
+    if (tmux_session is None):
+        tmux_session = TmuxSession(name = data['metadata']['tmux']['session_name']).save()
+        host.session.connect(tmux_session)
+
+    pane = tmux_session.pane.get_or_none(pane_id = int(data['metadata']['tmux']['pane_id']))
+
+    if (pane is None):
+        pane = TmuxPane(pane_id = int(data['metadata']['tmux']['pane_id'])).save()
+        tmux_session.pane.connect(pane)
+
+    io_context = pane.getActiveIoContext()
+
+    if io_context is None:
+        io_context = IoContext(name = data['message']['ps1_start']).save()
+        pane.io_context.connect(io_context)
+
+
+
+    io_document = IoDocument(
+        input = data['message']['input'],
+        output = data['message']['output']
+    ).save()
+
+
+    if (data['message']['ps1_start'] != data['message']['ps1_end']):
+        # context change
+        print("New context")
+        new_io_context = IoContext(name = data['message']['ps1_end']).save()
+        pane.switchActiveIoContext(new_io_context)
+        #io_context.io_context.connect(new_io_context)
+        io_document.io_context.connect(new_io_context)
+    else:
+        #pane.getActiveIoContext().connect(io_context)
+        pane.switchActiveIoContext(io_context)
+
+
+    pane.getActiveIoContext().io_document.connect(io_document)
+
+rabbitmq_channel.basic_consume(queue=queue_name, on_message_callback=callback, auto_ack=True)
+print("Waiting for messages. To exit, press Ctrl+C")
+rabbitmq_channel.start_consuming()

files/neomodel/docker_run.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+docker run \
+    -e NEO4J_AUTH=neo4j/password \
+    -e NEO4J_apoc_export_file_enabled=true \
+    -e NEO4J_apoc_import_file_enabled=true \
+    -e NEO4J_apoc_import_file_use__neo4j__config=true \
+    -e NEO4JLABS_PLUGINS=\[\"apoc\"\] \
+   --publish=7474:7474 --publish=7687:7687 \
+   --volume=`pwd`/plugins:/plugins \
+   neo4j:4.4.0

files/neomodel/requirements.txt

@@ -0,0 +1,12 @@
+neomodel
+pika
+neo4j
+#neographviz
+neographviz
+py2neo
+ipython
+jinja2
+pygraphviz
+pydot
+networkx
+matplotlib

inventory/targets/neomodel.yml

@@ -0,0 +1,171 @@
+---
+classes:
+  - common
+  - mindwm
+  - vim
+  - tmuxinator
+  - bash.kapitan
+  - bash.script
+  - tmuxinator
+  - tmuxinator.kapitan
+  - kapitan.bash.compile-fetch
+
+parameters:
+
+  bash:
+    functions:
+      session_record_send: |
+        TMUX_HOST="\${1:-sun}"
+        TMUX_SESSION_NAME="\${2:-test}"
+        TMUX_PANE_ID="\${3:-42}"
+        SHELL_PID="\${4:-8484}"
+        sh -c "perl /home/bebebeko/mindwm/compiled/vector/pipe-pane2hexstream.pl | VECTOR_UPSTREAM='"`minikube ip`":31399' SHELL_PID='"\${SHELL_PID}"' TMUX_SESSION_NAME='"\${TMUX_SESSION_NAME}"' TMUX_PANE_ID='"\${TMUX_PANE_ID}"' vector --config /home/bebebeko/mindwm/compiled/vector/vector/tmux-bytestream.yaml"
+
+      export_json: |
+        curl -X POST -H 'Accept:application/json' -H 'Content-Type:application/json' -d "{\"statements\":[{\"statement\":\"MATCH (n) RETURN n\"}]}" -u neo4j:password -v http://`minikube ip`:31527/db/neo4j/tx/commit  | jq
+
+  target_name: neomodel
+
+
+  knowledge_graph:
+    IoDocument:
+      input:
+        type: string
+        required: true
+      output:
+        type: string
+        required: True
+      io_context: &io_context_rel_to
+        rel_to:
+          class: IoContext
+          type: HAS_IO_CONTEXT
+
+    IoContext:
+      name:
+        type: string
+        required: True
+        unique_index: True
+      io_context: *io_context_rel_to
+      io_document:
+        rel_to:
+          class: IoDocument
+          type: HAS_IO_DOCUMENT
+
+  mindwm2:
+    knowledge-graph-io-context:
+      requirements:
+        - neomodel
+        - pika
+        - neo4j
+        - neographviz
+        - py2neo
+        - ipython
+        - jinja2
+        - pygraphviz
+        - pydot
+        - networkx
+        - matplotlib
+      import:
+        - pika
+        - pprint
+        - json
+        - os
+        - from neomodel import config
+        - from neo4j import GraphDatabase
+      in:
+        rabbitmq:
+          io-context: # exchange
+            - io-context # queue-name
+
+      code: |
+        data = json.loads(body.decode())
+        pprint.pprint(data)
+
+        host = TmuxHost.get_or_create(
+            {
+                "name": data['host']
+            }
+        )[0]
+
+        tmux_session = host.session.get_or_none(name = data['metadata']['tmux']['session_name'])
+
+        if (tmux_session is None):
+            tmux_session = TmuxSession(name = data['metadata']['tmux']['session_name']).save()
+            host.session.connect(tmux_session)
+
+        pane = tmux_session.pane.get_or_none(pane_id = int(data['metadata']['tmux']['pane_id']))
+
+        if (pane is None):
+            pane = TmuxPane(pane_id = int(data['metadata']['tmux']['pane_id'])).save()
+            tmux_session.pane.connect(pane)
+
+        io_context = pane.getActiveIoContext()
+
+        if io_context is None:
+            io_context = IoContext(name = data['message']['ps1_start']).save()
+            pane.io_context.connect(io_context)
+
+
+
+        io_document = IoDocument(
+            input = data['message']['input'],
+            output = data['message']['output']
+        ).save()
+
+
+        if (data['message']['ps1_start'] != data['message']['ps1_end']):
+            new_io_context = IoContext(name = data['message']['ps1_end']).save()
+            pane.switchActiveIoContext(new_io_context)
+            io_document.io_context.connect(new_io_context)
+        else:
+            pane.switchActiveIoContext(io_context)
+
+        pane.getActiveIoContext().io_document.connect(io_document)
+
+  kapitan:
+    compile:
+      - input_type: jinja2
+        input_paths:
+          - templates/neomodel/classes.py
+        output_path: knowledge_graph
+      - input_type: jsonnet
+        output_path: knowledge_graph
+        input_paths:
+          - jsonnet/processing/dockerfile.jsonnet
+          - jsonnet/processing/processing.jsonnet
+          - jsonnet/processing/requirements.jsonnet
+          - jsonnet/processing/makefile.jsonnet
+        output_type: plain
+
+
+  tmuxinator:
+    windows:
+      neomodel-consumer:
+        panes:
+          -
+            - cd ${kapitan_root}/files/neomodel
+            - . venv/bin/activate
+            - . ./env
+            - vim context-graph-model.py
+          -
+            - cd ${kapitan_root}/files/neomodel
+            - . venv/bin/activate
+            - . ./env
+            - |
+              while :; do
+                python ./context-graph-model.py
+                sleep 5
+              done
+          -
+            - cd ${kapitan_root}/compiled/vector/
+            - . functions.bash
+            - tmux-pipe-stream
+            - id
+
+      neo4j_docker:
+        pre:
+          - cd ${kapitan_root}/files/neomodel
+        panes:
+          -
+            - bash ./docker_run.sh
+          -