Feature/fhir json schema + Remove Elasticsearch support from Grip

.github/workflows/tests.yml

@@ -130,30 +130,6 @@ jobs:
           sleep 5
           make test-conformance
 
-
-  elasticTest:
-    needs: build
-    name: Elastic Test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-      - name: Python Dependencies for Conformance
-        run: pip install requests numpy
-      - name: Download grip
-        uses: actions/download-artifact@v4
-        with:
-          name: gripBin
-      - name: Elastic Conformance
-        run: |
-          chmod +x grip
-          make start-elastic
-          sleep 15
-          ./grip server --rpc-port 18202 --http-port 18201 --config ./test/elastic.yml &
-          sleep 5
-          make test-conformance
-
-
   postgresTest:
     needs: build
     name: Postgres Test

.gitignore

@@ -35,3 +35,4 @@ gripql/R/*.tar.gz
 # dev tools
 .idea/
 .vscode/
+.DS_Store

Makefile

@@ -132,10 +132,6 @@ start-mongo:
 	@docker rm -f grip-mongodb-test > /dev/null 2>&1 || echo
 	docker run -d --name grip-mongodb-test -p 27017:27017 mongo:7.0.13-rc0-jammy > /dev/null
 
-start-elastic:
-	@docker rm -f grip-es-test > /dev/null 2>&1 || echo
-	docker run -d --name grip-es-test -p 19200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" docker.elastic.co/elasticsearch/elasticsearch:5.6.3 > /dev/null
-
 start-postgres:
 	@docker rm -f grip-postgres-test > /dev/null 2>&1 || echo
 	docker run -d --name grip-postgres-test -p 15432:5432 -e POSTGRES_PASSWORD= -e POSTGRES_USER=postgres postgres:10.4 > /dev/null

README.md

@@ -7,7 +7,7 @@
 
 https://bmeg.github.io/grip/
 
-GRIP stands for GRaph Integration Platform. It provides a graph interface on top of a variety of existing database technologies including: MongoDB, Elasticsearch, PostgreSQL, MySQL, MariaDB, Badger, and LevelDB.
+GRIP stands for GRaph Integration Platform. It provides a graph interface on top of a variety of existing database technologies including: MongoDB, PostgreSQL, MySQL, MariaDB, Badger, and LevelDB.
 
 Properties of an GRIP graph:
 

accounts/bulk_write_filter.go

@@ -52,3 +52,47 @@ func (bw *BulkWriteFilter) RecvMsg(m interface{}) error {
 		}
 	}
 }
+
+type BulkWriteRawFilter struct {
+	SS     grpc.ServerStream
+	User   string
+	Access Access
+}
+
+func (bw *BulkWriteRawFilter) SetHeader(m metadata.MD) error {
+	return bw.SS.SendHeader(m)
+}
+
+func (bw *BulkWriteRawFilter) SendHeader(m metadata.MD) error {
+	return bw.SS.SendHeader(m)
+}
+
+func (bw *BulkWriteRawFilter) SetTrailer(m metadata.MD) {
+	bw.SS.SetTrailer(m)
+}
+
+func (bw *BulkWriteRawFilter) Context() context.Context {
+	return bw.SS.Context()
+}
+
+func (bw *BulkWriteRawFilter) SendMsg(m interface{}) error {
+	return bw.SS.SendMsg(m)
+}
+
+func (bw *BulkWriteRawFilter) RecvMsg(m interface{}) error {
+	for {
+		var ge gripql.RawJson
+		err := bw.SS.RecvMsg(&ge)
+		if err != nil {
+			return err
+		}
+		err = bw.Access.Enforce(bw.User, ge.Graph, Write)
+		if err == nil {
+			mPtr := m.(*gripql.RawJson)
+			*mPtr = ge
+			return nil
+		} else {
+			log.Infof("Graph write error: %s", ge.Graph)
+		}
+	}
+}

accounts/interface.go

@@ -33,18 +33,20 @@ var MethodMap = map[string]Operation{
 	"/gripql.Job/ViewJob":    Read,
 	"/gripql.Job/ResumeJob":  Exec,
 
-	"/gripql.Edit/AddVertex":    Write,
-	"/gripql.Edit/AddEdge":      Write,
-	"/gripql.Edit/BulkAdd":      Write,
-	"/gripql.Edit/BulkDelete":   Write,
-	"/gripql.Edit/AddGraph":     Write,
-	"/gripql.Edit/DeleteGraph":  Write,
-	"/gripql.Edit/DeleteVertex": Write,
-	"/gripql.Edit/DeleteEdge":   Write,
-	"/gripql.Edit/AddIndex":     Write,
-	"/gripql.Edit/AddSchema":    Write,
-	"/gripql.Edit/AddMapping":   Write,
-	"/gripql.Edit/SampleSchema": Write, //Maybe exec?
+	"/gripql.Edit/AddVertex":     Write,
+	"/gripql.Edit/AddEdge":       Write,
+	"/gripql.Edit/BulkAdd":       Write,
+	"/gripql.Edit/BulkAddRaw":    Write,
+	"/gripql.Edit/BulkDelete":    Write,
+	"/gripql.Edit/AddGraph":      Write,
+	"/gripql.Edit/DeleteGraph":   Write,
+	"/gripql.Edit/DeleteVertex":  Write,
+	"/gripql.Edit/DeleteEdge":    Write,
+	"/gripql.Edit/AddIndex":      Write,
+	"/gripql.Edit/AddSchema":     Write,
+	"/gripql.Edit/AddJsonSchema": Write,
+	"/gripql.Edit/AddMapping":    Write,
+	"/gripql.Edit/SampleSchema":  Write, //Maybe exec?
 
 	"/gripql.Configure/StartPlugin": Admin,
 	"/gripql.Configure/ListPlugin":  Admin,

accounts/util.go

@@ -143,6 +143,8 @@ func streamAuthInterceptor(auth Authenticate, access Access) grpc.StreamServerIn
 				return handler(srv, &BulkWriteFilter{ss, user, access})
 			} else if info.FullMethod == "/gripql.Edit/BulkDelete" {
 				return handler(srv, &BulkWriteFilter{ss, user, access})
+			} else if info.FullMethod == "/gripql.Edit/BulkAddRaw" {
+				return handler(srv, &BulkWriteRawFilter{ss, user, access})
 			} else {
 				log.Errorf("Unknown input streaming op %#v!!!", info)
 				return handler(srv, ss)
@@ -189,6 +191,9 @@ func getUnaryRequestGraph(req interface{}, info *grpc.UnaryServerInfo) (string,
 	case "/gripql.Edit/AddSchema", "/gripql.Edit/AddMapping":
 		o := req.(*gripql.Graph)
 		return o.Graph, nil
+	case "/gripql.Edit/AddJsonSchema":
+		o := req.(*gripql.RawJson)
+		return o.Graph, nil
 	case "/gripql.Edit/SampleSchema":
 		o := req.(*gripql.GraphID)
 		return o.Graph, nil

cmd/jsonload/main.go

@@ -0,0 +1,88 @@
+package jsonload
+
+import (
+	"encoding/json"
+
+	"github.com/bmeg/grip/gripql"
+	"github.com/bmeg/grip/log"
+	"github.com/bmeg/grip/util"
+	"github.com/bmeg/grip/util/rpc"
+	"github.com/spf13/cobra"
+)
+
+var host = "localhost:8202"
+var NdJsonFile string
+var workerCount = 5
+var graph string
+var ExtraArgs string
+var logRate = 10000
+
+var Cmd = &cobra.Command{
+	Use:   "jsonload <NdJsonFile> <graph> <ExtraArgs>",
+	Short: "Load, Validate NdJson data into grip graph",
+	Long:  ``,
+	Args:  cobra.ExactArgs(3),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		NdJsonFile = args[0]
+		graph = args[1]
+		ExtraArgs = args[2]
+
+		var args_map map[string]any
+		if ExtraArgs != "" {
+			err := json.Unmarshal([]byte(ExtraArgs), &args_map)
+			if err != nil {
+				return err
+			}
+		}
+
+		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
+		if err != nil {
+			return err
+		}
+		resp, err := conn.ListGraphs()
+		if err != nil {
+			return err
+		}
+		found := false
+		for _, g := range resp.Graphs {
+			if graph == g {
+				found = true
+			}
+		}
+		if !found {
+			log.WithFields(log.Fields{"graph": graph}).Info("creating graph")
+			err := conn.AddGraph(graph)
+			if err != nil {
+				return err
+			}
+		}
+		elemChan := make(chan *gripql.RawJson)
+		wait := make(chan bool)
+		go func() {
+			_, err := conn.BulkAddRaw(elemChan)
+			if err != nil {
+				log.Errorf("bulk add raw error: %v", err)
+			}
+			wait <- false
+		}()
+
+		jsonChan, err := util.StreamRawJsonFromFile(NdJsonFile, workerCount, graph, args_map)
+		if err != nil {
+			return err
+		}
+		count := 0
+		for j := range jsonChan {
+			count++
+			if count%logRate == 0 {
+				log.Infof("Loaded %d vertices", count)
+			}
+			elemChan <- j
+		}
+
+		close(elemChan)
+		<-wait
+
+		log.WithFields(log.Fields{"graph": graph}).Info("loading data")
+		return nil
+	},
+}

cmd/load/main.go

@@ -6,6 +6,7 @@ import (
 	"github.com/bmeg/grip/cmd/load/example"
 	"github.com/bmeg/grip/gripql"
 	"github.com/bmeg/grip/log"
+	"github.com/bmeg/grip/schema"
 	"github.com/bmeg/grip/util"
 	"github.com/bmeg/grip/util/rpc"
 	"github.com/spf13/cobra"
@@ -153,7 +154,7 @@ var Cmd = &cobra.Command{
 
 		if jsonFile != "" {
 			log.Infof("Loading json file: %s", jsonFile)
-			graphs, err := gripql.ParseJSONGraphsFile(jsonFile)
+			graphs, err := schema.ParseJSONGraphsFile(jsonFile)
 			if err != nil {
 				return err
 			}
@@ -171,7 +172,7 @@ var Cmd = &cobra.Command{
 
 		if yamlFile != "" {
 			log.Infof("Loading YAML file: %s", yamlFile)
-			graphs, err := gripql.ParseYAMLGraphsFile(yamlFile)
+			graphs, err := schema.ParseYAMLGraphsFile(yamlFile)
 			if err != nil {
 				return err
 			}

cmd/mapping/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	"github.com/bmeg/grip/gripql"
+	graphSchema "github.com/bmeg/grip/schema"
 	"github.com/bmeg/grip/util/rpc"
 	"github.com/spf13/cobra"
 )
@@ -43,9 +44,9 @@ var getCmd = &cobra.Command{
 
 		var txt string
 		if yaml {
-			txt, err = gripql.GraphToYAMLString(schema)
+			txt, err = graphSchema.GraphToYAMLString(schema)
 		} else {
-			txt, err = gripql.GraphToJSONString(schema)
+			txt, err = graphSchema.GraphToJSONString(schema)
 		}
 		if err != nil {
 			return err
@@ -78,9 +79,9 @@ var postCmd = &cobra.Command{
 				if err != nil {
 					return err
 				}
-				graphs, err = gripql.ParseJSONGraphs(bytes)
+				graphs, err = graphSchema.ParseJSONGraphs(bytes)
 			} else {
-				graphs, err = gripql.ParseJSONGraphsFile(jsonFile)
+				graphs, err = graphSchema.ParseJSONGraphsFile(jsonFile)
 			}
 			if err != nil {
 				return err
@@ -101,9 +102,9 @@ var postCmd = &cobra.Command{
 				if err != nil {
 					return err
 				}
-				graphs, err = gripql.ParseYAMLGraphs(bytes)
+				graphs, err = graphSchema.ParseYAMLGraphs(bytes)
 			} else {
-				graphs, err = gripql.ParseYAMLGraphsFile(yamlFile)
+				graphs, err = graphSchema.ParseYAMLGraphsFile(yamlFile)
 			}
 			if err != nil {
 				return err

cmd/root.go

@@ -13,6 +13,7 @@ import (
 	"github.com/bmeg/grip/cmd/erclient"
 	"github.com/bmeg/grip/cmd/info"
 	"github.com/bmeg/grip/cmd/job"
+	"github.com/bmeg/grip/cmd/jsonload"
 	"github.com/bmeg/grip/cmd/kvload"
 	"github.com/bmeg/grip/cmd/list"
 	"github.com/bmeg/grip/cmd/load"
@@ -60,6 +61,7 @@ func init() {
 	RootCmd.AddCommand(job.Cmd)
 	RootCmd.AddCommand(load.Cmd)
 	RootCmd.AddCommand(mongoload.Cmd)
+	RootCmd.AddCommand(jsonload.Cmd)
 	RootCmd.AddCommand(query.Cmd)
 	RootCmd.AddCommand(erclient.Cmd)
 	RootCmd.AddCommand(rdf.Cmd)