diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2f15e4fd..226630d4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -130,30 +130,6 @@ jobs: sleep 5 make test-conformance - - elasticTest: - needs: build - name: Elastic Test - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Python Dependencies for Conformance - run: pip install requests numpy - - name: Download grip - uses: actions/download-artifact@v4 - with: - name: gripBin - - name: Elastic Conformance - run: | - chmod +x grip - make start-elastic - sleep 15 - ./grip server --rpc-port 18202 --http-port 18201 --config ./test/elastic.yml & - sleep 5 - make test-conformance - - postgresTest: needs: build name: Postgres Test diff --git a/.gitignore b/.gitignore index a7c43db8..75eb8e5c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ gripql/R/*.tar.gz # dev tools .idea/ .vscode/ +.DS_Store \ No newline at end of file diff --git a/Makefile b/Makefile index e8d047d8..0af8254c 100644 --- a/Makefile +++ b/Makefile @@ -132,10 +132,6 @@ start-mongo: @docker rm -f grip-mongodb-test > /dev/null 2>&1 || echo docker run -d --name grip-mongodb-test -p 27017:27017 mongo:7.0.13-rc0-jammy > /dev/null -start-elastic: - @docker rm -f grip-es-test > /dev/null 2>&1 || echo - docker run -d --name grip-es-test -p 19200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" docker.elastic.co/elasticsearch/elasticsearch:5.6.3 > /dev/null - start-postgres: @docker rm -f grip-postgres-test > /dev/null 2>&1 || echo docker run -d --name grip-postgres-test -p 15432:5432 -e POSTGRES_PASSWORD= -e POSTGRES_USER=postgres postgres:10.4 > /dev/null diff --git a/README.md b/README.md index 3bc84476..faa2232e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ https://bmeg.github.io/grip/ -GRIP stands for GRaph Integration Platform. It provides a graph interface on top of a variety of existing database technologies including: MongoDB, Elasticsearch, PostgreSQL, MySQL, MariaDB, Badger, and LevelDB. +GRIP stands for GRaph Integration Platform. It provides a graph interface on top of a variety of existing database technologies including: MongoDB, PostgreSQL, MySQL, MariaDB, Badger, and LevelDB. Properties of an GRIP graph: diff --git a/accounts/bulk_write_filter.go b/accounts/bulk_write_filter.go index dafa875c..5f7c704b 100644 --- a/accounts/bulk_write_filter.go +++ b/accounts/bulk_write_filter.go @@ -52,3 +52,47 @@ func (bw *BulkWriteFilter) RecvMsg(m interface{}) error { } } } + +type BulkWriteRawFilter struct { + SS grpc.ServerStream + User string + Access Access +} + +func (bw *BulkWriteRawFilter) SetHeader(m metadata.MD) error { + return bw.SS.SendHeader(m) +} + +func (bw *BulkWriteRawFilter) SendHeader(m metadata.MD) error { + return bw.SS.SendHeader(m) +} + +func (bw *BulkWriteRawFilter) SetTrailer(m metadata.MD) { + bw.SS.SetTrailer(m) +} + +func (bw *BulkWriteRawFilter) Context() context.Context { + return bw.SS.Context() +} + +func (bw *BulkWriteRawFilter) SendMsg(m interface{}) error { + return bw.SS.SendMsg(m) +} + +func (bw *BulkWriteRawFilter) RecvMsg(m interface{}) error { + for { + var ge gripql.RawJson + err := bw.SS.RecvMsg(&ge) + if err != nil { + return err + } + err = bw.Access.Enforce(bw.User, ge.Graph, Write) + if err == nil { + mPtr := m.(*gripql.RawJson) + *mPtr = ge + return nil + } else { + log.Infof("Graph write error: %s", ge.Graph) + } + } +} diff --git a/accounts/interface.go b/accounts/interface.go index 776aaed9..7f86438c 100644 --- a/accounts/interface.go +++ b/accounts/interface.go @@ -33,18 +33,20 @@ var MethodMap = map[string]Operation{ "/gripql.Job/ViewJob": Read, "/gripql.Job/ResumeJob": Exec, - "/gripql.Edit/AddVertex": Write, - "/gripql.Edit/AddEdge": Write, - "/gripql.Edit/BulkAdd": Write, - "/gripql.Edit/BulkDelete": Write, - "/gripql.Edit/AddGraph": Write, - "/gripql.Edit/DeleteGraph": Write, - "/gripql.Edit/DeleteVertex": Write, - "/gripql.Edit/DeleteEdge": Write, - "/gripql.Edit/AddIndex": Write, - "/gripql.Edit/AddSchema": Write, - "/gripql.Edit/AddMapping": Write, - "/gripql.Edit/SampleSchema": Write, //Maybe exec? + "/gripql.Edit/AddVertex": Write, + "/gripql.Edit/AddEdge": Write, + "/gripql.Edit/BulkAdd": Write, + "/gripql.Edit/BulkAddRaw": Write, + "/gripql.Edit/BulkDelete": Write, + "/gripql.Edit/AddGraph": Write, + "/gripql.Edit/DeleteGraph": Write, + "/gripql.Edit/DeleteVertex": Write, + "/gripql.Edit/DeleteEdge": Write, + "/gripql.Edit/AddIndex": Write, + "/gripql.Edit/AddSchema": Write, + "/gripql.Edit/AddJsonSchema": Write, + "/gripql.Edit/AddMapping": Write, + "/gripql.Edit/SampleSchema": Write, //Maybe exec? "/gripql.Configure/StartPlugin": Admin, "/gripql.Configure/ListPlugin": Admin, diff --git a/accounts/util.go b/accounts/util.go index fe13b9cd..c70a5afd 100644 --- a/accounts/util.go +++ b/accounts/util.go @@ -143,6 +143,8 @@ func streamAuthInterceptor(auth Authenticate, access Access) grpc.StreamServerIn return handler(srv, &BulkWriteFilter{ss, user, access}) } else if info.FullMethod == "/gripql.Edit/BulkDelete" { return handler(srv, &BulkWriteFilter{ss, user, access}) + } else if info.FullMethod == "/gripql.Edit/BulkAddRaw" { + return handler(srv, &BulkWriteRawFilter{ss, user, access}) } else { log.Errorf("Unknown input streaming op %#v!!!", info) return handler(srv, ss) @@ -189,6 +191,9 @@ func getUnaryRequestGraph(req interface{}, info *grpc.UnaryServerInfo) (string, case "/gripql.Edit/AddSchema", "/gripql.Edit/AddMapping": o := req.(*gripql.Graph) return o.Graph, nil + case "/gripql.Edit/AddJsonSchema": + o := req.(*gripql.RawJson) + return o.Graph, nil case "/gripql.Edit/SampleSchema": o := req.(*gripql.GraphID) return o.Graph, nil diff --git a/cmd/jsonload/main.go b/cmd/jsonload/main.go new file mode 100644 index 00000000..1bf6ecb7 --- /dev/null +++ b/cmd/jsonload/main.go @@ -0,0 +1,88 @@ +package jsonload + +import ( + "encoding/json" + + "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" + "github.com/bmeg/grip/util" + "github.com/bmeg/grip/util/rpc" + "github.com/spf13/cobra" +) + +var host = "localhost:8202" +var NdJsonFile string +var workerCount = 5 +var graph string +var ExtraArgs string +var logRate = 10000 + +var Cmd = &cobra.Command{ + Use: "jsonload ", + Short: "Load, Validate NdJson data into grip graph", + Long: ``, + Args: cobra.ExactArgs(3), + RunE: func(cmd *cobra.Command, args []string) error { + NdJsonFile = args[0] + graph = args[1] + ExtraArgs = args[2] + + var args_map map[string]any + if ExtraArgs != "" { + err := json.Unmarshal([]byte(ExtraArgs), &args_map) + if err != nil { + return err + } + } + + conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true) + if err != nil { + return err + } + resp, err := conn.ListGraphs() + if err != nil { + return err + } + found := false + for _, g := range resp.Graphs { + if graph == g { + found = true + } + } + if !found { + log.WithFields(log.Fields{"graph": graph}).Info("creating graph") + err := conn.AddGraph(graph) + if err != nil { + return err + } + } + elemChan := make(chan *gripql.RawJson) + wait := make(chan bool) + go func() { + _, err := conn.BulkAddRaw(elemChan) + if err != nil { + log.Errorf("bulk add raw error: %v", err) + } + wait <- false + }() + + jsonChan, err := util.StreamRawJsonFromFile(NdJsonFile, workerCount, graph, args_map) + if err != nil { + return err + } + count := 0 + for j := range jsonChan { + count++ + if count%logRate == 0 { + log.Infof("Loaded %d vertices", count) + } + elemChan <- j + } + + close(elemChan) + <-wait + + log.WithFields(log.Fields{"graph": graph}).Info("loading data") + return nil + }, +} diff --git a/cmd/load/main.go b/cmd/load/main.go index b2e18422..ec71c11e 100644 --- a/cmd/load/main.go +++ b/cmd/load/main.go @@ -6,6 +6,7 @@ import ( "github.com/bmeg/grip/cmd/load/example" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" + "github.com/bmeg/grip/schema" "github.com/bmeg/grip/util" "github.com/bmeg/grip/util/rpc" "github.com/spf13/cobra" @@ -153,7 +154,7 @@ var Cmd = &cobra.Command{ if jsonFile != "" { log.Infof("Loading json file: %s", jsonFile) - graphs, err := gripql.ParseJSONGraphsFile(jsonFile) + graphs, err := schema.ParseJSONGraphsFile(jsonFile) if err != nil { return err } @@ -171,7 +172,7 @@ var Cmd = &cobra.Command{ if yamlFile != "" { log.Infof("Loading YAML file: %s", yamlFile) - graphs, err := gripql.ParseYAMLGraphsFile(yamlFile) + graphs, err := schema.ParseYAMLGraphsFile(yamlFile) if err != nil { return err } diff --git a/cmd/mapping/main.go b/cmd/mapping/main.go index f4e8605a..60dc7325 100644 --- a/cmd/mapping/main.go +++ b/cmd/mapping/main.go @@ -6,6 +6,7 @@ import ( "os" "github.com/bmeg/grip/gripql" + graphSchema "github.com/bmeg/grip/schema" "github.com/bmeg/grip/util/rpc" "github.com/spf13/cobra" ) @@ -43,9 +44,9 @@ var getCmd = &cobra.Command{ var txt string if yaml { - txt, err = gripql.GraphToYAMLString(schema) + txt, err = graphSchema.GraphToYAMLString(schema) } else { - txt, err = gripql.GraphToJSONString(schema) + txt, err = graphSchema.GraphToJSONString(schema) } if err != nil { return err @@ -78,9 +79,9 @@ var postCmd = &cobra.Command{ if err != nil { return err } - graphs, err = gripql.ParseJSONGraphs(bytes) + graphs, err = graphSchema.ParseJSONGraphs(bytes) } else { - graphs, err = gripql.ParseJSONGraphsFile(jsonFile) + graphs, err = graphSchema.ParseJSONGraphsFile(jsonFile) } if err != nil { return err @@ -101,9 +102,9 @@ var postCmd = &cobra.Command{ if err != nil { return err } - graphs, err = gripql.ParseYAMLGraphs(bytes) + graphs, err = graphSchema.ParseYAMLGraphs(bytes) } else { - graphs, err = gripql.ParseYAMLGraphsFile(yamlFile) + graphs, err = graphSchema.ParseYAMLGraphsFile(yamlFile) } if err != nil { return err diff --git a/cmd/root.go b/cmd/root.go index 4ac259b2..b2a85c8d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,6 +13,7 @@ import ( "github.com/bmeg/grip/cmd/erclient" "github.com/bmeg/grip/cmd/info" "github.com/bmeg/grip/cmd/job" + "github.com/bmeg/grip/cmd/jsonload" "github.com/bmeg/grip/cmd/kvload" "github.com/bmeg/grip/cmd/list" "github.com/bmeg/grip/cmd/load" @@ -60,6 +61,7 @@ func init() { RootCmd.AddCommand(job.Cmd) RootCmd.AddCommand(load.Cmd) RootCmd.AddCommand(mongoload.Cmd) + RootCmd.AddCommand(jsonload.Cmd) RootCmd.AddCommand(query.Cmd) RootCmd.AddCommand(erclient.Cmd) RootCmd.AddCommand(rdf.Cmd) diff --git a/cmd/schema/main.go b/cmd/schema/main.go index fb3c570d..a24e6aa0 100644 --- a/cmd/schema/main.go +++ b/cmd/schema/main.go @@ -6,8 +6,8 @@ import ( "os" "github.com/bmeg/grip/gripql" - gripql_schema "github.com/bmeg/grip/gripql/schema" "github.com/bmeg/grip/log" + "github.com/bmeg/grip/schema" "github.com/bmeg/grip/util/rpc" "github.com/spf13/cobra" ) @@ -16,10 +16,9 @@ var host = "localhost:8202" var yaml = false var jsonFile string var yamlFile string -var sampleCount uint32 = 50 -var excludeLabels []string - -var manual bool +var graphName string +var jsonSchemaFile string +var yamlSchemaPath string // Cmd line declaration var Cmd = &cobra.Command{ @@ -40,16 +39,16 @@ var getCmd = &cobra.Command{ return err } - schema, err := conn.GetSchema(graph) + gripqlschema, err := conn.GetSchema(graph) if err != nil { return err } var txt string if yaml { - txt, err = gripql.GraphToYAMLString(schema) + txt, err = schema.GraphToYAMLString(gripqlschema) } else { - txt, err = gripql.GraphToJSONString(schema) + txt, err = schema.GraphToJSONString(gripqlschema) } if err != nil { return err @@ -59,13 +58,18 @@ var getCmd = &cobra.Command{ }, } +type Config struct { + DependencyOrder []string `yaml:"dependency_order"` +} + var postCmd = &cobra.Command{ - Use: "post", - Short: "Post graph schemas", + Use: "post [graph name]", + Short: "Post jsonschema graph schemas", Long: ``, - Args: cobra.NoArgs, + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if jsonFile == "" && yamlFile == "" { + graphName := args[0] + if jsonFile == "" && yamlFile == "" && jsonSchemaFile == "" && yamlSchemaPath == "" { return fmt.Errorf("no schema file was provided") } @@ -82,9 +86,9 @@ var postCmd = &cobra.Command{ if err != nil { return err } - graphs, err = gripql.ParseJSONGraphs(bytes) + graphs, err = schema.ParseJSONGraphs(bytes) } else { - graphs, err = gripql.ParseJSONGraphsFile(jsonFile) + graphs, err = schema.ParseJSONGraphsFile(jsonFile) } if err != nil { return err @@ -106,9 +110,9 @@ var postCmd = &cobra.Command{ if err != nil { return err } - graphs, err = gripql.ParseYAMLGraphs(bytes) + graphs, err = schema.ParseYAMLGraphs(bytes) } else { - graphs, err = gripql.ParseYAMLGraphsFile(yamlFile) + graphs, err = schema.ParseYAMLGraphsFile(yamlFile) } if err != nil { return err @@ -120,46 +124,34 @@ var postCmd = &cobra.Command{ } } } - return nil - }, -} - -var sampleCmd = &cobra.Command{ - Use: "sample ", - Short: "Sample graph and construct schema", - Long: ``, - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - graph := args[0] - - conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true) - if err != nil { - return err - } - - var schema *gripql.Graph - if manual { - schema, err = gripql_schema.ScanSchema(conn, graph, sampleCount, excludeLabels) + if jsonSchemaFile != "" { + log.Infof("Loading Json Schema file: %s", jsonSchemaFile) + graphs, err := schema.ParseJsonSchema(jsonSchemaFile, graphName) if err != nil { return err } - } else { - schema, err = conn.SampleSchema(graph) + for _, g := range graphs { + err := conn.AddSchema(g) + if err != nil { + return err + } + log.Debug("Posted schema: %s", g.Graph) + } + } + if yamlSchemaPath != "" { + log.Infof("Loading Yaml Schema path: %s", yamlSchemaPath) + graphs, err := schema.ParseYamlSchemaPath(yamlSchemaPath, graphName) if err != nil { return err } + for _, g := range graphs { + err := conn.AddSchema(g) + if err != nil { + return err + } + log.Debug("Posted schema: %s", g.Graph) + } } - var txt string - if yaml { - txt, err = gripql.GraphToYAMLString(schema) - } else { - txt, err = gripql.GraphToJSONString(schema) - } - if err != nil { - return err - } - fmt.Printf("%s\n", txt) - conn.Close() return nil }, } @@ -173,15 +165,9 @@ func init() { pflags.StringVar(&host, "host", host, "grip server url") pflags.StringVar(&jsonFile, "json", "", "JSON graph file") pflags.StringVar(&yamlFile, "yaml", "", "YAML graph file") - - sflags := sampleCmd.Flags() - sflags.StringVar(&host, "host", host, "grip server url") - sflags.Uint32Var(&sampleCount, "sample", sampleCount, "Number of elements to sample") - sflags.BoolVar(&yaml, "yaml", yaml, "output schema in YAML rather than JSON format") - sflags.BoolVar(&manual, "manual", manual, "Use client side schema sampling") - sflags.StringSliceVar(&excludeLabels, "exclude-label", excludeLabels, "exclude vertex/edge label from schema") + pflags.StringVar(&jsonSchemaFile, "jsonSchema", "", "Json Schema") + pflags.StringVar(&yamlSchemaPath, "yamlSchemaPath", "", "Name of the dir that contains YAML schemas") Cmd.AddCommand(getCmd) Cmd.AddCommand(postCmd) - Cmd.AddCommand(sampleCmd) } diff --git a/config/config.go b/config/config.go index b5d6f119..db684f29 100644 --- a/config/config.go +++ b/config/config.go @@ -8,13 +8,12 @@ import ( "strings" "time" - "github.com/bmeg/grip/elastic" esql "github.com/bmeg/grip/existing-sql" "github.com/bmeg/grip/gripper" - "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" "github.com/bmeg/grip/mongo" "github.com/bmeg/grip/psql" + "github.com/bmeg/grip/schema" "github.com/bmeg/grip/sqlite" "github.com/bmeg/grip/util" "github.com/bmeg/grip/util/duration" @@ -27,17 +26,16 @@ func init() { } type DriverConfig struct { - Grids *string - Badger *string - Bolt *string - Level *string - Pebble *string - Elasticsearch *elastic.Config - MongoDB *mongo.Config - PSQL *psql.Config - ExistingSQL *esql.Config - Sqlite *sqlite.Config - Gripper *gripper.Config + Grids *string + Badger *string + Bolt *string + Level *string + Pebble *string + MongoDB *mongo.Config + PSQL *psql.Config + ExistingSQL *esql.Config + Sqlite *sqlite.Config + Gripper *gripper.Config } // Config describes the configuration for Grip. @@ -134,10 +132,6 @@ func TestifyConfig(c *Config) { if d.MongoDB != nil { d.MongoDB.DBName = "gripdb-" + rand } - if d.Elasticsearch != nil { - d.Elasticsearch.DBName = "gripdb-" + rand - d.Elasticsearch.Synchronous = true - } if d.Sqlite != nil { d.Sqlite.DBName = "gripdb-" + rand } @@ -149,9 +143,6 @@ func (c *Config) SetDefaults() { if d.MongoDB != nil { d.MongoDB.SetDefaults() } - if d.Elasticsearch != nil { - d.Elasticsearch.SetDefaults() - } } } @@ -211,7 +202,7 @@ func ParseConfigFile(relpath string, conf *Config) error { if err != nil { return fmt.Errorf("failed to parse config at path %s: \n%v", path, err) } - graph, err := gripql.GraphMapToProto(data) + graph, err := schema.GraphMapToProto(data) if err != nil { return fmt.Errorf("failed to parse config at path %s: \n%v", path, err) } diff --git a/conformance/graphs/prompt-schema.json b/conformance/graphs/prompt-schema.json new file mode 100644 index 00000000..4518b99c --- /dev/null +++ b/conformance/graphs/prompt-schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://grip-schema.io/schema/0.0.1", + "$defs": { + "Prompt": { + "$id": "http://grip-schema.io/schema/0.0.1/Prompt", + "additionalProperties": false, + "description": "User generated prompt", + "links": [ + { + "href": "{id}", + "rel": "responses", + "targetHints": { + "direction": [ + "outbound" + ], + "multiplicity": [ + "has_many" + ], + "regex_match": [ + "response:*" + ] + }, + "targetSchema": { + "$ref": "http://grip-schema.io/schema/0.0.1/Response" + }, + "templatePointers": { + "id": "/response" + } + } + ], + "properties": { + "id": { + "type": "string" + }, + "text": { + "type": "string" + }, + "resourceType": { + "type": "string" + }, + "responses": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "Response": { + "$id": "http://grip-schema.io/schema/0.0.1/Response", + "additionalProperties": false, + "description": "User LLM generated response", + "links": [ + { + "href": "{id}", + "rel": "prompt", + "targetHints": { + "direction": [ + "outbound" + ], + "multiplicity": [ + "has_one" + ], + "regex_match": [ + "prompt:*" + ] + }, + "targetSchema": { + "$ref": "http://grip-schema.io/schema/0.0.1/Prompt" + }, + "templatePointers": { + "id": "/prompt" + } + } + ], + "properties": { + "id": { + "type": "string" + }, + "text": { + "type": "string" + }, + "resourceType": { + "type": "string" + }, + "prompt": { + "type": "string" + }, + "model": { + "type": "string" + } + } + } + } +} \ No newline at end of file diff --git a/conformance/graphs/prompt.yaml b/conformance/graphs/prompt.yaml new file mode 100644 index 00000000..0f58c6f5 --- /dev/null +++ b/conformance/graphs/prompt.yaml @@ -0,0 +1,29 @@ +id: http://grip-schema.io/schema/0.0.1/Prompt +additionalProperties: false +description: User generated prompt +links: +- href: '{id}' + rel: responses + targetHints: + direction: + - outbound + multiplicity: + - has_many + regex_match: + - response:* + targetSchema: + $ref: http://grip-schema.io/schema/0.0.1/Response + templatePointers: + id: /response + +properties: + id: + type: string + text: + type: string + resourceType: + type: string + responses: + type: array + items: + type: string diff --git a/conformance/graphs/response.yaml b/conformance/graphs/response.yaml new file mode 100644 index 00000000..311ef2c5 --- /dev/null +++ b/conformance/graphs/response.yaml @@ -0,0 +1,30 @@ +id: http://grip-schema.io/schema/0.0.1/Response +additionalProperties: false +description: User LLM generated response +links: +- href: '{id}' + rel: prompt + targetHints: + direction: + - outbound + multiplicity: + - has_one + regex_match: + - prompt:* + targetSchema: + $ref: http://grip-schema.io/schema/0.0.1/Prompt + templatePointers: + id: /prompt + +properties: + id: + type: string + text: + type: string + resourceType: + type: string + prompt: + type: string + model: + type: string + diff --git a/conformance/tests/ot_bulk_raw.py b/conformance/tests/ot_bulk_raw.py new file mode 100644 index 00000000..e98afdc3 --- /dev/null +++ b/conformance/tests/ot_bulk_raw.py @@ -0,0 +1,72 @@ +import json + +def load_json_schema(path): + with open(path, 'r') as file: + content = file.read() + return json.loads(content) + + +def test_bulk_add_raw(man): + errors = [] + + G = man.writeTest() + G.addJsonSchema(load_json_schema("conformance/graphs/prompt-schema.json")) + + fetchedSchema = G.getSchema() + len_vertices = len(fetchedSchema['vertices']) + if len_vertices != 2: + errors.append(f"incorrect number of vertices in schema {len_vertices} != 2") + + bulkRaw = G.bulkAddRaw() + bulkRaw.addJson(data={"id": "prompt:0", "resourceType":"Prompt", "text": "Identify the drug-target interactions in the passage given below (along with the interaction type among the following: 'inhibitor', 'agonist', 'modulator', 'activator', 'blocker', 'inducer', 'antagonist', 'cleavage', 'disruption', 'intercalation', 'inactivator', 'bind', 'binder', 'partial agonist', 'cofactor', 'substrate', 'ligand', 'chelator', 'downregulator', 'other', 'antibody', 'other/unknown'):\n\nInhibition of rat brain monoamine oxidase activities by psoralen and isopsoralen: implications for the treatment of affective disorders. Psoralen and isopsoralen, furocoumarins isolated from the plant Psoralea corylifolia L., were demonstrated to exhibit in vitro inhibitory actions on monoamine oxidase (MAO) activities in rat brain mitochondria, preferentially inhibiting MAO-A activity over MAO-B activity. This inhibition of enzyme activities was found to be dose-dependent and reversible. For MAO-A, the IC50 values are 15.2 +/- 1.3 microM psoralen and 9.0 +/- 0.6 microM isopsoralen. For MAO-B, the IC50 values are 61.8 +/- 4.3 microM psoralen and 12.8 +/- 0.5 microM isopsoralen. Lineweaver-Burk transformation of the inhibition data indicates that inhibition by both psoralen and isopsoralen is non-competitive for MAO-A. The Ki values were calculated to be 14.0 microM for psoralen and 6.5 microM for isopsoralen. On the other hand, inhibition by both psoralen and isopsoralen is competitive for MAO-B. The Ki values were calculated to be 58.1 microM for psoralen and 10.8 microM for isopsoralen. These inhibitory actions of psoralen and isopsoralen on rat brain mitochondrial MAO activities are discussed in relation to their toxicities and their potential applications to treat affective disorders.\n", "responses": ["response:0"]}) + bulkRaw.addJson(data={"id": "prompt:1", "resourceType":"Prompt", "text": "Identify the drug-target interactions in the passage given below (along with the interaction type among the following: 'inhibitor', 'agonist', 'modulator', 'activator', 'blocker', 'inducer', 'antagonist', 'cleavage', 'disruption', 'intercalation', 'inactivator', 'bind', 'binder', 'partial agonist', 'cofactor', 'substrate', 'ligand', 'chelator', 'downregulator', 'other', 'antibody', 'other/unknown'):\n\nSelective inhibitor of Janus tyrosine kinase 3, PNU156804, prolongs allograft survival and acts synergistically with cyclosporine but additively with rapamycin.\tJanus kinase 3 (Jak3) is a cytoplasmic tyrosine (Tyr) kinase associated with the interleukin-2 (IL-2) receptor common gamma chain (gamma(c)) that is activated by multiple T-cell growth factors (TCGFs) such as IL-2, -4, and -7. Using human T cells, it was found that a recently discovered variant of the undecylprodigiosin family of antibiotics, PNU156804, previously shown to inhibit IL-2-induced cell proliferation, also blocks IL-2-mediated Jak3 auto-tyrosine phosphorylation, activation of Jak3 substrates signal transducers and activators of transcription (Stat) 5a and Stat5b, and extracellular regulated kinase 1 (Erk1) and Erk2 (p44/p42). Although PNU156804 displayed similar efficacy in blocking Jak3-dependent T-cell proliferation by IL-2, -4, -7, or -15, it was more than 2-fold less effective in blocking Jak2-mediated cell growth, its most homologous Jak family member. A 14-day alternate-day oral gavage with 40 to 120 mg/kg PNU156804 extended the survival of heart allografts in a dose-dependent fashion. In vivo, PNU156804 acted synergistically with the signal 1 inhibitor cyclosporine A (CsA) and additively with the signal 3 inhibitor rapamycin to block allograft rejection. It is concluded that inhibition of signal 3 alone by targeting Jak3 in combination with a signal 1 inhibitor provides a unique strategy to achieve potent immunosuppression.\n", "responses": ["response:1"]}) + bulkRaw.addJson(data={"id": "response:0", "resourceType":"Response", "text": "\n\nDrug: Psoralen\nTarget: Monoamine oxidase (MAO)\nInteraction Type: Inhibitor\n\nDrug: Isopsoralen\nTarget: Monoamine oxidase (MAO)\nInteraction Type: Inhibitor", "prompt": "prompt:0"}) + bulkRaw.addJson(data={"id": "response:1", "resourceType":"Response", "text": "\n\nDrug-target interactions:\n- PNU156804: inhibitor of Jak3\n- Cyclosporine A: signal 1 inhibitor\n- Rapamycin: signal 3 inhibitor\n\nInteraction types:\n- PNU156804: inhibitor\n- Cyclosporine A: inhibitor\n- Rapamycin: inhibitor/signal transduction inhibitor (exact interaction type not specified in the passage)", "prompt": "prompt:1"}) + err = bulkRaw.execute() + + if err["insertCount"] != 4: + errors.append(f"Wrong number of inserted vertices {err['insertCount']} != 4") + if len(err["errors"]) != 0: + errors.append(f"Wrong number of errors {len(err['errors'])} != 0") + + vertex = G.getVertex("prompt:0")['data']['responses'] + if vertex != ['response:0']: + errors.append("prompt:0 responses != ['response:0']") + + labels = G.listLabels() + if not all(item in labels['vertexLabels'] for item in ['Prompt', 'Response']): + errors.append(f"After insert operations {labels} != expected {{'vertexLabels': ['Condition', 'Patient']}}") + + return errors + + + +def test_bulk_add_raw_validation_error(man): + errors = [] + G = man.writeTest() + G.addJsonSchema(load_json_schema("conformance/graphs/prompt-schema.json")) + + bulkRaw = G.bulkAddRaw() + bulkRaw.addJson(data={"id": "prompt:0", "resourceType":"Prompt", "text": "Identify the drug-target interactions in the passage given below (along with the interaction type among the following: 'inhibitor', 'agonist', 'modulator', 'activator', 'blocker', 'inducer', 'antagonist', 'cleavage', 'disruption', 'intercalation', 'inactivator', 'bind', 'binder', 'partial agonist', 'cofactor', 'substrate', 'ligand', 'chelator', 'downregulator', 'other', 'antibody', 'other/unknown'):\n\nInhibition of rat brain monoamine oxidase activities by psoralen and isopsoralen: implications for the treatment of affective disorders. Psoralen and isopsoralen, furocoumarins isolated from the plant Psoralea corylifolia L., were demonstrated to exhibit in vitro inhibitory actions on monoamine oxidase (MAO) activities in rat brain mitochondria, preferentially inhibiting MAO-A activity over MAO-B activity. This inhibition of enzyme activities was found to be dose-dependent and reversible. For MAO-A, the IC50 values are 15.2 +/- 1.3 microM psoralen and 9.0 +/- 0.6 microM isopsoralen. For MAO-B, the IC50 values are 61.8 +/- 4.3 microM psoralen and 12.8 +/- 0.5 microM isopsoralen. Lineweaver-Burk transformation of the inhibition data indicates that inhibition by both psoralen and isopsoralen is non-competitive for MAO-A. The Ki values were calculated to be 14.0 microM for psoralen and 6.5 microM for isopsoralen. On the other hand, inhibition by both psoralen and isopsoralen is competitive for MAO-B. The Ki values were calculated to be 58.1 microM for psoralen and 10.8 microM for isopsoralen. These inhibitory actions of psoralen and isopsoralen on rat brain mitochondrial MAO activities are discussed in relation to their toxicities and their potential applications to treat affective disorders.\n", "responses": ["response:0"]}) + bulkRaw.addJson(data={"id": ["prompt:1"], "resourceType":"Prompt", "text": "Identify the drug-target interactions in the passage given below (along with the interaction type among the following: 'inhibitor', 'agonist', 'modulator', 'activator', 'blocker', 'inducer', 'antagonist', 'cleavage', 'disruption', 'intercalation', 'inactivator', 'bind', 'binder', 'partial agonist', 'cofactor', 'substrate', 'ligand', 'chelator', 'downregulator', 'other', 'antibody', 'other/unknown'):\n\nSelective inhibitor of Janus tyrosine kinase 3, PNU156804, prolongs allograft survival and acts synergistically with cyclosporine but additively with rapamycin.\tJanus kinase 3 (Jak3) is a cytoplasmic tyrosine (Tyr) kinase associated with the interleukin-2 (IL-2) receptor common gamma chain (gamma(c)) that is activated by multiple T-cell growth factors (TCGFs) such as IL-2, -4, and -7. Using human T cells, it was found that a recently discovered variant of the undecylprodigiosin family of antibiotics, PNU156804, previously shown to inhibit IL-2-induced cell proliferation, also blocks IL-2-mediated Jak3 auto-tyrosine phosphorylation, activation of Jak3 substrates signal transducers and activators of transcription (Stat) 5a and Stat5b, and extracellular regulated kinase 1 (Erk1) and Erk2 (p44/p42). Although PNU156804 displayed similar efficacy in blocking Jak3-dependent T-cell proliferation by IL-2, -4, -7, or -15, it was more than 2-fold less effective in blocking Jak2-mediated cell growth, its most homologous Jak family member. A 14-day alternate-day oral gavage with 40 to 120 mg/kg PNU156804 extended the survival of heart allografts in a dose-dependent fashion. In vivo, PNU156804 acted synergistically with the signal 1 inhibitor cyclosporine A (CsA) and additively with the signal 3 inhibitor rapamycin to block allograft rejection. It is concluded that inhibition of signal 3 alone by targeting Jak3 in combination with a signal 1 inhibitor provides a unique strategy to achieve potent immunosuppression.\n", "responses": ["response:1"]}) + bulkRaw.addJson(data={"id": "response:0", "resourceType":"Response", "text": "\n\nDrug: Psoralen\nTarget: Monoamine oxidase (MAO)\nInteraction Type: Inhibitor\n\nDrug: Isopsoralen\nTarget: Monoamine oxidase (MAO)\nInteraction Type: Inhibitor", "prompt": "prompt:0"}) + bulkRaw.addJson(data={"id": "response:1", "resourceType":"Response", "text": "\n\nDrug-target interactions:\n- PNU156804: inhibitor of Jak3\n- Cyclosporine A: signal 1 inhibitor\n- Rapamycin: signal 3 inhibitor\n\nInteraction types:\n- PNU156804: inhibitor\n- Cyclosporine A: inhibitor\n- Rapamycin: inhibitor/signal transduction inhibitor (exact interaction type not specified in the passage)", "prompt": "prompt:1"}) + + err = bulkRaw.execute() + if err['insertCount'] != 3 or len(err['errors']) != 1: + errors.append(f"validation error causes -1 insertCount +1 error: {err['insertCount']} != 3 or {len(err['errors'])} != 1") + + return errors + + +def test_bulk_add_raw_no_schema(man): + errors = [] + + G = man.writeTest() + bulkRaw = G.bulkAddRaw() + bulkRaw.addJson(data={"category":[{"coding":[{"code":"encounter-diagnosis","display":"Encounter Diagnosis","system":"http://terminology.hl7.org/CodeSystem/condition-category"}]}],"clinicalStatus":{"coding":[{"code":"active","system":"http://terminology.hl7.org/CodeSystem/condition-clinical"}]},"code":{"coding":[{"code":"230690007","display":"Stroke","system":"http://snomed.info/sct"}],"text":"Stroke"},"encounter":{"reference":"Encounter/f78a1442-683a-4ea2-adca-161902be19cb"},"id":"838e42fb-a65d-4039-9f83-59c37b1ae889","links":[{"href":"Patient/45c11dad-2b38-4c8e-822e-7abff8a1ee1d","rel":"subject_Patient"}],"meta":{"lastUpdated":"2023-01-26T14:21:56.658+00:00","profile":["http://hl7.org/fhir/us/core/StructureDefinition/us-core-condition"],"source":"#DmW9sueQ4yuQdyA9","versionId":"1"},"onsetDateTime":"2013-08-24T15:40:53-04:00","recordedDate":"2013-08-24T15:40:53-04:00","resourceType":"Condition","subject":{"reference":"Patient/45c11dad-2b38-4c8e-822e-7abff8a1ee1d"},"verificationStatus":{"coding":[{"code":"confirmed","system":"http://terminology.hl7.org/CodeSystem/condition-ver-status"}]}}) + err = bulkRaw.execute() + if len(err["errors"]) != 1: + errors.append("No schema was provided so error count should equal 1") + + return [] diff --git a/conformance/tests/ot_bulk_raw_yaml.py b/conformance/tests/ot_bulk_raw_yaml.py new file mode 100644 index 00000000..bc40a17b --- /dev/null +++ b/conformance/tests/ot_bulk_raw_yaml.py @@ -0,0 +1,42 @@ +import json +import yaml + +def test_post_yaml_schema(man): + errors = [] + G = man.writeTest() + + prompt = process_graph_schema("conformance/graphs/prompt.yaml") + response = process_graph_schema("conformance/graphs/response.yaml") + + G.addSchema(vertices=prompt) + G.addSchema(vertices=response) + + fetchedSchema = G.getSchema() + len_vertices = len(fetchedSchema['vertices']) + if len_vertices != 2: + errors.append(f"incorrect number of vertices in schema {len_vertices} != 2") + + bulkRaw = G.bulkAddRaw() + bulkRaw.addJson(data={"id": "prompt:0", "resourceType":"Prompt", "text": "Identify the drug-target interactions in the passage given below (along with the interaction type among the following: 'inhibitor', 'agonist', 'modulator', 'activator', 'blocker', 'inducer', 'antagonist', 'cleavage', 'disruption', 'intercalation', 'inactivator', 'bind', 'binder', 'partial agonist', 'cofactor', 'substrate', 'ligand', 'chelator', 'downregulator', 'other', 'antibody', 'other/unknown'):\n\nInhibition of rat brain monoamine oxidase activities by psoralen and isopsoralen: implications for the treatment of affective disorders. Psoralen and isopsoralen, furocoumarins isolated from the plant Psoralea corylifolia L., were demonstrated to exhibit in vitro inhibitory actions on monoamine oxidase (MAO) activities in rat brain mitochondria, preferentially inhibiting MAO-A activity over MAO-B activity. This inhibition of enzyme activities was found to be dose-dependent and reversible. For MAO-A, the IC50 values are 15.2 +/- 1.3 microM psoralen and 9.0 +/- 0.6 microM isopsoralen. For MAO-B, the IC50 values are 61.8 +/- 4.3 microM psoralen and 12.8 +/- 0.5 microM isopsoralen. Lineweaver-Burk transformation of the inhibition data indicates that inhibition by both psoralen and isopsoralen is non-competitive for MAO-A. The Ki values were calculated to be 14.0 microM for psoralen and 6.5 microM for isopsoralen. On the other hand, inhibition by both psoralen and isopsoralen is competitive for MAO-B. The Ki values were calculated to be 58.1 microM for psoralen and 10.8 microM for isopsoralen. These inhibitory actions of psoralen and isopsoralen on rat brain mitochondrial MAO activities are discussed in relation to their toxicities and their potential applications to treat affective disorders.\n", "responses": ["response:0"]}) + bulkRaw.addJson(data={"id": "prompt:1", "resourceType":"Prompt", "text": "Identify the drug-target interactions in the passage given below (along with the interaction type among the following: 'inhibitor', 'agonist', 'modulator', 'activator', 'blocker', 'inducer', 'antagonist', 'cleavage', 'disruption', 'intercalation', 'inactivator', 'bind', 'binder', 'partial agonist', 'cofactor', 'substrate', 'ligand', 'chelator', 'downregulator', 'other', 'antibody', 'other/unknown'):\n\nSelective inhibitor of Janus tyrosine kinase 3, PNU156804, prolongs allograft survival and acts synergistically with cyclosporine but additively with rapamycin.\tJanus kinase 3 (Jak3) is a cytoplasmic tyrosine (Tyr) kinase associated with the interleukin-2 (IL-2) receptor common gamma chain (gamma(c)) that is activated by multiple T-cell growth factors (TCGFs) such as IL-2, -4, and -7. Using human T cells, it was found that a recently discovered variant of the undecylprodigiosin family of antibiotics, PNU156804, previously shown to inhibit IL-2-induced cell proliferation, also blocks IL-2-mediated Jak3 auto-tyrosine phosphorylation, activation of Jak3 substrates signal transducers and activators of transcription (Stat) 5a and Stat5b, and extracellular regulated kinase 1 (Erk1) and Erk2 (p44/p42). Although PNU156804 displayed similar efficacy in blocking Jak3-dependent T-cell proliferation by IL-2, -4, -7, or -15, it was more than 2-fold less effective in blocking Jak2-mediated cell growth, its most homologous Jak family member. A 14-day alternate-day oral gavage with 40 to 120 mg/kg PNU156804 extended the survival of heart allografts in a dose-dependent fashion. In vivo, PNU156804 acted synergistically with the signal 1 inhibitor cyclosporine A (CsA) and additively with the signal 3 inhibitor rapamycin to block allograft rejection. It is concluded that inhibition of signal 3 alone by targeting Jak3 in combination with a signal 1 inhibitor provides a unique strategy to achieve potent immunosuppression.\n", "responses": ["response:1"]}) + bulkRaw.addJson(data={"id": "response:0", "resourceType":"Response", "text": "\n\nDrug: Psoralen\nTarget: Monoamine oxidase (MAO)\nInteraction Type: Inhibitor\n\nDrug: Isopsoralen\nTarget: Monoamine oxidase (MAO)\nInteraction Type: Inhibitor", "prompt": "prompt:0"}) + bulkRaw.addJson(data={"id": "response:1", "resourceType":"Response", "text": "\n\nDrug-target interactions:\n- PNU156804: inhibitor of Jak3\n- Cyclosporine A: signal 1 inhibitor\n- Rapamycin: signal 3 inhibitor\n\nInteraction types:\n- PNU156804: inhibitor\n- Cyclosporine A: inhibitor\n- Rapamycin: inhibitor/signal transduction inhibitor (exact interaction type not specified in the passage)", "prompt": "prompt:1"}) + err = bulkRaw.execute() + + if err["insertCount"] != 4: + errors.append(f"Wrong number of inserted vertices {err['insertCount']} != 4") + if len(err["errors"]) != 0: + errors.append(f"Wrong number of errors {len(err['errors'])} != 0") + + vertex = G.getVertex("prompt:0")['data']['responses'] + if vertex != ['response:0']: + errors.append("prompt:0 responses != ['response:0']") + + return errors + +def process_graph_schema(path): + with open(path, 'r') as file: + content = file.read() + vertex = yaml.safe_load(content) + id = vertex["id"].split("/")[-1] + return [{"data":vertex, "label": id, "gid": vertex["id"] }] diff --git a/conformance/tests/ot_schema.py b/conformance/tests/ot_schema.py index 2493a9f0..18ae04fa 100644 --- a/conformance/tests/ot_schema.py +++ b/conformance/tests/ot_schema.py @@ -1,4 +1,5 @@ +import json def test_getscheama(man): errors = [] @@ -26,5 +27,25 @@ def test_getscheama(man): errors.append("Incorrect labels returned from sampling: %s != %s " % (eLabels, eExpectedLabels) ) + return errors + + +def test_post_json_schema(man): + errors = [] + G = man.setGraph("swapi") + G.addJsonSchema(load_json_schema("conformance/graphs/prompt-schema.json")) + fetched_schema = G.getSchema() + len_vertices = len(fetched_schema['vertices']) + if len_vertices != 2: + errors.append(f"incorrect number of vertices in schema {len_vertices} != 2") + len_edges = len(fetched_schema['edges']) + if len_edges != 0: + errors.append(f"incorrect number of edges in schema {len_vertices} != 0") return errors + + +def load_json_schema(path): + with open(path, 'r') as file: + content = file.read() + return json.loads(content) diff --git a/conformance/tests/ot_transform.py b/conformance/tests/ot_transform.py index f3c9fdc4..77c385fe 100644 --- a/conformance/tests/ot_transform.py +++ b/conformance/tests/ot_transform.py @@ -1,4 +1,5 @@ +from re import I import gripql def test_count(man): @@ -8,6 +9,7 @@ def test_count(man): q = G.query().V().hasLabel("Planet").unwind("terrain").aggregate(gripql.term("t", "terrain")) count = 0 + for row in q: if row['key'] not in ['rainforests', 'desert', 'mountains', 'jungle', 'rainforests', 'grasslands']: errors.append("Incorrect value %s returned" % row['key']) @@ -19,3 +21,28 @@ def test_count(man): errors.append("Incorrect # elements returned") return errors + + +def test_unwind(man): + errors = [] + G = man.writeTest() + bulk = G.bulkAdd() + bulk.addVertex("1", "Observation", {"resourceType": "Observation", "id": "e87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-9", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + err = bulk.execute() + + q = G.query().V().hasLabel("Observation").unwind("component") + count = 0 + for row in q: + #print("ROW: ", row) + count +=1 + if count != 3: + errors.append("There should be 3 vertices after unwind process") + + q = G.query().V().hasLabel("Observation").unwind("component").has(gripql.gt("component.valueQuantity.value", 1)) + count = 0 + for r in q: + count +=1 + if count != 2: + errors.append("There should be 2 vertices after unwind process and filter") + + return errors diff --git a/docs/categories/index.xml b/docs/categories/index.xml index bb6d02fd..8f7de393 100644 --- a/docs/categories/index.xml +++ b/docs/categories/index.xml @@ -4,11 +4,8 @@ Categories on GRIP https://bmeg.github.io/grip/categories/ Recent content in Categories on GRIP - Hugo -- gohugo.io + Hugo en-us - - - - + - \ No newline at end of file + diff --git a/docs/docs/commands/drop/index.html b/docs/docs/commands/drop/index.html index 5401498c..6f949ce1 100644 --- a/docs/docs/commands/drop/index.html +++ b/docs/docs/commands/drop/index.html @@ -32,7 +32,7 @@