feat: connect different schemas in registry by giving in reference while creating entity

Makefile

@@ -42,7 +42,7 @@ test:
 	@docker-compose down
 	@rm -rf db-data-2 || echo "no permission to delete"
 	# test with native search service
-	@RELEASE_VERSION=latest KEYCLOAK_IMPORT_DIR=java/apitest/src/test/resources KEYCLOAK_SECRET=a52c5f4a-89fd-40b9-aea2-3f711f14c889 DB_DIR=db-data-3 SEARCH_PROVIDER_NAME=dev.sunbirdrc.registry.service.NativeSearchService docker-compose up -d db keycloak registry certificate-signer certificate-api
+	@RELEASE_VERSION=latest KEYCLOAK_IMPORT_DIR=java/apitest/src/test/resources KEYCLOAK_SECRET=a52c5f4a-89fd-40b9-aea2-3f711f14c889 DB_DIR=db-data-3 SEARCH_PROVIDER_NAME=dev.sunbirdrc.registry.service.NativeSearchService EXPAND_REFERENCE=false docker-compose up -d db keycloak registry certificate-signer certificate-api
 	@echo "Starting the test" && sh build/wait_for_port.sh 8080
 	@echo "Starting the test" && sh build/wait_for_port.sh 8081
 	@docker-compose ps
@@ -59,6 +59,15 @@ test:
 	@cd java/apitest && MODE=async ../mvnw -Pe2e test || echo 'Tests failed'
 	@docker-compose down
 	@rm -rf db-data-4 || echo "no permission to delete"
+	# test with elastic search service
+	@RELEASE_VERSION=latest KEYCLOAK_IMPORT_DIR=java/apitest/src/test/resources KEYCLOAK_SECRET=a52c5f4a-89fd-40b9-aea2-3f711f14c889 DB_DIR=db-data-5 SEARCH_PROVIDER_NAME=dev.sunbirdrc.registry.service.ElasticSearchService EXPAND_REFERENCE=false docker-compose up -d db keycloak registry certificate-signer certificate-api
+	@echo "Starting the test" && sh build/wait_for_port.sh 8080
+	@echo "Starting the test" && sh build/wait_for_port.sh 8081
+	@docker-compose ps
+	@curl -v http://localhost:8081/health
+	@cd java/apitest && ../mvnw -Pe2e test || echo 'Tests failed'
+	@docker-compose down
+	@rm -rf db-data-5 || echo "no permission to delete"
 	make -C services/certificate-signer test
 	make -C services/public-key-service test
 	make -C services/context-proxy-service test

docker-compose.yml

@@ -73,6 +73,7 @@ services:
       - redis_host=redis
       - redis_port=6379
       - manager_type=${MANAGER_TYPE-DefinitionsManager}
+      - expand_reference=${EXPAND_REFERENCE-false}
     ports:
       - "8081:8081"
     depends_on:

java/apitest/src/test/java/e2e/registry/inviteFlow.json

@@ -11,12 +11,13 @@
   },
   "instituteSchema": {
     "name": "Institute",
-    "schema": "{\n  \"$schema\": \"http://json-schema.org/draft-07/schema\",\n  \"type\": \"object\",\n  \"properties\": {\n    \"Institute\": {\n      \"$ref\": \"#/definitions/Institute\"\n    }\n  },\n  \"required\": [\n    \"Institute\"\n  ],\n  \"title\":\"Institute\",\n  \"definitions\": {\n    \"Institute\": {\n      \"$id\": \"#/properties/Institute\",\n      \"type\": \"object\",\n      \"title\": \"The Institute Schema\",\n      \"required\": [\n\n      ],\n      \"properties\": {\n        \"name\": {\n          \"type\": \"string\"\n        },\n        \"email\": {\n          \"type\": \"string\"\n        },\n        \"mobile\": {\n          \"type\": \"string\"\n        }\n      }\n    }\n  },\n  \"_osConfig\": {\n    \"osComment\": [\"This section contains the OpenSABER specific configuration information\",\n      \"privateFields: Optional; list of field names to be encrypted and stored in database\",\n      \"signedFields: Optional; list of field names that must be pre-signed\",\n      \"indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, studentCode) is a composite index and studentName is a single column index.\",\n      \"uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name\",\n      \"systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid\"],\n\n    \"privateFields\": [\n\n    ],\n    \"signedFields\": [],\n    \"indexFields\": [],\n    \"uniqueIndexFields\": [],\n    \"systemFields\": [\n      \"_osCreatedAt\",\n      \"_osUpdatedAt\",\n      \"_osCreatedBy\",\n      \"_osUpdatedBy\",\n      \"_osAttestedData\",\n      \"_osClaimId\",\n      \"_osState\"\n    ],\n    \"inviteRoles\":[\"Board\"],\n    \"enableLogin\": true,\n    \"ownershipAttributes\": [\n      {\n        \"email\": \"/email\",\n        \"mobile\": \"/mobile\",\n        \"userId\": \"/mobile\"\n      }\n    ]\n  }\n}",
+    "schema": "{\n  \"$schema\": \"http://json-schema.org/draft-07/schema\",\n  \"type\": \"object\",\n  \"properties\": {\n    \"Institute\": {\n      \"$ref\": \"#/definitions/Institute\"\n    }\n  },\n  \"required\": [\n    \"Institute\"\n  ],\n  \"title\":\"Institute\",\n  \"definitions\": {\n    \"Institute\": {\n      \"$id\": \"#/properties/Institute\",\n      \"type\": \"object\",\n      \"title\": \"The Institute Schema\",\n      \"required\": [\n\n      ],\n      \"properties\": {\n        \"name\": {\n          \"type\": \"string\"\n        },\n        \"email\": {\n          \"type\": \"string\"\n        },\n   \"references\": {\n \"type\": \"string\"\n}, \n     \"mobile\": {\n          \"type\": \"string\"\n        }\n      }\n    }\n  },\n  \"_osConfig\": {\n    \"osComment\": [\"This section contains the OpenSABER specific configuration information\",\n      \"privateFields: Optional; list of field names to be encrypted and stored in database\",\n      \"signedFields: Optional; list of field names that must be pre-signed\",\n      \"indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, studentCode) is a composite index and studentName is a single column index.\",\n      \"uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name\",\n      \"systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid\"],\n\n    \"privateFields\": [\n\n    ],\n    \"signedFields\": [],\n    \"indexFields\": [],\n    \"uniqueIndexFields\": [],\n    \"systemFields\": [\n      \"_osCreatedAt\",\n      \"_osUpdatedAt\",\n      \"_osCreatedBy\",\n      \"_osUpdatedBy\",\n      \"_osAttestedData\",\n      \"_osClaimId\",\n      \"_osState\"\n    ],\n    \"inviteRoles\":[\"Board\"],\n    \"enableLogin\": true,\n    \"ownershipAttributes\": [\n      {\n        \"email\": \"/email\",\n        \"mobile\": \"/mobile\",\n        \"userId\": \"/mobile\"\n      }\n    ]\n  }\n}",
     "status": "PUBLISHED"
   },
   "instituteRequest": {
     "name": "test_institute",
     "email": "test_institute@mail.com",
-    "mobile": "456"
+    "mobile": "456",
+    "references": "did:Board:123"
   }
 }

java/apitest/src/test/java/e2e/registry/registry.feature

@@ -274,6 +274,7 @@ Feature: Registry api tests
     And request sample.boardInviteRequest
     When method post
     Then status 200
+    * def boardOsid = response.result.Board.osid
   #  get board token
     * url authUrl
     * path 'auth/realms/sunbird-rc/protocol/openid-connect/token'
@@ -299,7 +300,9 @@ Feature: Registry api tests
   # invite institute with token
     Given url registryUrl
     And path 'api/v1/Institute/invite'
-    And request sample.instituteRequest
+    * def requestBody = sample.instituteRequest
+    * requestBody.references = 'did:Board:' + boardOsid
+    And request requestBody
     And header Authorization = board_token
     When method post
     Then status 200
@@ -324,4 +327,5 @@ Feature: Registry api tests
     When method get
     Then status 200
     And response[0].osid.length > 0
+    And match response[0].references == '#present'
 

java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/Constants.java

@@ -125,6 +125,7 @@ public class Constants {
 	public static final String MOBILE = "mobile";
 	public static final String SVG_MEDIA_TYPE = "image/svg+xml";
 	public static final String CONNECTION_FAILURE = "CONNECTION_FAILURE";
+	public static final String DID_TYPE = "did";
 
     public enum GraphDatabaseProvider {
 		NEO4J("NEO4J"), ORIENTDB("ORIENTDB"), SQLG("SQLG"), CASSANDRA("CASSANDRA"), TINKERGRAPH("TINKERGRAPH");

java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java

@@ -17,6 +17,7 @@
 
 public class RegistryDaoImpl implements IRegistryDao {
     public String uuidPropertyName;
+    private final boolean expandReferenceObj;
     private IDefinitionsManager definitionsManager;
     private DatabaseProvider databaseProvider;
     private List<String> privatePropertyList;
@@ -33,10 +34,11 @@ public void setPrivatePropertyList(List<String> privatePropertyList) {
         this.privatePropertyList = privatePropertyList;
     }
 
-    public RegistryDaoImpl(DatabaseProvider dbProvider, IDefinitionsManager defnManager, String uuidPropName) {
+    public RegistryDaoImpl(DatabaseProvider dbProvider, IDefinitionsManager defnManager, String uuidPropName, boolean expandReferenceObj) {
         databaseProvider = dbProvider;
         definitionsManager = defnManager;
         uuidPropertyName = uuidPropName;
+        this.expandReferenceObj = expandReferenceObj;
     }
 
     public DatabaseProvider getDatabaseProvider() {
@@ -64,7 +66,7 @@ public String addEntity(Graph graph, JsonNode rootNode) {
      */
     public JsonNode getEntity(Graph graph, String entityType, String uuid, ReadConfigurator readConfigurator) throws Exception {
 
-        VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager);
+        VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager, expandReferenceObj);
         JsonNode result = vr.read(entityType, uuid);
 
         return result;
@@ -73,7 +75,7 @@ public JsonNode getEntity(Graph graph, String entityType, String uuid, ReadConfi
 
     public JsonNode getEntity(Graph graph, Vertex vertex, ReadConfigurator readConfigurator, boolean expandInternal) throws Exception {
 
-        VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager);
+        VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager, expandReferenceObj);
         ObjectNode constructObject = vr.constructObject(vertex);
         if (expandInternal) {
             String entityType = (String) ValueType.getValue(constructObject.get(TypePropertyHelper.getTypeName()));

java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java

@@ -8,12 +8,7 @@
 import dev.sunbirdrc.registry.middleware.util.Constants;
 import dev.sunbirdrc.registry.middleware.util.JSONUtil;
 import dev.sunbirdrc.registry.sink.DatabaseProvider;
-import dev.sunbirdrc.registry.util.ArrayHelper;
-import dev.sunbirdrc.registry.util.Definition;
-import dev.sunbirdrc.registry.util.IDefinitionsManager;
-import dev.sunbirdrc.registry.util.ReadConfigurator;
-import dev.sunbirdrc.registry.util.RefLabelHelper;
-import dev.sunbirdrc.registry.util.TypePropertyHelper;
+import dev.sunbirdrc.registry.util.*;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -32,6 +27,8 @@
 import java.util.NoSuchElementException;
 import java.util.Set;
 
+import static dev.sunbirdrc.registry.middleware.util.Constants.DID_TYPE;
+
 /**
  * Given a vertex from the graph, constructs a json out it
  */
@@ -46,15 +43,18 @@ public class VertexReader {
     private Vertex rootVertex;
     private LinkedHashMap<String, Vertex> uuidVertexMap = new LinkedHashMap<>();
 
+    private boolean expandReferenceObj;
     private Logger logger = LoggerFactory.getLogger(VertexReader.class);
 
+
     public VertexReader(DatabaseProvider databaseProvider, Graph graph, ReadConfigurator configurator, String uuidPropertyName,
-                        IDefinitionsManager definitionsManager) {
+                        IDefinitionsManager definitionsManager, boolean expandReferenceObj) {
         this.databaseProvider = databaseProvider;
         this.graph = graph;
         this.configurator = configurator;
         this.uuidPropertyName = uuidPropertyName;
         this.definitionsManager = definitionsManager;
+        this.expandReferenceObj = expandReferenceObj;
     }
 
     /**
@@ -460,7 +460,7 @@ public JsonNode readInternal(Vertex rootVertex) throws Exception {
             throw new RecordNotFoundException("entity status is inactive");
         }
         ObjectNode rootNode = constructObject(rootVertex);
-        entityType = (String) ValueType.getValue(rootNode.get(TypePropertyHelper.getTypeName()));
+        String entityType = (String) ValueType.getValue(rootNode.get(TypePropertyHelper.getTypeName()));
 
         // Set the type for the root node, so as to wrap.
         populateMaps(rootNode, rootVertex);
@@ -485,14 +485,39 @@ public JsonNode readInternal(Vertex rootVertex) throws Exception {
         // objects.
         // The properties could exist anywhere. Refer to the local arrMap.
         expandChildObject(rootNode, 0);
-
+        if(expandReferenceObj)
+            expandReferenceNodes(rootNode);
         entityNode.set(entityType, rootNode);
 
+
         // After reading the entire type, now trim the @type property
         trimAttributes(entityNode);
         return entityNode;
     }
 
+    private void expandReferenceNodes(ObjectNode rootNode) {
+        rootNode.fields().forEachRemaining(entry -> {
+            if(entry.getValue().isValueNode() && entry.getValue().asText().startsWith(DID_TYPE)) {
+                String[] dids = entry.getValue().asText().split(":");
+                String osid = RecordIdentifier.parse(dids[2]).getUuid();
+                Iterator<Vertex> vertexIterator = graph.traversal().clone().V().hasLabel(dids[1]).has(uuidPropertyName, osid);
+                while (vertexIterator.hasNext()) {
+                    Vertex dependent = vertexIterator.next();
+                    ObjectNode references;
+                    try {
+                        references = (ObjectNode) readInternal(dependent);
+                        references = (ObjectNode) references.get(references.fieldNames().next());
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                    if(references != null) {
+                        entry.setValue(references);
+                    }
+                }
+            }
+        });
+    }
+
     /**
      * Trims out local helper attributes like the type, uuid depending on the
      * ReadConfigurator

java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java

@@ -2,10 +2,12 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import dev.sunbirdrc.registry.middleware.util.Constants;
 import dev.sunbirdrc.registry.sink.DatabaseProvider;
 import dev.sunbirdrc.registry.util.ArrayHelper;
+import dev.sunbirdrc.registry.util.RecordIdentifier;
 import dev.sunbirdrc.registry.util.RefLabelHelper;
 import dev.sunbirdrc.registry.util.TypePropertyHelper;
 import org.apache.tinkerpop.gremlin.process.traversal.P;
@@ -16,12 +18,15 @@
 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import static dev.sunbirdrc.registry.middleware.util.Constants.DID_TYPE;
+
 /**
  * Helps in writing a vertex, edge into the database
  */
@@ -31,7 +36,6 @@ public class VertexWriter {
     private DatabaseProvider databaseProvider;
     private String parentOSid;
     private static final String EMPTY_STR = "";
-
     private Logger logger = LoggerFactory.getLogger(VertexWriter.class);
 
     public VertexWriter(Graph graph, DatabaseProvider databaseProvider, String uuidPropertyName) {
@@ -215,7 +219,6 @@ private Vertex processNode(String label, JsonNode jsonObject) {
         jsonObject.fields().forEachRemaining(entry -> {
             JsonNode entryValue = entry.getValue();
             logger.debug("Processing {} -> {}", entry.getKey(), entry.getValue());
-
             if (entryValue.isValueNode()) {
                 // Directly add under the vertex as a property
                 vertex.property(entry.getKey(), ValueType.getValue(entryValue));