Merge pull request #316 from holashchand/issue-986

Added pagination for search and list apis

docker-compose-v1.yml

@@ -2,7 +2,7 @@ version: '2.4'
 
 services:
   es:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.13
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23
     volumes:
       - ./${ES_DIR-es-data}:/usr/share/elasticsearch/data/*
     environment:

docker-compose.yml

@@ -2,7 +2,7 @@ version: '2.4'
 
 services:
   es:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.13
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23
     volumes:
       - ./${ES_DIR-es-data}:/usr/share/elasticsearch/data/*
     environment:

java/apitest/src/test/java/e2e/registry/fusionauth-registry.feature

@@ -57,4 +57,4 @@ Feature: Registry api tests
         And header Authorization = student_token
         When method get
         Then status 200
-        And response[0].osid.length > 0
+        And response.data[0].osid.length > 0

java/apitest/src/test/java/e2e/registry/registry.feature

@@ -151,8 +151,8 @@ Feature: Registry api tests
     And header Authorization = student_token
     When method get
     Then status 200
-    And response[0].osid.length > 0
-    * def studentOsid = response[0].osid
+    And response.data[0].osid.length > 0
+    * def studentOsid = response.data[0].osid
   # update student info
     Given url registryUrl
     And path 'api/v1/Student/' + studentOsid
@@ -168,7 +168,7 @@ Feature: Registry api tests
     And header Authorization = student_token
     When method get
     Then status 200
-    And response[0].name == "xyz"
+    And response.data[0].name == "xyz"
   # get student
     Given url registryUrl
     And path 'api/v1/Student/search'
@@ -177,8 +177,8 @@ Feature: Registry api tests
     Then status 200
     * print response
     And response.length == 1
-    And match response[0].contact == '#notpresent'
-    And match response[0].favoriteSubject == '#notpresent'
+    And match response.data[0].contact == '#notpresent'
+    And match response.data[0].favoriteSubject == '#notpresent'
   # delete student info
     Given url registryUrl
     And path 'api/v1/Student/' + studentOsid
@@ -191,7 +191,8 @@ Feature: Registry api tests
     And path 'api/v1/Student'
     And header Authorization = student_token
     When method get
-    Then status 404
+    Then status 200
+    And response.totalCount == 0
 
   @env=async
   Scenario: Create a teacher schema and create teacher entity asynchronously
@@ -240,17 +241,17 @@ Feature: Registry api tests
     When method post
     Then status 200
     * print response
-    And response.length == 1
-    And response[0].contact == '#notpresent'
+    And response.totalCount == 1
+    And response.data[0].contact == '#notpresent'
   # get teacher info
     Given url registryUrl
     And path 'api/v1/Teacher/search'
     And request {"filters":{ "name":  { "endsWith":  "abc" }}}
     When method post
     Then status 200
     * print response
-    And response.length == 1
-    And response[0].contact == '#notpresent'
+    And response.totalCount == 1
+    And response.data[0].contact == '#notpresent'
 
   @envnot=fusionauth
   Scenario: Create Board and invite institutes
@@ -316,7 +317,7 @@ Feature: Registry api tests
     And header Authorization = board_token
     When method get
     Then status 200
-    And response[0].osid.length > 0
+    And response.data[0].osid.length > 0
 
   # invite institute with token
     Given url registryUrl
@@ -347,9 +348,9 @@ Feature: Registry api tests
     And header Authorization = institute_token
     When method get
     Then status 200
-    And response[0].osid.length > 0
-    * def instituteOsid = response[0].osid
-    * def address = response[0].address
+    And response.data[0].osid.length > 0
+    * def instituteOsid = response.data[0].osid
+    * def address = response.data[0].address
 
   # update property of institute
     Given url registryUrl
@@ -366,8 +367,8 @@ Feature: Registry api tests
     And header Authorization = institute_token
     When method get
     Then status 200
-    And assert response[0].address[0].phoneNo.length == 1
-    And assert response[0].address[0].phoneNo[0] == "444"
+    And assert response.data[0].address[0].phoneNo.length == 1
+    And assert response.data[0].address[0].phoneNo[0] == "444"
 
   @envnot=fusionauth
   Scenario: write a api test, to test the schema not found error

java/elastic-search/src/main/java/dev/sunbirdrc/elastic/ElasticServiceImpl.java

@@ -4,6 +4,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import dev.sunbirdrc.pojos.ComponentHealthInfo;
 import dev.sunbirdrc.pojos.Filter;
 import dev.sunbirdrc.pojos.FilterOperators;
@@ -49,8 +50,7 @@
 import org.springframework.retry.annotation.Backoff;
 import org.springframework.retry.annotation.Retryable;
 
-import static dev.sunbirdrc.registry.middleware.util.Constants.CONNECTION_FAILURE;
-import static dev.sunbirdrc.registry.middleware.util.Constants.SUNBIRD_ELASTIC_SERVICE_NAME;
+import static dev.sunbirdrc.registry.middleware.util.Constants.*;
 
 public class ElasticServiceImpl implements IElasticService {
     private static Map<String, RestHighLevelClient> esClient = new HashMap<String, RestHighLevelClient>();
@@ -283,21 +283,25 @@ public JsonNode search(String index, SearchQuery searchQuery) throws IOException
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
                 .query(query)
                 .size(searchQuery.getLimit())
-                .from(searchQuery.getOffset());
+                .from(searchQuery.getOffset())
+                .trackTotalHits(true);
         SearchRequest searchRequest = new SearchRequest(index).source(sourceBuilder);
-        ArrayNode resultArray = JsonNodeFactory.instance.arrayNode();
+        ObjectNode resultNode = JsonNodeFactory.instance.objectNode();
+        ArrayNode dataArray = JsonNodeFactory.instance.arrayNode();
         ObjectMapper mapper = new ObjectMapper();
-            SearchResponse searchResponse = getClient(index).search(searchRequest, RequestOptions.DEFAULT);
-            for (SearchHit hit : searchResponse.getHits()) {
-                JsonNode node = mapper.readValue(hit.getSourceAsString(), JsonNode.class);
-                // TODO: Add draft mode condition
-                if(node.get("_status") == null || node.get("_status").asBoolean()) {
-                    resultArray.add(node);
-                }
+        SearchResponse searchResponse = getClient(index).search(searchRequest, RequestOptions.DEFAULT);
+        for (SearchHit hit : searchResponse.getHits()) {
+            JsonNode node = mapper.readValue(hit.getSourceAsString(), JsonNode.class);
+            // TODO: Add draft mode condition
+            if(node.get(STATUS_KEYWORD) == null || node.get(STATUS_KEYWORD).asBoolean()) {
+                dataArray.add(node);
             }
-            logger.debug("Total search records found " + resultArray.size());
+        }
+        resultNode.set(ENTITY_LIST, dataArray);
+        resultNode.put(TOTAL_COUNT, searchResponse.getHits().getTotalHits());
+        logger.debug("Total search records found " + dataArray.size());
 
-        return resultArray;
+        return resultNode;
 
     }
 

java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/Constants.java

@@ -43,6 +43,10 @@ public class Constants {
 	public static final String END_DATE="endDate";
 	public static final String LIMIT="limit";
 	public static final String OFFSET="offset";
+	public static final String NEXT_PAGE="nextPage";
+	public static final String PREV_PAGE="prevPage";
+	public static final String TOTAL_COUNT="totalCount";
+	public static final String ENTITY_LIST="data";
 
 	// JSON LD specific
 	public static final String CONTEXT_KEYWORD = "@context";

java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/JSONUtil.java

@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.lang.reflect.Type;
+import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -30,7 +31,8 @@
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.core.NestedExceptionUtils;
+
+import static dev.sunbirdrc.registry.middleware.util.Constants.*;
 
 public class JSONUtil {
 
@@ -545,4 +547,28 @@ public static JsonNode extractPropertyDataFromEntity(JsonNode entityNode, Map<St
 		}
 		return result;
 	}
+
+	public static ObjectNode getSearchPageUrls(JsonNode inputNode, long defaultLimit, long defaultOffset, long totalCount, String url) throws IOException {
+		ObjectNode result = JsonNodeFactory.instance.objectNode();
+		JsonNode searchNode = objectMapper.readTree(inputNode.toString());
+		long limit = searchNode.get(LIMIT) == null ? defaultLimit : searchNode.get(LIMIT).asLong(defaultLimit);
+		long offset = searchNode.get(OFFSET) == null ? defaultOffset : searchNode.get(OFFSET).asLong(defaultOffset);
+		((ObjectNode) searchNode).set(OFFSET, JsonNodeFactory.instance.numberNode(offset - limit));
+		String prevPageToken = Base64.getEncoder().encodeToString(searchNode.toString().getBytes(StandardCharsets.UTF_8));
+		((ObjectNode) searchNode).set(OFFSET, JsonNodeFactory.instance.numberNode(offset + limit));
+		String nextPageToken = Base64.getEncoder().encodeToString(searchNode.toString().getBytes(StandardCharsets.UTF_8));
+		if(offset - limit >=0) result.put(PREV_PAGE, url + "?search=" + prevPageToken);
+		if(offset + limit <= totalCount) result.put(NEXT_PAGE, url + "?search=" + nextPageToken);
+		return result;
+	}
+
+	public static ObjectNode parseSearchToken(String endcodedValue) {
+		try {
+			byte[] decoded = Base64.getDecoder().decode(endcodedValue);
+			return (ObjectNode) objectMapper.readTree(decoded);
+		} catch (Exception ignored) {
+			logger.warn("Unable to parse next page token");
+		}
+		return null;
+	}
 }

java/registry/src/main/java/dev/sunbirdrc/registry/config/SchemaLoader.java

@@ -21,8 +21,7 @@
 import java.util.ArrayList;
 
 import static dev.sunbirdrc.registry.Constants.Schema;
-import static dev.sunbirdrc.registry.middleware.util.Constants.ENTITY_TYPE;
-import static dev.sunbirdrc.registry.middleware.util.Constants.FILTERS;
+import static dev.sunbirdrc.registry.middleware.util.Constants.*;
 
 @Component
 public class SchemaLoader implements ApplicationListener<ContextRefreshedEvent> {
@@ -46,14 +45,14 @@ private void loadSchemasFromDB() {
 		objectNode.set(FILTERS, JsonNodeFactory.instance.objectNode());
 		try {
 			JsonNode searchResults = searchService.search(objectNode, "");
-			for (JsonNode schemaNode : searchResults.get(Schema)) {
+			for (JsonNode schemaNode : searchResults.get(Schema).get(ENTITY_LIST)) {
 				try {
 					schemaService.addSchema(JsonNodeFactory.instance.objectNode().set(Schema, schemaNode));
 				} catch (Exception e) {
 					logger.error("Failed loading schema to definition manager: {}", ExceptionUtils.getStackTrace(e));
 				}
 			}
-			logger.error("Loaded {} schema from DB", searchResults.get(Schema).size());
+			logger.error("Loaded {} schema from DB", searchResults.get(Schema).get(TOTAL_COUNT));
 		} catch (IOException e) {
 			logger.error("Exception occurred while loading schema from db: {}", ExceptionUtils.getStackTrace(e));
 		} catch (Exception e) {

java/registry/src/main/java/dev/sunbirdrc/registry/controller/RegistryController.java

@@ -56,7 +56,7 @@ public ResponseEntity<Response> searchEntity(@RequestHeader HttpHeaders header)
 
         try {
             watch.start("RegistryController.searchEntity");
-            JsonNode result = registryHelper.searchEntity(payload, null);
+            JsonNode result = registryHelper.searchEntity(payload, null, false);
 
             response.setResult(result);
             responseParams.setStatus(Response.Status.SUCCESSFUL);

java/registry/src/main/java/dev/sunbirdrc/registry/controller/RegistryEntityController.java

@@ -48,12 +48,14 @@
 import org.springframework.web.bind.annotation.*;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.BadRequestException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.*;
 
 import static dev.sunbirdrc.registry.Constants.*;
 import static dev.sunbirdrc.registry.helper.RegistryHelper.ServiceNotEnabledResponse;
 import static dev.sunbirdrc.registry.middleware.util.Constants.ENTITY_TYPE;
+import static dev.sunbirdrc.registry.middleware.util.Constants.TOTAL_COUNT;
 
 @RestController
 public class RegistryEntityController extends AbstractController {
@@ -83,6 +85,10 @@ public class RegistryEntityController extends AbstractController {
     boolean securityEnabled;
     @Value("${certificate.enableExternalTemplates:false}")
     boolean externalTemplatesEnabled;
+    @Value("${search.offset:0}")
+    private int searchOffset;
+    @Value("${search.limit:2000}")
+    private int searchLimit;
 
     @RequestMapping(value = "/api/v1/{entityName}/invite", method = RequestMethod.POST)
     public ResponseEntity<Object> invite(
@@ -191,8 +197,11 @@ public ResponseEntity<Object> deleteEntity(
         }
     }
 
-    @RequestMapping(value = "/api/v1/{entityName}/search", method = RequestMethod.POST)
-    public ResponseEntity<Object> searchEntity(@PathVariable String entityName, @RequestHeader HttpHeaders header, @RequestBody ObjectNode searchNode) {
+    @RequestMapping(value = "/api/v1/{entityName}/search", method = {RequestMethod.POST, RequestMethod.GET})
+    public ResponseEntity<Object> searchEntity(@PathVariable String entityName,
+                                               HttpServletRequest request,
+                                               @RequestHeader HttpHeaders header, @RequestBody(required = false) ObjectNode searchNode,
+                                               @RequestParam(value = "search", required = false) String searchQueryString) {
 
         ResponseParams responseParams = new ResponseParams();
         Response response = new Response(Response.API_ID.SEARCH, "OK", responseParams);
@@ -201,12 +210,21 @@ public ResponseEntity<Object> searchEntity(@PathVariable String entityName, @Req
             watch.start("RegistryController.searchEntity");
             ArrayNode entity = JsonNodeFactory.instance.arrayNode();
             entity.add(entityName);
+            if(searchNode == null && (searchQueryString == null || searchQueryString.isEmpty())) {
+                throw new BadRequestException("Search Request body not found");
+            }
+            if(searchNode == null) {
+                searchNode = JsonNodeFactory.instance.objectNode();
+                registryHelper.addSearchTokenToQuery(searchQueryString, searchNode);
+            }
             searchNode.set(ENTITY_TYPE, entity);
             checkEntityNameInDefinitionManager(entityName);
             if (definitionsManager.getDefinition(entityName).getOsSchemaConfiguration().getEnableSearch()) {
-                JsonNode result = registryHelper.searchEntity(searchNode, null);
+                JsonNode result = registryHelper.searchEntity(searchNode, null, false).get(entityName);
+                ObjectNode pageUrls = JSONUtil.getSearchPageUrls(searchNode, searchLimit, searchOffset, result.get(TOTAL_COUNT).asLong(), request.getRequestURL().toString());
+                ((ObjectNode) result).setAll(pageUrls);
                 watch.stop("RegistryController.searchEntity");
-                return new ResponseEntity<>(result.get(entityName), HttpStatus.OK);
+                return new ResponseEntity<>(result, HttpStatus.OK);
             } else {
                 watch.stop("RegistryController.searchEntity");
                 logger.error("Searching on entity {} not allowed", entityName);
@@ -669,17 +687,21 @@ private JsonNode getEntityJsonNode(@PathVariable String entityName, @PathVariabl
 
     @RequestMapping(value = "/api/v1/{entityName}", method = RequestMethod.GET)
     public ResponseEntity<Object> getEntityByToken(@PathVariable String entityName, HttpServletRequest request,
-                                                   @RequestHeader(required = false) String viewTemplateId) throws RecordNotFoundException {
+                                                   @RequestHeader(required = false) String viewTemplateId,
+                                                   @RequestParam(value = "search", required = false) String searchToken) throws RecordNotFoundException {
         ResponseParams responseParams = new ResponseParams();
         Response response = new Response(Response.API_ID.GET, "OK", responseParams);
         try {
             checkEntityNameInDefinitionManager(entityName);
             String userId = registryHelper.getUserId(entityName);
             if (!Strings.isEmpty(userId)) {
-                JsonNode responseFromDb = registryHelper.searchEntitiesByUserId(entityName, userId, viewTemplateId);
-                JsonNode entities = responseFromDb.get(entityName);
-                if (!entities.isEmpty()) {
-                    return new ResponseEntity<>(entities, HttpStatus.OK);
+                JsonNode searchQuery = registryHelper.searchQueryByUserId(entityName, userId, searchToken, viewTemplateId);
+                JsonNode responseFromDb = registryHelper.searchEntity(searchQuery, userId, true);
+                JsonNode results = responseFromDb.get(entityName);
+                if (!results.isEmpty()) {
+                    ObjectNode pageUrls = JSONUtil.getSearchPageUrls(searchQuery, searchLimit, searchOffset, results.get(TOTAL_COUNT).asLong(), request.getRequestURL().toString());
+                    ((ObjectNode) results).setAll(pageUrls);
+                    return new ResponseEntity<>(results, HttpStatus.OK);
                 } else {
                     responseParams.setErrmsg("No record found");
                     responseParams.setStatus(Response.Status.UNSUCCESSFUL);

java/registry/src/main/java/dev/sunbirdrc/registry/dao/SearchDaoImpl.java

@@ -24,6 +24,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static dev.sunbirdrc.registry.middleware.util.Constants.ENTITY_LIST;
+import static dev.sunbirdrc.registry.middleware.util.Constants.TOTAL_COUNT;
 import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.has;
 import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.hasNot;
 
@@ -47,10 +49,15 @@ public JsonNode search(Graph graphFromStore, SearchQuery searchQuery, boolean ex
             GraphTraversal<Vertex, Vertex> parentTraversal = resultGraphTraversal.asAdmin();
 
             resultGraphTraversal = getFilteredResultTraversal(resultGraphTraversal, filterList)
-                    .or(hasNot(Constants.STATUS_KEYWORD), has(Constants.STATUS_KEYWORD, Constants.STATUS_ACTIVE))
-                    .range(offset, offset + searchQuery.getLimit()).limit(searchQuery.getLimit());
+                    .or(hasNot(Constants.STATUS_KEYWORD), has(Constants.STATUS_KEYWORD, Constants.STATUS_ACTIVE));
+            GraphTraversal<Vertex, Vertex> resultGraphTraversalCount = resultGraphTraversal.asAdmin().clone();
+            resultGraphTraversal.range(offset, offset + searchQuery.getLimit());
             JsonNode result = getResult(graphFromStore, resultGraphTraversal, parentTraversal, expandInternal);
-            resultNode.set(entity, result);
+            ObjectNode response = JsonNodeFactory.instance.objectNode();
+            response.set(ENTITY_LIST, result);
+            long count = resultGraphTraversalCount.count().next();
+            response.set(TOTAL_COUNT, JsonNodeFactory.instance.numberNode(count));
+            resultNode.set(entity, response);
         }
 
         return resultNode;