Simple Query Cursor support (#390)

* Add  integration tests to be passed

* Add cluster settings for cursor - enabled, fetch_size, keep_alive

* Add fetch_size and cursor params. fetch_size validation

* new SqlRequest constructor for cursor

* Add logic to open scroll based on settings, fetch_size and limit values

* Add cursor close endpoint

* Remove date formatting changes

* Fix unit and integ tests, Ignored date format tests for a while, synced previous cursor changes

* Add cursor generation

* Add test helper methods

* Cursor close API

* Remove commented code and add partial date formatting change

* Add error metrics when not able to close cursor

* Add indexname and fieldAliasMap to cursor context

* Remove ignored test cases affected by date formatting changes

* Remove unneeded interface, refactor CursorType enum

* Remove logs, unneeded fields, comments, refactor

* Disable cursor by default

* Fix cursor for parameterized request, add integration test for same

* LIMIT changes

* Changes to handle different LIMIT cases

* Add default cursor metrics

* Add integration test on explain cursor

* Update monitoring, settings and endpoint docs

* Refactor cursor classes to separate package

* Add Lombok for DefaultCursor

* Add unit test for DefaultCursor

* Update doc

* Unit tests, bug fix , refactoring

docs/user/admin/monitoring.rst

@@ -24,19 +24,23 @@ Description
 
 The meaning of fields in the response is as follows:
 
-+---------------------------+---------------------------------------------------------------+
-|                 Field name|                                                    Description|
-+===========================+===============================================================+
-|              request_total|                                         Total count of request|
-+---------------------------+---------------------------------------------------------------+
-|              request_count|                     Total count of request within the interval|
-+---------------------------+---------------------------------------------------------------+
-|failed_request_count_syserr|Count of failed request due to system error within the interval|
-+---------------------------+---------------------------------------------------------------+
-|failed_request_count_cuserr| Count of failed request due to bad request within the interval|
-+---------------------------+---------------------------------------------------------------+
-|    failed_request_count_cb| Indicate if plugin is being circuit broken within the interval|
-+---------------------------+---------------------------------------------------------------+
++----------------------------+---------------------------------------------------------------+
+|                  Field name|                                                    Description|
++============================+===============================================================+
+|               request_total|                                         Total count of request|
++----------------------------+---------------------------------------------------------------+
+|               request_count|                     Total count of request within the interval|
++----------------------------+---------------------------------------------------------------+
+|default_cursor_request_total|                           Total count of simple cursor request|
++----------------------------+---------------------------------------------------------------+
+|default_cursor_request_count|       Total count of simple cursor request within the interval|
++----------------------------+---------------------------------------------------------------+
+| failed_request_count_syserr|Count of failed request due to system error within the interval|
++----------------------------+---------------------------------------------------------------+
+| failed_request_count_cuserr| Count of failed request due to bad request within the interval|
++----------------------------+---------------------------------------------------------------+
+|     failed_request_count_cb| Indicate if plugin is being circuit broken within the interval|
++----------------------------+---------------------------------------------------------------+
 
 
 Example
@@ -50,9 +54,11 @@ Result set::
 
 	{
 	  "failed_request_count_cb" : 0,
+	  "default_cursor_request_count" : 10,
+	  "default_cursor_request_total" : 3,
 	  "failed_request_count_cuserr" : 0,
 	  "circuit_breaker" : 0,
-	  "request_total" : 49,
+	  "request_total" : 70,
 	  "request_count" : 0,
 	  "failed_request_count_syserr" : 0
 	}

docs/user/admin/settings.rst

@@ -384,3 +384,129 @@ Result set::
 	  "timed_out" : false
 	}
 
+opendistro.sql.cursor.enabled
+=============================
+
+Description
+-----------
+
+User can enable/disable pagination for all queries that are supported.
+
+1. The default value is false.
+2. This setting is node scope.
+3. This setting can be updated dynamically.
+
+
+Example
+-------
+
+You can update the setting with a new value like this.
+
+SQL query::
+
+	>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_opendistro/_sql/settings -d '{
+	  "transient" : {
+	    "opendistro.sql.cursor.enabled" : "true"
+	  }
+	}'
+
+Result set::
+
+	{
+	  "acknowledged" : true,
+	  "persistent" : { },
+	  "transient" : {
+	    "opendistro" : {
+	      "sql" : {
+	        "cursor" : {
+	          "enabled" : "true"
+	        }
+	      }
+	    }
+	  }
+	}
+
+opendistro.sql.cursor.fetch_size
+================================
+
+Description
+-----------
+
+User can set the default fetch_size for all queries that are supported by pagination. Explicit `fetch_size` passed in request will override this value
+
+1. The default value is 1000.
+2. This setting is node scope.
+3. This setting can be updated dynamically.
+
+
+Example
+-------
+
+You can update the setting with a new value like this.
+
+SQL query::
+
+	>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_opendistro/_sql/settings -d '{
+	  "transient" : {
+	    "opendistro.sql.cursor.fetch_size" : "50"
+	  }
+	}'
+
+Result set::
+
+	{
+	  "acknowledged" : true,
+	  "persistent" : { },
+	  "transient" : {
+	    "opendistro" : {
+	      "sql" : {
+	        "cursor" : {
+	          "fetch_size" : "50"
+	        }
+	      }
+	    }
+	  }
+	}
+
+opendistro.sql.cursor.keep_alive
+================================
+
+Description
+-----------
+
+User can set this value to indicate how long the cursor context should be kept open. Cursor contexts are resource heavy, and a lower value should be used if possible.
+
+1. The default value is 1m.
+2. This setting is node scope.
+3. This setting can be updated dynamically.
+
+
+Example
+-------
+
+You can update the setting with a new value like this.
+
+SQL query::
+
+	>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_opendistro/_sql/settings -d '{
+	  "transient" : {
+	    "opendistro.sql.cursor.keep_alive" : "5m"
+	  }
+	}'
+
+Result set::
+
+	{
+	  "acknowledged" : true,
+	  "persistent" : { },
+	  "transient" : {
+	    "opendistro" : {
+	      "sql" : {
+	        "cursor" : {
+	          "keep_alive" : "5m"
+	        }
+	      }
+	    }
+	  }
+	}
+

docs/user/interfaces/endpoint.rst

@@ -91,3 +91,62 @@ Explain::
 	  }
 	}
 
+Cursor
+======
+
+Description
+-----------
+
+To get paginated response for a query, user needs to provide `fetch_size` parameter as part of normal query. The value of `fetch_size` should be greater than `0`. In absence of `fetch_size`, default value of 1000 is used. A value of `0` will fallback to non-paginated response. This feature is only available over `jdbc` format for now.
+
+Example
+-------
+
+SQL query::
+
+	>> curl -H 'Content-Type: application/json' -X POST localhost:9200/_opendistro/_sql -d '{
+	  "fetch_size" : 5,
+	  "query" : "SELECT firstname, lastname FROM accounts WHERE age > 20 ORDER BY state ASC"
+	}'
+
+Result set::
+
+    {
+      "schema": [
+        {
+          "name": "firstname",
+          "type": "text"
+        },
+        {
+          "name": "lastname",
+          "type": "text"
+        }
+      ],
+      "cursor": "d:eyJhIjp7fSwicyI6IkRYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQU1XZWpkdFRFRkZUMlpTZEZkeFdsWnJkRlZoYnpaeVVRPT0iLCJjIjpbeyJuYW1lIjoiZmlyc3RuYW1lIiwidHlwZSI6InRleHQifSx7Im5hbWUiOiJsYXN0bmFtZSIsInR5cGUiOiJ0ZXh0In1dLCJmIjo1LCJpIjoiYWNjb3VudHMiLCJsIjo5NTF9",
+      "total": 956,
+      "datarows": [
+        [
+          "Cherry",
+          "Carey"
+        ],
+        [
+          "Lindsey",
+          "Hawkins"
+        ],
+        [
+          "Sargent",
+          "Powers"
+        ],
+        [
+          "Campos",
+          "Olsen"
+        ],
+        [
+          "Savannah",
+          "Kirby"
+        ]
+      ],
+      "size": 5,
+      "status": 200
+    }
+

src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/Cursor.java

@@ -0,0 +1,30 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.cursor;
+
+
+public interface Cursor {
+
+    NullCursor NULL_CURSOR = new NullCursor();
+
+    /**
+     * All cursor's are of the form <cursorType>:<base64 encoded cursor>
+     * The serialized form before encoding is upto Cursor implementation
+     */
+    String generateCursorId();
+
+    CursorType getType();
+}

src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/CursorType.java

@@ -0,0 +1,53 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.cursor;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Different types queries for which cursor is supported.
+ * The result execution, and cursor genreation/parsing will depend on the cursor type.
+ * NullCursor is the placeholder implementation in case of non-cursor query.
+ */
+public enum CursorType {
+    NULL(null),
+    DEFAULT("d"),
+    AGGREGATION("a"),
+    JOIN("j");
+
+    public String id;
+
+    CursorType(String id) {
+        this.id = id;
+    }
+
+    public String getId() {
+        return this.id;
+    }
+
+    public static final Map<String, CursorType> LOOKUP = new HashMap<>();
+
+    static {
+        for (CursorType type : CursorType.values()) {
+            LOOKUP.put(type.getId(), type);
+        }
+    }
+
+    public static CursorType getById(String id) {
+        return LOOKUP.getOrDefault(id, NULL);
+    }
+}