Backporting latest master commit to opendistro 1.4.0 branch

.github/workflows/release-workflow.yml

@@ -0,0 +1,43 @@
+name: Release SQL Artifacts
+# This workflow is triggered on creating tags to master or an opendistro release branch
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        java: [12]
+
+    name: Build and Release SQL Plugin
+    runs-on: [ubuntu-16.04]
+
+    steps:
+      - name: Checkout SQL
+        uses: actions/checkout@v1
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Run build
+        run: |
+          ./gradlew buildPackages --refresh-dependencies --console=plain -Dbuild.snapshot=false
+          artifact=`ls build/distributions/*.zip`
+          rpm_artifact=`ls build/distributions/*.rpm`
+          deb_artifact=`ls build/distributions/*.deb`
+
+          aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-plugins/opendistro-sql/
+          aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-sql/
+          aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-sql/
+          aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/downloads/*"

README.md

@@ -29,9 +29,6 @@ The package uses the [Gradle](https://docs.gradle.org/4.10.2/userguide/userguide
 To use the feature, send requests to the `_opendistro/_sql` URI. You can use a request parameter or the request body (recommended).
 
 * Simple query
-```
-GET https://<host>:<port>/_opendistro/_sql?sql=select * from my-index limit 50
-```
 
 ```
 POST https://<host>:<port>/_opendistro/_sql
@@ -186,7 +183,7 @@ If you discover a potential security issue in this project we ask that you notif
 
 ## Licensing
 
-See the [LICENSE](./LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
+See the [LICENSE](./LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
 
 
 ## Copyright

build.gradle

@@ -63,7 +63,9 @@ version = "${opendistroVersion}.0"
 
 apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'jacoco'
-apply from: 'build-tools/sqlplugin-coverage.gradle'
+if (!System.properties.containsKey('tests.rest.cluster') && !System.properties.containsKey('tests.cluster')){
+    apply from: 'build-tools/sqlplugin-coverage.gradle'
+}
 apply plugin: 'antlr'
 
 jacoco {
@@ -134,6 +136,15 @@ integTestRunner {
     // allows integration test classes to access test resource from project root path
     systemProperty('project.root', project.rootDir.absolutePath)
 
+    // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
+    // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
+    doFirst { systemProperty 'cluster.debug', integTestCluster.debug }
+
+    // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
+    if (System.getProperty("test.debug") != null) {
+        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
+    }
+
     // Run different task based on test type. "exclude" is required for each task.
     def testType = System.getProperty("testType")
     if (testType == 'doctest') { // Doctest to generate documentation
@@ -155,9 +166,9 @@ integTestRunner {
         systemProperty "otherDbUrls", System.getProperty("otherDbUrls")
         systemProperty "queries", System.getProperty("queries")
 
-    } else { // Run all other integration tests and doctest by default
+    } else { // Run all other integration tests. Skip doctest for now due to randomness in our stats API.
         include 'com/amazon/opendistroforelasticsearch/sql/esintgtest/**/*IT.class'
-        include 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class'
+        exclude 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class'
         exclude 'com/amazon/opendistroforelasticsearch/sql/correctness/**'
     }
 }
@@ -167,6 +178,10 @@ integTestCluster {
     distribution = "oss-zip"
 }
 
+run {
+    distribution = "oss-zip"
+}
+
 generateGrammarSource {
     arguments += ['-visitor', '-package', 'com.amazon.opendistroforelasticsearch.sql.antlr.parser']
     source = sourceSets.main.antlr
@@ -223,6 +238,7 @@ dependencies {
     compile group: "org.elasticsearch.plugin", name: 'reindex-client', version: "${es_version}"
     compile group: 'com.google.guava', name: 'guava', version:'15.0'
     compile group: 'org.json', name: 'json', version:'20180813'
+    compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
 
     // ANTLR gradle plugin and runtime dependency
     antlr "org.antlr:antlr4:4.7.1"

docs/dev/Architecture.md

@@ -0,0 +1,39 @@
+# OpenDistro SQL Engine Architecture
+
+---
+## 1.Overview
+
+The OpenDistro SQL (OD-SQL) project is developed based on NLPChina project (https://github.com/NLPchina/elasticsearch-sql) which has been deprecated now ([attributions](https://github.com/opendistro-for-elasticsearch/sql/blob/master/docs/attributions.md)). Over the one year in development, a lot of features have been added to the OD-SQL project on top of the existing older NLPChina project. The purpose of this document is to explain the OD-SQL current architecture going ahead.
+
+---
+## 2.High Level View
+
+In the high level, the OD-SQL Engine could be divided into four major sub-module.
+
+* *Parser*: Currently, there are two Lex&Parser coexists. The Druid Lex&Parser is the original one from NLPChina. The input AST of Core Engine is from the Druid Lex&Parser. The [ANTLR](https://github.com/opendistro-for-elasticsearch/sql/blob/master/src/main/antlr/OpenDistroSqlParser.g4) Lex&Parser is added by us to customized the verification and exception handling.
+* *Analyzer*: The analyzer module take the output from ANTLR Lex&Parser then perform syntax and semantic analyze.
+* *Core Engine*: The QueryAction take the output from Druid Lex&Parser and translate to the Elasticsearch DSL if possible. This is an NLPChina original module. The QueryPlanner Builder is added by us to support the JOIN and Post-processing logic. The QueryPlanner will take the take the output from Druid Lex&Parser and build the PhysicalPlan
+* *Execution*: The execution module execute QueryAction or QueryPlanner and return the response to the client. Different from the Frontend, Analyzer and Core Engine which running on the Transport Thread and can’t do any blocking operation. The Execution module running on the client threadpool and can perform the blocking operation.
+
+There are also others modules include in the OD-SQL engine.
+
+* _Documentation_: it is used to auto-generated documentation.
+* _Metrics_: it is used to collect OD-SQL related metrics.
+* _Resource Manager_:  it is used to monitor the memory consumption when performing join operation to avoid the impact to Elasticsearch availability.
+
+![Architecture Overview](img/architecture-overview.png)
+
+---
+## 3.Journey of the query in OD-SQL engine.
+
+The following diagram take a sample query and explain how the query flow within different modules.
+
+![Architecture Journey](img/architecture-journey.png)
+
+1. The ANTRL parser based on grammar file (https://github.com/opendistro-for-elasticsearch/sql/blob/master/src/main/antlr/OpenDistroSqlParser.g4) to auto generate the AST. 
+2. The Syntax and Semantic Analyzer will walk through the AST and verify whether the query is follow the grammar and supported by the OD-SQL. e.g. *SELECT * FROM semantics WHERE LOG(age, city) = 1, *will throw exception with message* Function [LOG] cannot work with [INTEGER, KEYWORD]. *and sample usage message* Usage: LOG(NUMBER T) → DOUBLE.
+3. The Druid Lex&Parser takes the input query and generate the druid AST which is different from the AST generated by the ANTRL. This module is the open source library (https://github.com/alibaba/druid) used by NLPChina originally.
+4. The QueryPlanner Builder take the AST as input and generate the LogicalPlan from it. Then it optimize the LogicalPlan to PhysicalPlan.(In current implementation, only rule-based model is implemented). The major part of PhysicalPlan generation use NLPChina’s original logic to translate the SQL expression in AST to Elasticsearch DSL.
+5. The QueryPlanner executor execute the PhysicalPlan in worker thread.
+6. The formatter will reformat the response data to the required format. The default format is JDBC format.
+