Opendistro Release 1.9.0 (#532)

* prepare odfe 1.9

* Fix all ES 7.8 compile and build errors

* Revert changes as Lombok is working now

* Update CustomExternalTestCluster.java

* Fix license headers check

* Use splitFieldsByMetadata to separate fields when calling SearchHit constructor

* More fixes for ODFE 1.9

* Remove todo statement

* Add ODFE 1.9.0 release notes

build.gradle

@@ -15,7 +15,7 @@
 
 buildscript {
     ext {
-        es_version = "7.7.0"
+        es_version = "7.8.0"
     }
     // This isn't applying from repositories.gradle so repeating it here
     repositories {
@@ -40,7 +40,7 @@ repositories {
 }
 
 ext {
-    opendistroVersion = '1.8.0'
+    opendistroVersion = '1.9.0'
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
 }
 
@@ -97,6 +97,10 @@ licenseHeaders {
     excludes = ['com/amazon/opendistroforelasticsearch/sql/antlr/parser/**']
 }
 
+tasks.withType(licenseHeaders.class) {
+    additionalLicense 'AL   ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
+}
+
 // TODO: need to fix java doc to enable JavaDoc
 javadoc.enabled = false
 esplugin {
@@ -272,14 +276,16 @@ dependencies {
     testCompile group: "org.elasticsearch.client", name: 'transport', version: "${es_version}"
 
     // JDBC drivers for comparison test. Somehow Apache Derby throws security permission exception.
-    testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.3.0.0'
+    testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.8.0.0'
     testCompile group: 'com.h2database', name: 'h2', version: '1.4.200'
     testCompile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0'
     //testCompile group: 'org.apache.derby', name: 'derby', version: '10.15.1.3'
 }
 
 apply plugin: 'nebula.ospackage'
 
+validateNebulaPom.enabled = false
+
 // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
 afterEvaluate {
     ospackage {

gradle.properties

@@ -13,4 +13,4 @@
 #   permissions and limitations under the License.
 #
 
-version=1.8.0
+version=1.9.0

gradle/wrapper/gradle-wrapper.properties

@@ -13,7 +13,7 @@
 #   permissions and limitations under the License.
 #
 
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists

release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.md

@@ -0,0 +1,16 @@
+## 2020-06-23 Version 1.9.0.0
+
+### Features
+#### Elasticsearch Compatibility
+* Feature [#532](https://github.com/opendistro-for-elasticsearch/sql/pull/532): Elasticsearch 7.8.0 compatibility (issue: [#531](https://github.com/opendistro-for-elasticsearch/sql/issues/531))
+
+#### Documentation
+* Feature [#486](https://github.com/opendistro-for-elasticsearch/sql/pull/486): Add Github badges to README
+
+### Enhancements
+#### SQL Features
+* Feature [#473](https://github.com/opendistro-for-elasticsearch/sql/pull/473): Support Integration Tests for Security enabled ODFE cluster
+
+### Bugfixes
+* Bugfix [#489](https://github.com/opendistro-for-elasticsearch/sql/pull/489): Fix ANTLR grammar for negative integer and floating point number (issue: [#488](https://github.com/opendistro-for-elasticsearch/sql/issues/488))
+* Bugfix [#522](https://github.com/opendistro-for-elasticsearch/sql/pull/522): Bug fix, support long type for aggregation (issue: [#521](https://github.com/opendistro-for-elasticsearch/sql/issues/521))

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java

@@ -109,11 +109,11 @@ public void setClusterService(ClusterService clusterService) {
         this.clusterService = clusterService;
 
         clusterService.addListener(event -> {
-            if (event.metaDataChanged()) {
+            if (event.metadataChanged()) {
                 // State in cluster service is already changed to event.state() before listener fired
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("Metadata in cluster state changed: {}",
-                            new IndexMappings(clusterService.state().metaData()));
+                            new IndexMappings(clusterService.state().metadata()));
                 }
                 cache.invalidateAll();
             }
@@ -169,8 +169,8 @@ public IndexMappings getFieldMappings(String[] indices, String[] types) {
     }
 
     /**
-     * Get field mappings by index expressions, type and field filter. Because IndexMetaData/MappingMetaData
-     * is hard to convert to FieldMappingMetaData, custom mapping domain objects are being used here. In future,
+     * Get field mappings by index expressions, type and field filter. Because IndexMetadata/MappingMetadata
+     * is hard to convert to FieldMappingMetadata, custom mapping domain objects are being used here. In future,
      * it should be moved to domain model layer for all ES specific knowledge.
      * <p>
      * Note that cluster state may be change inside ES so it's possible to read different state in 2 accesses
@@ -222,7 +222,7 @@ private IndexMappings findMappings(ClusterState state, String[] indices, String[
                                        Function<String, Predicate<String>> fieldFilter) throws IOException {
         LOG.debug("Cache didn't help. Load and parse mapping in cluster state");
         return new IndexMappings(
-                state.metaData().findMappings(indices, types, fieldFilter)
+                state.metadata().findMappings(indices, types, fieldFilter)
         );
     }
 

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java

@@ -21,7 +21,7 @@
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
-import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata;
 
 /**
  * Field mapping that parses native ES mapping.
@@ -39,7 +39,7 @@ public class FieldMapping {
     /**
      * Native mapping information returned from ES
      */
-    private final Map<String, FieldMappingMetaData> typeMappings;
+    private final Map<String, FieldMappingMetadata> typeMappings;
 
     /**
      * Maps a field name to Field object that specified in query explicitly
@@ -51,7 +51,7 @@ public FieldMapping(String fieldName) {
     }
 
     public FieldMapping(String fieldName,
-                        Map<String, FieldMappingMetaData> typeMappings,
+                        Map<String, FieldMappingMetadata> typeMappings,
                         Map<String, Field> specifiedFieldByNames) {
 
         this.fieldName = fieldName;
@@ -119,16 +119,16 @@ public String path() {
     }
 
     /**
-     * Used to retrieve the type of fields from metaData map structures for both regular and nested fields
+     * Used to retrieve the type of fields from metadata map structures for both regular and nested fields
      */
     @SuppressWarnings("unchecked")
     public String type() {
-        FieldMappingMetaData metaData = typeMappings.get(fieldName);
-        Map<String, Object> source = metaData.sourceAsMap();
+        FieldMappingMetadata metadata = typeMappings.get(fieldName);
+        Map<String, Object> source = metadata.sourceAsMap();
         String[] fieldPath = fieldName.split("\\.");
 
         /*
-         * When field is not nested the metaData source is fieldName -> type
+         * When field is not nested the metadata source is fieldName -> type
          * When it is nested or contains "." in general (ex. fieldName.nestedName) the source is nestedName -> type
          */
         String root = (fieldPath.length == 1) ? fieldName : fieldPath[1];

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java

@@ -15,7 +15,7 @@
 
 package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;
 
-import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.json.JSONObject;
 
 import java.util.HashMap;
@@ -61,7 +61,7 @@ public class FieldMappings implements Mappings<Map<String, Object>> {
      */
     private final Map<String, Object> fieldMappings;
 
-    public FieldMappings(MappingMetaData mappings) {
+    public FieldMappings(MappingMetadata mappings) {
         fieldMappings = mappings.sourceAsMap();
     }
 

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java

@@ -15,8 +15,8 @@
 
 package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;
 
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 
 import java.util.Map;
@@ -55,12 +55,12 @@ public IndexMappings() {
         this.indexMappings = emptyMap();
     }
 
-    public IndexMappings(MetaData metaData) {
-        this.indexMappings = buildMappings(metaData.indices(),
-                indexMetaData -> new TypeMappings(indexMetaData.getMappings()));
+    public IndexMappings(Metadata metadata) {
+        this.indexMappings = buildMappings(metadata.indices(),
+                indexMetadata -> new TypeMappings(indexMetadata.getMappings()));
     }
 
-    public IndexMappings(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings) {
+    public IndexMappings(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings) {
         this.indexMappings = buildMappings(mappings, TypeMappings::new);
     }
 

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java

@@ -15,7 +15,7 @@
 
 package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;
 
-import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 
 import java.util.Map;
@@ -36,7 +36,7 @@ public class TypeMappings implements Mappings<FieldMappings> {
      */
     private final Map<String, FieldMappings> typeMappings;
 
-    public TypeMappings(ImmutableOpenMap<String, MappingMetaData> mappings) {
+    public TypeMappings(ImmutableOpenMap<String, MappingMetadata> mappings) {
         typeMappings = buildMappings(mappings, FieldMappings::new);
     }
 

src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java

@@ -143,7 +143,7 @@ private void async(Client client, Map<String, String> params, QueryAction queryA
 
         // Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions
         threadPool.schedule(
-                threadPool.preserveContext(LogUtils.withCurrentContext(runnable)),
+                LogUtils.withCurrentContext(runnable),
                 new TimeValue(0L),
                 SQL_WORKER_THREAD_POOL_NAME
         );

src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java

@@ -19,8 +19,8 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -77,22 +77,22 @@ public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBui
         return new BytesRestResponse(RestStatus.OK, builder);
     }
 
-    private void writeAliases(List<AliasMetaData> aliases, XContentBuilder builder, ToXContent.Params params)
+    private void writeAliases(List<AliasMetadata> aliases, XContentBuilder builder, ToXContent.Params params)
             throws IOException {
         builder.startObject(Fields.ALIASES);
         if (aliases != null) {
-            for (AliasMetaData alias : aliases) {
-                AliasMetaData.Builder.toXContent(alias, builder, params);
+            for (AliasMetadata alias : aliases) {
+                AliasMetadata.Builder.toXContent(alias, builder, params);
             }
         }
         builder.endObject();
     }
 
-    private void writeMappings(ImmutableOpenMap<String, MappingMetaData> mappings,
+    private void writeMappings(ImmutableOpenMap<String, MappingMetadata> mappings,
                                XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject(Fields.MAPPINGS);
         if (mappings != null) {
-            for (ObjectObjectCursor<String, MappingMetaData> typeEntry : mappings) {
+            for (ObjectObjectCursor<String, MappingMetadata> typeEntry : mappings) {
                 builder.field(typeEntry.key);
                 builder.map(typeEntry.value.sourceAsMap());
             }
@@ -114,4 +114,4 @@ static class Fields {
         static final String SETTINGS = "settings";
         static final String WARMERS = "warmers";
     }
-}
+}

src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java

@@ -93,7 +93,7 @@ private void async(Client client, Map<String, String> params, RestChannel channe
 
         // Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions
         threadPool.schedule(
-                threadPool.preserveContext(LogUtils.withCurrentContext(runnable)),
+                LogUtils.withCurrentContext(runnable),
                 new TimeValue(0L),
                 SQL_WORKER_THREAD_POOL_NAME
         );

src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java

@@ -22,7 +22,7 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 
 import java.util.ArrayList;
@@ -89,33 +89,33 @@ private List<Column> loadColumns() {
     private List<Row> loadRows() {
         List<Row> rows = new ArrayList<>();
         GetIndexResponse indexResponse = (GetIndexResponse) queryResult;
-        ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> indexMappings = indexResponse.getMappings();
+        ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> indexMappings = indexResponse.getMappings();
 
         // Iterate through indices in indexMappings
-        for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexCursor : indexMappings) {
+        for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetadata>> indexCursor : indexMappings) {
             String index = indexCursor.key;
 
             // Check to see if index matches given pattern
             if (matchesPattern(index, statement.getIndexPattern())) {
-                ImmutableOpenMap<String, MappingMetaData> typeMapping = indexCursor.value;
+                ImmutableOpenMap<String, MappingMetadata> typeMapping = indexCursor.value;
                 // Assuming ES 6.x, iterate through the only type of the index to get mapping data
-                for (ObjectObjectCursor<String, MappingMetaData> typeCursor : typeMapping) {
-                    MappingMetaData mappingMetaData = typeCursor.value;
+                for (ObjectObjectCursor<String, MappingMetadata> typeCursor : typeMapping) {
+                    MappingMetadata mappingMetadata = typeCursor.value;
                     // Load rows for each field in the mapping
-                    rows.addAll(loadIndexData(index, mappingMetaData.getSourceAsMap()));
+                    rows.addAll(loadIndexData(index, mappingMetadata.getSourceAsMap()));
                 }
             }
         }
         return rows;
     }
 
     @SuppressWarnings("unchecked")
-    private List<Row> loadIndexData(String index, Map<String, Object> mappingMetaData) {
+    private List<Row> loadIndexData(String index, Map<String, Object> mappingMetadata) {
         List<Row> rows = new ArrayList<>();
 
-        Map<String, String> flattenedMetaData = flattenMappingMetaData(mappingMetaData, "", new HashMap<>());
+        Map<String, String> flattenedMetadata = flattenMappingMetadata(mappingMetadata, "", new HashMap<>());
         int position = 1; // Used as an arbitrary ORDINAL_POSITION value for the time being
-        for (Entry<String, String> entry : flattenedMetaData.entrySet()) {
+        for (Entry<String, String> entry : flattenedMetadata.entrySet()) {
             String columnPattern = statement.getColumnPattern();
 
             // Check to see if column name matches pattern, if given
@@ -153,21 +153,21 @@ private Map<String, Object> loadRowData(String index, String column, String type
      * 'GetIndexRequestBuilder' that was used in the old ShowQueryAction. Since the format of the resulting meta data
      * is different, this method is being used to flatten and retrieve types.
      * <p>
-     * In the future, should look for a way to generalize this since Schema is currently using FieldMappingMetaData
-     * whereas here we are using MappingMetaData.
+     * In the future, should look for a way to generalize this since Schema is currently using FieldMappingMetadata
+     * whereas here we are using MappingMetadata.
      */
     @SuppressWarnings("unchecked")
-    private Map<String, String> flattenMappingMetaData(Map<String, Object> mappingMetaData,
+    private Map<String, String> flattenMappingMetadata(Map<String, Object> mappingMetadata,
                                                        String currPath,
                                                        Map<String, String> flattenedMapping) {
-        Map<String, Object> properties = (Map<String, Object>) mappingMetaData.get("properties");
+        Map<String, Object> properties = (Map<String, Object>) mappingMetadata.get("properties");
         for (Entry<String, Object> entry : properties.entrySet()) {
-            Map<String, Object> metaData = (Map<String, Object>) entry.getValue();
+            Map<String, Object> metadata = (Map<String, Object>) entry.getValue();
 
             String fullPath = addToPath(currPath, entry.getKey());
-            flattenedMapping.put(fullPath, (String) metaData.getOrDefault("type", DEFAULT_OBJECT_DATATYPE));
-            if (metaData.containsKey("properties")) {
-                flattenedMapping = flattenMappingMetaData(metaData, fullPath, flattenedMapping);
+            flattenedMapping.put(fullPath, (String) metadata.getOrDefault("type", DEFAULT_OBJECT_DATATYPE));
+            if (metadata.containsKey("properties")) {
+                flattenedMapping = flattenMappingMetadata(metadata, fullPath, flattenedMapping);
             }
         }