Merge remote-tracking branch 'origin/master' into closed-index-replic…

…ation

build.gradle

@@ -53,9 +53,23 @@ subprojects {
   description = "Elasticsearch subproject ${project.path}"
 }
 
+apply plugin: 'nebula.info-scm'
+String licenseCommit
+if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
+  licenseCommit = scminfo.change ?: "master" // leniency for non git builds
+} else {
+  licenseCommit = "v${version}"
+}
+String elasticLicenseUrl = "https://raw.githubusercontent.com/elastic/elasticsearch/${licenseCommit}/licenses/ELASTIC-LICENSE.txt"
+
 subprojects {
+  // Default to the apache license
   project.ext.licenseName = 'The Apache Software License, Version 2.0'
   project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+
+  // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
+  project.ext.elasticLicenseUrl = elasticLicenseUrl
+
   // we only use maven publish to add tasks for pom generation
   plugins.withType(MavenPublishPlugin).whenPluginAdded {
     publishing {
@@ -131,7 +145,7 @@ task verifyVersions {
     new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
         xml = new XmlParser().parse(s)
     }
-    Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) })
+    Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }.collect { Version.fromString(it) })
 
     // Limit the known versions to those that should be index compatible, and are not future versions
     knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }

buildSrc/src/main/resources/checkstyle.xml

@@ -26,13 +26,9 @@
     </module>
 
     <module name="AvoidStarImport" />
-    <!-- Doesn't pass but we could make it pass pretty quick.
-    <module name="UnusedImports">
-      The next property is optional. If we remove it then imports that are
-      only referenced by Javadoc cause the check to fail.
-      <property name="processJavadoc" value="true" />
-    </module>
-    -->
+
+    <!-- Unused imports are forbidden -->
+    <module name="UnusedImports" />
 
     <!-- Non-inner classes must be in files that match their names. -->
     <module name="OuterTypeFilename" />

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0-snapshot-0a7c3f462f
+lucene            = 7.4.0-snapshot-518d303506
 
 # optional dependencies
 spatial4j         = 0.7

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -57,6 +57,8 @@
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -661,6 +663,36 @@ public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, Re
             PutIndexTemplateResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Validate a potentially expensive query without executing it.
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-validate.html"> Validate Query API
+     * on elastic.co</a>
+     * @param validateQueryRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public ValidateQueryResponse validateQuery(ValidateQueryRequest validateQueryRequest, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(validateQueryRequest, RequestConverters::validateQuery, options,
+            ValidateQueryResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously validate a potentially expensive query without executing it.
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-validate.html"> Validate Query API
+     * on elastic.co</a>
+     * @param validateQueryRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void validateQueryAsync(ValidateQueryRequest validateQueryRequest, RequestOptions options,
+                                   ActionListener<ValidateQueryResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(validateQueryRequest, RequestConverters::validateQuery, options,
+            ValidateQueryResponse::fromXContent, listener, emptySet());
+    }
+
     /**
      * Gets index templates using the Index Templates API
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -58,6 +58,7 @@
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
@@ -856,6 +857,20 @@ static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) thro
         return request;
     }
 
+    static Request validateQuery(ValidateQueryRequest validateQueryRequest) throws IOException {
+        String[] indices = validateQueryRequest.indices() == null ? Strings.EMPTY_ARRAY : validateQueryRequest.indices();
+        String[] types = validateQueryRequest.types() == null || indices.length <= 0 ? Strings.EMPTY_ARRAY : validateQueryRequest.types();
+        String endpoint = endpoint(indices, types, "_validate/query");
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Params params = new Params(request);
+        params.withIndicesOptions(validateQueryRequest.indicesOptions());
+        params.putParam("explain", Boolean.toString(validateQueryRequest.explain()));
+        params.putParam("all_shards", Boolean.toString(validateQueryRequest.allShards()));
+        params.putParam("rewrite", Boolean.toString(validateQueryRequest.rewrite()));
+        request.setEntity(createEntity(validateQueryRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request getAlias(GetAliasesRequest getAliasesRequest) {
         String[] indices = getAliasesRequest.indices() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.indices();
         String[] aliases = getAliasesRequest.aliases() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.aliases();

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.client;
 
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
@@ -63,6 +64,8 @@
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
@@ -80,6 +83,8 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -1155,6 +1160,40 @@ public void testPutTemplateBadRequests() throws Exception {
         assertThat(unknownSettingError.getDetailedMessage(), containsString("unknown setting [index.this-setting-does-not-exist]"));
     }
 
+    public void testValidateQuery() throws IOException{
+        String index = "some_index";
+        createIndex(index, Settings.EMPTY);
+        QueryBuilder builder = QueryBuilders
+            .boolQuery()
+            .must(QueryBuilders.queryStringQuery("*:*"))
+            .filter(QueryBuilders.termQuery("user", "kimchy"));
+        ValidateQueryRequest request = new ValidateQueryRequest(index).query(builder);
+        request.explain(randomBoolean());
+        ValidateQueryResponse response = execute(request, highLevelClient().indices()::validateQuery,
+            highLevelClient().indices()::validateQueryAsync);
+        assertTrue(response.isValid());
+    }
+
+    public void testInvalidValidateQuery() throws IOException{
+        String index = "shakespeare";
+
+        createIndex(index, Settings.EMPTY);
+        Request postDoc = new Request(HttpPost.METHOD_NAME, "/" + index + "/1");
+        postDoc.setJsonEntity(
+            "{\"type\":\"act\",\"line_id\":1,\"play_name\":\"Henry IV\", \"speech_number\":\"\"," +
+                "\"line_number\":\"\",\"speaker\":\"\",\"text_entry\":\"ACT I\"}");
+        assertOK(client().performRequest(postDoc));
+
+        QueryBuilder builder = QueryBuilders
+            .queryStringQuery("line_id:foo")
+            .lenient(false);
+        ValidateQueryRequest request = new ValidateQueryRequest(index).query(builder);
+        request.explain(true);
+        ValidateQueryResponse response = execute(request, highLevelClient().indices()::validateQuery,
+            highLevelClient().indices()::validateQueryAsync);
+        assertFalse(response.isValid());
+    }
+
     public void testGetIndexTemplate() throws Exception {
         RestHighLevelClient client = highLevelClient();
 

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -60,6 +60,7 @@
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -1895,6 +1896,40 @@ public void testPutTemplateRequest() throws Exception {
         assertToXContentBody(putTemplateRequest, request.getEntity());
     }
 
+    public void testValidateQuery() throws Exception {
+        String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5);
+        String[] types = randomBoolean() ? generateRandomStringArray(5, 5, false, false) : null;
+        ValidateQueryRequest validateQueryRequest;
+        if (randomBoolean()) {
+            validateQueryRequest = new ValidateQueryRequest(indices);
+        } else {
+            validateQueryRequest = new ValidateQueryRequest();
+            validateQueryRequest.indices(indices);
+        }
+        validateQueryRequest.types(types);
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomIndicesOptions(validateQueryRequest::indicesOptions, validateQueryRequest::indicesOptions, expectedParams);
+        validateQueryRequest.explain(randomBoolean());
+        validateQueryRequest.rewrite(randomBoolean());
+        validateQueryRequest.allShards(randomBoolean());
+        expectedParams.put("explain", Boolean.toString(validateQueryRequest.explain()));
+        expectedParams.put("rewrite", Boolean.toString(validateQueryRequest.rewrite()));
+        expectedParams.put("all_shards", Boolean.toString(validateQueryRequest.allShards()));
+        Request request = RequestConverters.validateQuery(validateQueryRequest);
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        if (indices != null && indices.length > 0) {
+            endpoint.add(String.join(",", indices));
+            if (types != null && types.length > 0) {
+                endpoint.add(String.join(",", types));
+            }
+        }
+        endpoint.add("_validate/query");
+        assertThat(request.getEndpoint(), equalTo(endpoint.toString()));
+        assertThat(request.getParameters(), equalTo(expectedParams));
+        assertToXContentBody(validateQueryRequest, request.getEntity());
+        assertThat(request.getMethod(), equalTo(HttpGet.METHOD_NAME));
+    }
+
     public void testGetTemplateRequest() throws Exception {
         Map<String, String> encodes = new HashMap<>();
         encodes.put("log", "log");

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
+
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
@@ -607,7 +608,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(7, namedXContents.size());
+        assertEquals(8, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -625,9 +626,10 @@ public void testProvidedNamedXContents() {
         assertTrue(names.contains(PrecisionAtK.NAME));
         assertTrue(names.contains(DiscountedCumulativeGain.NAME));
         assertTrue(names.contains(MeanReciprocalRank.NAME));
-        assertEquals(Integer.valueOf(2), categories.get(MetricDetail.class));
+        assertEquals(Integer.valueOf(3), categories.get(MetricDetail.class));
         assertTrue(names.contains(PrecisionAtK.NAME));
         assertTrue(names.contains(MeanReciprocalRank.NAME));
+        assertTrue(names.contains(DiscountedCumulativeGain.NAME));
     }
 
     private static class TrackingActionListener implements ActionListener<Integer> {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java

@@ -62,6 +62,9 @@
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.validate.query.QueryExplanation;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -81,6 +84,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 
@@ -2128,4 +2132,83 @@ public void onFailure(Exception e) {
 
         assertTrue(latch.await(30L, TimeUnit.SECONDS));
     }
+
+    public void testValidateQuery() throws IOException, InterruptedException {
+        RestHighLevelClient client = highLevelClient();
+
+        String index = "some_index";
+        createIndex(index, Settings.EMPTY);
+
+        // tag::validate-query-request
+        ValidateQueryRequest request = new ValidateQueryRequest(index); // <1>
+        // end::validate-query-request
+
+        // tag::validate-query-request-query
+        QueryBuilder builder = QueryBuilders
+            .boolQuery() // <1>
+            .must(QueryBuilders.queryStringQuery("*:*"))
+            .filter(QueryBuilders.termQuery("user", "kimchy"));
+        request.query(builder); // <2>
+        // end::validate-query-request-query
+
+        // tag::validate-query-request-explain
+        request.explain(true); // <1>
+        // end::validate-query-request-explain
+
+        // tag::validate-query-request-allShards
+        request.allShards(true); // <1>
+        // end::validate-query-request-allShards
+
+        // tag::validate-query-request-rewrite
+        request.rewrite(true); // <1>
+        // end::validate-query-request-rewrite
+
+        // tag::validate-query-execute
+        ValidateQueryResponse response = client.indices().validateQuery(request, RequestOptions.DEFAULT); // <1>
+        // end::validate-query-execute
+
+        // tag::validate-query-response
+        boolean isValid = response.isValid(); // <1>
+        int totalShards = response.getTotalShards(); // <2>
+        int successfulShards = response.getSuccessfulShards(); // <3>
+        int failedShards = response.getFailedShards(); // <4>
+        if (failedShards > 0) {
+            for(DefaultShardOperationFailedException failure: response.getShardFailures()) { // <5>
+                String failedIndex = failure.index(); // <6>
+                int shardId = failure.shardId(); // <7>
+                String reason = failure.reason(); // <8>
+            }
+        }
+        for(QueryExplanation explanation: response.getQueryExplanation()) { // <9>
+            String explanationIndex = explanation.getIndex(); // <10>
+            int shardId = explanation.getShard(); // <11>
+            String explanationString = explanation.getExplanation(); // <12>
+        }
+        // end::validate-query-response
+
+        // tag::validate-query-execute-listener
+        ActionListener<ValidateQueryResponse> listener =
+            new ActionListener<ValidateQueryResponse>() {
+                @Override
+                public void onResponse(ValidateQueryResponse validateQueryResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+        // end::validate-query-execute-listener
+
+        // Replace the empty listener by a blocking listener in test
+        final CountDownLatch latch = new CountDownLatch(1);
+        listener = new LatchedActionListener<>(listener, latch);
+
+        // tag::validate-query-execute-async
+        client.indices().validateQueryAsync(request, RequestOptions.DEFAULT, listener); // <1>
+        // end::validate-query-execute-async
+
+        assertTrue(latch.await(30L, TimeUnit.SECONDS));
+    }
 }