Merge branch 'master' into ccr_clear_fetch_errors_empty_fetch_response

* master: (63 commits)
  [Build] randomizedtesting: Allow property values to be closures (elastic#34319)
  Feature/hlrc ml docs cleanup (elastic#34316)
  Docs: DRY up CRUD docs (elastic#34203)
  Minor corrections in geo-queries.asciidoc (elastic#34314)
  [DOCS] Remove beta label from normalizers (elastic#34326)
  Adjust size of BigArrays in circuit breaker test
  Adapt bwc version after backport
  Follow stats structure (elastic#34301)
  Rename CCR stats implementation (elastic#34300)
  Add max_children limit to nested sort (elastic#33587)
  MINOR: Remove Dead Code from Netty4Transport (elastic#34134)
  Rename clsuterformation -> testclusters (elastic#34299)
  [Build] make sure there are no duplicate classes in third party audit (elastic#34213)
  BWC Build: Read CI properties to determine java version (elastic#34295)
  [DOCS] Fix typo and add [float]
  Allow User/Password realms to disable authc (elastic#34033)
  Enable security automaton caching (elastic#34028)
  Preserve thread context during authentication. (elastic#34290)
  [ML] Allow asynchronous job deletion (elastic#34058)
  HLRC: ML Adding get datafeed stats API (elastic#34271)
  ...

.ci/java-versions.properties

@@ -4,5 +4,5 @@
 # build and test Elasticsearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-ES_BUILD_JAVA=java10
+ES_BUILD_JAVA=java11
 ES_RUNTIME_JAVA=java8

.ci/matrix-build-javas.yml

@@ -6,5 +6,4 @@
 # or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA:
-  - java10
   - java11

.ci/matrix-runtime-javas.yml

@@ -8,5 +8,4 @@
 ES_RUNTIME_JAVA:
   - java8
   - java8fips
-  - java10
   - java11

CONTRIBUTING.md

@@ -320,7 +320,7 @@ have to test Elasticsearch.
 #### Configurations
 
 Gradle organizes dependencies and build artifacts into "configurations" and
-allows you to use these configurations arbitrarilly. Here are some of the most
+allows you to use these configurations arbitrarily. Here are some of the most
 common configurations in our build and how we use them:
 
 <dl>

TESTING.asciidoc

@@ -250,7 +250,7 @@ Pass arbitrary jvm arguments.
 
 Running backwards compatibility tests is disabled by default since it
 requires a release version of elasticsearch to be present on the test system.
-To run backwards compatibilty tests untar or unzip a release and run the tests
+To run backwards compatibility tests untar or unzip a release and run the tests
 with the following command:
 
 ---------------------------------------------------------------------------

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy

@@ -264,7 +264,11 @@ class RandomizedTestingTask extends DefaultTask {
                         throw new InvalidUserDataException('Seed should be ' +
                             'set on the project instead of a system property')
                     }
-                    sysproperty key: prop.getKey(), value: prop.getValue().toString()
+                    if (prop.getValue() instanceof Closure) {
+                        sysproperty key: prop.getKey(), value: (prop.getValue() as Closure).call().toString()
+                    } else {
+                        sysproperty key: prop.getKey(), value: prop.getValue().toString()
+                    }
                 }
                 systemProperty 'tests.seed', project.testSeed
                 for (Map.Entry<String, Object> envvar : environmentVariables) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy

@@ -122,7 +122,7 @@ class VersionCollection {
                         if (isReleased(version) == false) {
                             // caveat 1 - This should only ever contain 2 non released branches in flight. An example is 6.x is frozen,
                             // and 6.2 is cut but not yet released there is some simple logic to make sure that in the case of more than 2,
-                            // it will bail. The order is that the minor snapshot is fufilled first, and then the staged minor snapshot
+                            // it will bail. The order is that the minor snapshot is fulfilled first, and then the staged minor snapshot
                             if (nextMinorSnapshot == null) {
                                 // it has not been set yet
                                 nextMinorSnapshot = replaceAsSnapshot(version)

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -72,7 +72,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
 
     /**
      * Root directory containing all the files generated by this task. It is
-     * contained withing testRoot.
+     * contained within testRoot.
      */
     File outputRoot() {
         return new File(testRoot, '/rest-api-spec/test')
@@ -226,10 +226,10 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             } else {
                 current.println('---')
                 current.println("\"line_$test.start\":")
-                /* The Elasticsearch test runner doesn't support the warnings
-                 * construct unless you output this skip. Since we don't know
-                 * if this snippet will use the warnings construct we emit this
-                 * warning every time. */
+                /* The Elasticsearch test runner doesn't support quite a few
+                 * constructs unless we output this skip. We don't know if
+                 * we're going to use these constructs, but we might so we
+                 * output the skip just in case. */
                 current.println("  - skip:")
                 current.println("      features: ")
                 current.println("        - default_shards")
@@ -250,13 +250,13 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                     }
                 }
             }
-            if (test.skipTest) {
+            if (test.skip) {
                 if (test.continued) {
                     throw new InvalidUserDataException("Continued snippets "
                         + "can't be skipped")
                 }
                 current.println("        - always_skip")
-                current.println("      reason: $test.skipTest")
+                current.println("      reason: $test.skip")
             }
             if (test.setup != null) {
                 // Insert a setup defined outside of the docs
@@ -274,9 +274,11 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         }
 
         private void response(Snippet response) {
-            current.println("  - match: ")
-            current.println("      \$body: ")
-            response.contents.eachLine { current.println("        $it") }
+            if (null == response.skip) {
+                current.println("  - match: ")
+                current.println("      \$body: ")
+                response.contents.eachLine { current.println("        $it") }
+            }
         }
 
         void emitDo(String method, String pathAndQuery, String body,

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -122,7 +122,9 @@ public class SnippetsTask extends DefaultTask {
                             + "contain `curl`.")
                     }
                 }
-                if (snippet.testResponse && snippet.language == 'js') {
+                if (snippet.testResponse
+                        && 'js' == snippet.language
+                        && null == snippet.skip) {
                     String quoted = snippet.contents
                         // quote values starting with $
                         .replaceAll(/([:,])\s*(\$[^ ,\n}]+)/, '$1 "$2"')
@@ -216,7 +218,7 @@ public class SnippetsTask extends DefaultTask {
                                 return
                             }
                             if (it.group(4) != null) {
-                                snippet.skipTest = it.group(4)
+                                snippet.skip = it.group(4)
                                 return
                             }
                             if (it.group(5) != null) {
@@ -249,7 +251,7 @@ public class SnippetsTask extends DefaultTask {
                             substitutions = []
                         }
                         String loc = "$file:$lineNumber"
-                        parse(loc, matcher.group(2), /(?:$SUBSTITUTION|$CAT) ?/) {
+                        parse(loc, matcher.group(2), /(?:$SUBSTITUTION|$CAT|$SKIP) ?/) {
                             if (it.group(1) != null) {
                                 // TESTRESPONSE[s/adsf/jkl/]
                                 substitutions.add([it.group(1), it.group(2)])
@@ -259,6 +261,9 @@ public class SnippetsTask extends DefaultTask {
                                 substitutions.add(['\n$', '\\\\s*/'])
                                 substitutions.add(['( +)', '$1\\\\s+'])
                                 substitutions.add(['\n', '\\\\s*\n '])
+                            } else if (it.group(4) != null) {
+                                // TESTRESPONSE[skip:reason]
+                                snippet.skip = it.group(4)
                             }
                         }
                     }
@@ -312,7 +317,7 @@ public class SnippetsTask extends DefaultTask {
         boolean test = false
         boolean testResponse = false
         boolean testSetup = false
-        String skipTest = null
+        String skip = null
         boolean continued = false
         String language = null
         String catchPart = null
@@ -337,8 +342,8 @@ public class SnippetsTask extends DefaultTask {
                 if (catchPart) {
                     result += "[catch: $catchPart]"
                 }
-                if (skipTest) {
-                    result += "[skip=$skipTest]"
+                if (skip) {
+                    result += "[skip=$skip]"
                 }
                 if (continued) {
                     result += '[continued]'
@@ -352,6 +357,9 @@ public class SnippetsTask extends DefaultTask {
             }
             if (testResponse) {
                 result += '// TESTRESPONSE'
+                if (skip) {
+                    result += "[skip=$skip]"
+                }
             }
             if (testSetup) {
                 result += '// TESTSETUP'

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -337,7 +337,7 @@ class NodeInfo {
             case 'deb':
                 return new File(baseDir, "${distro}-extracted/etc/elasticsearch")
             default:
-                throw new InvalidUserDataException("Unkown distribution: ${distro}")
+                throw new InvalidUserDataException("Unknown distribution: ${distro}")
         }
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java

@@ -43,7 +43,7 @@ private void scanForJDKJarHell(Path root) throws IOException {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
                 String entry = root.relativize(file).toString().replace('\\', '/');
-                if (entry.endsWith(".class")) {
+                if (entry.endsWith(".class") && entry.endsWith("module-info.class") == false) {
                     if (ext.getResource(entry) != null) {
                         detected.add(
                             entry

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -20,12 +20,12 @@
 
 import org.apache.commons.io.output.NullOutputStream;
 import org.elasticsearch.gradle.JdkJarHellCheck;
-import org.elasticsearch.test.NamingConventionsCheck;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.FileCollection;
+import org.gradle.api.file.FileTree;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
@@ -47,6 +47,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 public class ThirdPartyAuditTask extends DefaultTask {
 
@@ -171,19 +172,38 @@ private void extractJars(FileCollection jars) {
         File jarExpandDir = getJarExpandDir();
         // We need to clean up to make sure old dependencies don't linger
         getProject().delete(jarExpandDir);
-        jars.forEach(jar ->
+
+        jars.forEach(jar -> {
+            FileTree jarFiles = getProject().zipTree(jar);
             getProject().copy(spec -> {
+                spec.from(jarFiles);
+                spec.into(jarExpandDir);
+                // exclude classes from multi release jars
+                spec.exclude("META-INF/versions/**");
+            });
+            // Deal with multi release jars:
+            // The order is important, we iterate here so we don't depend on the order in which Gradle executes the spec
+            // We extract multi release jar classes ( if these exist ) going from 9 - the first to support them, to the
+            // current `targetCompatibility` version.
+            // Each extract will overwrite the top level classes that existed before it, the result is that we end up
+            // with a single version of the class in `jarExpandDir`.
+            // This will be the closes version to `targetCompatibility`, the same class that would be loaded in a JVM
+            // that has `targetCompatibility` version.
+            // This means we only scan classes that would be loaded into `targetCompatibility`, and don't look at any
+            // pther version specific implementation of said classes.
+            IntStream.rangeClosed(
+                Integer.parseInt(JavaVersion.VERSION_1_9.getMajorVersion()),
+                Integer.parseInt(targetCompatibility.getMajorVersion())
+            ).forEach(majorVersion -> getProject().copy(spec -> {
                 spec.from(getProject().zipTree(jar));
                 spec.into(jarExpandDir);
-                // Exclude classes for multi release jars above target
-                for (int i = Integer.parseInt(targetCompatibility.getMajorVersion()) + 1;
-                     i <= Integer.parseInt(JavaVersion.VERSION_HIGHER.getMajorVersion());
-                     i++
-                ) {
-                  spec.exclude("META-INF/versions/" + i + "/**");
-                }
-            })
-        );
+                String metaInfPrefix = "META-INF/versions/" + majorVersion;
+                spec.include(metaInfPrefix + "/**");
+                // Drop the version specific prefix
+                spec.eachFile(details -> details.setPath(details.getPath().replace(metaInfPrefix, "")));
+                spec.setIncludeEmptyDirs(false);
+            }));
+        });
     }
 
     private void assertNoJarHell(Set<String> jdkJarHellClasses) {
@@ -276,9 +296,9 @@ private String formatClassList(Set<String> classList) {
     private Set<String> runJdkJarHellCheck() throws IOException {
         ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
         ExecResult execResult = getProject().javaexec(spec -> {
-            URL location = NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation();
+            URL location = JdkJarHellCheck.class.getProtectionDomain().getCodeSource().getLocation();
             if (location.getProtocol().equals("file") == false) {
-                throw new GradleException("Unexpected location for NamingConventionCheck class: " + location);
+                throw new GradleException("Unexpected location for JdkJarHellCheck class: " + location);
             }
             try {
                 spec.classpath(

buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ElasticsearchConfiguration.java → buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchConfiguration.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.clusterformation;
+package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.gradle.Distribution;
 import org.elasticsearch.gradle.Version;

buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ElasticsearchNode.java → buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.clusterformation;
+package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.GradleServicesAdapter;
 import org.elasticsearch.gradle.Distribution;

buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ClusterformationPlugin.java → buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.clusterformation;
+package org.elasticsearch.gradle.testclusters;
 
 import groovy.lang.Closure;
 import org.elasticsearch.GradleServicesAdapter;
@@ -37,12 +37,12 @@
 import java.util.List;
 import java.util.Map;
 
-public class ClusterformationPlugin implements Plugin<Project> {
+public class TestClustersPlugin implements Plugin<Project> {
 
     public static final String LIST_TASK_NAME = "listElasticSearchClusters";
     public static final String EXTENSION_NAME = "elasticSearchClusters";
 
-    private final Logger logger =  Logging.getLogger(ClusterformationPlugin.class);
+    private final Logger logger =  Logging.getLogger(TestClustersPlugin.class);
 
     @Override
     public void apply(Project project) {

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.testclusters.properties

@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.testclusters.TestClustersPlugin

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -42,11 +42,6 @@
   <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]StoredScriptsDocumentationIT.java" id="SnippetLength" />
   <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]TasksClientDocumentationIT.java" id="SnippetLength" />
   <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]WatcherDocumentationIT.java" id="SnippetLength" />
-  <!--
-    This one is in plugins/examples/script-expert-scoring but we need to
-    suppress it like this because we build that project twice, once in for
-    real and once as a test for our build system. -->
-  <suppress files="src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]example[/\\]expertscript[/\\]ExpertScriptPlugin.java" id="SnippetLength" />
   <suppress files="modules[/\\]reindex[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]ReindexDocumentationIT.jav" id="SnippetLength" />
 
   <!-- Hopefully temporary suppression of LineLength on files that don't pass it. We should remove these when we the

buildSrc/src/main/resources/minimumCompilerVersion

@@ -1 +1 @@
-1.10
+1.11