Merge branch 'main' into lucene_snapshot

cbuescher · Oct 11, 2024 · edb88a0 · edb88a0
2 parents 34ed721 + a91a9a5
commit edb88a0
Show file tree

Hide file tree

Showing 145 changed files with 3,171 additions and 1,232 deletions.
diff --git a/docs/changelog/114303.yaml b/docs/changelog/114303.yaml
@@ -0,0 +1,6 @@
+pr: 114303
+summary: Fix TDigestState.read CB leaks
+area: ES|QL
+type: bug
+issues:
+ - 114194
diff --git a/docs/changelog/114309.yaml b/docs/changelog/114309.yaml
@@ -0,0 +1,6 @@
+pr: 114309
+summary: Upgrade to AWS SDK v2
+area: Machine Learning
+type: enhancement
+issues:
+ - 110590
diff --git a/docs/changelog/114386.yaml b/docs/changelog/114386.yaml
@@ -0,0 +1,5 @@
+pr: 114386
+summary: Improve handling of failure to create persistent task
+area: Task Management
+type: bug
+issues: []
diff --git a/docs/changelog/114429.yaml b/docs/changelog/114429.yaml
@@ -0,0 +1,5 @@
+pr: 114429
+summary: Add chunking settings configuration to `ElasticsearchService/ELSER`
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/114464.yaml b/docs/changelog/114464.yaml
@@ -0,0 +1,5 @@
+pr: 114464
+summary: Stream Azure Completion
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/114512.yaml b/docs/changelog/114512.yaml
@@ -0,0 +1,5 @@
+pr: 114512
+summary: Ensure clean thread context in `MasterService`
+area: Cluster Coordination
+type: bug
+issues: []
diff --git a/docs/changelog/114527.yaml b/docs/changelog/114527.yaml
@@ -0,0 +1,5 @@
+pr: 114527
+summary: Verify Maxmind database types in the geoip processor
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc
@@ -162,7 +162,7 @@ node.
 ====== Maximum number of retries exceeded
 
 The following response contains an allocation explanation for an unassigned
-primary shard that has reached the maximum number of allocation retry attempts. 
+primary shard that has reached the maximum number of allocation retry attempts.
 
 [source,js]
 ----
@@ -204,9 +204,10 @@ primary shard that has reached the maximum number of allocation retry attempts.
 ----
 // NOTCONSOLE
 
-If decider message indicates a transient allocation issue, use 
-<<cluster-reroute,the cluster reroute API>> to retry allocation. 
+If decider message indicates a transient allocation issue, use
+the <<cluster-reroute,cluster reroute>> API to retry allocation.
 
+[[no-valid-shard-copy]]
 ====== No valid shard copy
 
 The following response contains an allocation explanation for an unassigned
@@ -334,7 +335,7 @@ queued to allocate but currently waiting on other queued shards.
 ----
 // NOTCONSOLE
 
-This is a transient message that might appear when a large amount of shards are allocating. 
+This is a transient message that might appear when a large amount of shards are allocating.
 
 ===== Assigned shard
 
@@ -437,7 +438,7 @@ cluster balance.
 ===== No arguments
 
 If you call the API with no arguments, {es} retrieves an allocation explanation
-for an arbitrary unassigned primary or replica shard, returning any unassigned primary shards first. 
+for an arbitrary unassigned primary or replica shard, returning any unassigned primary shards first.
 
 [source,console]
 ----

diff --git a/docs/reference/security/ccs-clients-integrations/index.asciidoc b/docs/reference/security/ccs-clients-integrations/index.asciidoc
@@ -13,6 +13,7 @@ be secured as well, or at least communicate with the cluster in a secured way:
 * <<hadoop, Apache Hadoop>>
 * {auditbeat-ref}/securing-auditbeat.html[Auditbeat]
 * {filebeat-ref}/securing-filebeat.html[Filebeat]
+* {fleet-guide}/secure.html[{fleet} & {agent}]
 * {heartbeat-ref}/securing-heartbeat.html[Heartbeat]
 * {kibana-ref}/using-kibana-with-security.html[{kib}]
 * {logstash-ref}/ls-security.html[Logstash]

diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -1456,6 +1456,11 @@
             <sha256 value="dfa817a156ea263aa9ad8364a2e226527665c9722aca40a7945f228c2c14f1da" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.netty" name="netty-transport-classes-epoll" version="4.1.109.Final">
+         <artifact name="netty-transport-classes-epoll-4.1.109.Final.jar">
+            <sha256 value="02ce34ba9ab6ecc62de667ed35a8680202a902ec6e90bed587bea2e574fae38d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.netty" name="netty-transport-native-epoll" version="4.1.42.Final">
          <artifact name="netty-transport-native-epoll-4.1.42.Final.jar">
             <sha256 value="3c7d659b3bd773e0ea9b7517d2d6baffa275a3d2ae8eb4c10cb8f0a7724b11d5" origin="Generated by Gradle"/>
@@ -4222,6 +4227,11 @@
             <sha256 value="f75ca597789b3dac58f61857b9ac2e1034a68fa672db35055a8fb4509e325f28" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.reactivestreams" name="reactive-streams-tck" version="1.0.4">
+         <artifact name="reactive-streams-tck-1.0.4.jar">
+            <sha256 value="8423b5062b3f3c5600a85cfad5a6ccbda88129a822be82b2e364d737d53c9836" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.redline-rpm" name="redline" version="1.2.10">
          <artifact name="redline-1.2.10.jar">
             <sha256 value="2e074e7bd42174eaeac98cebc782f0d7bcc8dc2a41617932ac9c79d56bc86ecd" origin="Generated by Gradle"/>
@@ -4435,6 +4445,136 @@
             <sha256 value="880c9d896e4b74a06c549c15ca496450165d6909fa15d7e662bee8f6a66d7afa" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="annotations" version="2.28.13">
+         <artifact name="annotations-2.28.13.jar">
+            <sha256 value="f8f0df5ee1fcfef0381d167ae50d85ce635b7e5b32d5d620bbb8019f183c6b41" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="auth" version="2.28.13">
+         <artifact name="auth-2.28.13.jar">
+            <sha256 value="494db83a2a06f09ba6717bb7fff07d50eb85b0b0d51904bf76601ee48e728741" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="aws-core" version="2.28.13">
+         <artifact name="aws-core-2.28.13.jar">
+            <sha256 value="8dd71bedb30d0a4a857cb9e5b30a48e81a66dd9381c13537c2f4639248adba60" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="aws-json-protocol" version="2.28.13">
+         <artifact name="aws-json-protocol-2.28.13.jar">
+            <sha256 value="63adac3a637c67f779cc56099e264f1cdd2fc4ac85c27e281b2cad53a693f7d2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="bedrockruntime" version="2.28.13">
+         <artifact name="bedrockruntime-2.28.13.jar">
+            <sha256 value="9ff1571e87a11114407eade316e4439b63275283ff49b6aaf52549c37d8e6a92" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="checksums" version="2.28.13">
+         <artifact name="checksums-2.28.13.jar">
+            <sha256 value="b4d452f8ceddf0e9a4874a9cfd65e62c257fef4c2b3b942893f590e09e945eca" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="checksums-spi" version="2.28.13">
+         <artifact name="checksums-spi-2.28.13.jar">
+            <sha256 value="20dfb45d582c175e48aa50237fd44704e31e91418b5d3da1092508dbcb9a4d11" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="endpoints-spi" version="2.28.13">
+         <artifact name="endpoints-spi-2.28.13.jar">
+            <sha256 value="b18dd1d66f03bf5e192ab51d7f3a8139e5bf1e7bab27501b00338f1d8e260f61" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="http-auth" version="2.28.13">
+         <artifact name="http-auth-2.28.13.jar">
+            <sha256 value="aeb18af3ba2c8b60b1012122737dc613e1b8d68558d69da301ba1c6a19e12593" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="http-auth-aws" version="2.28.13">
+         <artifact name="http-auth-aws-2.28.13.jar">
+            <sha256 value="cd0303e19ae51e8bb4dc1e454ef6e46e54ef865cf80988d775cfa8edb48ca975" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="http-auth-aws-eventstream" version="2.28.13">
+         <artifact name="http-auth-aws-eventstream-2.28.13.jar">
+            <sha256 value="55da99bf20179255f1da5551f41192095bf36eb8ab97b08f1a430f50bbb61733" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="http-auth-spi" version="2.28.13">
+         <artifact name="http-auth-spi-2.28.13.jar">
+            <sha256 value="61a142852865dc050006ff6ab61e59780d296e7b2721d578db255c4dc31be083" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="http-client-spi" version="2.28.13">
+         <artifact name="http-client-spi-2.28.13.jar">
+            <sha256 value="827763cbf43131d9542a58546a5252a4f6c4f32b861f755e339cb4c85e74f4c8" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="identity-spi" version="2.28.13">
+         <artifact name="identity-spi-2.28.13.jar">
+            <sha256 value="8baf158caf32cbab7cfdc2fabf48bac90e737917703c2a6e0502f46c46e3ef71" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="json-utils" version="2.28.13">
+         <artifact name="json-utils-2.28.13.jar">
+            <sha256 value="369ed42586213a33bc7f94e9d21594ee64fec1152819476c24c82b312b27b170" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="metrics-spi" version="2.28.13">
+         <artifact name="metrics-spi-2.28.13.jar">
+            <sha256 value="5ba10e6cf4882455eb27f6cb3b72832ea01d9bdf260c2c7fe80442b997af951b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="netty-nio-client" version="2.28.13">
+         <artifact name="netty-nio-client-2.28.13.jar">
+            <sha256 value="d382c2fc88121faa4a523e48683d22694f47638428741a1cf67985dd893de560" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="profiles" version="2.28.13">
+         <artifact name="profiles-2.28.13.jar">
+            <sha256 value="1889896c4a2714c3ae9ebd9c7be430c8b7add52fcd7561b4cdca70f41d225902" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="protocol-core" version="2.28.13">
+         <artifact name="protocol-core-2.28.13.jar">
+            <sha256 value="86bd64f9c897058578a7153d7d1dbc1ce4fcf7025b5b2651f4b3404144dbeba4" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="regions" version="2.28.13">
+         <artifact name="regions-2.28.13.jar">
+            <sha256 value="97051ca69715150e1bc3f12f89e05692585a0f1e8a6901699d1fe07cad9a348f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="retries" version="2.28.13">
+         <artifact name="retries-2.28.13.jar">
+            <sha256 value="3bf13cc6ba9dabf41009fcf8b9f31d936340d8b565b2bf83aa030d12494e8d18" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="retries-spi" version="2.28.13">
+         <artifact name="retries-spi-2.28.13.jar">
+            <sha256 value="d30595d38eb17929afe724290667873828f9c4c79dd5d5b8d3ae902318e6495f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="sdk-core" version="2.28.13">
+         <artifact name="sdk-core-2.28.13.jar">
+            <sha256 value="1072f71dd15f89596c63d30bdb779e0956eb4380f94c8b75d675d01846711df3" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="third-party-jackson-core" version="2.28.13">
+         <artifact name="third-party-jackson-core-2.28.13.jar">
+            <sha256 value="3782d94baf0d396f7687d186b3fe0f3ffdbab9d610f0099b782ae9d802387183" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.awssdk" name="utils" version="2.28.13">
+         <artifact name="utils-2.28.13.jar">
+            <sha256 value="c0ab5446f8009bde04a09de880804c5bda30cf8f69ce857116c1f99606c67069" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="software.amazon.eventstream" name="eventstream" version="1.0.1">
+         <artifact name="eventstream-1.0.1.jar">
+            <sha256 value="0c37d8e696117f02c302191b8110b0d0eb20fa412fce34c3a269ec73c16ce822" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="ua.net.nlp" name="morfologik-ukrainian-search" version="3.7.5">
          <artifact name="morfologik-ukrainian-search-3.7.5.jar">
             <sha256 value="1c7e5f7158e02bd35924a73a85958313c95a00d0973f830d0d19a8268a35a668" origin="Generated by Gradle"/>

diff --git a/.../javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java b/.../javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java
@@ -276,6 +276,33 @@ private Response query(final SearchSourceBuilder search, final Supplier<String>
         return client.performRequest(request);
     }
 
+    public Response esqlBaseline(final String query) throws IOException {
+        return esql(query, this::getBaselineDataStreamName);
+    }
+
+    public Response esqlContender(final String query) throws IOException {
+        return esql(query, this::getContenderDataStreamName);
+    }
+
+    private Response esql(final String query, final Supplier<String> dataStreamNameSupplier) throws IOException {
+        final Request request = new Request("POST", "/_query");
+        request.setJsonEntity("{\"query\": \"" + query.replace("$index", dataStreamNameSupplier.get()) + "\"}");
+        return client.performRequest(request);
+    }
+
+    public Response fieldCapsBaseline() throws IOException {
+        return fieldCaps(this::getBaselineDataStreamName);
+    }
+
+    public Response fieldCapsContender() throws IOException {
+        return fieldCaps(this::getContenderDataStreamName);
+    }
+
+    private Response fieldCaps(final Supplier<String> dataStreamNameSupplier) throws IOException {
+        final Request request = new Request("GET", "/" + dataStreamNameSupplier.get() + "/_field_caps?fields=*");
+        return client.performRequest(request);
+    }
+
     public String getBaselineDataStreamName() {
         return baselineDataStreamName;
     }

diff --git a/...a/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/...a/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java
@@ -42,6 +42,7 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -271,6 +272,51 @@ public void testDateHistogramAggregation() throws IOException {
         assertTrue(matchResult.getMessage(), matchResult.isMatch());
     }
 
+    public void testEsqlSource() throws IOException {
+        int numberOfDocuments = ESTestCase.randomIntBetween(100, 200);
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
+
+        indexDocuments(documents);
+
+        final String query = "FROM $index METADATA _source, _id | KEEP _source, _id | LIMIT " + numberOfDocuments;
+        final MatchResult matchResult = Matcher.matchSource()
+            .mappings(getContenderMappings(), getBaselineMappings())
+            .settings(getContenderSettings(), getBaselineSettings())
+            .expected(getEsqlSourceResults(esqlBaseline(query)))
+            .ignoringSort(true)
+            .isEqualTo(getEsqlSourceResults(esqlContender(query)));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
+    }
+
+    public void testEsqlTermsAggregation() throws IOException {
+        int numberOfDocuments = ESTestCase.randomIntBetween(100, 200);
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
+
+        indexDocuments(documents);
+
+        final String query = "FROM $index | STATS count(*) BY host.name | SORT host.name | LIMIT " + numberOfDocuments;
+        final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings())
+            .settings(getContenderSettings(), getBaselineSettings())
+            .expected(getEsqlStatsResults(esqlBaseline(query)))
+            .ignoringSort(true)
+            .isEqualTo(getEsqlStatsResults(esqlContender(query)));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
+    }
+
+    public void testFieldCaps() throws IOException {
+        int numberOfDocuments = ESTestCase.randomIntBetween(20, 50);
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
+
+        indexDocuments(documents);
+
+        final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings())
+            .settings(getContenderSettings(), getBaselineSettings())
+            .expected(getFields(fieldCapsBaseline()))
+            .ignoringSort(true)
+            .isEqualTo(getFields(fieldCapsContender()));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
+    }
+
     @Override
     public Response indexBaselineDocuments(CheckedSupplier<List<XContentBuilder>, IOException> documentsSupplier) throws IOException {
         var response = super.indexBaselineDocuments(documentsSupplier);
@@ -329,6 +375,40 @@ private static List<Map<String, Object>> getQueryHits(final Response response) t
             .toList();
     }
 
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> getFields(final Response response) throws IOException {
+        final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true);
+        final Map<String, Object> fields = (Map<String, Object>) map.get("fields");
+        assertThat(fields.size(), greaterThan(0));
+        return new TreeMap<>(fields);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static List<Map<String, Object>> getEsqlSourceResults(final Response response) throws IOException {
+        final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true);
+        final List<List<Object>> values = (List<List<Object>>) map.get("values");
+        assertThat(values.size(), greaterThan(0));
+
+        // Results contain a list of [source, id] lists.
+        return values.stream()
+            .sorted(Comparator.comparingInt((List<Object> value) -> Integer.parseInt((String) value.get(1))))
+            .map(value -> (Map<String, Object>) value.get(0))
+            .toList();
+    }
+
+    @SuppressWarnings("unchecked")
+    private static List<Map<String, Object>> getEsqlStatsResults(final Response response) throws IOException {
+        final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true);
+        final List<List<Object>> values = (List<List<Object>>) map.get("values");
+        assertThat(values.size(), greaterThan(0));
+
+        // Results contain a list of [agg value, group name] lists.
+        return values.stream()
+            .sorted(Comparator.comparing((List<Object> value) -> (String) value.get(1)))
+            .map(value -> Map.of((String) value.get(1), value.get(0)))
+            .toList();
+    }
+
     @SuppressWarnings("unchecked")
     private static List<Map<String, Object>> getAggregationBuckets(final Response response, final String aggName) throws IOException {
         final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true);