Merge branch 'main' into array-chunked-xcontent-builder

.buildkite/pipelines/dra-workflow.yml

@@ -2,6 +2,7 @@ steps:
   - command: .buildkite/scripts/dra-workflow.sh
     env:
       USE_DRA_CREDENTIALS: "true"
+      USE_PROD_DOCKER_CREDENTIALS: "true"
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2204
@@ -18,4 +19,5 @@ steps:
       branch: "${BUILDKITE_BRANCH}"
       env:
         DRA_WORKFLOW: staging
+        USE_PROD_DOCKER_CREDENTIALS: "true"
     if: build.env('DRA_WORKFLOW') == 'staging'

.buildkite/pipelines/periodic-packaging.template.yml

@@ -27,7 +27,8 @@ steps:
           image: family/elasticsearch-{{matrix.image}}
           diskSizeGb: 350
           machineType: n1-standard-8
-        env: {}
+        env:
+          USE_PROD_DOCKER_CREDENTIALS: "true"
   - group: packaging-tests-upgrade
     steps: $BWC_STEPS
   - group: packaging-tests-windows

.buildkite/pipelines/periodic-packaging.yml

@@ -28,7 +28,8 @@ steps:
           image: family/elasticsearch-{{matrix.image}}
           diskSizeGb: 350
           machineType: n1-standard-8
-        env: {}
+        env:
+          USE_PROD_DOCKER_CREDENTIALS: "true"
   - group: packaging-tests-upgrade
     steps:
       - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade"

.buildkite/pipelines/pull-request/packaging-tests-unix-sample.yml

@@ -24,4 +24,5 @@ steps:
           diskSizeGb: 350
           machineType: custom-16-32768
         env:
+          USE_PROD_DOCKER_CREDENTIALS: "true"
           PACKAGING_TASK: "{{matrix.PACKAGING_TASK}}"

build-tools-internal/build.gradle

@@ -384,6 +384,15 @@ tasks.named("jar") {
   exclude("classpath.index")
 }
 
+spotless {
+  java {
+    // IDEs can sometimes run annotation processors that leave files in
+    // here, causing Spotless to complain. Even though this path ought not
+    // to exist, exclude it anyway in order to avoid spurious failures.
+    toggleOffOn()
+  }
+}
+
 def resolveMainWrapperVersion() {
   new URL("https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/minimumGradleVersion").text.trim()
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -29,11 +29,13 @@ public enum DockerBase {
     CLOUD_ESS(null, "-cloud-ess", "apt-get"),
 
     // Chainguard based wolfi image with latest jdk
-    WOLFI(
-        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0",
+    // This is usually updated via renovatebot
+    // spotless:off
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0",
         "-wolfi",
         "apk"
     ),
+    // spotless:on
 
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the Wolfi image.

build-tools-internal/src/main/resources/changelog-schema.json

@@ -284,6 +284,7 @@
         "Cluster and node setting",
         "Command line tool",
         "CRUD",
+        "ES|QL",
         "Index setting",
         "Ingest",
         "JVM option",

docs/changelog/112567.yaml

@@ -0,0 +1,5 @@
+pr: 112567
+summary: Track shard snapshot progress during node shutdown
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/113563.yaml

@@ -0,0 +1,5 @@
+pr: 113563
+summary: Use ELSER By Default For Semantic Text
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/113897.yaml

@@ -0,0 +1,6 @@
+pr: 113897
+summary: "Add chunking settings configuration to `CohereService,` `AmazonBedrockService,`\
+  \ and `AzureOpenAiService`"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/113967.yaml

@@ -0,0 +1,13 @@
+pr: 113967
+summary: "ESQL: Entirely remove META FUNCTIONS"
+area: ES|QL
+type: breaking
+issues: []
+breaking:
+  title: "ESQL: Entirely remove META FUNCTIONS"
+  area: ES|QL
+  details: |
+    Removes an undocumented syntax from ESQL: META FUNCTION. This was never
+    reliable or really useful. Consult the documentation instead.
+  impact: "Removes an undocumented syntax from ESQL: META FUNCTION"
+  notable: false

docs/changelog/114128.yaml

@@ -0,0 +1,5 @@
+pr: 114128
+summary: Adding `index_template_substitutions` to the simulate ingest API
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/114157.yaml

@@ -0,0 +1,6 @@
+pr: 114157
+summary: Add a `terminate` ingest processor
+area: Ingest Node
+type: feature
+issues:
+ - 110218

docs/changelog/114264.yaml

@@ -0,0 +1,5 @@
+pr: 114264
+summary: "Fix analyzed wildcard query in simple_query_string when disjunctions is empty"
+area: Search
+type: bug
+issues: [114185]

docs/reference/esql/multivalued-fields.asciidoc

@@ -177,6 +177,37 @@ POST /_query
 ----
 // TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
 
+[discrete]
+[[esql-multivalued-nulls]]
+==== `null` in a list
+
+`null` values in a list are not preserved at the storage layer:
+
+[source,console,id=esql-multivalued-fields-multivalued-nulls]
+----
+POST /mv/_doc?refresh
+{ "a": [2, null, 1] }
+
+POST /_query
+{
+  "query": "FROM mv | LIMIT 1"
+}
+----
+
+[source,console-result]
+----
+{
+  "took": 28,
+  "columns": [
+    { "name": "a", "type": "long"},
+  ],
+  "values": [
+    [[1, 2]],
+  ]
+}
+----
+// TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
+
 [discrete]
 [[esql-multivalued-fields-functions]]
 ==== Functions

docs/reference/indices/put-index-template.asciidoc

@@ -85,6 +85,8 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 [[put-index-template-api-request-body]]
 ==== {api-request-body-title}
 
+// tag::request-body[]
+
 `composed_of`::
 (Optional, array of strings)
 An ordered list of component template names. Component templates are merged in the order
@@ -102,7 +104,7 @@ See <<create-index-template,create an index template>>.
 +
 .Properties of `data_stream`
 [%collapsible%open]
-====
+=====
 `allow_custom_routing`::
 (Optional, Boolean) If `true`, the data stream supports
 <<mapping-routing-field,custom routing>>. Defaults to `false`.
@@ -117,7 +119,7 @@ See <<create-index-template,create an index template>>.
 +
 If `time_series`, each backing index has an `index.mode` index setting of
 `time_series`.
-====
+=====
 
 `index_patterns`::
 (Required, array of strings)
@@ -146,7 +148,7 @@ Template to be applied. It may optionally include an `aliases`, `mappings`, or
 +
 .Properties of `template`
 [%collapsible%open]
-====
+=====
 `aliases`::
 (Optional, object of objects) Aliases to add.
 +
@@ -161,7 +163,7 @@ include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props]
 include::{docdir}/rest-api/common-parms.asciidoc[tag=mappings]
 
 include::{docdir}/rest-api/common-parms.asciidoc[tag=settings]
-====
+=====
 
 `version`::
 (Optional, integer)
@@ -174,6 +176,7 @@ Marks this index template as deprecated.
 When creating or updating a non-deprecated index template that uses deprecated components,
 {es} will emit a deprecation warning.
 // end::index-template-api-body[]
+// end::request-body[]
 
 [[put-index-template-api-example]]
 ==== {api-examples-title}

docs/reference/ingest/apis/simulate-ingest.asciidoc

@@ -102,13 +102,21 @@ POST /_ingest/_simulate
         }
       }
     }
+  },
+  "index_template_substitutions": { <3>
+    "my-index-template": {
+      "index_patterns": ["my-index-*"],
+      "composed_of": ["component_template_1", "component_template_2"]
+    }
   }
 }
 ----
 
 <1> This replaces the existing `my-pipeline` pipeline with the contents given here for the duration of this request.
 <2> This replaces the existing `my-component-template` component template with the contents given here for the duration of this request.
 These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result.
+<3> This replaces the existing `my-index-template` index template with the contents given here for the duration of this request.
+These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result.
 
 [[simulate-ingest-api-request]]
 ==== {api-request-title}
@@ -225,6 +233,19 @@ include::{es-ref-dir}/indices/put-component-template.asciidoc[tag=template]
 
 ====
 
+`index_template_substitutions`::
+(Optional, map of strings to objects)
+Map of index template names to substitute index template definition objects.
++
+.Properties of index template definition objects
+[%collapsible%open]
+
+====
+
+include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=request-body]
+
+====
+
 [[simulate-ingest-api-example]]
 ==== {api-examples-title}
 

docs/reference/ingest/processors/terminate.asciidoc

@@ -0,0 +1,30 @@
+[[terminate-processor]]
+=== Terminate processor
+
+++++
+<titleabbrev>Terminate</titleabbrev>
+++++
+
+Terminates the current ingest pipeline, causing no further processors to be run.
+This will normally be executed conditionally, using the `if` option.
+
+If this pipeline is being called from another pipeline, the calling pipeline is *not* terminated.
+
+[[terminate-options]]
+.Terminate Options
+[options="header"]
+|======
+| Name        | Required | Default | Description
+include::common-options.asciidoc[]
+|======
+
+[source,js]
+--------------------------------------------------
+{
+  "description" : "terminates the current pipeline if the error field is present",
+  "terminate": {
+    "if": "ctx.error != null"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE