Merge remote-tracking branch 'upstream/8.x' into exception-json-warning

thecoop · Oct 17, 2024 · 4532932 · 4532932
2 parents f2886fd + f7ecddb
commit 4532932
Show file tree

Hide file tree

Showing 172 changed files with 879 additions and 542 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.25", "8.15.3", "8.16.0", "8.17.0"]
+            BWC_VERSION: ["7.17.25", "8.15.4", "8.16.0", "8.17.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -559,8 +559,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.3
+      - label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -573,7 +573,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.3
+          BWC_VERSION: 8.15.4
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -629,8 +629,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.3#bwcTest
+      - label: 8.15.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -639,7 +639,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.3
+          BWC_VERSION: 8.15.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -752,7 +752,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.25", "8.15.3", "8.16.0", "8.17.0"]
+            BWC_VERSION: ["7.17.25", "8.15.4", "8.16.0", "8.17.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -800,7 +800,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.25", "8.15.3", "8.16.0", "8.17.0"]
+            BWC_VERSION: ["7.17.25", "8.15.4", "8.16.0", "8.17.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/scripts/dra-workflow.sh b/.buildkite/scripts/dra-workflow.sh
@@ -22,14 +22,18 @@ if [[ "$BRANCH" == "main" ]]; then
 fi
 
 ES_VERSION=$(grep elasticsearch build-tools-internal/version.properties | sed "s/elasticsearch *= *//g")
+echo "ES_VERSION=$ES_VERSION"
 
 VERSION_SUFFIX=""
 if [[ "$WORKFLOW" == "snapshot" ]]; then
   VERSION_SUFFIX="-SNAPSHOT"
 fi
 
 BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
+echo "BEATS_BUILD_ID=$BEATS_BUILD_ID"
+
 ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
+echo "ML_CPP_BUILD_ID=$ML_CPP_BUILD_ID"
 
 LICENSE_KEY_ARG=""
 BUILD_SNAPSHOT_ARG=""

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -32,6 +32,6 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.3"
+  - "8.15.4"
   - "8.16.0"
   - "8.17.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.25"
-  - "8.15.3"
+  - "8.15.4"
   - "8.16.0"
   - "8.17.0"
diff --git a/docs/changelog/111684.yaml b/docs/changelog/111684.yaml
diff --git a/docs/changelog/112761.yaml b/docs/changelog/112761.yaml
diff --git a/docs/changelog/113123.yaml b/docs/changelog/113123.yaml
diff --git a/docs/changelog/113129.yaml b/docs/changelog/113129.yaml
diff --git a/docs/changelog/113437.yaml b/docs/changelog/113437.yaml
diff --git a/docs/changelog/113697.yaml b/docs/changelog/113697.yaml
diff --git a/docs/changelog/113699.yaml b/docs/changelog/113699.yaml
diff --git a/docs/changelog/113846.yaml b/docs/changelog/113846.yaml
diff --git a/docs/changelog/113869.yaml b/docs/changelog/113869.yaml
diff --git a/docs/changelog/113961.yaml b/docs/changelog/113961.yaml
diff --git a/docs/changelog/114116.yaml b/docs/changelog/114116.yaml
diff --git a/docs/changelog/114264.yaml b/docs/changelog/114264.yaml
diff --git a/docs/changelog/114337.yaml b/docs/changelog/114337.yaml
diff --git a/docs/reference/connector/docs/connectors-content-extraction.asciidoc b/docs/reference/connector/docs/connectors-content-extraction.asciidoc
@@ -90,7 +90,7 @@ include::_connectors-list-local-content-extraction.asciidoc[]
 Self-hosted content extraction is handled by a *separate* extraction service.
 
 The versions for the extraction service do not align with the Elastic stack.
-For version `8.11.x`, you should use extraction service version `0.3.x`.
+For versions after `8.11.x` (including {version}), you should use extraction service version `0.3.x`.
 
 You can run the service with the following command:
 

diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc
@@ -21,6 +21,7 @@ the following APIs to manage {infer} models and perform {infer}:
 * <<get-inference-api>>
 * <<post-inference-api>>
 * <<put-inference-api>>
+* <<update-inference-api>>
 
 [[inference-landscape]]
 .A representation of the Elastic inference landscape
@@ -39,6 +40,7 @@ include::delete-inference.asciidoc[]
 include::get-inference.asciidoc[]
 include::post-inference.asciidoc[]
 include::put-inference.asciidoc[]
+include::update-inference.asciidoc[]
 include::service-alibabacloud-ai-search.asciidoc[]
 include::service-amazon-bedrock.asciidoc[]
 include::service-anthropic.asciidoc[]

diff --git a/docs/reference/inference/update-inference.asciidoc b/docs/reference/inference/update-inference.asciidoc
@@ -0,0 +1,87 @@
+[role="xpack"]
+[[update-inference-api]]
+=== Update inference API
+
+experimental[]
+
+Updates an {infer} endpoint.
+
+IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or Hugging Face.
+For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models.
+However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
+
+
+[discrete]
+[[update-inference-api-request]]
+==== {api-request-title}
+
+`POST _inference/<inference_id>/_update`
+
+`POST _inference/<task_type>/<inference_id>/_update`
+
+
+[discrete]
+[[update-inference-api-prereqs]]
+==== {api-prereq-title}
+
+* Requires the `manage_inference` <<privileges-list-cluster,cluster privilege>> (the built-in inference_admin role grants this privilege)
+* Requires an existing {infer} endpoint, created by using the <<put-inference-api>>
+
+
+[discrete]
+[[update-inference-api-desc]]
+==== {api-description-title}
+
+The update inference API enables you to update the task_settings, secrets, and/or num_allocations of an existing {infer} endpoint.
+
+To use the update API, you can modify `task_settings`, secrets (within `service_settings`), or `num_allocations`, depending on the specific endpoint service and task_type you've created.
+To view the updatable `task_settings`, the field names of secrets (specific to each service), and the services where `num_allocations` is applicable (only for the `elasticsearch` service), refer to the following list of services available through the {infer} API.
+You will find the available task types next to each service name.
+Click the links to review the service configuration details:
+
+* <<infer-service-alibabacloud-ai-search,AlibabaCloud AI Search>> (`completion`, `rerank`, `sparse_embedding`, `text_embedding`)
+* <<infer-service-amazon-bedrock,Amazon Bedrock>> (`completion`, `text_embedding`)
+* <<infer-service-anthropic,Anthropic>> (`completion`)
+* <<infer-service-azure-ai-studio,Azure AI Studio>> (`completion`, `text_embedding`)
+* <<infer-service-azure-openai,Azure OpenAI>> (`completion`, `text_embedding`)
+* <<infer-service-cohere,Cohere>> (`completion`, `rerank`, `text_embedding`)
+* <<infer-service-elasticsearch,Elasticsearch>> (`rerank`, `sparse_embedding`, `text_embedding` - this service is for built-in models and models uploaded through Eland)
+* <<infer-service-elser,ELSER>> (`sparse_embedding`)
+* <<infer-service-google-ai-studio,Google AI Studio>> (`completion`, `text_embedding`)
+* <<infer-service-google-vertex-ai,Google Vertex AI>> (`rerank`, `text_embedding`) 
+* <<infer-service-hugging-face,Hugging Face>> (`text_embedding`)
+* <<infer-service-mistral,Mistral>> (`text_embedding`)
+* <<infer-service-openai,OpenAI>> (`completion`, `text_embedding`)
+
+
+[discrete]
+[[update-inference-api-path-params]]
+==== {api-path-parms-title}
+
+`<inference_id>`::
+(Required, string)
+The unique identifier of the {infer} endpoint.
+
+
+`<task_type>`::
+(Optional, string)
+The type of {infer} task that the model performs.
+Refer to the service list in the <<put-inference-api-desc,API description section>> for the available task types.
+
+
+[discrete]
+[[update-inference-api-example]]
+==== {api-examples-title}
+
+The following example shows how to update an API key of an {infer} endpoint called `my-inference-endpoint`:
+
+[source,console]
+------------------------------------------------------------
+POST _inference/my-inference-endpoint/_update
+{
+ "service_settings": {
+   "api_key": "<API_KEY>"
+ }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
@@ -3,7 +3,6 @@ Adaptive allocations configuration object.
 If enabled, the number of allocations of the model is set based on the current load the process gets.
 When the load is high, a new model allocation is automatically created (respecting the value of `max_number_of_allocations` if it's set).
 When the load is low, a model allocation is automatically removed (respecting the value of `min_number_of_allocations` if it's set).
-The number of model allocations cannot be scaled down to less than `1` this way.
 If `adaptive_allocations` is enabled, do not set the number of allocations manually.
 end::adaptive-allocation[]
 

diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
@@ -8,6 +8,7 @@ This section summarizes the changes in each release.
 
 * <<release-notes-8.17.0>>
 * <<release-notes-8.16.0>>
+* <<release-notes-8.15.3>>
 * <<release-notes-8.15.2>>
 * <<release-notes-8.15.1>>
 * <<release-notes-8.15.0>>
@@ -76,6 +77,7 @@ This section summarizes the changes in each release.
 
 include::release-notes/8.17.0.asciidoc[]
 include::release-notes/8.16.0.asciidoc[]
+include::release-notes/8.15.3.asciidoc[]
 include::release-notes/8.15.2.asciidoc[]
 include::release-notes/8.15.1.asciidoc[]
 include::release-notes/8.15.0.asciidoc[]

diff --git a/docs/reference/release-notes/8.15.3.asciidoc b/docs/reference/release-notes/8.15.3.asciidoc
@@ -0,0 +1,47 @@
+[[release-notes-8.15.3]]
+== {es} version 8.15.3
+
+Also see <<breaking-changes-8.15,Breaking changes in 8.15>>.
+
+[[bug-8.15.3]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Don't validate internal stats if they are empty {es-pull}113846[#113846] (issue: {es-issue}113811[#113811])
+* Fix `needsScore` computation in `GlobalOrdCardinalityAggregator` {es-pull}113129[#113129] (issue: {es-issue}112975[#112975])
+
+Authentication::
+* Enables cluster state role mapper, to include ECK operator-defined role mappings in role resolution {es-pull}114337[#114337]
+
+ES|QL::
+* ES|QL: Ensure minimum capacity for `PlanStreamInput` caches {es-pull}114116[#114116]
+* ES|QL: Skip CASE function from `InferIsNotNull` rule checks {es-pull}113123[#113123] (issue: {es-issue}112704[#112704])
+* [ESQL] Fix init value in max float aggregation {es-pull}113699[#113699]
+* [ESQL] Support datetime data type in Least and Greatest functions {es-pull}113961[#113961]
+
+Machine Learning::
+* Fix check on E5 model platform compatibility {es-pull}113437[#113437] (issue: {es-issue}113577[#113577])
+* Handle parsing ingest processors where definition is not a object {es-pull}113697[#113697] (issue: {es-issue}113615[#113615])
+* [ML][backport] Warn for model load failures if they have a status code <500 {es-pull}113410[#113410]
+* [M] Fix error message formatting {es-pull}113266[#113266]
+
+Search::
+* Fix analyzed wildcard query in simple_query_string when disjunctions is empty {es-pull}114264[#114264] (issue: {es-issue}114185[#114185])
+* Fix collapse interaction with stored fields {es-pull}112761[#112761] (issue: {es-issue}112646[#112646])
+
+[[enhancement-8.15.3]]
+[float]
+=== Enhancements
+
+Machine Learning::
+* Write downloaded model parts async {es-pull}111684[#111684]
+
+[[upgrade-8.15.3]]
+[float]
+=== Upgrades
+
+Snapshot/Restore::
+* Upgrade protobufer to 3.25.5 {es-pull}113869[#113869]
+
+