Merge branch 'main' of github.com:elastic/elasticsearch into trainedM…

…odelsQueueCapacity
elastic · Oct 18, 2024 · 4ece775 · 4ece775
2 parents 1701999 + 8a613e7
commit 4ece775
Show file tree

Hide file tree

Showing 151 changed files with 5,447 additions and 633 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -16,10 +16,10 @@ export COMPOSE_HTTP_TIMEOUT
 JOB_BRANCH="$BUILDKITE_BRANCH"
 export JOB_BRANCH
 
-GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+GRADLEW="./gradlew --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
 export GRADLEW
 
-GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+GRADLEW_BAT="./gradlew.bat --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
 export GRADLEW_BAT
 
 export $(cat .ci/java-versions.properties | grep '=' | xargs)

diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh
@@ -78,5 +78,5 @@ sudo -E env \
   --unset=JAVA_HOME \
   SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \
   DOCKER_CONFIG="${HOME}/.docker" \
-  ./gradlew -g $HOME/.gradle --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@
+  ./gradlew -g $HOME/.gradle --console=plain --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@
 
diff --git a/.../elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java b/.../elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java
@@ -21,7 +21,7 @@ public abstract class AbstractDocValuesForUtilBenchmark {
     protected final int blockSize;
 
     public AbstractDocValuesForUtilBenchmark() {
-        this.forUtil = new DocValuesForUtil();
+        this.forUtil = new DocValuesForUtil(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
         this.blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
     }
 

diff --git a/.../src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java b/.../src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.ByteArrayDataOutput;
 import org.apache.lucene.store.DataOutput;
-import org.elasticsearch.index.codec.tsdb.DocValuesForUtil;
 import org.openjdk.jmh.infra.Blackhole;
 
 import java.io.IOException;
@@ -44,7 +43,7 @@ public void setupInvocation(int bitsPerValue) {
 
     @Override
     public void benchmark(int bitsPerValue, Blackhole bh) throws IOException {
-        DocValuesForUtil.decode(bitsPerValue, this.dataInput, this.output);
+        forUtil.decode(bitsPerValue, this.dataInput, this.output);
         bh.consume(this.output);
     }
 }
diff --git a/docs/changelog/114601.yaml b/docs/changelog/114601.yaml
@@ -0,0 +1,6 @@
+pr: 114601
+summary: Support semantic_text in object fields
+area: Vector Search
+type: bug
+issues:
+ - 114401
diff --git a/docs/changelog/114899.yaml b/docs/changelog/114899.yaml
@@ -0,0 +1,5 @@
+pr: 114899
+summary: "ES|QL: Fix stats by constant expression"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/115031.yaml b/docs/changelog/115031.yaml
@@ -0,0 +1,5 @@
+pr: 115031
+summary: Bool query early termination should also consider `must_not` clauses
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/115048.yaml b/docs/changelog/115048.yaml
@@ -0,0 +1,5 @@
+pr: 115048
+summary: Add timeout and cancellation check to rescore phase
+area: Ranking
+type: enhancement
+issues: []
diff --git a/docs/plugins/mapper-annotated-text.asciidoc b/docs/plugins/mapper-annotated-text.asciidoc
@@ -155,11 +155,6 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`annotated_text` fields support {ref}/mapping-source-field.html#synthetic-source[synthetic `_source`] if they have
-a {ref}/keyword.html#keyword-synthetic-source[`keyword`] sub-field that supports synthetic
-`_source` or if the `annotated_text` field sets `store` to `true`. Either way, it may
-not have {ref}/copy-to.html[`copy_to`].
-
 If using a sub-`keyword` field then the values are sorted in the same way as
 a `keyword` field's values are sorted. By default, that means sorted with
 duplicates removed. So:

diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc
@@ -159,6 +159,7 @@ node.
 <5> The decider which led to the `no` decision for the node.
 <6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision. In this example, a newly created index has <<indices-get-settings,an index setting>> that requires that it only be allocated to a node named `nonexistent_node`, which does not exist, so the index is unable to allocate.
 
+[[maximum-number-of-retries-exceeded]]
 ====== Maximum number of retries exceeded
 
 The following response contains an allocation explanation for an unassigned
@@ -195,17 +196,19 @@ primary shard that has reached the maximum number of allocation retry attempts.
         {
           "decider": "max_retry",
           "decision" : "NO",
-          "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-07-30T21:04:12.166Z], failed_attempts[5], failed_nodes[[mEKjwwzLT1yJVb8UxT6anw]], delayed=false, details[failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException], allocation_status[deciders_no]]]"
+          "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [POST /_cluster/reroute?retry_failed] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-07-30T21:04:12.166Z], failed_attempts[5], failed_nodes[[mEKjwwzLT1yJVb8UxT6anw]], delayed=false, details[failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException], allocation_status[deciders_no]]]"
         }
       ]
     }
   ]
 }
 ----
 // NOTCONSOLE
-
-If decider message indicates a transient allocation issue, use
-the <<cluster-reroute,cluster reroute>> API to retry allocation.
+When Elasticsearch is unable to allocate a shard, it will attempt to retry allocation up to
+the maximum number of retries allowed. After this, Elasticsearch will stop attempting to
+allocate the shard in order to prevent infinite retries which may impact cluster
+performance. Run the <<cluster-reroute,cluster reroute>> API to retry allocation, which
+will allocate the shard if the issue preventing allocation has been resolved.
 
 [[no-valid-shard-copy]]
 ====== No valid shard copy

diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc
@@ -59,8 +59,7 @@ since it relies on separate data structures to perform the search. Before
 using the <<include-exclude, `excludes`>> parameter, make sure to review the
 downsides of omitting fields from `_source`.
 
-Another option is to use  <<synthetic-source,synthetic `_source`>> if all
-your index fields support it.
+Another option is to use  <<synthetic-source,synthetic `_source`>>.
 
 [discrete]
 === Ensure data nodes have enough memory

diff --git a/docs/reference/inference/images/inference-landscape.jpg b/docs/reference/inference/images/inference-landscape.jpg
diff --git a/docs/reference/inference/images/inference-landscape.png b/docs/reference/inference/images/inference-landscape.png
diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc
@@ -25,7 +25,7 @@ the following APIs to manage {infer} models and perform {infer}:
 
 [[inference-landscape]]
 .A representation of the Elastic inference landscape
-image::images/inference-landscape.png[A representation of the Elastic inference landscape,align="center"]
+image::images/inference-landscape.jpg[A representation of the Elastic inference landscape,align="center"]
 
 An {infer} endpoint enables you to use the corresponding {ml} model without
 manual deployment and apply it to your data at ingestion time through

diff --git a/docs/reference/mapping/types/aggregate-metric-double.asciidoc b/docs/reference/mapping/types/aggregate-metric-double.asciidoc
@@ -259,9 +259,6 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`aggregate_metric-double` fields support <<synthetic-source,synthetic `_source`>> in their default
-configuration.
-
 For example:
 [source,console,id=synthetic-source-aggregate-metric-double-example]
 ----

diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc
@@ -63,7 +63,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`binary` fields support <<synthetic-source,synthetic `_source`>> only when <<doc-values,`doc_values`>> are enabled. Synthetic source always sorts `binary` values in order of their byte representation. For example:
+Synthetic source may sort `binary` values in order of their byte representation. For example:
 [source,console,id=synthetic-source-binary-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
@@ -241,10 +241,9 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `boolean` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
+default configuration.
 
-Synthetic source always sorts `boolean` fields. For example:
+Synthetic source may sort `boolean` field values. For example:
 [source,console,id=synthetic-source-boolean-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc
@@ -239,11 +239,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`date` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `date` fields. For example:
+Synthetic source may sort `date` field values. For example:
 [source,console,id=synthetic-source-date-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/date_nanos.asciidoc b/docs/reference/mapping/types/date_nanos.asciidoc
@@ -150,12 +150,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`date_nanos` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>>, <<ignore-malformed,`ignore_malformed`>> set to true
-or with <<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `date_nanos` fields. For example:
+Synthetic source may sort `date_nanos` field values. For example:
 [source,console,id=synthetic-source-date-nanos-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc
@@ -325,10 +325,9 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 Flattened fields support <<synthetic-source,synthetic`_source`>> in their default
-configuration. Synthetic `_source` cannot be used with <<doc-values,`doc_values`>>
-disabled.
+configuration.
 
-Synthetic source always sorts alphabetically and de-duplicates flattened fields.
+Synthetic source may sort `flattened` field values and remove duplicates.
 For example:
 [source,console,id=synthetic-source-flattened-sorting-example]
 ----

diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
@@ -219,11 +219,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`geo_point` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with <<copy-to,`copy_to`>> or with
-<<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `geo_point` fields (first by latitude and then
+Synthetic source may sort `geo_point` fields (first by latitude and then
 longitude) and reduces them to their stored precision. For example:
 [source,console,id=synthetic-source-geo-point-example]
 ----

diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc
@@ -502,6 +502,3 @@ synthetic `_source` is in technical preview. Features in technical preview may
 be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
-
-`geo_shape` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration.
diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc
@@ -79,7 +79,7 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `histogram` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with <<copy-to,`copy_to`>>.
+default configuration.
 
 NOTE: To save space, zero-count buckets are not stored in the histogram doc values.
 As a result, when indexing a histogram field in an index with synthetic source enabled,

diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
@@ -161,11 +161,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`ip` fields support <<synthetic-source,synthetic `_source`>> in their default
-configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `ip` fields and removes duplicates. For example:
+Synthetic source may sort `ip` field values and remove duplicates. For example:
 [source,console,id=synthetic-source-ip-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
@@ -178,11 +178,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`keyword` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-a <<normalizer,`normalizer`>> or <<copy-to,`copy_to`>>.
-
-By default, synthetic source sorts `keyword` fields and removes duplicates.
+Synthetic source may sort `keyword` fields and remove duplicates.
 For example:
 [source,console,id=synthetic-source-keyword-example-default]
 ----

diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
@@ -254,7 +254,7 @@ All numeric fields support <<synthetic-source,synthetic
 together with <<copy-to,`copy_to`>>, or
 with <<doc-values,`doc_values`>> disabled.
 
-Synthetic source always sorts numeric fields. For example:
+Synthetic source may sort numeric field values. For example:
 [source,console,id=synthetic-source-numeric-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc
@@ -247,9 +247,9 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `range` fields support <<synthetic-source,synthetic `_source`>> in their default
-configuration. Synthetic `_source` cannot be used with <<doc-values,`doc_values`>> disabled.
+configuration.
 
-Synthetic source always sorts values and removes duplicates for all `range` fields except `ip_range`. Ranges are sorted by their lower bound and then by upper bound. For example:
+Synthetic source may sort `range` field values and remove duplicates for all `range` fields except `ip_range`. Ranges are sorted by their lower bound and then by upper bound. For example:
 [source,console,id=synthetic-source-range-sorting-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/search-as-you-type.asciidoc b/docs/reference/mapping/types/search-as-you-type.asciidoc
@@ -266,5 +266,4 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `search_as_you_type` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>>.
+default configuration.
diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc
@@ -134,10 +134,6 @@ The following parameters are accepted by `text` fields:
 
     Whether the field value should be stored and retrievable separately from
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false` (default).
-    This parameter will be automatically set to `true` for TSDB indices
-    (indices that have `index.mode` set to `time_series`)
-    if there is no <<keyword-synthetic-source, `keyword`>>
-    sub-field that supports synthetic `_source`.
 
 <<search-analyzer,`search_analyzer`>>::
 

diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc
@@ -103,5 +103,4 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `token_count` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>>.
+default configuration.
diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc
@@ -77,10 +77,10 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`version` fields support <<synthetic-source,synthetic `_source`>> so long as they don't
-declare <<copy-to,`copy_to`>>.
+`version` fields support <<synthetic-source,synthetic `_source`>> in their
+default configuration..
 
-Synthetic source always sorts `version` fields and removes duplicates. For example:
+Synthetic source may sort `version` field values and remove duplicates. For example:
 [source,console,id=synthetic-source-version-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
@@ -133,10 +133,8 @@ The following parameters are accepted by `wildcard` fields:
 
 [[wildcard-synthetic-source]]
 ==== Synthetic `_source`
-`wildcard` fields support <<synthetic-source,synthetic `_source`>> so long as they don't
-declare <<copy-to,`copy_to`>>.
 
-Synthetic source always sorts `wildcard` fields. For example:
+Synthetic source may sort `wildcard` field values. For example:
 [source,console,id=synthetic-source-wildcard-example]
 ----
 PUT idx

diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc
@@ -248,10 +248,11 @@ that you have an archive copy of its contents that you can use to recreate the
 repository in its current state at a later date.
 
 You must ensure that {es} does not write to the repository while you are taking
-the backup of its contents. You can do this by unregistering it, or registering
-it with `readonly: true`, on all your clusters. If {es} writes any data to the
-repository during the backup then the contents of the backup may not be
-consistent and it may not be possible to recover any data from it in future.
+the backup of its contents. If {es} writes any data to the repository during
+the backup then the contents of the backup may not be consistent and it may not
+be possible to recover any data from it in future. Prevent writes to the
+repository by unregistering the repository from the cluster which has write
+access to it.
 
 Alternatively, if your repository supports it, you may take an atomic snapshot
 of the underlying filesystem and then take a backup of this filesystem