Merge branch 'main' into kderusso/query-rules-retriever

.backportrc.json

@@ -1,10 +1,10 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.x", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.0.0$" : "main",
-    "^v8.16.0$" : "8.x",
+    "^v8.17.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.template.yml

@@ -75,6 +75,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
+    branches: "main 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -304,6 +304,22 @@ steps:
         env:
           BWC_VERSION: 8.16.0
 
+      - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.17.0
+
       - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -325,6 +325,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.17.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 8.17.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
       - label: 9.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
         timeout_in_minutes: 300
@@ -410,7 +429,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -452,7 +471,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -554,7 +573,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.ci/bwcVersions

@@ -16,4 +16,5 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.3"
   - "8.16.0"
+  - "8.17.0"
   - "9.0.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,5 @@
 BWC_VERSION:
   - "8.15.3"
   - "8.16.0"
+  - "8.17.0"
   - "9.0.0"

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.16"
+    },
     {
       "branch": "8.x"
     },

docs/changelog/113237.yaml

@@ -0,0 +1,5 @@
+pr: 113237
+summary: Retry throttled snapshot deletions
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/114271.yaml

@@ -0,0 +1,5 @@
+pr: 114271
+summary: "[ES|QL] Skip validating remote cluster index names in parser"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/114784.yaml

@@ -0,0 +1,5 @@
+pr: 114784
+summary: "[ES|QL] make named parameter for identifier and pattern snapshot"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/114813.yaml

@@ -0,0 +1,5 @@
+pr: 114813
+summary: Retry `S3BlobContainer#getRegister` on all exceptions
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/114836.yaml

@@ -0,0 +1,6 @@
+pr: 114836
+summary: Support multi-valued fields in compute engine for ST_DISTANCE
+area: ES|QL
+type: enhancement
+issues:
+ - 112910

docs/changelog/114837.yaml

@@ -0,0 +1,5 @@
+pr: 114837
+summary: Add warning headers for ingest pipelines containing special characters
+area: Ingest Node
+type: bug
+issues: [ 104411 ]

docs/changelog/114854.yaml

@@ -0,0 +1,10 @@
+pr: 114854
+summary: Adding deprecation warnings for rrf using rank and `sub_searches`
+area: Search
+type: deprecation
+issues: []
+deprecation:
+  title: Adding deprecation warnings for rrf using rank and `sub_searches`
+  area: REST API
+  details: Search API parameter `sub_searches` will no longer be a supported and will be removed in future releases. Similarly, `rrf` can only be used through the specified `retriever` and no longer though the `rank` parameter
+  impact: Requests specifying rrf through `rank` and/or `sub_searches` elements will be disallowed in a future version. Users should instead utilize the new `retriever` parameter.

docs/changelog/114856.yaml

@@ -0,0 +1,5 @@
+pr: 114856
+summary: "OTel mappings: avoid metrics to be rejected when attributes are malformed"
+area: Data streams
+type: bug
+issues: []

docs/changelog/114869.yaml

@@ -0,0 +1,5 @@
+pr: 114869
+summary: Standardize error code when bulk body is invalid
+area: CRUD
+type: bug
+issues: []

docs/changelog/114888.yaml

@@ -0,0 +1,6 @@
+pr: 114888
+summary: Fix ST_CENTROID_AGG when no records are aggregated
+area: ES|QL
+type: bug
+issues:
+ - 106025

docs/changelog/114924.yaml

@@ -0,0 +1,5 @@
+pr: 114924
+summary: Reducing error-level stack trace logging for normal events in `GeoIpDownloader`
+area: Ingest Node
+type: bug
+issues: []

docs/plugins/mapper-annotated-text.asciidoc

@@ -167,8 +167,16 @@ duplicates removed. So:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "text": {
         "type": "annotated_text",
@@ -215,8 +223,16 @@ are preserved.
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "text": { "type": "annotated_text", "store": true }
     }

docs/reference/cat/indices.asciidoc

@@ -6,8 +6,8 @@
 
 [IMPORTANT]
 ====
-cat APIs are only intended for human consumption using the command line or {kib} 
-console. They are _not_ intended for use by applications. For application 
+cat APIs are only intended for human consumption using the command line or {kib}
+console. They are _not_ intended for use by applications. For application
 consumption, use the <<indices-get-index,get index API>>.
 ====
 
@@ -50,6 +50,10 @@ indexing and search. As a result, all document counts include hidden
 To get an accurate count of {es} documents, use the <<cat-count,cat count>> or
 <<search-count,count>> APIs.
 
+Note that information such as document count, deleted document count and store size are not shown for
+indices restored from <<snapshots-source-only-repository,source-only snapshots>> since these indices
+do not contain the relevant data structures to retrieve this information from.
+
 
 [[cat-indices-api-path-params]]
 ==== {api-path-parms-title}

docs/reference/how-to/size-your-shards.asciidoc

@@ -208,6 +208,7 @@ index can be <<indices-delete-index,removed>>. You may then consider setting
 <<indices-add-alias,Create Alias>> against the destination index for the source 
 index's name to point to it for continuity. 
 
+See this https://www.youtube.com/watch?v=sHyNYnwbYro[fixing shard sizes video] for an example troubleshooting walkthrough.
 
 [discrete]
 [[shard-count-recommendation]]
@@ -571,6 +572,8 @@ PUT _cluster/settings
 }
 ----
 
+For more information, see <<troubleshooting-shards-capacity-issues,Troubleshooting shards capacity>>.
+
 [discrete]
 [[troubleshooting-max-docs-limit]]
 ==== Number of documents in the shard cannot exceed [2147483519]

docs/reference/ilm/error-handling.asciidoc

@@ -8,6 +8,9 @@ When this happens, {ilm-init} moves the index to an `ERROR` step.
 If {ilm-init} cannot resolve the error automatically, execution is halted  
 until you resolve the underlying issues with the policy, index, or cluster.
 
+See this https://www.youtube.com/watch?v=VCIqkji3IwY[{ilm-init} health video] 
+for example troubleshooting walkthrough.
+
 For example, you might have a `shrink-index` policy that shrinks an index to four shards once it
 is at least five days old: 
 
@@ -183,6 +186,8 @@ The rollover action then manages setting and updating the alias to
 
 Do not explicitly configure this same alias in the aliases section of an index template.
 
+See this https://www.youtube.com/watch?v=Ww5POq4zZtY[resolving `duplicate alias` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== index.lifecycle.rollover_alias [x] does not point to index [y]
 
@@ -191,20 +196,26 @@ Either the index is using the wrong alias or the alias does not exist.
 Check the `index.lifecycle.rollover_alias` <<indices-get-settings, index setting>>.
 To see what aliases are configured, use <<cat-alias, _cat/aliases>>.
 
+See this https://www.youtube.com/watch?v=NKSe67x7aw8[resolving `not point to index` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== Setting [index.lifecycle.rollover_alias] for index [y] is empty or not defined
 
 The `index.lifecycle.rollover_alias` setting must be configured for the rollover action to work.
 
 Update the index settings to set `index.lifecycle.rollover_alias`.
 
+See this https://www.youtube.com/watch?v=LRpMC2GS_FQ[resolving `empty or not defined` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== Alias [x] has more than one write index [y,z]
 
 Only one index can be designated as the write index for a particular alias.
 
 Use the <<indices-aliases, aliases>> API to set `is_write_index:false` for all but one index.
 
+See this https://www.youtube.com/watch?v=jCUvZCT5Hm4[resolving `more than one write index` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== index name [x] does not match pattern ^.*-\d+
 
@@ -214,6 +225,8 @@ For example, `my-index` does not match the pattern requirement.
 
 Append a numeric value to the index name, for example `my-index-000001`.
 
+See this https://www.youtube.com/watch?v=9sp1zF6iL00[resolving `does not match pattern` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== CircuitBreakingException: [x] data too large, data for [y]
 
@@ -227,8 +240,7 @@ For more information, see <<circuit-breaker-errors>>.
 
 This indicates that the cluster is running out of disk space.
 This can happen when you don't have {ilm} set up to roll over from hot to warm nodes.
-
-Consider adding nodes, upgrading your hardware, or deleting unneeded indices.
+For more information, see <<fix-watermark-errors,Fix watermark errors>>.
 
 [discrete]
 ==== security_exception: action [<action-name>] is unauthorized for user [<user-name>] with roles [<role-name>], this action is granted by the index privileges [manage_follow_index,manage,all]

docs/reference/inference/service-elser.asciidoc

@@ -80,12 +80,13 @@ Must be a power of 2. Max allowed value is 32.
 [[inference-example-elser]]
 ==== ELSER service example
 
-The following example shows how to create an {infer} endpoint called
-`my-elser-model` to perform a `sparse_embedding` task type.
+The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type.
 Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more info.
 
-The request below will automatically download the ELSER model if it isn't
-already downloaded and then deploy the model.
+NOTE: If you want to optimize your ELSER endpoint for ingest, set the number of threads to `1` (`"num_threads": 1`).
+If you want to optimize your ELSER endpoint for search, set the number of threads to greater than `1`.
+
+The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
 
 [source,console]
 ------------------------------------------------------------
@@ -100,7 +101,6 @@ PUT _inference/sparse_embedding/my-elser-model
 ------------------------------------------------------------
 // TEST[skip:TBD]
 
-
 Example response:
 
 [source,console-result]
@@ -130,12 +130,12 @@ If using the Python client, you can set the `timeout` parameter to a higher valu
 [[inference-example-elser-adaptive-allocation]]
 ==== Setting adaptive allocation for the ELSER service
 
-The following example shows how to create an {infer} endpoint called
-`my-elser-model` to perform a `sparse_embedding` task type and configure
-adaptive allocations.
+NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation.
+To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page.
+
+The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type and configure adaptive allocations.
 
-The request below will automatically download the ELSER model if it isn't
-already downloaded and then deploy the model.
+The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
 
 [source,console]
 ------------------------------------------------------------