Merge branch 'main' into test/99778/mappings-update-race-condition

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v8.12.0$" : "main",
+    "^v8.13.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0"]
+            BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1761,6 +1761,22 @@ steps:
         env:
           BWC_VERSION: 8.12.0
 
+      - label: "{{matrix.image}} / 8.13.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.13.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic.yml

@@ -658,7 +658,7 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.16
@@ -1082,6 +1082,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.0
+      - label: 8.13.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.13.0
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
     timeout_in_minutes: 420

.ci/bwcVersions

@@ -107,3 +107,4 @@ BWC_VERSION:
   - "8.11.1"
   - "8.11.2"
   - "8.12.0"
+  - "8.13.0"

.ci/snapshotBwcVersions

@@ -2,3 +2,4 @@ BWC_VERSION:
   - "7.17.16"
   - "8.11.2"
   - "8.12.0"
+  - "8.13.0"

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.12"
+    },
     {
       "branch": "8.11"
     },

build-tools-internal/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 8.12.0
+elasticsearch     = 8.13.0
 lucene            = 9.9.0
 
 bundled_jdk_vendor = openjdk

distribution/docker/src/docker/Dockerfile

@@ -22,7 +22,7 @@
 <% if (docker_base == 'iron_bank') { %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.2
+ARG BASE_TAG=9.3
 <% } %>
 
 ################################################################################

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -14,7 +14,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.2"
+  BASE_TAG: "9.3"
 
 # Docker image labels
 labels:

docs/changelog/102559.yaml

@@ -0,0 +1,5 @@
+pr: 102559
+summary: "Prune unnecessary information from TransportNodesStatsAction.NodeStatsRequest"
+area: Network
+type: enhancement
+issues: [100878]

docs/changelog/102741.yaml

@@ -0,0 +1,6 @@
+pr: 102741
+summary: "[ILM] More resilient when a policy is added to searchable snapshot"
+area: ILM+SLM
+type: bug
+issues:
+ - 101958

docs/changelog/102843.yaml

@@ -0,0 +1,5 @@
+pr: 102843
+summary: Restore `SharedBytes.IO` refcounting on reads & writes
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/102902.yaml

@@ -0,0 +1,5 @@
+pr: 102902
+summary: Fast path for reading single doc with ordinals
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/103031.yaml

@@ -0,0 +1,9 @@
+pr: 103031
+summary: Collect warnings in compute service
+area: ES|QL
+type: bug
+issues:
+ - 100163
+ - 103028
+ - 102871
+ - 102982

docs/changelog/103033.yaml

@@ -0,0 +1,5 @@
+pr: 103033
+summary: "X-pack/plugin/core: rename `double_metrics` template"
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/103047.yaml

@@ -0,0 +1,5 @@
+pr: 103047
+summary: Ensure `dynamicMapping` updates are handled in insertion order
+area: Mapping
+type: bug
+issues: []

docs/changelog/103116.yaml

@@ -0,0 +1,6 @@
+pr: 103116
+summary: Fix `frequent_item_sets` aggregation on empty index
+area: Machine Learning
+type: bug
+issues:
+ - 103067

docs/reference/ilm/actions/ilm-delete.asciidoc

@@ -16,6 +16,11 @@ Defaults to `true`.
 This option is applicable when the <<ilm-searchable-snapshot,searchable
 snapshot>> action is used in any previous phase.
 
+WARNING: If a policy with a searchable snapshot action is applied on an existing searchable snapshot index,
+the snapshot backing this index will NOT be deleted because it was not created by this policy. If you want
+to clean this snapshot, please delete it manually after the index is deleted using the <<delete-snapshot-api, delete snapshot API>>, you
+can find the repository and snapshot name using the <<indices-get-index, get index API>>.
+
 [[ilm-delete-action-ex]]
 ==== Example
 

docs/reference/migration/index.asciidoc

@@ -1,5 +1,6 @@
 include::migration_intro.asciidoc[]
 
+* <<migrating-8.13,Migrating to 8.13>>
 * <<migrating-8.12,Migrating to 8.12>>
 * <<migrating-8.11,Migrating to 8.11>>
 * <<migrating-8.10,Migrating to 8.10>>
@@ -14,6 +15,7 @@ include::migration_intro.asciidoc[]
 * <<migrating-8.1,Migrating to 8.1>>
 * <<migrating-8.0,Migrating to 8.0>>
 
+include::migrate_8_13.asciidoc[]
 include::migrate_8_12.asciidoc[]
 include::migrate_8_11.asciidoc[]
 include::migrate_8_10.asciidoc[]

docs/reference/migration/migrate_8_13.asciidoc

@@ -0,0 +1,20 @@
+[[migrating-8.13]]
+== Migrating to 8.13
+++++
+<titleabbrev>8.13</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to {es} 8.13.
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+coming::[8.13.0]
+
+
+[discrete]
+[[breaking-changes-8.13]]
+=== Breaking changes
+
+There are no breaking changes in {es} 8.13.
+

docs/reference/query-dsl/rule-query.asciidoc

@@ -10,6 +10,7 @@ preview::[]
 Applies <<query-rules-apis,query rules>> to the query before returning results.
 This feature is used to promote documents in the manner of a <<query-dsl-pinned-query>> based on matching defined rules.
 If no matching query rules are defined, the "organic" matches for the query are returned.
+All matching rules are applied in the order in which they appear in the query ruleset.
 
 [NOTE]
 ====

docs/reference/query-rules/apis/put-query-ruleset.asciidoc

@@ -29,6 +29,9 @@ Requires the `manage_search_query_rules` privilege.
 `rules`::
 (Required, array of objects) The specific rules included in this query ruleset.
 
+There is a limit of 100 rules per ruleset.
+This can be increased up to 1000 using the `xpack.applications.rules.max_rules_per_ruleset` cluster setting.
+
 Each rule must have the following information:
 
 - `rule_id` (Required, string) A unique identifier for this rule.

docs/reference/release-notes.asciidoc

@@ -6,6 +6,7 @@
 
 This section summarizes the changes in each release.
 
+* <<release-notes-8.13.0>>
 * <<release-notes-8.12.0>>
 * <<release-notes-8.11.1>>
 * <<release-notes-8.11.0>>
@@ -55,6 +56,7 @@ This section summarizes the changes in each release.
 
 --
 
+include::release-notes/8.13.0.asciidoc[]
 include::release-notes/8.12.0.asciidoc[]
 include::release-notes/8.11.1.asciidoc[]
 include::release-notes/8.11.0.asciidoc[]

docs/reference/release-notes/8.13.0.asciidoc

@@ -0,0 +1,8 @@
+[[release-notes-8.13.0]]
+== {es} version 8.13.0
+
+coming[8.13.0]
+
+Also see <<breaking-changes-8.13,Breaking changes in 8.13>>.
+
+

docs/reference/release-notes/highlights.asciidoc

@@ -11,7 +11,8 @@ For detailed information about this release, see the <<es-release-notes>> and
 // Add previous release to the list
 Other versions:
 
-{ref-bare}/8.11/release-highlights.html[8.11]
+{ref-bare}/8.12/release-highlights.html[8.12]
+| {ref-bare}/8.11/release-highlights.html[8.11]
 | {ref-bare}/8.10/release-highlights.html[8.10]
 | {ref-bare}/8.9/release-highlights.html[8.9]
 | {ref-bare}/8.8/release-highlights.html[8.8]

docs/reference/search/search-your-data/search-using-query-rules.asciidoc

@@ -174,6 +174,9 @@ PUT /_query_rules/my-ruleset
 
 The API response returns a results of `created` or `updated` depending on whether this was a new or edited ruleset.
 
+NOTE: There is a limit of 100 rules per ruleset.
+This can be increased up to 1000 using the `xpack.applications.rules.max_rules_per_ruleset` cluster setting.
+
 [source,console-result]
 ----
 {
@@ -217,3 +220,8 @@ GET /my-index-000001/_search
 
 This rule query will match against `rule1` in the defined query ruleset, and will convert the organic query into a pinned query with `id1` and `id2` pinned as the top hits.
 Any other matches from the organic query will be returned below the pinned results.
+
+It's possible to have multiple rules in a ruleset match a single `rule_query`. In this case, the pinned documents are returned in the following order:
+
+- Where the matching rule appears in the ruleset
+- If multiple documents are specified in a single rule, in the order they are specified

docs/reference/settings/security-settings.asciidoc

@@ -1253,6 +1253,20 @@ As per `attribute_patterns.principal`, but for the _mail_ property.
 As per `attribute_patterns.principal`, but for the _dn_ property.
 // end::saml-attributes-patterns-dn-tag[]
 
+// tag::saml-attributes-delimiters-groups-tag[]
+`attribute_delimiters.groups` {ess-icon}::
+(<<static-cluster-setting,Static>>)
+A plain string that is used as a delimiter to split a single-valued SAML
+attribute specified by attributes.groups before it is applied to the user's
+groups property. For example, splitting the SAML attribute value
+engineering,elasticsearch-admins,employees on a delimiter value of , will
+result in engineering, elasticsearch-admins, and employees as the list of
+groups for the user. The delimiter will always be split on, regardless of
+escaping in the input string. This setting does not support multi-valued SAML
+attributes. It cannot be used together with the attribute_patterns setting.
+You can only configure this setting for the groups attribute.
+// end::saml-attributes-delimiters-groups-tag[]
+
 // tag::saml-nameid-format-tag[]
 `nameid_format` {ess-icon}::
 (<<static-cluster-setting,Static>>)

server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java

@@ -97,7 +97,7 @@ void syncFlush(String syncId) throws IOException {
             // make sure that background merges won't happen; otherwise, IndexWriter#hasUncommittedChanges can become true again
             forceMerge(false, 1, false, UUIDs.randomBase64UUID());
             assertNotNull(indexWriter);
-            try (ReleasableLock ignored = writeLock.acquire()) {
+            try (ReleasableLock ignored = readLock.acquire()) {
                 assertThat(getTranslogStats().getUncommittedOperations(), equalTo(0));
                 Map<String, String> userData = new HashMap<>(getLastCommittedSegmentInfos().userData);
                 SequenceNumbers.CommitInfo commitInfo = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userData.entrySet());

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -191,6 +191,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion MISSED_INDICES_UPDATE_EXCEPTION_ADDED = def(8_558_00_0);
     public static final TransportVersion INFERENCE_SERVICE_EMBEDDING_SIZE_ADDED = def(8_559_00_0);
     public static final TransportVersion ENRICH_ELASTICSEARCH_VERSION_REMOVED = def(8_560_00_0);
+    public static final TransportVersion NODE_STATS_REQUEST_SIMPLIFIED = def(8_561_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/Version.java

@@ -158,7 +158,8 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_11_1 = new Version(8_11_01_99);
     public static final Version V_8_11_2 = new Version(8_11_02_99);
     public static final Version V_8_12_0 = new Version(8_12_00_99);
-    public static final Version CURRENT = V_8_12_0;
+    public static final Version V_8_13_0 = new Version(8_13_00_99);
+    public static final Version CURRENT = V_8_13_0;
 
     private static final NavigableMap<Integer, Version> VERSION_IDS;
     private static final Map<String, Version> VERSION_STRINGS;

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java

@@ -47,8 +47,12 @@ public NodesStatsRequest(StreamInput in) throws IOException {
      * for all nodes will be returned.
      */
     public NodesStatsRequest(String... nodesIds) {
+        this(new NodesStatsRequestParameters(), nodesIds);
+    }
+
+    public NodesStatsRequest(NodesStatsRequestParameters nodesStatsRequestParameters, String... nodesIds) {
         super(nodesIds);
-        nodesStatsRequestParameters = new NodesStatsRequestParameters();
+        this.nodesStatsRequestParameters = nodesStatsRequestParameters;
     }
 
     /**
@@ -180,4 +184,7 @@ public void writeTo(StreamOutput out) throws IOException {
         nodesStatsRequestParameters.writeTo(out);
     }
 
+    public NodesStatsRequestParameters getNodesStatsRequestParameters() {
+        return nodesStatsRequestParameters;
+    }
 }