merging main

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v8.12.0$" : "main",
+    "^v8.13.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0"]
+            BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1761,6 +1761,22 @@ steps:
         env:
           BWC_VERSION: 8.12.0
 
+      - label: "{{matrix.image}} / 8.13.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.13.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic.yml

@@ -658,7 +658,7 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.16
@@ -1082,6 +1082,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.0
+      - label: 8.13.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.13.0
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
     timeout_in_minutes: 420

.ci/bwcVersions

@@ -107,3 +107,4 @@ BWC_VERSION:
   - "8.11.1"
   - "8.11.2"
   - "8.12.0"
+  - "8.13.0"

.ci/snapshotBwcVersions

@@ -2,3 +2,4 @@ BWC_VERSION:
   - "7.17.16"
   - "8.11.2"
   - "8.12.0"
+  - "8.13.0"

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.12"
+    },
     {
       "branch": "8.11"
     },

build-tools-internal/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 8.12.0
+elasticsearch     = 8.13.0
 lucene            = 9.9.0
 
 bundled_jdk_vendor = openjdk

distribution/docker/src/docker/Dockerfile

@@ -22,7 +22,7 @@
 <% if (docker_base == 'iron_bank') { %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.2
+ARG BASE_TAG=9.3
 <% } %>
 
 ################################################################################

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -14,7 +14,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.2"
+  BASE_TAG: "9.3"
 
 # Docker image labels
 labels:

docs/changelog/102741.yaml

@@ -0,0 +1,6 @@
+pr: 102741
+summary: "[ILM] More resilient when a policy is added to searchable snapshot"
+area: ILM+SLM
+type: bug
+issues:
+ - 101958

docs/changelog/102843.yaml

@@ -0,0 +1,5 @@
+pr: 102843
+summary: Restore `SharedBytes.IO` refcounting on reads & writes
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/102902.yaml

@@ -0,0 +1,5 @@
+pr: 102902
+summary: Fast path for reading single doc with ordinals
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/102925.yaml

@@ -0,0 +1,5 @@
+pr: 102925
+summary: Add ldap user metadata mappings for full name and email
+area: Authentication
+type: enhancement
+issues: []

docs/changelog/102937.yaml

@@ -0,0 +1,5 @@
+pr: 102937
+summary: "ESQL: New telemetry commands"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/103031.yaml

@@ -0,0 +1,9 @@
+pr: 103031
+summary: Collect warnings in compute service
+area: ES|QL
+type: bug
+issues:
+ - 100163
+ - 103028
+ - 102871
+ - 102982

docs/changelog/103047.yaml

@@ -0,0 +1,5 @@
+pr: 103047
+summary: Ensure `dynamicMapping` updates are handled in insertion order
+area: Mapping
+type: bug
+issues: []

docs/reference/ilm/actions/ilm-delete.asciidoc

@@ -16,6 +16,11 @@ Defaults to `true`.
 This option is applicable when the <<ilm-searchable-snapshot,searchable
 snapshot>> action is used in any previous phase.
 
+WARNING: If a policy with a searchable snapshot action is applied on an existing searchable snapshot index,
+the snapshot backing this index will NOT be deleted because it was not created by this policy. If you want
+to clean this snapshot, please delete it manually after the index is deleted using the <<delete-snapshot-api, delete snapshot API>>, you
+can find the repository and snapshot name using the <<indices-get-index, get index API>>.
+
 [[ilm-delete-action-ex]]
 ==== Example
 

docs/reference/migration/index.asciidoc

@@ -1,5 +1,6 @@
 include::migration_intro.asciidoc[]
 
+* <<migrating-8.13,Migrating to 8.13>>
 * <<migrating-8.12,Migrating to 8.12>>
 * <<migrating-8.11,Migrating to 8.11>>
 * <<migrating-8.10,Migrating to 8.10>>
@@ -14,6 +15,7 @@ include::migration_intro.asciidoc[]
 * <<migrating-8.1,Migrating to 8.1>>
 * <<migrating-8.0,Migrating to 8.0>>
 
+include::migrate_8_13.asciidoc[]
 include::migrate_8_12.asciidoc[]
 include::migrate_8_11.asciidoc[]
 include::migrate_8_10.asciidoc[]

docs/reference/migration/migrate_8_13.asciidoc

@@ -0,0 +1,20 @@
+[[migrating-8.13]]
+== Migrating to 8.13
+++++
+<titleabbrev>8.13</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to {es} 8.13.
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+coming::[8.13.0]
+
+
+[discrete]
+[[breaking-changes-8.13]]
+=== Breaking changes
+
+There are no breaking changes in {es} 8.13.
+

docs/reference/query-dsl/rule-query.asciidoc

@@ -10,6 +10,7 @@ preview::[]
 Applies <<query-rules-apis,query rules>> to the query before returning results.
 This feature is used to promote documents in the manner of a <<query-dsl-pinned-query>> based on matching defined rules.
 If no matching query rules are defined, the "organic" matches for the query are returned.
+All matching rules are applied in the order in which they appear in the query ruleset.
 
 [NOTE]
 ====

docs/reference/query-rules/apis/put-query-ruleset.asciidoc

@@ -29,6 +29,9 @@ Requires the `manage_search_query_rules` privilege.
 `rules`::
 (Required, array of objects) The specific rules included in this query ruleset.
 
+There is a limit of 100 rules per ruleset.
+This can be increased up to 1000 using the `xpack.applications.rules.max_rules_per_ruleset` cluster setting.
+
 Each rule must have the following information:
 
 - `rule_id` (Required, string) A unique identifier for this rule.

docs/reference/release-notes.asciidoc

@@ -6,6 +6,7 @@
 
 This section summarizes the changes in each release.
 
+* <<release-notes-8.13.0>>
 * <<release-notes-8.12.0>>
 * <<release-notes-8.11.1>>
 * <<release-notes-8.11.0>>
@@ -55,6 +56,7 @@ This section summarizes the changes in each release.
 
 --
 
+include::release-notes/8.13.0.asciidoc[]
 include::release-notes/8.12.0.asciidoc[]
 include::release-notes/8.11.1.asciidoc[]
 include::release-notes/8.11.0.asciidoc[]

docs/reference/release-notes/8.13.0.asciidoc

@@ -0,0 +1,8 @@
+[[release-notes-8.13.0]]
+== {es} version 8.13.0
+
+coming[8.13.0]
+
+Also see <<breaking-changes-8.13,Breaking changes in 8.13>>.
+
+

docs/reference/release-notes/highlights.asciidoc

@@ -11,7 +11,8 @@ For detailed information about this release, see the <<es-release-notes>> and
 // Add previous release to the list
 Other versions:
 
-{ref-bare}/8.11/release-highlights.html[8.11]
+{ref-bare}/8.12/release-highlights.html[8.12]
+| {ref-bare}/8.11/release-highlights.html[8.11]
 | {ref-bare}/8.10/release-highlights.html[8.10]
 | {ref-bare}/8.9/release-highlights.html[8.9]
 | {ref-bare}/8.8/release-highlights.html[8.8]

docs/reference/rest-api/usage.asciidoc

@@ -220,7 +220,15 @@ GET /_xpack/usage
       "grok" : 0,
       "limit" : 0,
       "where" : 0,
-      "sort" : 0
+      "sort" : 0,
+      "drop" : 0,
+      "show" : 0,
+      "rename" : 0,
+      "mv_expand" : 0,
+      "keep" : 0,
+      "enrich" : 0,
+      "from" : 0,
+      "row" : 0
     },
     "queries" : {
       "rest" : {

docs/reference/search/search-your-data/search-using-query-rules.asciidoc

@@ -174,6 +174,9 @@ PUT /_query_rules/my-ruleset
 
 The API response returns a results of `created` or `updated` depending on whether this was a new or edited ruleset.
 
+NOTE: There is a limit of 100 rules per ruleset.
+This can be increased up to 1000 using the `xpack.applications.rules.max_rules_per_ruleset` cluster setting.
+
 [source,console-result]
 ----
 {
@@ -217,3 +220,8 @@ GET /my-index-000001/_search
 
 This rule query will match against `rule1` in the defined query ruleset, and will convert the organic query into a pinned query with `id1` and `id2` pinned as the top hits.
 Any other matches from the organic query will be returned below the pinned results.
+
+It's possible to have multiple rules in a ruleset match a single `rule_query`. In this case, the pinned documents are returned in the following order:
+
+- Where the matching rule appears in the ruleset
+- If multiple documents are specified in a single rule, in the order they are specified

docs/reference/settings/security-settings.asciidoc

@@ -431,6 +431,16 @@ Specifies the attribute to examine on the user for group membership.
 If any `group_search` settings are specified, this setting is ignored. Defaults
 to `memberOf`.
 
+`user_full_name_attribute`::
+(<<static-cluster-setting,Static>>)
+Specifies the attribute to examine on the user for the full name of the user.
+Defaults to `cn`.
+
+`user_email_attribute`::
+(<<static-cluster-setting,Static>>)
+Specifies the attribute to examine on the user for the email address of the user.
+Defaults to `mail`.
+
 `user_search.base_dn`::
 (<<static-cluster-setting,Static>>)
 Specifies a container DN to search for users. Required

docs/reference/setup/sysconfig/tcpretries.asciidoc

@@ -17,7 +17,7 @@ Most Linux distributions default to retransmitting any lost packets 15 times.
 Retransmissions back off exponentially, so these 15 retransmissions take over
 900 seconds to complete. This means it takes Linux many minutes to detect a
 network partition or a failed node with this method. Windows defaults to just 5
-retransmissions which corresponds with a timeout of around 6 seconds.
+retransmissions which corresponds with a timeout of around 13 seconds.
 
 The Linux default allows for communication over networks that may experience
 very long periods of packet loss, but this default is excessive and even harmful
@@ -32,7 +32,7 @@ therefore reduce the maximum number of TCP retransmissions.
 
 You can decrease the maximum number of TCP retransmissions to `5` by running the
 following command as `root`. Five retransmissions corresponds with a timeout of
-around six seconds.
+around 13 seconds.
 
 [source,sh]
 -------------------------------------

docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc

@@ -159,6 +159,7 @@ The cluster state includes:
 * <<indices-templates-v1,Legacy index templates>>
 * <<ingest,Ingest pipelines>>
 * <<index-lifecycle-management,{ilm-init} policies>>
+* <<script-stored-scripts,Stored scripts>>
 * For snapshots taken after 7.12.0, <<feature-state,feature states>>
 // end::cluster-state-contents[]
 
@@ -260,12 +261,12 @@ You may want to restore an index in-place, for example when no alternative
 options surface after the <<cluster-allocation-explain>> API reports
 `no_valid_shard_copy`.
 
-The following request <<indices-close,closes>> `index_1` and then restores it 
+The following request <<indices-close,closes>> `index_1` and then restores it
 in-place from the `snapshot_2` snapshot in the `my_repository` repository.
 
 [source,console]
 ----
-POST index_1/_close 
+POST index_1/_close
 
 POST /_snapshot/my_repository/snapshot_2/_restore?wait_for_completion=true
 {

plugins/repository-hdfs/build.gradle

@@ -59,12 +59,6 @@ dependencies {
     runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
     //  runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714
 
-    // Set the keytab files in the classpath so that we can access them from test code without the security manager
-    // freaking out.
-    if (isEclipse == false) {
-        javaRestTestRuntimeOnly project(path: ':test:fixtures:krb5kdc-fixture', configuration:'krb5KeytabsHdfsDir')
-    }
-
     krb5Keytabs project(path: ':test:fixtures:krb5kdc-fixture', configuration: 'krb5KeytabsHdfsDir')
     krb5Config project(path: ':test:fixtures:krb5kdc-fixture', configuration: 'krb5ConfHdfsFile')
 }
@@ -187,7 +181,11 @@ for (int hadoopVersion = minTestedHadoopVersion; hadoopVersion <= maxTestedHadoo
                 }
             }
             testClassesDirs = sourceSets.javaRestTest.output.classesDirs
-            classpath = sourceSets.javaRestTest.runtimeClasspath + files(portsFileDir)
+            // Set the keytab files in the classpath so that we can access them from test code without the security manager
+            // freaking out.
+            classpath = sourceSets.javaRestTest.runtimeClasspath +
+              configurations.krb5Keytabs +
+              files(portsFileDir)
         }
     }
 

server/src/main/java/org/elasticsearch/Version.java

@@ -158,7 +158,8 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_11_1 = new Version(8_11_01_99);
     public static final Version V_8_11_2 = new Version(8_11_02_99);
     public static final Version V_8_12_0 = new Version(8_12_00_99);
-    public static final Version CURRENT = V_8_12_0;
+    public static final Version V_8_13_0 = new Version(8_13_00_99);
+    public static final Version CURRENT = V_8_13_0;
 
     private static final NavigableMap<Integer, Version> VERSION_IDS;
     private static final Map<String, Version> VERSION_STRINGS;