Merge branch 'main' into multiproject/split-custom

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v8.15.0$" : "main",
+    "^v8.16.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.yml

@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.23", "8.14.2", "8.15.0"]
+            BWC_VERSION: ["7.17.23", "8.14.3", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -577,8 +577,8 @@ steps:
         env:
           BWC_VERSION: 8.13.4
 
-      - label: "{{matrix.image}} / 8.14.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.2
+      - label: "{{matrix.image}} / 8.14.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -592,7 +592,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.14.2
+          BWC_VERSION: 8.14.3
 
       - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0
@@ -611,6 +611,23 @@ steps:
         env:
           BWC_VERSION: 8.15.0
 
+      - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+          diskSizeGb: 250
+        env:
+          BWC_VERSION: 8.16.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic.yml

@@ -642,8 +642,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.14.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.2#bwcTest
+      - label: 8.14.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -653,7 +653,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.14.2
+          BWC_VERSION: 8.14.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -682,6 +682,26 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.16.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+          diskSizeGb: 250
+        env:
+          BWC_VERSION: 8.16.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
@@ -751,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.23", "8.14.2", "8.15.0"]
+            BWC_VERSION: ["7.17.23", "8.14.3", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -801,7 +821,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.23", "8.14.2", "8.15.0"]
+            BWC_VERSION: ["7.17.23", "8.14.3", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -31,5 +31,6 @@ BWC_VERSION:
   - "8.11.4"
   - "8.12.2"
   - "8.13.4"
-  - "8.14.2"
+  - "8.14.3"
   - "8.15.0"
+  - "8.16.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,5 @@
 BWC_VERSION:
   - "7.17.23"
-  - "8.14.2"
+  - "8.14.3"
   - "8.15.0"
+  - "8.16.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java

@@ -71,7 +71,7 @@ public class TermsReduceBenchmark {
     private final SearchPhaseController controller = new SearchPhaseController((task, req) -> new AggregationReduceContext.Builder() {
         @Override
         public AggregationReduceContext forPartialReduction() {
-            return new AggregationReduceContext.ForPartial(null, null, task, builder);
+            return new AggregationReduceContext.ForPartial(null, null, task, builder, b -> {});
         }
 
         @Override

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.15"
+    },
     {
       "branch": "8.14"
     },

build-tools-internal/build.gradle

@@ -274,10 +274,7 @@ dependencies {
     // ensuring brought asm version brought in by spock is up-to-date
     testImplementation buildLibs.asm
     integTestImplementation buildLibs.asm
-    integTestImplementation('org.ow2.asm:asm:9.6')
-    api("org.yaml:snakeyaml") {
-      version { strictly(versions.snakeyaml) }
-    }
+    api(buildLibs.snakeyaml)
   }
   // Forcefully downgrade the jackson platform as used in production
   api enforcedPlatform(buildLibs.jackson.platform)
@@ -314,7 +311,7 @@ dependencies {
   compileOnly buildLibs.checkstyle
   compileOnly buildLibs.reflections
 
-  implementation 'com.github.javaparser:javaparser-core:3.18.0'
+  implementation buildLibs.javaparser
 
   runtimeOnly "org.elasticsearch.gradle:reaper:$version"
   testImplementation buildLibs.checkstyle

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -26,13 +26,10 @@ develocity {
     if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') {
       publishing.onlyIf { true }
       server = 'https://gradle-enterprise.elastic.co'
-    } else {
-      publishing.onlyIf {
-        server.isPresent();
-      }
+    } else if( server.isPresent() == false) {
+      publishing.onlyIf { false }
     }
 
-
     background {
       tag OS.current().name()
       tag Architecture.current().name()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -25,6 +25,7 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.testing.Test;
@@ -33,16 +34,21 @@
 import java.util.List;
 import java.util.Map;
 
+import javax.inject.Inject;
+
 import static org.elasticsearch.gradle.util.FileUtils.mkdirs;
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
 
 /**
  * Applies commonly used settings to all Test tasks in the project
  */
-public class ElasticsearchTestBasePlugin implements Plugin<Project> {
+public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
 
     public static final String DUMP_OUTPUT_ON_FAILURE_PROP_NAME = "dumpOutputOnFailure";
 
+    @Inject
+    protected abstract ProviderFactory getProviderFactory();
+
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply(GradleTestPolicySetupPlugin.class);
@@ -150,13 +156,11 @@ public void execute(Task t) {
             // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
             nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
 
+            test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("tests.").get());
+            test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("es.").get());
+
             // TODO: remove setting logging level via system property
             test.systemProperty("tests.logger.level", "WARN");
-            System.getProperties().entrySet().forEach(entry -> {
-                if ((entry.getKey().toString().startsWith("tests.") || entry.getKey().toString().startsWith("es."))) {
-                    test.systemProperty(entry.getKey().toString(), entry.getValue());
-                }
-            });
 
             // TODO: remove this once ctx isn't added to update script params in 7.0
             test.systemProperty("es.scripting.update.ctx_in_params", "false");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java

@@ -88,7 +88,7 @@ public String url(String os, String arch, String extension) {
     List<JdkBuild> builds = List.of(
         getBundledJdkBuild(),
         // 23 early access
-        new EarlyAccessJdkBuild(JavaLanguageVersion.of(23), "23", "23")
+        new EarlyAccessJdkBuild(JavaLanguageVersion.of(23), "23", "24")
     );
 
     private JdkBuild getBundledJdkBuild() {

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
-elasticsearch     = 8.15.0
-lucene            = 9.11.0
+elasticsearch     = 8.16.0
+lucene            = 9.11.1
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.11.0
-:lucene_version_path:   9_11_0
+:lucene_version:        9.11.1
+:lucene_version_path:   9_11_1
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/build.gradle

@@ -1815,6 +1815,20 @@ setups['setup-snapshots'] = setups['setup-repository'] + '''
             "run_as": [ "other_user" ],
             "metadata" : {"version": 1}
           }
+'''
+  setups['user_role'] = '''
+  - do:
+      security.put_role:
+        name: "my_user_role"
+        body: >
+          {
+            "description": "Grants user access to some indicies.",
+            "indices": [
+              {"names": ["index1", "index2" ], "privileges": ["all"], "field_security" : {"grant" : [ "title", "body" ]}}
+              ],
+            "metadata" : {"version": 1}
+          }
+
 '''
   setups['jacknich_user'] = '''
   - do:

docs/changelog/106520.yaml

@@ -0,0 +1,6 @@
+pr: 106520
+summary: Updated the transport CA name in Security Auto-Configuration.
+area: Security
+type: bug
+issues:
+ - 106455

docs/changelog/107191.yaml

@@ -0,0 +1,17 @@
+pr: 107191
+summary: Stricter failure handling in multi-repo get-snapshots request handling
+area: Snapshot/Restore
+type: bug
+issues: []
+highlight:
+  title: Stricter failure handling in multi-repo get-snapshots request handling
+  body: |
+    If a multi-repo get-snapshots request encounters a failure in one of the
+    targeted repositories then earlier versions of Elasticsearch would proceed
+    as if the faulty repository did not exist, except for a per-repository
+    failure report in a separate section of the response body. This makes it
+    impossible to paginate the results properly in the presence of failures. In
+    versions 8.15.0 and later this API's failure handling behaviour has been
+    made stricter, reporting an overall failure if any targeted repository's
+    contents cannot be listed.
+  notable: true

docs/changelog/108395.yaml

@@ -0,0 +1,5 @@
+pr: 108395
+summary: "ESQL: change from quoting from backtick to quote"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/108733.yaml

@@ -0,0 +1,5 @@
+pr: 108733
+summary: Query Roles API
+area: Security
+type: feature
+issues: []

docs/changelog/109084.yaml

@@ -0,0 +1,5 @@
+pr: 109084
+summary: Add AVX-512 optimised vector distance functions for int7 on x64
+area: Search
+type: enhancement
+issues: []

docs/changelog/109501.yaml

@@ -0,0 +1,14 @@
+pr: 109501
+summary: Reflect latest changes in synthetic source documentation
+area: Mapping
+type: enhancement
+issues: []
+highlight:
+  title: Synthetic `_source` improvements
+  body: |-
+    There are multiple improvements to synthetic `_source` functionality:
+
+    * Synthetic `_source` is now supported for all field types including `nested` and `object`. `object` fields are supported with `enabled` set to `false`.
+
+    * Synthetic `_source` can be enabled together with `ignore_malformed` and `ignore_above` parameters for all field types that support them.
+  notable: false