Merge branch 'main' into RealmsAuthenticator-log-all-causes

.buildkite/scripts/gradle-cache-validation.sh → .buildkite/scripts/gradle-build-cache-validation.sh

@@ -12,17 +12,18 @@ curl -s -L -O https://github.com/gradle/gradle-enterprise-build-validation-scrip
 tmpOutputFile=$(mktemp)
 trap "rm $tmpOutputFile" EXIT
 
+set +e
 gradle-enterprise-gradle-build-validation/03-validate-local-build-caching-different-locations.sh -r https://github.com/elastic/elasticsearch.git -b $BUILDKITE_BRANCH --gradle-enterprise-server https://gradle-enterprise.elastic.co -t precommit --fail-if-not-fully-cacheable | tee $tmpOutputFile
-
 # Capture the return value
 retval=$?
+set -e
 
 # Now read the content from the temporary file into a variable
 perfOutput=$(cat $tmpOutputFile | sed -n '/Performance Characteristics/,/See https:\/\/gradle.com\/bvs\/main\/Gradle.md#performance-characteristics for details./p' | sed '$d' | sed 's/\x1b\[[0-9;]*m//g')
 investigationOutput=$(cat $tmpOutputFile | sed -n '/Investigation Quick Links/,$p' | sed 's/\x1b\[[0-9;]*m//g')
 
 # Initialize HTML output variable
-summaryHtml="<h4>Performance Characteristics</h4>"
+summaryHtml="<h4>Build Cache Performance Characteristics</h4>"
 summaryHtml+="<ul>"
 
 # Process each line of the string

.buildkite/scripts/gradle-configuration-cache-validation.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# TODO/ FIXIT without a full resolved gradle home, we see issues configuration cache reuse
+./gradlew --max-workers=8 --parallel --scan --no-daemon precommit
+
+./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+
+# Create a temporary file
+tmpOutputFile=$(mktemp)
+trap "rm $tmpOutputFile" EXIT
+
+echo "2nd run"
+# TODO run-gradle.sh script causes issues because of init script handling
+./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+
+# Check if the command was successful
+if grep -q "Configuration cache entry reused." $tmpOutputFile; then
+    echo "Gradle configuration cache reused"
+    exit 0
+else
+    echo "Failed to reuse Gradle configuration cache."
+    exit 1
+fi
+
+

README.asciidoc

@@ -33,76 +33,144 @@ https://www.elastic.co/downloads/elasticsearch[elastic.co/downloads/elasticsearc
 === Run Elasticsearch locally
 
 //// 
-IMPORTANT: This content is replicated in the Elasticsearch guide. 
-If you make changes, you must also update setup/set-up-local-dev-deployment.asciidoc.
+IMPORTANT: This content is replicated in the Elasticsearch guide. See `run-elasticsearch-locally.asciidoc`.
+Both will soon be replaced by a quickstart script.
 ////
 
-To try out Elasticsearch on your own machine, we recommend using Docker
-and running both Elasticsearch and Kibana.
-Docker images are available from the https://www.docker.elastic.co[Elastic Docker registry].
+[WARNING]
+==== 
+DO NOT USE THESE INSTRUCTIONS FOR PRODUCTION DEPLOYMENTS.
 
-NOTE: Starting in Elasticsearch 8.0, security is enabled by default. 
-The first time you start Elasticsearch, TLS encryption is configured automatically, 
-a password is generated for the `elastic` user, 
-and a Kibana enrollment token is created so you can connect Kibana to your secured cluster.
+This setup is intended for local development and testing only.
+====
 
-For other installation options, see the
-https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html[Elasticsearch installation documentation].
+The following commands help you very quickly spin up a single-node Elasticsearch cluster, together with Kibana in Docker.
+Use this setup for local development or testing.
 
-**Start Elasticsearch**
+==== Prerequisites
 
-. Install and start https://www.docker.com/products/docker-desktop[Docker
-Desktop]. Go to **Preferences > Resources > Advanced** and set Memory to at least 4GB.
+If you don't have Docker installed, https://www.docker.com/products/docker-desktop[download and install Docker Desktop] for your operating system.
 
-. Start an Elasticsearch container:
-+
+==== Set environment variables
+
+Configure the following environment variables.
+
+[source,sh]
+----
+export ELASTIC_PASSWORD="<ES_PASSWORD>"  # password for "elastic" username
+export KIBANA_PASSWORD="<KIB_PASSWORD>"   # Used internally by Kibana, must be at least 6 characters long
+----
+
+==== Create a Docker network
+
+To run both Elasticsearch and Kibana, you'll need to create a Docker network:
+
+[source,sh]
 ----
-docker network create elastic
-docker pull docker.elastic.co/elasticsearch/elasticsearch:{version} <1>
-docker run --name elasticsearch --net elastic -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -t docker.elastic.co/elasticsearch/elasticsearch:{version}
+docker network create elastic-net
 ----
-<1> Replace {version} with the version of Elasticsearch you want to run.
-+
-When you start Elasticsearch for the first time, the generated `elastic` user password and
-Kibana enrollment token are output to the terminal.
-+
-NOTE: You might need to scroll back a bit in the terminal to view the password 
-and enrollment token.
 
-. Copy the generated password and enrollment token and save them in a secure 
-location. These values are shown only when you start Elasticsearch for the first time.
-You'll use these to enroll Kibana with your Elasticsearch cluster and log in.
+==== Run Elasticsearch
+
+Start the Elasticsearch container with the following command:
 
-**Start Kibana**
+[source,sh]
+----
+docker run -p 127.0.0.1:9200:9200 -d --name elasticsearch --network elastic-net \
+  -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD \
+  -e "discovery.type=single-node" \
+  -e "xpack.security.http.ssl.enabled=false" \
+  -e "xpack.license.self_generated.type=trial" \
+  docker.elastic.co/elasticsearch/elasticsearch:{version}
+----
 
-Kibana enables you to easily send requests to Elasticsearch and analyze, visualize, and manage data interactively.
+==== Run Kibana (optional)
 
-. In a new terminal session, start Kibana and connect it to your Elasticsearch container:
-+
+To run Kibana, you must first set the `kibana_system` password in the Elasticsearch container.
+
+[source,sh]
 ----
-docker pull docker.elastic.co/kibana/kibana:{version} <1>
-docker run --name kibana --net elastic -p 5601:5601 docker.elastic.co/kibana/kibana:{version}
+# configure the Kibana password in the ES container
+curl -u elastic:$ELASTIC_PASSWORD \
+  -X POST \
+  http://localhost:9200/_security/user/kibana_system/_password \
+  -d '{"password":"'"$KIBANA_PASSWORD"'"}' \
+  -H 'Content-Type: application/json'
 ----
-<1> Replace {version} with the version of Kibana you want to run.
-+
-When you start Kibana, a unique URL is output to your terminal.
+// NOTCONSOLE
 
-. To access Kibana, open the generated URL in your browser.
+Start the Kibana container with the following command:
 
-  .. Paste the enrollment token that you copied when starting
-  Elasticsearch and click the button to connect your Kibana instance with Elasticsearch.
+[source,sh]
+----
+docker run -p 127.0.0.1:5601:5601 -d --name kibana --network elastic-net \
+  -e ELASTICSEARCH_URL=http://elasticsearch:9200 \
+  -e ELASTICSEARCH_HOSTS=http://elasticsearch:9200 \
+  -e ELASTICSEARCH_USERNAME=kibana_system \
+  -e ELASTICSEARCH_PASSWORD=$KIBANA_PASSWORD \
+  -e "xpack.security.enabled=false" \
+  -e "xpack.license.self_generated.type=trial" \
+  docker.elastic.co/kibana/kibana:{version}
+----
 
-  .. Log in to Kibana as the `elastic` user with the password that was generated
-  when you started Elasticsearch.
+.Trial license
+[%collapsible]
+====
+The service is started with a trial license. The trial license enables all features of Elasticsearch for a trial period of 30 days. After the trial period expires, the license is downgraded to a basic license, which is free forever. If you prefer to skip the trial and use the basic license, set the value of the `xpack.license.self_generated.type` variable to basic instead. For a detailed feature comparison between the different licenses, refer to our https://www.elastic.co/subscriptions[subscriptions page].
+====
 
-**Send requests to Elasticsearch**
+==== Send requests to Elasticsearch
 
 You send data and other requests to Elasticsearch through REST APIs. 
 You can interact with Elasticsearch using any client that sends HTTP requests, 
 such as the https://www.elastic.co/guide/en/elasticsearch/client/index.html[Elasticsearch
 language clients] and https://curl.se[curl]. 
+
+===== Using curl 
+
+Here's an example curl command to create a new Elasticsearch index, using basic auth:
+
+[source,sh]
+----
+curl -u elastic:$ELASTIC_PASSWORD \
+  -X PUT \
+  http://localhost:9200/my-new-index \
+  -H 'Content-Type: application/json'
+----
+// NOTCONSOLE
+
+===== Using a language client
+
+To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable.
+
+You'll use the following connection details:
+
+* **Elasticsearch endpoint**: `http://localhost:9200`
+* **Username**: `elastic`
+* **Password**: `$ELASTIC_PASSWORD` (Value you set in the environment variable)
+
+For example, to connect with the Python `elasticsearch` client:
+
+[source,python]
+----
+import os
+from elasticsearch import Elasticsearch
+
+username = 'elastic'
+password = os.getenv('ELASTIC_PASSWORD') # Value you set in the environment variable
+
+client = Elasticsearch(
+    "http://localhost:9200",
+    basic_auth=(username, password)
+)
+
+print(client.info())
+----
+
+===== Using the Dev Tools Console
+
 Kibana's developer console provides an easy way to experiment and test requests. 
-To access the console, go to **Management > Dev Tools**.
+To access the console, open Kibana, then go to **Management** > **Dev Tools**.
 
 **Add data**
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -61,7 +61,7 @@ public void apply(Project target) {
             : System.getenv("BUILDKITE_BUILD_NUMBER");
         String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST");
         if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) {
-            File targetFile = target.file("build/" + buildNumber + ".tar.bz2");
+            File targetFile = calculateTargetFile(target, buildNumber);
             File projectDir = target.getProjectDir();
             File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");
             DevelocityConfiguration extension = target.getExtensions().getByType(DevelocityConfiguration.class);
@@ -86,9 +86,19 @@ public void apply(Project target) {
         }
     }
 
+    private File calculateTargetFile(Project target, String buildNumber) {
+        File uploadFile = target.file("build/" + buildNumber + ".tar.bz2");
+        int artifactIndex = 1;
+        while (uploadFile.exists()) {
+            uploadFile = target.file("build/" + buildNumber + "-" + artifactIndex++ + ".tar.bz2");
+        }
+        return uploadFile;
+    }
+
     private List<File> resolveProjectLogs(File projectDir) {
         var projectDirFiles = getFileOperations().fileTree(projectDir);
         projectDirFiles.include("**/*.hprof");
+        projectDirFiles.include("**/build/reports/configuration-cache/**");
         projectDirFiles.include("**/build/test-results/**/*.xml");
         projectDirFiles.include("**/build/testclusters/**");
         projectDirFiles.include("**/build/testrun/*/temp/**");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java

@@ -43,17 +43,16 @@ void impl(String implName, Project implProject) {
         });
 
         String manifestTaskName = "generate" + capitalName + "ProviderManifest";
-        Provider<Directory> generatedResourcesDir = project.getLayout().getBuildDirectory().dir("generated-resources");
+        Provider<Directory> generatedResourcesRoot = project.getLayout().getBuildDirectory().dir("generated-resources");
         var generateProviderManifest = project.getTasks().register(manifestTaskName, GenerateProviderManifest.class);
         generateProviderManifest.configure(t -> {
-            t.getManifestFile().set(generatedResourcesDir.map(d -> d.file("LISTING.TXT")));
+            t.getManifestFile().set(generatedResourcesRoot.map(d -> d.dir(manifestTaskName).file("LISTING.TXT")));
             t.getProviderImplClasspath().from(implConfig);
         });
-
         String implTaskName = "generate" + capitalName + "ProviderImpl";
         var generateProviderImpl = project.getTasks().register(implTaskName, Sync.class);
         generateProviderImpl.configure(t -> {
-            t.into(generatedResourcesDir);
+            t.into(generatedResourcesRoot.map(d -> d.dir(implTaskName)));
             t.into("IMPL-JARS/" + implName, spec -> {
                 spec.from(implConfig);
                 spec.from(generateProviderManifest);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SourceDirectoryCommandLineArgumentProvider.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.util;
+
+import org.gradle.api.file.Directory;
+import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
+import org.gradle.process.CommandLineArgumentProvider;
+
+import java.util.Arrays;
+
+public class SourceDirectoryCommandLineArgumentProvider implements CommandLineArgumentProvider {
+
+    private final Directory sourceDirectory;
+
+    public SourceDirectoryCommandLineArgumentProvider(Directory sourceDirectory) {
+        this.sourceDirectory = sourceDirectory;
+    }
+
+    public Iterable<String> asArguments() {
+        return Arrays.asList("-s", sourceDirectory.getAsFile().getAbsolutePath());
+    }
+
+    @InputDirectory
+    @PathSensitive(PathSensitivity.RELATIVE)
+    public Directory getSourceDirectory() {
+        return sourceDirectory;
+    }
+}

docs/changelog/110216.yaml

@@ -0,0 +1,5 @@
+pr: 110216
+summary: Register SLM run before snapshotting to save stats
+area: ILM+SLM
+type: enhancement
+issues: []

docs/changelog/110901.yaml

@@ -0,0 +1,15 @@
+pr: 110901
+summary: Set lenient to true by default when using updateable synonyms
+area: Analysis
+type: breaking
+issues: []
+breaking:
+  title: Set lenient to true by default when using updateable synonyms
+  area: Analysis
+  details: |
+    When a `synonym` or `synonym_graph` token filter is configured with `updateable: true`, the default `lenient`
+    value will now be `true`.
+  impact: |
+    `synonym` or `synonym_graph` token filters configured with `updateable: true` will ignore invalid synonyms by
+    default. This prevents shard initialization errors on invalid synonyms.
+  notable: true

docs/changelog/111285.yaml

@@ -0,0 +1,5 @@
+pr: 111285
+summary: "[Bugfix] Add `accessDeclaredMembers` permission to allow search application templates to parse floats"
+area: Relevance
+type: bug
+issues: []

docs/changelog/111315.yaml

@@ -0,0 +1,5 @@
+pr: 111315
+summary: Add link to flood-stage watermark exception message
+area: Allocation
+type: enhancement
+issues: []

docs/changelog/111369.yaml

@@ -0,0 +1,5 @@
+pr: 111369
+summary: Improve security-crypto threadpool overflow handling
+area: Authentication
+type: bug
+issues: []

docs/changelog/111420.yaml

@@ -0,0 +1,5 @@
+pr: 111420
+summary: "[Query rules] Add `exclude` query rule type"
+area: Relevance
+type: feature
+issues: []

docs/changelog/111552.yaml

@@ -0,0 +1,5 @@
+pr: 111552
+summary: Siem ea 9521 improve test
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/111576.yaml

@@ -0,0 +1,6 @@
+pr: 111576
+summary: Execute shard snapshot tasks in shard-id order
+area: Snapshot/Restore
+type: enhancement
+issues:
+ - 108739

docs/changelog/111600.yaml

@@ -0,0 +1,5 @@
+pr: 111600
+summary: Make ecs@mappings work with OTel attributes
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/111624.yaml

@@ -0,0 +1,6 @@
+pr: 111624
+summary: Extend logging for dropped warning headers
+area: Infra/Core
+type: enhancement
+issues:
+ - 90527