Merge branch 'main' into exposing-ClientHelper

docs/changelog/116739.yaml

@@ -0,0 +1,5 @@
+pr: 116739
+summary: Change default Docker image to be based on UBI minimal instead of Ubuntu
+area: Infra/Core
+type: enhancement
+issues: []

docs/reference/connector/docs/connectors-content-syncs.asciidoc

@@ -52,7 +52,7 @@ However, a fast, accessible third-party data source that stores huge amounts of
 
 [NOTE]
 ====
-Incremental syncs for the SharePoint Online connector use specific logic.
+Incremental syncs for <<es-connectors-sharepoint-online,SharePoint Online>> and <<es-connectors-google-drive,Google Drive>> connectors use specific logic.
 All other connectors use the same shared connector framework logic for incremental syncs.
 ====
 

docs/reference/intro.asciidoc

@@ -85,7 +85,7 @@ You can deploy {es} in various ways.
 **Hosted options**
 
 * {cloud}/ec-getting-started-trial.html[*Elastic Cloud Hosted*]: {es} is available as part of the hosted Elastic Stack offering, deployed in the cloud with your provider of choice. Sign up for a https://cloud.elastic.co/registration[14-day free trial].
-* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless* (technical preview)]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial].
+* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless*]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial].
 
 **Advanced options**
 

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java

@@ -33,7 +33,7 @@
 public class InternalAdjacencyMatrix extends InternalMultiBucketAggregation<InternalAdjacencyMatrix, InternalAdjacencyMatrix.InternalBucket>
     implements
         AdjacencyMatrix {
-    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements AdjacencyMatrix.Bucket {
+    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucketWritable implements AdjacencyMatrix.Bucket {
 
         private final String key;
         private final long docCount;

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java

@@ -34,7 +34,7 @@ public class InternalTimeSeries extends InternalMultiBucketAggregation<InternalT
     /**
      * A bucket associated with a specific time series (identified by its key)
      */
-    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket {
+    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucketWritable {
         protected long bucketOrd;
         protected final BytesRef key;
         // TODO: make computing docCount optional

modules/repository-s3/build.gradle

@@ -46,6 +46,7 @@ dependencies {
   yamlRestTestImplementation project(":test:framework")
   yamlRestTestImplementation project(':test:fixtures:s3-fixture')
   yamlRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
+  yamlRestTestImplementation project(':test:fixtures:aws-sts-fixture')
   yamlRestTestImplementation project(':test:fixtures:minio-fixture')
   internalClusterTestImplementation project(':test:fixtures:minio-fixture')
 

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java

@@ -35,7 +35,14 @@ public class RepositoryS3RestReloadCredentialsIT extends ESRestTestCase {
     private static final String BUCKET = "RepositoryS3RestReloadCredentialsIT-bucket-" + HASHED_SEED;
     private static final String BASE_PATH = "RepositoryS3RestReloadCredentialsIT-base-path-" + HASHED_SEED;
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, "ignored");
+    private static volatile String repositoryAccessKey;
+
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        BUCKET,
+        BASE_PATH,
+        S3HttpFixture.mutableAccessKey(() -> repositoryAccessKey)
+    );
 
     private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();
 
@@ -68,7 +75,7 @@ public void testReloadCredentialsFromKeystore() throws IOException {
 
         // Set up initial credentials
         final var accessKey1 = randomIdentifier();
-        s3Fixture.setAccessKey(accessKey1);
+        repositoryAccessKey = accessKey1;
         keystoreSettings.put("s3.client.default.access_key", accessKey1);
         keystoreSettings.put("s3.client.default.secret_key", randomIdentifier());
         cluster.updateStoredSecureSettings();
@@ -79,14 +86,14 @@ public void testReloadCredentialsFromKeystore() throws IOException {
 
         // Rotate credentials in blob store
         final var accessKey2 = randomValueOtherThan(accessKey1, ESTestCase::randomIdentifier);
-        s3Fixture.setAccessKey(accessKey2);
+        repositoryAccessKey = accessKey2;
 
         // Ensure that initial credentials now invalid
         final var accessDeniedException2 = expectThrows(ResponseException.class, () -> client().performRequest(verifyRequest));
         assertThat(accessDeniedException2.getResponse().getStatusLine().getStatusCode(), equalTo(500));
         assertThat(
             accessDeniedException2.getMessage(),
-            allOf(containsString("Bad access key"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
+            allOf(containsString("Access denied"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
         );
 
         // Set up refreshed credentials

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java

@@ -10,8 +10,8 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
-import fixture.s3.S3HttpFixtureWithSessionToken;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
@@ -34,27 +34,30 @@ public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3Clien
 
     private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
     private static final String TEMPORARY_SESSION_TOKEN = "session_token-" + HASHED_SEED;
-    private static final String IMDS_ACCESS_KEY = "imds-access-key-" + HASHED_SEED;
-    private static final String IMDS_SESSION_TOKEN = "imds-session-token-" + HASHED_SEED;
 
     private static final S3HttpFixture s3Fixture = new S3HttpFixture();
 
-    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken(
+    private static final S3HttpFixture s3HttpFixtureWithSessionToken = new S3HttpFixture(
+        true,
         "session_token_bucket",
         "session_token_base_path_integration_tests",
-        System.getProperty("s3TemporaryAccessKey"),
-        TEMPORARY_SESSION_TOKEN
+        S3HttpFixture.fixedAccessKeyAndToken(System.getProperty("s3TemporaryAccessKey"), TEMPORARY_SESSION_TOKEN)
     );
 
-    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithImdsSessionToken = new S3HttpFixtureWithSessionToken(
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        dynamicS3Credentials::addValidCredentials,
+        Set.of()
+    );
+
+    private static final S3HttpFixture s3HttpFixtureWithImdsSessionToken = new S3HttpFixture(
+        true,
         "ec2_bucket",
         "ec2_base_path",
-        IMDS_ACCESS_KEY,
-        IMDS_SESSION_TOKEN
+        dynamicS3Credentials::isAuthorized
     );
 
-    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(IMDS_ACCESS_KEY, IMDS_SESSION_TOKEN, Set.of());
-
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .keystore("s3.client.integration_test_permanent.access_key", System.getProperty("s3PermanentAccessKey"))

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java

@@ -10,12 +10,12 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
-import fixture.s3.S3HttpFixtureWithSessionToken;
+import fixture.s3.DynamicS3Credentials;
+import fixture.s3.S3HttpFixture;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.junit.ClassRule;
@@ -26,23 +26,20 @@
 
 public class RepositoryS3EcsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
 
-    private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
-    private static final String ECS_ACCESS_KEY = "ecs-access-key-" + HASHED_SEED;
-    private static final String ECS_SESSION_TOKEN = "ecs-session-token-" + HASHED_SEED;
-
-    private static final S3HttpFixtureWithSessionToken s3Fixture = new S3HttpFixtureWithSessionToken(
-        "ecs_bucket",
-        "ecs_base_path",
-        ECS_ACCESS_KEY,
-        ECS_SESSION_TOKEN
-    );
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
-        ECS_ACCESS_KEY,
-        ECS_SESSION_TOKEN,
+        dynamicS3Credentials::addValidCredentials,
         Set.of("/ecs_credentials_endpoint")
     );
 
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        "ecs_bucket",
+        "ecs_base_path",
+        dynamicS3Credentials::isAuthorized
+    );
+
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .setting("s3.client.integration_test_ecs.endpoint", s3Fixture::getAddress)

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsClientYamlTestSuiteIT.java

@@ -9,8 +9,9 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.aws.sts.AwsStsHttpFixture;
+import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
-import fixture.s3.S3HttpFixtureWithSTS;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
@@ -24,13 +25,27 @@
 
 public class RepositoryS3StsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture();
-    private static final S3HttpFixtureWithSTS s3Sts = new S3HttpFixtureWithSTS();
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+
+    private static final S3HttpFixture s3HttpFixture = new S3HttpFixture(
+        true,
+        "sts_bucket",
+        "sts_base_path",
+        dynamicS3Credentials::isAuthorized
+    );
+
+    private static final AwsStsHttpFixture stsHttpFixture = new AwsStsHttpFixture(dynamicS3Credentials::addValidCredentials, """
+        Atza|IQEBLjAsAhRFiXuWpUXuRvQ9PZL3GMFcYevydwIUFAHZwXZXXXXXXXXJnrulxKDHwy87oGKPznh0D6bEQZTSCzyoCtL_8S07pLpr0zMbn6w1lfVZKNTBdDans\
+        FBmtGnIsIapjI6xKR02Yc_2bQ8LZbUXSGm6Ry6_BG7PrtLZtj_dfCTj92xNGed-CrKqjG7nPBjNIL016GGvuS5gSvPRUxWES3VYfm1wl7WTI7jn-Pcb6M-buCgHhFO\
+        zTQxod27L9CqnOLio7N3gZAGpsp6n1-AJBOCJckcyXe2c6uD0srOJeZlKUm2eTDVMf8IehDVI0r1QOnTV6KzzAI3OY87Vd_cVMQ""");
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
-        .setting("s3.client.integration_test_sts.endpoint", s3Sts::getAddress)
-        .systemProperty("com.amazonaws.sdk.stsMetadataServiceEndpointOverride", () -> s3Sts.getAddress() + "/assume-role-with-web-identity")
+        .setting("s3.client.integration_test_sts.endpoint", s3HttpFixture::getAddress)
+        .systemProperty(
+            "com.amazonaws.sdk.stsMetadataServiceEndpointOverride",
+            () -> stsHttpFixture.getAddress() + "/assume-role-with-web-identity"
+        )
         .configFile("repository-s3/aws-web-identity-token-file", Resource.fromClasspath("aws-web-identity-token-file"))
         .environment("AWS_WEB_IDENTITY_TOKEN_FILE", System.getProperty("awsWebIdentityTokenExternalLocation"))
         // // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the
@@ -40,7 +55,7 @@ public class RepositoryS3StsClientYamlTestSuiteIT extends AbstractRepositoryS3Cl
         .build();
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Sts).around(cluster);
+    public static TestRule ruleChain = RuleChain.outerRule(s3HttpFixture).around(stsHttpFixture).around(cluster);
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() throws Exception {

muted-tests.yml

@@ -240,6 +240,9 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/117524
 - class: org.elasticsearch.repositories.s3.RepositoryS3EcsClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/117525
+- class: org.elasticsearch.xpack.esql.qa.mixed.FieldExtractorIT
+  method: testConstantKeywordField
+  issue: https://github.com/elastic/elasticsearch/issues/117531
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.hamcrest.Matcher;
 
@@ -31,6 +32,7 @@
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.test.MockLog.assertThatLogger;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -211,4 +213,25 @@ public void testLogLocalHotThreads() {
             )
         );
     }
+
+    @TestLogging(reason = "testing logging at various levels", value = "org.elasticsearch.action.admin.HotThreadsIT:TRACE")
+    public void testLogLocalCurrentThreadsInPlainText() {
+        final var level = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        assertThatLogger(
+            () -> HotThreads.logLocalCurrentThreads(logger, level, getTestName()),
+            HotThreadsIT.class,
+            new MockLog.SeenEventExpectation(
+                "Should log hot threads header in plain text",
+                HotThreadsIT.class.getCanonicalName(),
+                level,
+                "testLogLocalCurrentThreadsInPlainText: Hot threads at"
+            ),
+            new MockLog.SeenEventExpectation(
+                "Should log hot threads cpu usage in plain text",
+                HotThreadsIT.class.getCanonicalName(),
+                level,
+                "cpu usage by thread"
+            )
+        );
+    }
 }

server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java

@@ -398,7 +398,7 @@ public void testErrorCanRecoverOnRestart() throws Exception {
 
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
 
-        assertTrue(masterFileSettingsService.watching());
+        assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
         assertFalse(dataFileSettingsService.watching());
 
         writeJSONFile(masterNode, testErrorJSON, logger, versionCounter.incrementAndGet());
@@ -434,7 +434,7 @@ public void testNewErrorOnRestartReprocessing() throws Exception {
 
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
 
-        assertTrue(masterFileSettingsService.watching());
+        assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
         assertFalse(dataFileSettingsService.watching());
 
         writeJSONFile(masterNode, testErrorJSON, logger, versionCounter.incrementAndGet());

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -206,7 +206,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INGEST_PIPELINE_CONFIGURATION_AS_MAP = def(8_797_00_0);
     public static final TransportVersion INDEXING_PRESSURE_THROTTLING_STATS = def(8_798_00_0);
     public static final TransportVersion REINDEX_DATA_STREAMS = def(8_799_00_0);
-
+    public static final TransportVersion ESQL_REMOVE_NODE_LEVEL_PLAN = def(8_800_00_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -131,6 +131,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion ADD_ROLE_MAPPING_CLEANUP_MIGRATION = def(8_518_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT_BACKPORT = def(8_519_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID_BACKPORT = def(8_520_00_0, Version.LUCENE_9_12_0);
+    public static final IndexVersion V8_DEPRECATE_SOURCE_MODE_MAPPER = def(8_521_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0);

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -484,8 +484,7 @@ public boolean isStored() {
     }
 
     public static boolean onOrAfterDeprecateModeVersion(IndexVersion version) {
-        return version.onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER);
-        // Adjust versions after backporting.
-        // || version.between(IndexVersions.BACKPORT_DEPRECATE_SOURCE_MODE_MAPPER, IndexVersions.UPGRADE_TO_LUCENE_10_0_0);
+        return version.onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER)
+            || version.between(IndexVersions.V8_DEPRECATE_SOURCE_MODE_MAPPER, IndexVersions.UPGRADE_TO_LUCENE_10_0_0);
     }
 }