Merge from main

.github/CODEOWNERS

@@ -65,6 +65,30 @@ server/src/main/java/org/elasticsearch/bootstrap @elastic/es-core-infra
 server/src/main/java/org/elasticsearch/node @elastic/es-core-infra
 server/src/main/java/org/elasticsearch/plugins @elastic/es-core-infra
 server/src/main/java/org/elasticsearch/threadpool @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/breaker @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/bytes @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/cli @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/collect @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/component @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/compress @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/document @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/file @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/hash @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/io @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/logging @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/metrics @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/network @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/path @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/recycler @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/regex @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/scheduler @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/settings @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/text @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/time @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/transport @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/unit @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/util @elastic/es-core-infra
+server/src/main/java/org/elasticsearch/common/xcontent @elastic/es-core-infra
 
 # Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege @elastic/es-security

docs/changelog/115655.yaml

@@ -0,0 +1,5 @@
+pr: 115655
+summary: Better sizing `BytesRef` for Strings in Queries
+area: Search
+type: enhancement
+issues: []

docs/changelog/115792.yaml

@@ -0,0 +1,5 @@
+pr: 115792
+summary: Add ES|QL `bit_length` function
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/116021.yaml

@@ -0,0 +1,6 @@
+pr: 116021
+summary: Fields caps does not honour ignore_unavailable
+area: Search
+type: bug
+issues:
+ - 107767

docs/changelog/116292.yaml

@@ -0,0 +1,5 @@
+pr: 116292
+summary: Add missing header in `put_data_lifecycle` rest-api-spec
+area: Data streams
+type: bug
+issues: []

docs/changelog/116346.yaml

@@ -0,0 +1,5 @@
+pr: 116346
+summary: "[ESQL] Fix Binary Comparisons on Date Nanos"
+area: ES|QL
+type: bug
+issues: []

docs/reference/connector/docs/_connectors-create-native.asciidoc

@@ -22,7 +22,7 @@ PUT _connector/my-{service-name-stub}-connector
   "index_name": "my-elasticsearch-index",
   "name": "Content synced from {service-name}",
   "service_type": "{service-name-stub}",
-  "is_native": "true"
+  "is_native": true
 }
 ----
 // TEST[skip:can't test in isolation]

docs/reference/esql/functions/string-functions.asciidoc

@@ -8,6 +8,7 @@
 {esql} supports these string functions:
 
 // tag::string_list[]
+* <<esql-bit_length>>
 * <<esql-concat>>
 * <<esql-ends_with>>
 * <<esql-from_base64>>
@@ -30,6 +31,7 @@
 * <<esql-trim>>
 // end::string_list[]
 
+include::layout/bit_length.asciidoc[]
 include::layout/concat.asciidoc[]
 include::layout/ends_with.asciidoc[]
 include::layout/from_base64.asciidoc[]

docs/reference/search/retriever.asciidoc

@@ -554,8 +554,8 @@ It then re-ranks the results based on semantic similarity to the text in the `in
 [[rule-retriever]]
 ==== Query Rules Retriever
 
-The `rule` retriever enables fine-grained control over search results by applying contextual <<query-rules>> to pin or exclude documents for specific queries.
-This retriever has similar functionality to the <<query-dsl-rule-query>>, but works out of the box with other retrievers.
+The `rule` retriever enables fine-grained control over search results by applying contextual <<query-rules,query rules>> to pin or exclude documents for specific queries.
+This retriever has similar functionality to the <<query-dsl-rule-query, rule query>>, but works out of the box with other retrievers.
 
 ===== Prerequisites
 

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ServiceTests.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.AmazonWebServiceRequest;
+import com.amazonaws.retry.PredefinedRetryPolicies;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
@@ -19,6 +20,7 @@
 
 import java.io.IOException;
 
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.mockito.Mockito.mock;
 
 public class S3ServiceTests extends ESTestCase {
@@ -47,19 +49,33 @@ public void testRetryOn403RetryPolicy() {
         e.setStatusCode(403);
         e.setErrorCode("InvalidAccessKeyId");
 
-        // Retry on 403 invalid access key id
+        // AWS default retry condition does not retry on 403
+        assertFalse(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION.shouldRetry(mock(AmazonWebServiceRequest.class), e, between(0, 9)));
+
+        // The retryable 403 condition retries on 403 invalid access key id
         assertTrue(
             S3Service.RETRYABLE_403_RETRY_POLICY.getRetryCondition().shouldRetry(mock(AmazonWebServiceRequest.class), e, between(0, 9))
         );
 
-        // Not retry if not 403 or not invalid access key id
         if (randomBoolean()) {
+            // Random for another error status that is not 403
             e.setStatusCode(randomValueOtherThan(403, () -> between(0, 600)));
+            // Retryable 403 condition delegates to the AWS default retry condition. Its result must be consistent with the decision
+            // by the AWS default, e.g. some error status like 429 is retryable by default, the retryable 403 condition respects it.
+            boolean actual = S3Service.RETRYABLE_403_RETRY_POLICY.getRetryCondition()
+                .shouldRetry(mock(AmazonWebServiceRequest.class), e, between(0, 9));
+            boolean expected = PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION.shouldRetry(
+                mock(AmazonWebServiceRequest.class),
+                e,
+                between(0, 9)
+            );
+            assertThat(actual, equalTo(expected));
         } else {
+            // Not retry for 403 with error code that is not invalid access key id
             e.setErrorCode(randomAlphaOfLength(10));
+            assertFalse(
+                S3Service.RETRYABLE_403_RETRY_POLICY.getRetryCondition().shouldRetry(mock(AmazonWebServiceRequest.class), e, between(0, 9))
+            );
         }
-        assertFalse(
-            S3Service.RETRYABLE_403_RETRY_POLICY.getRetryCondition().shouldRetry(mock(AmazonWebServiceRequest.class), e, between(0, 9))
-        );
     }
 }

muted-tests.yml

@@ -160,9 +160,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/inference_crud/Test delete given model referenced by pipeline}
   issue: https://github.com/elastic/elasticsearch/issues/115970
-- class: org.elasticsearch.repositories.s3.S3ServiceTests
-  method: testRetryOn403RetryPolicy
-  issue: https://github.com/elastic/elasticsearch/issues/115986
 - class: org.elasticsearch.search.slice.SearchSliceIT
   method: testPointInTime
   issue: https://github.com/elastic/elasticsearch/issues/115988
@@ -210,9 +207,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/inference_crud/Test delete given model with alias referenced by pipeline}
   issue: https://github.com/elastic/elasticsearch/issues/116133
-- class: org.elasticsearch.xpack.inference.InferenceRestIT
-  method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint}
-  issue: https://github.com/elastic/elasticsearch/issues/114412
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {categorize.Categorize SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113054
@@ -240,18 +234,12 @@ tests:
 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
   method: testSearchWithRandomDisconnects
   issue: https://github.com/elastic/elasticsearch/issues/116175
-- class: org.elasticsearch.indexing.IndexActionIT
-  method: testAutoGenerateIdNoDuplicates
-  issue: https://github.com/elastic/elasticsearch/issues/115716
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/start_stop_datafeed/Test start datafeed given index pattern with no matching indices}
   issue: https://github.com/elastic/elasticsearch/issues/116220
 - class: org.elasticsearch.search.basic.SearchWhileRelocatingIT
   method: testSearchAndRelocateConcurrentlyRandomReplicas
   issue: https://github.com/elastic/elasticsearch/issues/116145
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
-  method: testProcessFileChanges
-  issue: https://github.com/elastic/elasticsearch/issues/115280
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/filter_crud/Test update filter}
   issue: https://github.com/elastic/elasticsearch/issues/116271
@@ -264,24 +252,27 @@ tests:
 - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT
   method: testEveryActionIsEitherOperatorOnlyOrNonOperator
   issue: https://github.com/elastic/elasticsearch/issues/102992
-- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/116291
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/114723
-- class: org.elasticsearch.xpack.search.AsyncSearchSecurityIT
-  issue: https://github.com/elastic/elasticsearch/issues/116293
-- class: org.elasticsearch.xpack.downsample.DownsampleRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/116326
-- class: org.elasticsearch.xpack.downsample.DownsampleWithBasicRestIT
-  issue: https://github.com/elastic/elasticsearch/issues/116327
-- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/116328
 - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
   method: testBottomFieldSort
   issue: https://github.com/elastic/elasticsearch/issues/116249
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set}
   issue: https://github.com/elastic/elasticsearch/issues/116332
+- class: org.elasticsearch.xpack.shutdown.NodeShutdownIT
+  method: testAllocationPreventedForRemoval
+  issue: https://github.com/elastic/elasticsearch/issues/116363
+- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
+  method: test {categorize.Categorize ASYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/116373
+- class: org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsIntegTests
+  method: testCreateAndRestoreSearchableSnapshot
+  issue: https://github.com/elastic/elasticsearch/issues/116377
+- class: org.elasticsearch.threadpool.SimpleThreadPoolIT
+  method: testThreadPoolMetrics
+  issue: https://github.com/elastic/elasticsearch/issues/108320
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=ml/jobs_crud/Test put job deprecated bucket span}
+  issue: https://github.com/elastic/elasticsearch/issues/116419
 
 # Examples:
 #

rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_lifecycle.json

@@ -7,7 +7,8 @@
     "stability":"stable",
     "visibility":"public",
     "headers":{
-      "accept": [ "application/json"]
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
     },
     "url": {
       "paths": [

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -14,13 +14,15 @@
 import org.apache.logging.log4j.Level;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.fieldcaps.TransportFieldCapabilitiesAction;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Cancellable;
 import org.elasticsearch.client.Request;
@@ -45,6 +47,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.FieldPredicate;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -319,6 +322,63 @@ public void testWithIndexAlias() {
         assertEquals(response1, response2);
     }
 
+    public void testNoIndices() {
+        boolean ignoreUnavailable = false;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIndicesOptions(options).get();
+        assertIndices(response, "new_index");
+    }
+
+    public void testNoIndicesIgnoreUnavailable() {
+        boolean ignoreUnavailable = true;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIndicesOptions(options).get();
+        assertIndices(response, "new_index");
+    }
+
+    public void testOneClosedIndex() {
+        boolean ignoreUnavailable = false;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        IndexClosedException ex = expectThrows(
+            IndexClosedException.class,
+            client().prepareFieldCaps("old_index").setFields("*").setIndicesOptions(options)
+        );
+        assertEquals("closed", ex.getMessage());
+    }
+
+    public void testOneClosedIndexIgnoreUnavailable() {
+        boolean ignoreUnavailable = true;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index").setFields("*").setIndicesOptions(options).get();
+        assertIndices(response);
+    }
+
+    public void testTwoIndicesOneClosed() {
+        boolean ignoreUnavailable = false;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        IndexClosedException ex = expectThrows(
+            IndexClosedException.class,
+            client().prepareFieldCaps("old_index", "new_index").setFields("*").setIndicesOptions(options)
+        );
+        assertEquals("closed", ex.getMessage());
+    }
+
+    public void testTwoIndicesOneClosedIgnoreUnavailable() {
+        boolean ignoreUnavailable = true;
+        IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false);
+        client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index")
+            .setFields("*")
+            .setIndicesOptions(options)
+            .get();
+        assertIndices(response, "new_index");
+    }
+
     public void testWithIndexFilter() throws InterruptedException {
         assertAcked(prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"));
         assertAcked(prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long"));