Merge branch '8.16' into backport/8.16/commit-72aa17a0

docs/reference/connector/docs/_connectors-create-native.asciidoc

@@ -22,7 +22,7 @@ PUT _connector/my-{service-name-stub}-connector
   "index_name": "my-elasticsearch-index",
   "name": "Content synced from {service-name}",
   "service_type": "{service-name-stub}",
-  "is_native": "true"
+  "is_native": true
 }
 ----
 // TEST[skip:can't test in isolation]

docs/reference/modules/cluster/remote-clusters-migration.asciidoc

@@ -66,10 +66,11 @@ indices that were created from the auto-follow pattern.
 On the local cluster:
 
 . Enhance any roles used by local cluster users with the required
-<<roles-remote-indices-priv,remote indices privileges>> for {ccr} and {ccs}.
+<<roles-remote-indices-priv,remote indices privileges>> or 
+<<roles-remote-cluster-priv, remote cluster privileges>> for {ccr} and {ccs}.
 Refer to <<remote-clusters-privileges-api-key>>. Note:
 
-** You only need to assign additional `remote_indices` privileges to existing
+** You only need to assign additional `remote_indices` or `remote_cluster` privileges to existing
 roles used for cross-cluster operations. You should be able to copy these
 privileges from the original roles on the remote cluster, where they are defined
 under the certification based security model.
@@ -197,7 +198,7 @@ authentication.
 Resume any persistent tasks that you stopped earlier. Tasks should be restarted
 by the same user or API key that created the task before the migration. Ensure
 the roles of this user or API key have been updated with the required
-`remote_indices` privileges. For users, tasks capture the caller's credentials
+`remote_indices` or `remote_cluster` privileges. For users, tasks capture the caller's credentials
 when started and run in that user's security context. For API keys, restarting a
 task will update the task with the updated API key.
 
@@ -246,7 +247,7 @@ If you need to roll back, follow these steps on the local cluster:
 . Remove the remote cluster definition by setting the remote cluster settings to
 `null`.
 
-. Remove the `remote_indices` privileges from any roles that were updated during
+. Remove the `remote_indices` or 'remote_cluster' privileges from any roles that were updated during
 the migration.
 
 . On each node, remove the `remote_cluster_client.ssl.*` settings from

docs/reference/modules/cluster/remote-clusters-troubleshooting.asciidoc

@@ -399,7 +399,7 @@ This does not show up in any logs.
 
 ====== Resolution
 
-. Check that the local user has the necessary `remote_indices` privileges. Grant sufficient `remote_indices` privileges if necessary.
+. Check that the local user has the necessary `remote_indices` or `remote_cluster` privileges. Grant sufficient `remote_indices` or `remote_cluster` privileges if necessary.
 . If permission is not an issue locally, ask the remote cluster administrator to
 create and distribute a
 <<security-api-create-cross-cluster-api-key,cross-cluster API key>>. Replace the

docs/reference/rest-api/security/bulk-create-roles.asciidoc

@@ -75,7 +75,7 @@ that begin with `_` are reserved for system usage.
 For more information, see
 <<run-as-privilege>>.
 
-`remote_indices`:: beta:[] (list) A list of remote indices permissions entries.
+`remote_indices`:: (list) A list of remote indices permissions entries.
 +
 --
 NOTE: Remote indices are effective for <<remote-clusters-api-key,remote clusters configured with the API key based model>>.
@@ -94,6 +94,18 @@ have on the specified indices.
 read access to. A document within the specified indices must match this query in
 order for it to be accessible by the owners of the role.
 
+`remote_cluster`:: (list) A list of remote cluster permissions entries.
++
+--
+NOTE: Remote cluster permissions are effective for <<remote-clusters-api-key,remote clusters configured with the API key based model>>.
+They have no effect for remote clusters configured with the <<remote-clusters-cert,certificate based model>>.
+--
+`clusters` (required)::: (list) A list of cluster aliases to which the permissions
+in this entry apply.
+`privileges`(required)::: (list) The cluster level privileges that the owners of the role
+have in the specified clusters.
+
+
 For more information, see <<defining-roles>>.
 ====
 

docs/reference/rest-api/security/create-roles.asciidoc

@@ -96,6 +96,17 @@ have on the specified indices.
 read access to. A document within the specified indices must match this query in
 order for it to be accessible by the owners of the role.
 
+`remote_cluster`:: (list) A list of remote cluster permissions entries.
++
+--
+NOTE: Remote cluster permissions are effective for <<remote-clusters-api-key,remote clusters configured with the API key based model>>.
+They have no effect for remote clusters configured with the <<remote-clusters-cert,certificate based model>>.
+--
+`clusters` (required)::: (list) A list of cluster aliases to which the permissions
+in this entry apply.
+`privileges`(required)::: (list) The cluster level privileges that the owners of the role
+have in the specified clusters.
+
 For more information, see <<defining-roles>>.
 
 [[security-api-put-role-example]]

docs/reference/security/authentication/remote-clusters-privileges-api-key.asciidoc

@@ -2,7 +2,8 @@
 === Configure roles and users
 
 To use a remote cluster for {ccr} or {ccs}, you need to create user roles with
-<<roles-remote-indices-priv,remote indices privileges>> on the local cluster.
+<<roles-remote-indices-priv,remote indices privileges>> or
+<<roles-remote-cluster-priv, remote cluster privileges>> on the local cluster.
 
 You can manage users and roles from Stack Management in {kib} by selecting
 *Security > Roles* from the side navigation. You can also use the
@@ -80,7 +81,7 @@ POST /_security/role/remote-search
       "privileges": [
         "read",
         "read_cross_cluster",
-        "view_index_metadata" 
+        "view_index_metadata"
       ]
     }
   ]

muted-tests.yml

@@ -263,9 +263,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.rest.ServerSentEventsRestActionListenerTests
   method: testNoStream
   issue: https://github.com/elastic/elasticsearch/issues/114788
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
-  method: testProcessFileChanges
-  issue: https://github.com/elastic/elasticsearch/issues/115280
 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
   issue: https://github.com/elastic/elasticsearch/issues/111319
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
@@ -286,6 +283,9 @@ tests:
 - class: org.elasticsearch.xpack.deprecation.DeprecationHttpIT
   method: testDeprecatedSettingsReturnWarnings
   issue: https://github.com/elastic/elasticsearch/issues/108628
+- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
+  method: test {categorize.Categorize SYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/113054
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.action.support.ChannelActionListener;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.RefCountingRunnable;
+import org.elasticsearch.client.internal.RemoteClusterClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -113,23 +114,28 @@ public TransportFieldCapabilitiesAction(
 
     @Override
     protected void doExecute(Task task, FieldCapabilitiesRequest request, final ActionListener<FieldCapabilitiesResponse> listener) {
-        executeRequest(task, request, REMOTE_TYPE, listener);
+        executeRequest(
+            task,
+            request,
+            (remoteClient, remoteRequest, remoteListener) -> remoteClient.execute(REMOTE_TYPE, remoteRequest, remoteListener),
+            listener
+        );
     }
 
     public void executeRequest(
         Task task,
         FieldCapabilitiesRequest request,
-        RemoteClusterActionType<FieldCapabilitiesResponse> remoteAction,
+        RemoteRequestExecutor remoteRequestExecutor,
         ActionListener<FieldCapabilitiesResponse> listener
     ) {
         // workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can
-        searchCoordinationExecutor.execute(ActionRunnable.wrap(listener, l -> doExecuteForked(task, request, remoteAction, l)));
+        searchCoordinationExecutor.execute(ActionRunnable.wrap(listener, l -> doExecuteForked(task, request, remoteRequestExecutor, l)));
     }
 
     private void doExecuteForked(
         Task task,
         FieldCapabilitiesRequest request,
-        RemoteClusterActionType<FieldCapabilitiesResponse> remoteAction,
+        RemoteRequestExecutor remoteRequestExecutor,
         ActionListener<FieldCapabilitiesResponse> listener
     ) {
         if (ccsCheckCompatibility) {
@@ -282,8 +288,8 @@ private void doExecuteForked(
                         handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex);
                     }
                 });
-                remoteClusterClient.execute(
-                    remoteAction,
+                remoteRequestExecutor.executeRemoteRequest(
+                    remoteClusterClient,
                     remoteRequest,
                     // The underlying transport service may call onFailure with a thread pool other than search_coordinator.
                     // This fork is a workaround to ensure that the merging of field-caps always occurs on the search_coordinator.
@@ -298,6 +304,14 @@ private void doExecuteForked(
         }
     }
 
+    public interface RemoteRequestExecutor {
+        void executeRemoteRequest(
+            RemoteClusterClient remoteClient,
+            FieldCapabilitiesRequest remoteRequest,
+            ActionListener<FieldCapabilitiesResponse> remoteListener
+        );
+    }
+
     private static void checkIndexBlocks(ClusterState clusterState, String[] concreteIndices) {
         var blocks = clusterState.blocks();
         if (blocks.global().isEmpty() && blocks.indices().isEmpty()) {

server/src/test/java/org/elasticsearch/monitor/jvm/JvmStatsTests.java

@@ -53,7 +53,7 @@ public void testJvmStats() {
         assertThat(memoryPools, hasKey("Metaspace"));
         assertThat(memoryPools.keySet(), hasSize(greaterThan(3)));
         for (JvmStats.MemoryPool memoryPool : memoryPools.values()) {
-            assertThat(memoryPool.getUsed().getBytes(), greaterThan(0L));
+            assertThat("Memory pool: " + memoryPool.getName(), memoryPool.getUsed().getBytes(), greaterThanOrEqualTo(0L));
         }
 
         // Threads