[8.16] Expose cluster-state role mappings in APIs (#114951)

docs/changelog/114951.yaml

@@ -0,0 +1,5 @@
+pr: 114951
+summary: Expose cluster-state role mappings in APIs
+area: Authentication
+type: bug
+issues: []

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java

@@ -25,9 +25,12 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
@@ -106,6 +109,10 @@ public void testRoleMappingsAppliedOnUpgrade() throws IOException {
             );
             assertThat(roleMappings, is(not(nullValue())));
             assertThat(roleMappings.size(), equalTo(1));
+            assertThat(roleMappings, is(instanceOf(Map.class)));
+            @SuppressWarnings("unchecked")
+            Map<String, Object> roleMapping = (Map<String, Object>) roleMappings;
+            assertThat(roleMapping.keySet(), contains("everyone_kibana-read-only-operator-mapping"));
         }
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java

@@ -54,6 +54,18 @@
  */
 public class ExpressionRoleMapping implements ToXContentObject, Writeable {
 
+    /**
+     * Reserved suffix for read-only operator-defined role mappings.
+     * This suffix is added to the name of all cluster-state role mappings returned via
+     * the {@code TransportGetRoleMappingsAction} action.
+     */
+    public static final String READ_ONLY_ROLE_MAPPING_SUFFIX = "-read-only-operator-mapping";
+    /**
+     * Reserved metadata field to mark role mappings as read-only.
+     * This field is added to the metadata of all cluster-state role mappings returned via
+     * the {@code TransportGetRoleMappingsAction} action.
+     */
+    public static final String READ_ONLY_ROLE_MAPPING_METADATA_FLAG = "_read_only";
     private static final ObjectParser<Builder, String> PARSER = new ObjectParser<>("role-mapping", Builder::new);
 
     /**
@@ -136,6 +148,28 @@ public ExpressionRoleMapping(StreamInput in) throws IOException {
         this.metadata = in.readGenericMap();
     }
 
+    public static boolean hasReadOnlySuffix(String name) {
+        return name.endsWith(READ_ONLY_ROLE_MAPPING_SUFFIX);
+    }
+
+    public static void validateNoReadOnlySuffix(String name) {
+        if (hasReadOnlySuffix(name)) {
+            throw new IllegalArgumentException(
+                "Invalid mapping name [" + name + "]. [" + READ_ONLY_ROLE_MAPPING_SUFFIX + "] is not an allowed suffix"
+            );
+        }
+    }
+
+    public static String addReadOnlySuffix(String name) {
+        return name + READ_ONLY_ROLE_MAPPING_SUFFIX;
+    }
+
+    public static String removeReadOnlySuffixIfPresent(String name) {
+        return name.endsWith(READ_ONLY_ROLE_MAPPING_SUFFIX)
+            ? name.substring(0, name.length() - READ_ONLY_ROLE_MAPPING_SUFFIX.length())
+            : name;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java

@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.core.security.authz;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
@@ -26,8 +28,10 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -36,7 +40,11 @@
 
 public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
 
+    private static final Logger logger = LogManager.getLogger(RoleMappingMetadata.class);
+
     public static final String TYPE = "role_mappings";
+    public static final String METADATA_NAME_FIELD = "_es_reserved_role_mapping_name";
+    public static final String FALLBACK_NAME = "name_not_available_after_deserialization";
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<RoleMappingMetadata, Void> PARSER = new ConstructingObjectParser<>(
@@ -46,12 +54,7 @@ public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Cu
     );
 
     static {
-        PARSER.declareObjectArray(
-            constructorArg(),
-            // role mapping names are lost when the role mapping metadata is serialized
-            (p, c) -> ExpressionRoleMapping.parse("name_not_available_after_deserialization", p),
-            new ParseField(TYPE)
-        );
+        PARSER.declareObjectArray(constructorArg(), (p, c) -> parseWithNameFromMetadata(p), new ParseField(TYPE));
     }
 
     private static final RoleMappingMetadata EMPTY = new RoleMappingMetadata(Set.of());
@@ -153,4 +156,64 @@ public EnumSet<Metadata.XContentContext> context() {
         // are not persisted.
         return ALL_CONTEXTS;
     }
+
+    /**
+     * Ensures role mapping names are preserved when stored on disk using XContent format,
+     * which omits names. This method copies the role mapping's name into a reserved metadata field
+     * during serialization, allowing recovery during deserialization (e.g., after a master-node restart).
+     * {@link #parseWithNameFromMetadata(XContentParser)} restores the name during parsing.
+     */
+    public static ExpressionRoleMapping copyWithNameInMetadata(ExpressionRoleMapping roleMapping) {
+        Map<String, Object> metadata = new HashMap<>(roleMapping.getMetadata());
+        // note: can't use Maps.copyWith... since these create maps that don't support `null` values in map entries
+        if (metadata.put(METADATA_NAME_FIELD, roleMapping.getName()) != null) {
+            logger.error(
+                "Metadata field [{}] is reserved and will be overwritten with an internal system value. "
+                    + "Rename this field in your role mapping configuration.",
+                METADATA_NAME_FIELD
+            );
+        }
+        return new ExpressionRoleMapping(
+            roleMapping.getName(),
+            roleMapping.getExpression(),
+            roleMapping.getRoles(),
+            roleMapping.getRoleTemplates(),
+            metadata,
+            roleMapping.isEnabled()
+        );
+    }
+
+    /**
+     * If a role mapping does not yet have a name persisted in metadata, it will use a constant fallback name. This method checks if a
+     * role mapping has the fallback name.
+     */
+    public static boolean hasFallbackName(ExpressionRoleMapping expressionRoleMapping) {
+        return expressionRoleMapping.getName().equals(FALLBACK_NAME);
+    }
+
+    /**
+     * Parse a role mapping from XContent, restoring the name from a reserved metadata field.
+     * Used to parse a role mapping annotated with its name in metadata via @see {@link #copyWithNameInMetadata(ExpressionRoleMapping)}.
+     */
+    public static ExpressionRoleMapping parseWithNameFromMetadata(XContentParser parser) throws IOException {
+        ExpressionRoleMapping roleMapping = ExpressionRoleMapping.parse(FALLBACK_NAME, parser);
+        return new ExpressionRoleMapping(
+            getNameFromMetadata(roleMapping),
+            roleMapping.getExpression(),
+            roleMapping.getRoles(),
+            roleMapping.getRoleTemplates(),
+            roleMapping.getMetadata(),
+            roleMapping.isEnabled()
+        );
+    }
+
+    private static String getNameFromMetadata(ExpressionRoleMapping roleMapping) {
+        Map<String, Object> metadata = roleMapping.getMetadata();
+        if (metadata.containsKey(METADATA_NAME_FIELD) && metadata.get(METADATA_NAME_FIELD) instanceof String name) {
+            return name;
+        } else {
+            // This is valid the first time we recover from cluster-state: the old format metadata won't have a name stored in metadata yet
+            return FALLBACK_NAME;
+        }
+    }
 }

x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.TestSecurityClient;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -41,9 +42,7 @@
 public abstract class SecurityOnTrialLicenseRestTestCase extends ESRestTestCase {
     private TestSecurityClient securityClient;
 
-    @ClassRule
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .nodes(2)
+    public static LocalClusterConfigProvider commonTrialSecurityClusterConfig = cluster -> cluster.nodes(2)
         .distribution(DistributionType.DEFAULT)
         .setting("xpack.ml.enabled", "false")
         .setting("xpack.license.self_generated.type", "trial")
@@ -62,8 +61,10 @@ public abstract class SecurityOnTrialLicenseRestTestCase extends ESRestTestCase
         .user("admin_user", "admin-password", ROOT_USER_ROLE, true)
         .user("security_test_user", "security-test-password", "security_test_role", false)
         .user("x_pack_rest_user", "x-pack-test-password", ROOT_USER_ROLE, true)
-        .user("cat_test_user", "cat-test-password", "cat_test_role", false)
-        .build();
+        .user("cat_test_user", "cat-test-password", "cat_test_role", false);
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().apply(commonTrialSecurityClusterConfig).build();
 
     @Override
     protected String getTestRestCluster() {