[Backport 7.11] Add a smoke test for security realms

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -16,6 +16,7 @@
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -68,10 +69,12 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.GeneralSecurityException;
 import java.security.KeyManagementException;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
 import java.util.ArrayList;
@@ -81,6 +84,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
@@ -106,7 +110,13 @@
 public abstract class ESRestTestCase extends ESTestCase {
     public static final String TRUSTSTORE_PATH = "truststore.path";
     public static final String TRUSTSTORE_PASSWORD = "truststore.password";
+
     public static final String CERTIFICATE_AUTHORITIES = "certificate_authorities";
+
+    public static final String CLIENT_CERT_PATH = "client.cert.path";
+    public static final String CLIENT_KEY_PATH = "client.key.path";
+    public static final String CLIENT_KEY_PASSWORD = "client.key.password";
+
     public static final String CLIENT_SOCKET_TIMEOUT = "client.socket.timeout";
     public static final String CLIENT_PATH_PREFIX = "client.path.prefix";
 
@@ -1013,29 +1023,30 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE
     }
 
     protected static void configureClient(RestClientBuilder builder, Settings settings) throws IOException {
+        String truststorePath = settings.get(TRUSTSTORE_PATH);
         String certificateAuthorities = settings.get(CERTIFICATE_AUTHORITIES);
-        String keystorePath = settings.get(TRUSTSTORE_PATH);
+        String clientCertificatePath = settings.get(CLIENT_CERT_PATH);
 
-        if (certificateAuthorities != null && keystorePath != null) {
+        if (certificateAuthorities != null && truststorePath != null) {
             throw new IllegalStateException("Cannot set both " + CERTIFICATE_AUTHORITIES + " and " + TRUSTSTORE_PATH
                 + ". Please configure one of these.");
 
         }
-        if (keystorePath != null) {
+        if (truststorePath != null) {
             if (inFipsJvm()) {
-                throw new IllegalStateException("Keystore " + keystorePath + "cannot be used in FIPS 140 mode. Please configure "
+                throw new IllegalStateException("Keystore " + truststorePath + "cannot be used in FIPS 140 mode. Please configure "
                     + CERTIFICATE_AUTHORITIES + " with a PEM encoded trusted CA/certificate instead");
             }
             final String keystorePass = settings.get(TRUSTSTORE_PASSWORD);
             if (keystorePass == null) {
                 throw new IllegalStateException(TRUSTSTORE_PATH + " is provided but not " + TRUSTSTORE_PASSWORD);
             }
-            Path path = PathUtils.get(keystorePath);
-            if (!Files.exists(path)) {
+            Path path = PathUtils.get(truststorePath);
+            if (Files.exists(path) == false) {
                 throw new IllegalStateException(TRUSTSTORE_PATH + " is set but points to a non-existing file");
             }
             try {
-                final String keyStoreType = keystorePath.endsWith(".p12") ? "PKCS12" : "jks";
+                final String keyStoreType = truststorePath.endsWith(".p12") ? "PKCS12" : "jks";
                 KeyStore keyStore = KeyStore.getInstance(keyStoreType);
                 try (InputStream is = Files.newInputStream(path)) {
                     keyStore.load(is, keystorePass.toCharArray());
@@ -1048,21 +1059,35 @@ protected static void configureClient(RestClientBuilder builder, Settings settin
             }
         }
         if (certificateAuthorities != null) {
-            Path path = PathUtils.get(certificateAuthorities);
-            if (!Files.exists(path)) {
+            Path caPath = PathUtils.get(certificateAuthorities);
+            if (Files.exists(caPath) == false) {
                 throw new IllegalStateException(CERTIFICATE_AUTHORITIES + " is set but points to a non-existing file");
             }
             try {
                 KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
                 keyStore.load(null, null);
-                Certificate cert = PemUtils.readCertificates(Collections.singletonList(path)).get(0);
-                keyStore.setCertificateEntry(cert.toString(), cert);
-                SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(keyStore, null).build();
+                Certificate caCert = PemUtils.readCertificates(Collections.singletonList(caPath)).get(0);
+                keyStore.setCertificateEntry(caCert.toString(), caCert);
+                final SSLContextBuilder sslContextBuilder = SSLContexts.custom();
+                if (clientCertificatePath != null) {
+                    final Path certPath = PathUtils.get(clientCertificatePath);
+                    final Path keyPath = PathUtils.get(Objects.requireNonNull(settings.get(CLIENT_KEY_PATH), "No key provided"));
+                    final String password = settings.get(CLIENT_KEY_PASSWORD);
+                    final char[] passwordChars = password == null ? null : password.toCharArray();
+                    final PrivateKey key = PemUtils.readPrivateKey(keyPath, () -> passwordChars);
+                    final Certificate[] clientCertChain
+                        = PemUtils.readCertificates(Collections.singletonList(certPath)).toArray(new Certificate[1]);
+                    keyStore.setKeyEntry("client", key, passwordChars, clientCertChain);
+                    sslContextBuilder.loadKeyMaterial(keyStore, passwordChars);
+                }
+                SSLContext sslcontext = sslContextBuilder.loadTrustMaterial(keyStore, null).build();
                 SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslcontext);
                 builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy));
-            } catch (KeyStoreException | NoSuchAlgorithmException | KeyManagementException | CertificateException e) {
+            } catch (GeneralSecurityException e) {
                 throw new RuntimeException("Error setting up ssl", e);
             }
+        } else if (clientCertificatePath != null) {
+            throw new IllegalStateException("Client certificates are currently only supported when using a custom CA");
         }
         Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings);
         Header[] defaultHeaders = new Header[headers.size()];

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java

@@ -258,7 +258,7 @@ public static X509ExtendedTrustManager trustManager(Certificate[] certificates)
         return trustManager(store, TrustManagerFactory.getDefaultAlgorithm());
     }
 
-    static KeyStore trustStore(Certificate[] certificates)
+    public static KeyStore trustStore(Certificate[] certificates)
         throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
         assert certificates != null : "Cannot create trust store with null certificates";
         KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());

x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle

@@ -0,0 +1,85 @@
+/*
+ * This QA test is intended to smoke test all security realms with minimal dependencies.
+ * That is, it makes sure a node that has every realm configured can start, and tests those realms that can be tested without needing external services.
+ * This tradeoff is intentional because we want this set of tests to be very stable - failures in this QA suite should be an indicator that
+ *    something is broken in Elasticsearch (and not that an external docker fixture broke)
+ * This test is also intended to work correctly on FIPS mode because we also want to know if a realm breaks on FIPS.
+ */
+
+apply plugin: 'elasticsearch.java-rest-test'
+
+dependencies {
+  javaRestTestImplementation project(path: xpackModule('core'))
+  javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+testClusters.javaRestTest {
+  testDistribution = 'DEFAULT'
+  numberOfNodes = 2
+
+  extraConfigFile 'http-server.key', file('src/javaRestTest/resources/ssl/http-server.key')
+  extraConfigFile 'http-server.crt', file('src/javaRestTest/resources/ssl/http-server.crt')
+  extraConfigFile 'http-client-ca.crt', file('src/javaRestTest/resources/ssl/http-client-ca.crt')
+  extraConfigFile 'saml-metadata.xml', file('src/javaRestTest/resources/saml-metadata.xml')
+  extraConfigFile 'kerberos.keytab', file('src/javaRestTest/resources/kerberos.keytab')
+  extraConfigFile 'oidc-jwkset.json', file('src/javaRestTest/resources/oidc-jwkset.json')
+
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.authc.token.enabled', 'true'
+  setting 'xpack.security.authc.api_key.enabled', 'true'
+
+  // Need a trial license (not basic) to enable all realms
+  setting 'xpack.license.self_generated.type', 'trial'
+  // Need SSL to enable PKI realms
+  setting 'xpack.security.http.ssl.enabled', 'true'
+  setting 'xpack.security.http.ssl.certificate', 'http-server.crt'
+  setting 'xpack.security.http.ssl.key', 'http-server.key'
+  setting 'xpack.security.http.ssl.key_passphrase', 'http-password'
+  setting 'xpack.security.http.ssl.client_authentication', 'optional'
+  setting 'xpack.security.http.ssl.certificate_authorities', 'http-client-ca.crt'
+
+  // Don't need transport SSL, so leave it out
+  setting 'xpack.security.transport.ssl.enabled', 'false'
+
+  // Configure every realm type
+  //  - File
+  setting 'xpack.security.authc.realms.file.file0.order', '0'
+  //  - Native
+  setting 'xpack.security.authc.realms.native.native1.order', '1'
+  //  - LDAP (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.ldap.ldap2.order', '2'
+  setting 'xpack.security.authc.realms.ldap.ldap2.url', 'ldap://localhost:7777'
+  setting 'xpack.security.authc.realms.ldap.ldap2.user_search.base_dn', 'OU=users,DC=example,DC=com'
+  //  - AD (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.active_directory.ad3.order', '3'
+  setting 'xpack.security.authc.realms.active_directory.ad3.domain_name', 'localhost'
+  //  - PKI (works)
+  setting 'xpack.security.authc.realms.pki.pki4.order', '4'
+  //  - SAML (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.saml.saml5.order', '5'
+  setting 'xpack.security.authc.realms.saml.saml5.idp.metadata.path', 'saml-metadata.xml'
+  setting 'xpack.security.authc.realms.saml.saml5.idp.entity_id', 'http://idp.example.com/'
+  setting 'xpack.security.authc.realms.saml.saml5.sp.entity_id', 'http://kibana.example.net/'
+  setting 'xpack.security.authc.realms.saml.saml5.sp.acs', 'http://kibana.example.net/api/security/v1/saml'
+  setting 'xpack.security.authc.realms.saml.saml5.attributes.principal', 'uid'
+  //  - Kerberos (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.kerberos.kerb6.order', '6'
+  setting 'xpack.security.authc.realms.kerberos.kerb6.keytab.path', 'kerberos.keytab'
+  //  - OIDC (configured but won't work because we don't want external fixtures in this test suite)
+  setting 'xpack.security.authc.realms.oidc.openid7.order', '7'
+  setting 'xpack.security.authc.realms.oidc.openid7.rp.client_id', 'http://rp.example.net'
+  setting 'xpack.security.authc.realms.oidc.openid7.rp.response_type', 'id_token'
+  setting 'xpack.security.authc.realms.oidc.openid7.rp.redirect_uri', 'https://kibana.example.net/api/security/v1/oidc'
+  setting 'xpack.security.authc.realms.oidc.openid7.op.issuer', 'https://op.example.com/'
+  setting 'xpack.security.authc.realms.oidc.openid7.op.authorization_endpoint', 'https://op.example.com/auth'
+  setting 'xpack.security.authc.realms.oidc.openid7.op.jwkset_path', 'oidc-jwkset.json'
+  setting 'xpack.security.authc.realms.oidc.openid7.claims.principal', 'sub'
+  keystore 'xpack.security.authc.realms.oidc.openid7.rp.client_secret', 'this-is-my-secret'
+
+  extraConfigFile 'roles.yml', file('src/javaRestTest/resources/roles.yml')
+  user username: "admin_user", password: "admin-password"
+  user username: "security_test_user", password: "security-test-password", role: "security_test_role"
+}
+

x-pack/plugin/security/qa/smoke-test-all-realms/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/FileRealmAuthIT.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Integration Rest Test for testing authentication when all possible realms are configured
+ */
+public class FileRealmAuthIT extends SecurityRealmSmokeTestCase {
+
+    // Declared in build.gradle
+    private static final String USERNAME = "security_test_user";
+    private static final SecureString PASSWORD = new SecureString("security-test-password".toCharArray());
+    private static final String ROLE_NAME = "security_test_role";
+
+    public void testAuthenticationUsingFileRealm() throws IOException {
+        Map<String, Object> authenticate = super.authenticate(
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+                UsernamePasswordToken.basicAuthHeaderValue(USERNAME, PASSWORD))
+        );
+
+        assertUsername(authenticate, USERNAME);
+        assertRealm(authenticate, "file", "file0");
+        assertRoles(authenticate, ROLE_NAME);
+    }
+
+}

x-pack/plugin/security/qa/smoke-test-all-realms/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/NativeRealmAuthIT.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Integration Rest Test for testing authentication when all possible realms are configured
+ */
+public class NativeRealmAuthIT extends SecurityRealmSmokeTestCase {
+
+    private static final String USERNAME = "test_native_user";
+    private static final SecureString PASSWORD = new SecureString("native-user-password".toCharArray());
+    private static final String ROLE_NAME = "native_role";
+
+    @Before
+    public void createUsersAndRoles() throws IOException {
+        createUser(USERNAME, PASSWORD, Collections.singletonList(ROLE_NAME));
+        createRole("native_role", Collections.singleton("monitor"));
+    }
+
+    @After
+    public void cleanUp() throws IOException {
+        deleteUser(USERNAME);
+        deleteRole(ROLE_NAME);
+    }
+
+    public void testAuthenticationUsingNativeRealm() throws IOException {
+        Map<String, Object> authenticate = super.authenticate(
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+                UsernamePasswordToken.basicAuthHeaderValue(USERNAME, PASSWORD))
+        );
+
+        assertUsername(authenticate, USERNAME);
+        assertRealm(authenticate, "native", "native1");
+        assertRoles(authenticate, ROLE_NAME);
+    }
+
+}

x-pack/plugin/security/qa/smoke-test-all-realms/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/PkiRealmAuthIT.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Integration Rest Test for testing authentication when all possible realms are configured
+ */
+public class PkiRealmAuthIT extends SecurityRealmSmokeTestCase {
+
+    // Derived from certificate attributes (pki-auth.crt)
+    private static final String USERNAME = "pki-auth";
+
+    @Override
+    protected Settings restClientSettings() {
+        Settings.Builder builder = Settings.builder()
+            .put(super.restClientSettings())
+            .put(CLIENT_CERT_PATH, getDataPath("/ssl/pki-auth.crt"))
+            .put(CLIENT_KEY_PATH, getDataPath("/ssl/pki-auth.key"))
+            .put(CLIENT_KEY_PASSWORD, "http-password");
+        builder.remove(ThreadContext.PREFIX + ".Authorization");
+        return builder.build();
+    }
+
+    public void testAuthenticationUsingFileRealm() throws IOException {
+        Map<String, Object> authenticate = super.authenticate(RequestOptions.DEFAULT.toBuilder());
+
+        assertUsername(authenticate, USERNAME);
+        assertRealm(authenticate, "pki", "pki4");
+        assertRoles(authenticate, new String[0]);
+    }
+
+}

x-pack/plugin/security/qa/smoke-test-all-realms/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/RealmInfoIT.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Integration Rest Test for testing authentication when all possible realms are configured
+ */
+public class RealmInfoIT extends SecurityRealmSmokeTestCase {
+
+    public void testThatAllRealmTypesAreEnabled() throws IOException {
+        final Request request = new Request("GET", "_xpack/usage");
+        final Response response = client().performRequest(request);
+        Map<String, Object> usage = entityAsMap(response);
+
+        Map<String, Object> realms = ObjectPath.evaluate(usage, "security.realms");
+        realms.forEach((type, config) -> {
+            assertThat(config, Matchers.instanceOf(Map.class));
+            assertThat("Realm type [" + type + "] is not enabled",
+                ((Map<?, ?>) config).get("enabled"), Matchers.equalTo(true));
+        });
+    }
+
+}