Provide service accounts tokens to extensions

CHANGELOG.md

@@ -11,6 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
 - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
 - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
+- Added encryption-sdk lib to provide encryption and decryption capabilities ([#8466](https://github.com/opensearch-project/OpenSearch/pull/8466))
+- Added crypto-kms plugin to provide AWS KMS based key providers for encryption/decryption. ([#8465](https://github.com/opensearch-project/OpenSearch/pull/8465))
+- Implement service account issuance and fetching for extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0

plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java

@@ -64,7 +64,7 @@ public Optional<AuthenticationToken> translateAuthToken(org.opensearch.identity.
     public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) {
 
         String password = generatePassword();
-        final byte[] rawEncoded = Base64.getEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); // Make a new
+        final byte[] rawEncoded = Base64.getUrlEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); // Make a new
                                                                                                                        // ShiroSubject w/
                                                                                                                        // audience as name
         final String usernamePassword = new String(rawEncoded, UTF_8);
@@ -75,6 +75,19 @@ public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims)
         return token;
     }
 
+    @Override
+    public AuthToken issueServiceAccountToken(String audience) {
+
+        String password = generatePassword();
+        final byte[] rawEncoded = Base64.getEncoder().withoutPadding().encode((audience + ":" + password).getBytes(UTF_8)); // Make a new
+        final String usernamePassword = new String(rawEncoded, UTF_8);
+        final String header = "Basic " + usernamePassword;
+
+        BasicAuthToken token = new BasicAuthToken(header);
+        shiroTokenPasswordMap.put(token, password);
+        return token;
+    }
+
     @Override
     public Subject authenticateToken(AuthToken authToken) {
         return new NoopSubject();

plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java

@@ -167,6 +167,18 @@ public void testTokenNoopIssuance() {
         Subject subject = new NoopSubject();
         AuthToken token = tokenManager.issueOnBehalfOfToken(subject, claims);
         assertTrue(token instanceof AuthToken);
+        AuthToken serviceAccountToken = tokenManager.issueServiceAccountToken("test");
+        assertTrue(serviceAccountToken instanceof AuthToken);
+        assertEquals(serviceAccountToken.asAuthHeaderValue(), "noopToken");
     }
 
+    public void testShouldSucceedIssueServiceAccountToken() {
+        String audience = "testExtensionName";
+        BasicAuthToken authToken = (BasicAuthToken) shiroAuthTokenHandler.issueServiceAccountToken(audience);
+        assertTrue(authToken instanceof BasicAuthToken);
+        UsernamePasswordToken translatedToken = (UsernamePasswordToken) shiroAuthTokenHandler.translateAuthToken(authToken).get();
+        assertEquals(authToken.getPassword(), new String(translatedToken.getPassword()));
+        assertTrue(shiroAuthTokenHandler.getShiroTokenPasswordMap().containsKey(authToken));
+        assertEquals(shiroAuthTokenHandler.getShiroTokenPasswordMap().get(authToken), new String(translatedToken.getPassword()));
+    }
 }

server/src/main/java/org/opensearch/extensions/ExtensionsManager.java

@@ -33,6 +33,8 @@
 import org.opensearch.extensions.action.ExtensionActionRequest;
 import org.opensearch.extensions.action.ExtensionActionResponse;
 import org.opensearch.extensions.action.ExtensionTransportActionsHandler;
+import org.opensearch.extensions.action.IssueServiceAccountRequest;
+import org.opensearch.extensions.action.IssueServiceAccountResponse;
 import org.opensearch.extensions.action.RegisterTransportActionsRequest;
 import org.opensearch.extensions.action.RemoteExtensionActionResponse;
 import org.opensearch.extensions.action.TransportActionRequestFromExtension;
@@ -41,6 +43,7 @@
 import org.opensearch.extensions.settings.CustomSettingsRequestHandler;
 import org.opensearch.extensions.settings.RegisterCustomSettingsRequest;
 import org.opensearch.identity.IdentityService;
+import org.opensearch.identity.tokens.AuthToken;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.ConnectTransportException;
 import org.opensearch.transport.TransportException;
@@ -76,6 +79,7 @@ public class ExtensionsManager {
     public static final String REQUEST_REST_EXECUTE_ON_EXTENSION_ACTION = "internal:extensions/restexecuteonextensiontaction";
     public static final String REQUEST_EXTENSION_HANDLE_TRANSPORT_ACTION = "internal:extensions/handle-transportaction";
     public static final String REQUEST_EXTENSION_HANDLE_REMOTE_TRANSPORT_ACTION = "internal:extensions/handle-remote-transportaction";
+    public static final String REQUEST_EXTENSION_ISSUE_SERVICE_ACCOUNT = "internal:extensions/issue-service-account";
     public static final String TRANSPORT_ACTION_REQUEST_FROM_EXTENSION = "internal:extensions/request-transportaction-from-extension";
     public static final int EXTENSION_REQUEST_WAIT_TIMEOUT = 10;
     private static final Logger logger = LogManager.getLogger(ExtensionsManager.class);
@@ -101,14 +105,15 @@ public static enum OpenSearchRequestType {
     private Settings environmentSettings;
     private AddSettingsUpdateConsumerRequestHandler addSettingsUpdateConsumerRequestHandler;
     private NodeClient client;
+    private IdentityService identityService;
 
     /**
      * Instantiate a new ExtensionsManager object to handle requests and responses from extensions. This is called during Node bootstrap.
      *
      * @param additionalSettings  Additional settings to read in from extension initialization request
      * @throws IOException  If the extensions discovery file is not properly retrieved.
      */
-    public ExtensionsManager(Set<Setting<?>> additionalSettings) throws IOException {
+    public ExtensionsManager(Set<Setting<?>> additionalSettings, IdentityService identityService) throws IOException {
         logger.info("ExtensionsManager initialized");
         this.initializedExtensions = new HashMap<String, DiscoveryExtensionNode>();
         this.extensionIdMap = new HashMap<String, DiscoveryExtensionNode>();
@@ -123,6 +128,7 @@ public ExtensionsManager(Set<Setting<?>> additionalSettings) throws IOException
         }
         this.client = null;
         this.extensionTransportActionsHandler = null;
+        this.identityService = identityService;
     }
 
     /**
@@ -443,6 +449,55 @@ TransportResponse handleExtensionRequest(ExtensionRequest extensionRequest) thro
         }
     }
 
+    /**
+     * A separate transport action handler used to issue service accounts to extensions during initialization
+     * @param extension The extension to be issued a service account
+     */
+    public void issueServiceAccount(Extension extension) {
+        DiscoveryExtensionNode discoveryExtensionNode = extensionIdMap.get(extension.getUniqueId());
+        AuthToken serviceAccountToken = identityService.getTokenManager().issueServiceAccountToken(extension.getUniqueId());
+        String authTokenAsString = serviceAccountToken.asAuthHeaderValue();
+        final CompletableFuture<IssueServiceAccountResponse> inProgressFuture = new CompletableFuture<>();
+        final TransportResponseHandler<IssueServiceAccountResponse> issueServiceAccountResponseHandler = new TransportResponseHandler<
+            IssueServiceAccountResponse>() {
+
+            @Override
+            public IssueServiceAccountResponse read(StreamInput in) throws IOException {
+                return new IssueServiceAccountResponse(in);
+            }
+
+            @Override
+            public void handleResponse(IssueServiceAccountResponse response) {
+                for (DiscoveryExtensionNode extension : extensionIdMap.values()) {
+                    if (extension.getName().equals(response.getName())
+                        && (serviceAccountToken.equals(response.getServiceAccountString()))) {
+                        logger.info("Successfully issued service account token to extension: " + extension.getName());
+                        break;
+                    }
+                }
+                inProgressFuture.complete(response);
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                logger.error(new ParameterizedMessage("Issuance of service account token failed"), exp);
+                inProgressFuture.completeExceptionally(exp);
+            }
+
+            @Override
+            public String executor() {
+                return ThreadPool.Names.GENERIC;
+            }
+        };
+
+        transportService.sendRequest(
+            discoveryExtensionNode,
+            REQUEST_EXTENSION_ISSUE_SERVICE_ACCOUNT,
+            new IssueServiceAccountRequest(authTokenAsString),
+            issueServiceAccountResponseHandler
+        );
+    }
+
     static String getRequestExtensionActionName() {
         return REQUEST_EXTENSION_ACTION_NAME;
     }

server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java

@@ -20,6 +20,7 @@
 import org.opensearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 
@@ -31,7 +32,7 @@
 public class NoopExtensionsManager extends ExtensionsManager {
 
     public NoopExtensionsManager() throws IOException {
-        super(Set.of());
+        super(Set.of(), new IdentityService(Settings.EMPTY, List.of()));
     }
 
     @Override

server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java

@@ -0,0 +1,69 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.extensions.action;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.transport.TransportRequest;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * This class represents a Transport Request for issuing a service account to an extension.
+ */
+public class IssueServiceAccountRequest extends TransportRequest {
+
+    private final String serviceAccountToken;
+
+    /**
+     * This takes a string representing a service account token
+     * @param serviceAccountToken The string making up the service account token
+     */
+    public IssueServiceAccountRequest(String serviceAccountToken) {
+        this.serviceAccountToken = serviceAccountToken;
+    }
+
+    /**
+     * This takes a stream representing a service account token
+     * @param in The stream passing the token
+     */
+    public IssueServiceAccountRequest(StreamInput in) throws IOException {
+        super(in);
+        this.serviceAccountToken = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(serviceAccountToken);
+    }
+
+    public String getServiceAccountToken() {
+        return this.serviceAccountToken;
+    }
+
+    @Override
+    public String toString() {
+        return "IssueServiceAccountRequest {" + "serviceAccountToken=" + serviceAccountToken + "}";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        IssueServiceAccountRequest that = (IssueServiceAccountRequest) o;
+        return Objects.equals(serviceAccountToken, that.serviceAccountToken);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(serviceAccountToken);
+    }
+}

server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java

@@ -0,0 +1,81 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.extensions.action;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.transport.TransportResponse;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * This class represents a Transport Request for issuing a service account to an extension.
+ */
+public class IssueServiceAccountResponse extends TransportResponse {
+
+    private String name;
+    private String serviceAccountToken;
+
+    /**
+     * This takes in a name for the extension and the service account token string
+     * @param name The name of the extension
+     * @param serviceAccountToken A string encapsulating the service account token
+     */
+    public IssueServiceAccountResponse(String name, String serviceAccountToken) {
+        this.name = name;
+        this.serviceAccountToken = serviceAccountToken;
+    }
+
+    /**
+     * This takes in a stream containing for the extension and the service account token
+     * @param in the stream containing the extension name and the service account token
+     */
+    public IssueServiceAccountResponse(StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.serviceAccountToken = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeString(serviceAccountToken);
+    }
+
+    /**
+     * @return the node that is currently leading, according to the responding node.
+     */
+
+    public String getName() {
+        return this.name;
+    }
+
+    public String getServiceAccountString() {
+        return this.serviceAccountToken;
+    }
+
+    @Override
+    public String toString() {
+        return "IssueServiceAccountResponse{" + "name = " + name + " , " + "received service account = " + serviceAccountToken + "}";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        IssueServiceAccountResponse that = (IssueServiceAccountResponse) o;
+        return Objects.equals(name, that.name) && Objects.equals(serviceAccountToken, that.serviceAccountToken);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, serviceAccountToken);
+    }
+
+}

server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java

@@ -161,6 +161,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
         try {
             extensionsManager.loadExtension(extension);
             extensionsManager.initialize();
+            extensionsManager.issueServiceAccount(extension);
         } catch (CompletionException e) {
             Throwable cause = e.getCause();
             if (cause instanceof TimeoutException) {

server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java

@@ -37,6 +37,20 @@ public String asAuthHeaderValue() {
         };
     }
 
+    /**
+     * Issue a new Noop Token
+     * @return a new Noop Token
+     */
+    @Override
+    public AuthToken issueServiceAccountToken(final String audience) {
+        return new AuthToken() {
+            @Override
+            public String asAuthHeaderValue() {
+                return "noopToken";
+            }
+        };
+    }
+
     @Override
     public Subject authenticateToken(AuthToken authToken) {
         return null;

server/src/main/java/org/opensearch/identity/tokens/TokenManager.java

@@ -23,6 +23,14 @@ public interface TokenManager {
      */
     public AuthToken issueOnBehalfOfToken(final Subject subject, final OnBehalfOfClaims claims);
 
+    /**
+     * Create a new service account token
+     *
+     * @param audience: A string representing the unique id of the extension for which a service account token should be generated
+     * @return a new auth token
+     */
+    public AuthToken issueServiceAccountToken(final String audience);
+
     /**
      * Authenticates a provided authToken
      * @param authToken: The authToken to authenticate

server/src/main/java/org/opensearch/node/Node.java

@@ -494,7 +494,7 @@ protected Node(
                 for (ExtensionAwarePlugin extAwarePlugin : extensionAwarePlugins) {
                     additionalSettings.addAll(extAwarePlugin.getExtensionSettings());
                 }
-                this.extensionsManager = new ExtensionsManager(additionalSettings);
+                this.extensionsManager = new ExtensionsManager(additionalSettings, identityService);
             } else {
                 this.extensionsManager = new NoopExtensionsManager();
             }

server/src/test/java/org/opensearch/action/ActionModuleTests.java

@@ -143,7 +143,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() throws IOException {
             usageService,
             null,
             new IdentityService(Settings.EMPTY, new ArrayList<>()),
-            new ExtensionsManager(Set.of())
+            new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of()))
         );
         actionModule.initRestHandlers(null);
         // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail