Support use of IRSA for repository-s3 plugin credentials (#3475) (#3486)

* Support use of IRSA for repository-s3 plugin credentials

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Address code review comments

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Address code review comments

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
(cherry picked from commit 596d32a)

Co-authored-by: Andriy Redko <andriy.redko@aiven.io>

plugins/repository-s3/build.gradle

@@ -51,6 +51,7 @@ versions << [
 dependencies {
   api "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
   api "com.amazonaws:aws-java-sdk-core:${versions.aws}"
+  api "com.amazonaws:aws-java-sdk-sts:${versions.aws}"
   api "com.amazonaws:jmespath-java:${versions.aws}"
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"

plugins/repository-s3/licenses/aws-java-sdk-sts-1.11.749.jar.sha1

@@ -0,0 +1 @@
+724bd22c0ff41c496469e18f9bea12bdfb2f7540

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonS3Reference.java

@@ -32,17 +32,39 @@
 
 package org.opensearch.repositories.s3;
 
+import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
+
+import org.opensearch.common.Nullable;
 import org.opensearch.common.concurrent.RefCountedReleasable;
 
+import java.io.Closeable;
+import java.io.IOException;
+
 /**
  * Handles the shutdown of the wrapped {@link AmazonS3Client} using reference
  * counting.
  */
 public class AmazonS3Reference extends RefCountedReleasable<AmazonS3> {
-
     AmazonS3Reference(AmazonS3 client) {
-        super("AWS_S3_CLIENT", client, client::shutdown);
+        this(client, null);
+    }
+
+    AmazonS3Reference(AmazonS3WithCredentials client) {
+        this(client.client(), client.credentials());
+    }
+
+    AmazonS3Reference(AmazonS3 client, @Nullable AWSCredentialsProvider credentials) {
+        super("AWS_S3_CLIENT", client, () -> {
+            client.shutdown();
+            if (credentials instanceof Closeable) {
+                try {
+                    ((Closeable) credentials).close();
+                } catch (IOException e) {
+                    /* Do nothing here */
+                }
+            }
+        });
     }
 }

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonS3WithCredentials.java

@@ -0,0 +1,39 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+
+import org.opensearch.common.Nullable;
+
+/**
+ * The holder of the AmazonS3 and AWSCredentialsProvider
+ */
+final class AmazonS3WithCredentials {
+    private final AmazonS3 client;
+    private final AWSCredentialsProvider credentials;
+
+    private AmazonS3WithCredentials(final AmazonS3 client, @Nullable final AWSCredentialsProvider credentials) {
+        this.client = client;
+        this.credentials = credentials;
+    }
+
+    AmazonS3 client() {
+        return client;
+    }
+
+    AWSCredentialsProvider credentials() {
+        return credentials;
+    }
+
+    static AmazonS3WithCredentials create(final AmazonS3 client, @Nullable final AWSCredentialsProvider credentials) {
+        return new AmazonS3WithCredentials(client, credentials);
+    }
+}

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java

@@ -67,6 +67,29 @@ final class S3ClientSettings {
     /** Placeholder client name for normalizing client settings in the repository settings. */
     private static final String PLACEHOLDER_CLIENT = "placeholder";
 
+    // Properties to support using IAM Roles for Service Accounts (IRSA)
+
+    /** The identity token file for connecting to s3. */
+    static final Setting.AffixSetting<String> IDENTITY_TOKEN_FILE_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "identity_token_file",
+        key -> SecureSetting.simpleString(key, Property.NodeScope)
+    );
+
+    /** The role ARN (Amazon Resource Name) for connecting to s3. */
+    static final Setting.AffixSetting<SecureString> ROLE_ARN_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "role_arn",
+        key -> SecureSetting.secureString(key, null)
+    );
+
+    /** The role session name for connecting to s3. */
+    static final Setting.AffixSetting<SecureString> ROLE_SESSION_NAME_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "role_session_name",
+        key -> SecureSetting.secureString(key, null)
+    );
+
     /** The access key (ie login id) for connecting to s3. */
     static final Setting.AffixSetting<SecureString> ACCESS_KEY_SETTING = Setting.affixKeySetting(
         PREFIX,
@@ -189,6 +212,9 @@ final class S3ClientSettings {
     /** Credentials to authenticate with s3. */
     final S3BasicCredentials credentials;
 
+    /** Credentials to authenticate with s3 using IAM Roles for Service Accounts (IRSA). */
+    final IrsaCredentials irsaCredentials;
+
     /** The s3 endpoint the client should talk to, or empty string to use the default. */
     final String endpoint;
 
@@ -221,6 +247,7 @@ final class S3ClientSettings {
 
     private S3ClientSettings(
         S3BasicCredentials credentials,
+        IrsaCredentials irsaCredentials,
         String endpoint,
         Protocol protocol,
         int readTimeoutMillis,
@@ -233,6 +260,7 @@ private S3ClientSettings(
         ProxySettings proxySettings
     ) {
         this.credentials = credentials;
+        this.irsaCredentials = irsaCredentials;
         this.endpoint = endpoint;
         this.protocol = protocol;
         this.readTimeoutMillis = readTimeoutMillis;
@@ -301,6 +329,7 @@ S3ClientSettings refine(Settings repositorySettings) {
         validateInetAddressFor(newProxyHost);
         return new S3ClientSettings(
             newCredentials,
+            irsaCredentials,
             newEndpoint,
             newProtocol,
             newReadTimeoutMillis,
@@ -396,12 +425,27 @@ private static S3BasicCredentials loadCredentials(Settings settings, String clie
         }
     }
 
+    private static IrsaCredentials loadIrsaCredentials(Settings settings, String clientName) {
+        String identityTokenFile = getConfigValue(settings, clientName, IDENTITY_TOKEN_FILE_SETTING);
+        try (
+            SecureString roleArn = getConfigValue(settings, clientName, ROLE_ARN_SETTING);
+            SecureString roleSessionName = getConfigValue(settings, clientName, ROLE_SESSION_NAME_SETTING)
+        ) {
+            if (identityTokenFile.length() != 0 || roleArn.length() != 0 || roleSessionName.length() != 0) {
+                return new IrsaCredentials(identityTokenFile.toString(), roleArn.toString(), roleSessionName.toString());
+            }
+
+            return null;
+        }
+    }
+
     // pkg private for tests
     /** Parse settings for a single client. */
     static S3ClientSettings getClientSettings(final Settings settings, final String clientName) {
         final Protocol awsProtocol = getConfigValue(settings, clientName, PROTOCOL_SETTING);
         return new S3ClientSettings(
             S3ClientSettings.loadCredentials(settings, clientName),
+            S3ClientSettings.loadIrsaCredentials(settings, clientName),
             getConfigValue(settings, clientName, ENDPOINT_SETTING),
             awsProtocol,
             Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()),
@@ -482,7 +526,8 @@ public boolean equals(final Object o) {
             && proxySettings.equals(that.proxySettings)
             && Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding)
             && Objects.equals(region, that.region)
-            && Objects.equals(signerOverride, that.signerOverride);
+            && Objects.equals(signerOverride, that.signerOverride)
+            && Objects.equals(irsaCredentials, that.irsaCredentials);
     }
 
     @Override
@@ -512,4 +557,51 @@ private static <T> T getRepoSettingOrDefault(Setting.AffixSetting<T> setting, Se
         }
         return defaultValue;
     }
+
+    /**
+     * Class to store IAM Roles for Service Accounts (IRSA) credentials
+     * See please: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
+     */
+    static class IrsaCredentials {
+        private final String identityTokenFile;
+        private final String roleArn;
+        private final String roleSessionName;
+
+        IrsaCredentials(String identityTokenFile, String roleArn, String roleSessionName) {
+            this.identityTokenFile = Strings.isNullOrEmpty(identityTokenFile) ? null : identityTokenFile;
+            this.roleArn = Strings.isNullOrEmpty(roleArn) ? null : roleArn;
+            this.roleSessionName = Strings.isNullOrEmpty(roleSessionName) ? "s3-sdk-java-" + System.currentTimeMillis() : roleSessionName;
+        }
+
+        public String getIdentityTokenFile() {
+            return identityTokenFile;
+        }
+
+        public String getRoleArn() {
+            return roleArn;
+        }
+
+        public String getRoleSessionName() {
+            return roleSessionName;
+        }
+
+        @Override
+        public boolean equals(final Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            final IrsaCredentials that = (IrsaCredentials) o;
+            return Objects.equals(identityTokenFile, that.identityTokenFile)
+                && Objects.equals(roleArn, that.roleArn)
+                && Objects.equals(roleSessionName, that.roleSessionName);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(identityTokenFile, roleArn, roleSessionName);
+        }
+    }
 }

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java

@@ -132,7 +132,10 @@ public List<Setting<?>> getSettings() {
             S3Repository.ACCESS_KEY_SETTING,
             S3Repository.SECRET_KEY_SETTING,
             S3ClientSettings.SIGNER_OVERRIDE,
-            S3ClientSettings.REGION
+            S3ClientSettings.REGION,
+            S3ClientSettings.ROLE_ARN_SETTING,
+            S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING,
+            S3ClientSettings.ROLE_SESSION_NAME_SETTING
         );
     }