feat: AWS provider

README.md

@@ -16,7 +16,7 @@
 |  Local provider                                              | ✅ Implemented  |
 | [HashiCorp Vault](https://www.vaultproject.io)               | ✅ Implemented  |
 | [OpenBao](https://github.com/openbao/openbao)                | ✅ Implemented  |
-| [AWS Secrets Manager](https://aws.amazon.com/secrets-manager)| Upcoming       |
+| [AWS Secrets Manager](https://aws.amazon.com/secrets-manager)| ✅ Implemented  |
 
 ## Getting started
 

docker-compose.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   vault:
     container_name: secret-init-vault

env_store.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/bank-vaults/secret-init/pkg/common"
 	"github.com/bank-vaults/secret-init/pkg/provider"
+	"github.com/bank-vaults/secret-init/pkg/provider/aws"
 	"github.com/bank-vaults/secret-init/pkg/provider/bao"
 	"github.com/bank-vaults/secret-init/pkg/provider/file"
 	"github.com/bank-vaults/secret-init/pkg/provider/vault"
@@ -33,6 +34,7 @@ var supportedProviders = []string{
 	file.ProviderName,
 	vault.ProviderName,
 	bao.ProviderName,
+	aws.ProviderName,
 }
 
 // EnvStore is a helper for managing interactions between environment variables and providers,
@@ -57,45 +59,45 @@ func NewEnvStore(appConfig *common.Config) *EnvStore {
 	}
 }
 
-// GetProviderPaths returns a map of secret paths for each provider
-func (s *EnvStore) GetProviderPaths() map[string][]string {
-	providerPaths := make(map[string][]string)
+// GetSecretReferences returns a map of secret key=value pairs for each provider
+func (s *EnvStore) GetSecretReferences() map[string][]string {
+	secretReferences := make(map[string][]string)
 
-	for envKey, path := range s.data {
-		providerName, path := getProviderPath(path)
+	for envKey, envPath := range s.data {
+		providerName, envSecretReference := getProviderPath(envPath)
+		envSecretReference = envKey + "=" + envSecretReference
 		switch providerName {
 		case file.ProviderName:
-			providerPaths[file.ProviderName] = append(providerPaths[file.ProviderName], path)
+			secretReferences[file.ProviderName] = append(secretReferences[file.ProviderName], envSecretReference)
 
 		case vault.ProviderName:
-			// The injector function expects a map of key:value pairs
-			path = envKey + "=" + path
-			providerPaths[vault.ProviderName] = append(providerPaths[vault.ProviderName], path)
+			secretReferences[vault.ProviderName] = append(secretReferences[vault.ProviderName], envSecretReference)
 
 		case bao.ProviderName:
-			// The injector function expects a map of key:value pairs
-			path = envKey + "=" + path
-			providerPaths[bao.ProviderName] = append(providerPaths[bao.ProviderName], path)
+			secretReferences[bao.ProviderName] = append(secretReferences[bao.ProviderName], envSecretReference)
+
+		case aws.ProviderName:
+			secretReferences[aws.ProviderName] = append(secretReferences[aws.ProviderName], envSecretReference)
 		}
 	}
 
-	return providerPaths
+	return secretReferences
 }
 
 // LoadProviderSecrets creates a new provider for each detected provider using a specified config.
 // It then asynchronously loads secrets using each provider and it's corresponding paths.
-// The secrets from each provider are then placed into a map with the provider name as the key.
-func (s *EnvStore) LoadProviderSecrets(providerPaths map[string][]string) (map[string][]provider.Secret, error) {
+// The secrets from each provider are then placed into a single slice.
+func (s *EnvStore) LoadProviderSecrets(providerPaths map[string][]string) ([]provider.Secret, error) {
 	// At most, we will have one error per provider
 	errCh := make(chan error, len(supportedProviders))
-	providerSecrets := make(map[string][]provider.Secret)
+	var providerSecrets []provider.Secret
 
 	// Workaround for openBao
 	// Remove once openBao uses BAO_ADDR in their client, instead of VAULT_ADDR
 	vaultPaths, ok := providerPaths[vault.ProviderName]
 	if ok {
 		var err error
-		providerSecrets[vault.ProviderName], err = s.workaroundForBao(vaultPaths)
+		providerSecrets, err = s.workaroundForBao(vaultPaths)
 		if err != nil {
 			return nil, fmt.Errorf("failed to workaround for bao: %w", err)
 		}
@@ -126,7 +128,7 @@ func (s *EnvStore) LoadProviderSecrets(providerPaths map[string][]string) (map[s
 			}
 
 			mu.Lock()
-			providerSecrets[providerName] = secrets
+			providerSecrets = append(providerSecrets, secrets...)
 			mu.Unlock()
 		}(providerName, paths, errCh)
 	}
@@ -167,25 +169,11 @@ func (s *EnvStore) workaroundForBao(vaultPaths []string) ([]provider.Secret, err
 }
 
 // ConvertProviderSecrets converts the loaded secrets to environment variables
-func (s *EnvStore) ConvertProviderSecrets(providerSecrets map[string][]provider.Secret) ([]string, error) {
+func (s *EnvStore) ConvertProviderSecrets(providerSecrets []provider.Secret) ([]string, error) {
 	var secretsEnv []string
 
-	for providerName, secrets := range providerSecrets {
-		switch providerName {
-		case vault.ProviderName, bao.ProviderName:
-			// The Vault and Bao providers already returns the secrets with the environment variable keys
-			for _, secret := range secrets {
-				secretsEnv = append(secretsEnv, fmt.Sprintf("%s=%s", secret.Path, secret.Value))
-			}
-
-		default:
-			secrets, err := createSecretEnvsFrom(s.data, secrets)
-			if err != nil {
-				return nil, fmt.Errorf("failed to create secret environment variables: %w", err)
-			}
-
-			secretsEnv = append(secretsEnv, secrets...)
-		}
+	for _, secret := range providerSecrets {
+		secretsEnv = append(secretsEnv, fmt.Sprintf("%s=%s", secret.Key, secret.Value))
 	}
 
 	return secretsEnv, nil
@@ -194,24 +182,28 @@ func (s *EnvStore) ConvertProviderSecrets(providerSecrets map[string][]provider.
 // Returns the detected provider name and path with removed prefix
 func getProviderPath(path string) (string, string) {
 	if strings.HasPrefix(path, "file:") {
-		var fileProviderName = file.ProviderName
-		return fileProviderName, strings.TrimPrefix(path, "file:")
+		return file.ProviderName, path
 	}
 
 	// If the path contains some string formatted as "vault:{STR}#{STR}"
 	// it is most probably a vault path
 	if vault.ProviderEnvRegex.MatchString(path) {
-		// Do not remove the prefix since it will be processed during injection
 		return vault.ProviderName, path
 	}
 
 	// If the path contains some string formatted as "bao:{STR}#{STR}"
 	// it is most probably a vault path
 	if bao.ProviderEnvRegex.MatchString(path) {
-		// Do not remove the prefix since it will be processed during injection
 		return bao.ProviderName, path
 	}
 
+	// Example AWS prefixes:
+	// arn:aws:secretsmanager:us-west-2:123456789012:secret:my-secret
+	// arn:aws:ssm:us-west-2:123456789012:parameter/my-parameter
+	if strings.HasPrefix(path, "arn:aws:secretsmanager:") || strings.HasPrefix(path, "arn:aws:ssm:") {
+		return aws.ProviderName, path
+	}
+
 	return "", path
 }
 
@@ -249,33 +241,16 @@ func newProvider(providerName string, appConfig *common.Config) (provider.Provid
 		}
 		return provider, nil
 
-	default:
-		return nil, fmt.Errorf("provider %s is not supported", providerName)
-	}
-}
-
-func createSecretEnvsFrom(envs map[string]string, secrets []provider.Secret) ([]string, error) {
-	// Reverse the map so we can match
-	// the environment variable key to the secret
-	// by using the secret path
-	reversedEnvs := make(map[string]string)
-	for envKey, path := range envs {
-		providerName, path := getProviderPath(path)
-		if providerName != "" {
-			reversedEnvs[path] = envKey
+	case aws.ProviderName:
+		config, err := aws.LoadConfig()
+		if err != nil {
+			return nil, fmt.Errorf("failed to create aws config: %w", err)
 		}
-	}
 
-	var secretsEnv []string
-	for _, secret := range secrets {
-		path := secret.Path
-		key, ok := reversedEnvs[path]
-		if !ok {
-			return nil, fmt.Errorf("failed to find environment variable key for secret path: %s", path)
-		}
+		provider := aws.NewProvider(config)
+		return provider, nil
 
-		secretsEnv = append(secretsEnv, fmt.Sprintf("%s=%s", key, secret.Value))
+	default:
+		return nil, fmt.Errorf("provider %s is not supported", providerName)
 	}
-
-	return secretsEnv, nil
 }

env_store_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/bank-vaults/secret-init/pkg/provider"
 )
 
-func TestEnvStore_GetProviderPaths(t *testing.T) {
+func TestEnvStore_GetSecretReferences(t *testing.T) {
 	tests := []struct {
 		name      string
 		envs      map[string]string
@@ -38,7 +38,7 @@ func TestEnvStore_GetProviderPaths(t *testing.T) {
 			},
 			wantPaths: map[string][]string{
 				"file": {
-					"secret/data/test/aws",
+					"AWS_SECRET_ACCESS_KEY_ID=file:secret/data/test/aws",
 				},
 			},
 		},
@@ -65,20 +65,68 @@ func TestEnvStore_GetProviderPaths(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "bao provider",
+			envs: map[string]string{
+				"ACCOUNT_PASSWORD_1":              "bao:secret/data/account#password#1",
+				"ACCOUNT_PASSWORD":                "bao:secret/data/account#password",
+				"ROOT_CERT":                       ">>bao:pki/root/generate/internal#certificate",
+				"ROOT_CERT_CACHED":                ">>bao:pki/root/generate/internal#certificate",
+				"INLINE_SECRET":                   "scheme://${bao:secret/data/account#username}:${bao:secret/data/account#password}@127.0.0.1:8080",
+				"INLINE_SECRET_EMBEDDED_TEMPLATE": "scheme://${bao:secret/data/account#username}:${bao:secret/data/account#${.password | urlquery}}@127.0.0.1:8080",
+				"INLINE_DYNAMIC_SECRET":           "${>>bao:pki/root/generate/internal#certificate}__${>>bao:pki/root/generate/internal#certificate}",
+			},
+			wantPaths: map[string][]string{
+				"bao": {
+					"ACCOUNT_PASSWORD_1=bao:secret/data/account#password#1",
+					"ACCOUNT_PASSWORD=bao:secret/data/account#password",
+					"ROOT_CERT=>>bao:pki/root/generate/internal#certificate",
+					"ROOT_CERT_CACHED=>>bao:pki/root/generate/internal#certificate",
+					"INLINE_SECRET=scheme://${bao:secret/data/account#username}:${bao:secret/data/account#password}@127.0.0.1:8080",
+					"INLINE_SECRET_EMBEDDED_TEMPLATE=scheme://${bao:secret/data/account#username}:${bao:secret/data/account#${.password | urlquery}}@127.0.0.1:8080",
+					"INLINE_DYNAMIC_SECRET=${>>bao:pki/root/generate/internal#certificate}__${>>bao:pki/root/generate/internal#certificate}",
+				},
+			},
+		},
+		{
+			name: "aws provider",
+			envs: map[string]string{
+				"AWS_SECRET1": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-secret",
+				"AWS_SECRET2": "arn:aws:ssm:us-west-2:123456789012:parameter/my-parameter",
+			},
+			wantPaths: map[string][]string{
+				"aws": {
+					"AWS_SECRET1=arn:aws:secretsmanager:us-west-2:123456789012:secret:my-secret",
+					"AWS_SECRET2=arn:aws:ssm:us-west-2:123456789012:parameter/my-parameter",
+				},
+			},
+		},
 		{
 			name: "multi provider",
 			envs: map[string]string{
 				"AWS_SECRET_ACCESS_KEY_ID": "file:secret/data/test/aws",
 				"MYSQL_PASSWORD":           "vault:secret/data/test/mysql#MYSQL_PASSWORD",
 				"AWS_SECRET_ACCESS_KEY":    "vault:secret/data/test/aws#AWS_SECRET_ACCESS_KEY",
+				"RABBITMQ_USERNAME":        "bao:secret/data/test/rabbitmq#RABBITMQ_USERNAME",
+				"RABBITMQ_PASSWORD":        "bao:secret/data/test/rabbitmq#RABBITMQ_PASSWORD",
+				"AWS_SECRET1":              "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-secret",
+				"AWS_SECRET2":              "arn:aws:ssm:us-west-2:123456789012:parameter/my-parameter",
 			},
 			wantPaths: map[string][]string{
+				"file": {
+					"AWS_SECRET_ACCESS_KEY_ID=file:secret/data/test/aws",
+				},
 				"vault": {
 					"MYSQL_PASSWORD=vault:secret/data/test/mysql#MYSQL_PASSWORD",
 					"AWS_SECRET_ACCESS_KEY=vault:secret/data/test/aws#AWS_SECRET_ACCESS_KEY",
 				},
-				"file": {
-					"secret/data/test/aws",
+				"bao": {
+					"RABBITMQ_USERNAME=bao:secret/data/test/rabbitmq#RABBITMQ_USERNAME",
+					"RABBITMQ_PASSWORD=bao:secret/data/test/rabbitmq#RABBITMQ_PASSWORD",
+				},
+				"aws": {
+					"AWS_SECRET1=arn:aws:secretsmanager:us-west-2:123456789012:secret:my-secret",
+					"AWS_SECRET2=arn:aws:ssm:us-west-2:123456789012:parameter/my-parameter",
 				},
 			},
 		},
@@ -95,7 +143,7 @@ func TestEnvStore_GetProviderPaths(t *testing.T) {
 				os.Clearenv()
 			})
 
-			paths := NewEnvStore(&common.Config{}).GetProviderPaths()
+			paths := NewEnvStore(&common.Config{}).GetSecretReferences()
 
 			for key, expectedSlice := range ttp.wantPaths {
 				actualSlice, ok := paths[key]
@@ -113,23 +161,21 @@ func TestEnvStore_LoadProviderSecrets(t *testing.T) {
 	tests := []struct {
 		name                string
 		providerPaths       map[string][]string
-		wantProviderSecrets map[string][]provider.Secret
+		wantProviderSecrets []provider.Secret
 		addvault            bool
 		err                 error
 	}{
 		{
 			name: "Load secrets successfully",
 			providerPaths: map[string][]string{
 				"file": {
-					secretFile,
+					"AWS_SECRET_ACCESS_KEY_ID=file:" + secretFile,
 				},
 			},
-			wantProviderSecrets: map[string][]provider.Secret{
-				"file": {
-					{
-						Path:  secretFile,
-						Value: "secretId",
-					},
+			wantProviderSecrets: []provider.Secret{
+				{
+					Key:   "AWS_SECRET_ACCESS_KEY_ID",
+					Value: "secretId",
 				},
 			},
 			addvault: false,
@@ -138,7 +184,7 @@ func TestEnvStore_LoadProviderSecrets(t *testing.T) {
 			name: "Fail to create provider",
 			providerPaths: map[string][]string{
 				"invalid": {
-					secretFile,
+					"AWS_SECRET_ACCESS_KEY_ID=file:" + secretFile,
 				},
 			},
 			addvault: false,
@@ -168,39 +214,24 @@ func TestEnvStore_ConvertProviderSecrets(t *testing.T) {
 
 	tests := []struct {
 		name            string
-		providerSecrets map[string][]provider.Secret
+		providerSecrets []provider.Secret
 		wantSecretsEnv  []string
 		addvault        bool
 		err             error
 	}{
 		{
 			name: "Convert secrets successfully",
-			providerSecrets: map[string][]provider.Secret{
-				"file": {
-					{
-						Path:  secretFile,
-						Value: "secretId",
-					},
+			providerSecrets: []provider.Secret{
+				{
+					Key:   "AWS_SECRET_ACCESS_KEY_ID",
+					Value: "secretId",
 				},
 			},
 			wantSecretsEnv: []string{
 				"AWS_SECRET_ACCESS_KEY_ID=secretId",
 			},
 			addvault: false,
 		},
-		{
-			name: "Fail to convert secrets due to fail to find env-key",
-			providerSecrets: map[string][]provider.Secret{
-				"file": {
-					{
-						Path:  secretFile + "/invalid",
-						Value: "secretId",
-					},
-				},
-			},
-			addvault: false,
-			err:      fmt.Errorf("failed to create secret environment variables: failed to find environment variable key for secret path: " + secretFile + "/invalid"),
-		},
 	}
 
 	for _, tt := range tests {

examples/README.md

@@ -7,6 +7,7 @@ Discover a range of examples that highlight the functionalities of **secret-init
 - [File provider](file-provider.md)
 - [Vault provider](vault-provider.md)
 - [Bao provider](bao-provider.md)
+- [AWS provider](aws-provider.md)
 
 ## Multi provider use-case