feat: Implement an AWS Secrets Manager secrets provider + resolver #1545

 (#1575)

Please note that this is not integrated into the CLI and controller to
prevent potential conflicts with
#1473

Related issue: #1545

common/configuration/asm.go

@@ -0,0 +1,162 @@
+package configuration
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/TBD54566975/ftl/internal/slices"
+
+	. "github.com/alecthomas/types/optional" //nolint:stylecheck
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
+	"github.com/aws/smithy-go"
+)
+
+// ASM implements Resolver and Provider for AWS Secrets Manager (ASM).
+//
+// The resolver does a direct/proxy map from a Ref to a URL, module.name <-> asm://module.name and does not access ASM at all.
+type ASM struct {
+	Client secretsmanager.Client
+}
+
+var _ Resolver[Secrets] = &ASM{}
+var _ Provider[Secrets] = &ASM{}
+
+func asmURLForRef(ref Ref) *url.URL {
+	return &url.URL{
+		Scheme: "asm",
+		Host:   ref.String(),
+	}
+}
+
+func (ASM) Role() Secrets {
+	return Secrets{}
+}
+
+func (ASM) Key() string {
+	return "asm"
+}
+
+func (ASM) Get(ctx context.Context, ref Ref) (*url.URL, error) {
+	return asmURLForRef(ref), nil
+}
+
+func (ASM) Set(ctx context.Context, ref Ref, key *url.URL) error {
+	expectedKey := asmURLForRef(ref)
+	if key.String() != expectedKey.String() {
+		return fmt.Errorf("key does not match expected key for ref: %s", expectedKey)
+	}
+
+	return nil
+}
+
+// Unset does nothing because this resolver does not record any state.
+func (ASM) Unset(ctx context.Context, ref Ref) error {
+	return nil
+}
+
+// List all secrets in the account. This might require multiple calls to the AWS API if there are more than 100 secrets.
+func (a ASM) List(ctx context.Context) ([]Entry, error) {
+	nextToken := None[string]()
+	entries := []Entry{}
+	for {
+		out, err := a.Client.ListSecrets(ctx, &secretsmanager.ListSecretsInput{
+			MaxResults: aws.Int32(100),
+			NextToken:  nextToken.Ptr(),
+		})
+		if err != nil {
+			return nil, fmt.Errorf("unable to list secrets: %w", err)
+		}
+
+		var activeSecrets = slices.Filter(out.SecretList, func(s types.SecretListEntry) bool {
+			return s.DeletedDate == nil
+		})
+		page, err := slices.MapErr(activeSecrets, func(s types.SecretListEntry) (Entry, error) {
+			var ref Ref
+			ref, err = ParseRef(*s.Name)
+			if err != nil {
+				return Entry{}, fmt.Errorf("unable to parse ref: %w", err)
+			}
+
+			return Entry{
+				Ref:      ref,
+				Accessor: asmURLForRef(ref),
+			}, nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		entries = append(entries, page...)
+
+		nextToken = Ptr[string](out.NextToken)
+		if !nextToken.Ok() {
+			break
+		}
+	}
+
+	return entries, nil
+}
+
+// Load only supports loading "string" secrets, not binary secrets.
+func (a ASM) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) {
+	expectedKey := asmURLForRef(ref)
+	if key.String() != expectedKey.String() {
+		return nil, fmt.Errorf("key does not match expected key for ref: %s", expectedKey)
+	}
+
+	out, err := a.Client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{
+		SecretId: aws.String(ref.String()),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to retrieve secret: %w", err)
+	}
+
+	// Secret is a string
+	if out.SecretBinary != nil {
+		return nil, fmt.Errorf("secret is not a string")
+	}
+
+	return []byte(*out.SecretString), nil
+}
+
+// Store and if the secret already exists, update it.
+func (a ASM) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) {
+	_, err := a.Client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{
+		Name:         aws.String(ref.String()),
+		SecretString: aws.String(string(value)),
+	})
+
+	// https://github.com/aws/aws-sdk-go-v2/issues/1110#issuecomment-1054643716
+	var apiErr smithy.APIError
+	if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ResourceExistsException" {
+		_, err = a.Client.UpdateSecret(ctx, &secretsmanager.UpdateSecretInput{
+			SecretId:     aws.String(ref.String()),
+			SecretString: aws.String(string(value)),
+		})
+		if err != nil {
+			return nil, fmt.Errorf("unable to update secret: %w", err)
+		}
+
+	} else if err != nil {
+		return nil, fmt.Errorf("unable to store secret: %w", err)
+	}
+
+	return asmURLForRef(ref), nil
+}
+
+func (a ASM) Delete(ctx context.Context, ref Ref) error {
+	var t = true
+	_, err := a.Client.DeleteSecret(ctx, &secretsmanager.DeleteSecretInput{
+		SecretId:                   aws.String(ref.String()),
+		ForceDeleteWithoutRecovery: &t,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to delete secret: %w", err)
+	}
+
+	return nil
+}

common/configuration/asm_test.go

@@ -0,0 +1,126 @@
+package configuration
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/TBD54566975/ftl/internal/log"
+
+	"github.com/alecthomas/assert/v2"
+	. "github.com/alecthomas/types/optional"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+)
+
+func localstack(ctx context.Context, t *testing.T) ASM {
+	t.Helper()
+	cc := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("test", "test", ""))
+	cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(cc), config.WithRegion("us-west-2"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sm := secretsmanager.NewFromConfig(cfg, func(o *secretsmanager.Options) {
+		o.BaseEndpoint = aws.String("http://localhost:4566")
+	})
+	asm := ASM{Client: *sm}
+	return asm
+}
+
+func TestASMWorkflow(t *testing.T) {
+	ctx := log.ContextWithNewDefaultLogger(context.Background())
+	asm := localstack(ctx, t)
+	ref := Ref{Module: Some("foo"), Name: "bar"}
+	var mySecret = []byte("my secret")
+	manager, err := New(ctx, asm, []Provider[Secrets]{asm})
+	assert.NoError(t, err)
+
+	var gotSecret []byte
+	err = manager.Get(ctx, ref, &gotSecret)
+	assert.Error(t, err)
+
+	items, err := manager.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, items, []Entry{})
+
+	err = manager.Set(ctx, "asm", ref, mySecret)
+	assert.NoError(t, err)
+
+	items, err = manager.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, items, []Entry{{Ref: ref, Accessor: URL("asm://foo.bar")}})
+
+	err = manager.Get(ctx, ref, &gotSecret)
+	assert.NoError(t, err)
+
+	// Set again to make sure it updates.
+	mySecret = []byte("hunter1")
+	err = manager.Set(ctx, "asm", ref, mySecret)
+	assert.NoError(t, err)
+
+	err = manager.Get(ctx, ref, &gotSecret)
+	assert.NoError(t, err)
+	assert.Equal(t, gotSecret, mySecret)
+
+	err = manager.Unset(ctx, "asm", ref)
+	assert.NoError(t, err)
+
+	items, err = manager.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, items, []Entry{})
+
+	err = manager.Get(ctx, ref, &gotSecret)
+	assert.Error(t, err)
+}
+
+// Suggest not running this against a real AWS account (especially in CI) due to the cost. Maybe costs a few $.
+func TestASMPagination(t *testing.T) {
+	ctx := log.ContextWithNewDefaultLogger(context.Background())
+	asm := localstack(ctx, t)
+	manager, err := New(ctx, asm, []Provider[Secrets]{asm})
+	assert.NoError(t, err)
+
+	// Create 210 secrets, so we paginate at least twice.
+	for i := range 210 {
+		ref := NewRef("foo", fmt.Sprintf("bar%03d", i))
+		err := manager.Set(ctx, "asm", ref, []byte(fmt.Sprintf("hunter%03d", i)))
+		assert.NoError(t, err)
+	}
+
+	items, err := manager.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, len(items), 210)
+
+	// Check each secret.
+	sort.Slice(items, func(i, j int) bool {
+		return items[i].Ref.Name < items[j].Ref.Name
+	})
+	for i, item := range items {
+		assert.Equal(t, item.Ref.Name, fmt.Sprintf("bar%03d", i))
+
+		// Just to save on requests, skip by 10
+		if i%10 != 0 {
+			continue
+		}
+		var secret []byte
+		err := manager.Get(ctx, item.Ref, &secret)
+		assert.NoError(t, err)
+		assert.Equal(t, secret, []byte(fmt.Sprintf("hunter%03d", i)))
+	}
+
+	// Delete them
+	for i := range 210 {
+		ref := NewRef("foo", fmt.Sprintf("bar%03d", i))
+		err := manager.Unset(ctx, "asm", ref)
+		assert.NoError(t, err)
+	}
+
+	// Make sure they are all gone
+	items, err = manager.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, len(items), 0)
+}

docker-compose.yml

@@ -29,3 +29,10 @@ services:
     ports:
       - 4317:4317 # OTLP gRPC receiver
       - 4318:4318 # OTLP http receiver
+  localstack:
+    image: localstack/localstack
+    ports:
+      - 4566:4566
+    environment:
+      SERVICES: secretsmanager
+      DEBUG: 1

go.mod

@@ -16,6 +16,11 @@ require (
 	github.com/alecthomas/participle/v2 v2.1.1
 	github.com/alecthomas/types v0.16.0
 	github.com/amacneil/dbmate/v2 v2.16.0
+	github.com/aws/aws-sdk-go-v2 v1.27.0
+	github.com/aws/aws-sdk-go-v2/config v1.27.4
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.4
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.29.1
+	github.com/aws/smithy-go v1.20.2
 	github.com/beevik/etree v1.4.0
 	github.com/bmatcuk/doublestar/v4 v4.6.1
 	github.com/deckarep/golang-set/v2 v2.6.0
@@ -63,6 +68,15 @@ require (
 
 require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect