Add support for retrieving s3 credentials from AWS credentials file

.bazelci/system-test.sh

@@ -78,6 +78,7 @@ rm -rf $test_cache_dir
 	--s3.endpoint 127.0.0.1:9000 \
 	--s3.bucket bazel-remote \
 	--s3.prefix files \
+	--s3.auth_method access_key \
 	--s3.access_key_id minioadmin \
 	--s3.secret_access_key minioadmin \
 	--s3.disable_ssl \
@@ -134,6 +135,7 @@ rm -rf $test_cache_dir
 	--s3.endpoint 127.0.0.1:9000 \
 	--s3.bucket bazel-remote \
 	--s3.prefix files \
+	--s3.auth_method access_key \
 	--s3.access_key_id minioadmin \
 	--s3.secret_access_key minioadmin \
 	--s3.disable_ssl \
@@ -211,6 +213,7 @@ rm -rf $test_cache_dir
 	--s3.endpoint 127.0.0.1:9000 \
 	--s3.bucket bazel-remote \
 	--s3.prefix files \
+	--s3.auth_method access_key \
 	--s3.access_key_id minioadmin \
 	--s3.secret_access_key minioadmin \
 	--s3.disable_ssl \

README.md

@@ -232,19 +232,37 @@ OPTIONS:
    --s3.prefix value The S3/minio object prefix to use when using S3 proxy
       backend. [$BAZEL_REMOTE_S3_PREFIX]
 
+   --s3.auth_method value The S3/minio authentication method. This argument
+      is required when an s3 proxy backend is used. Allowed values: iam_role,
+      access_key, aws_credentials_file. [$BAZEL_REMOTE_S3_AUTH_METHOD]
+
    --s3.access_key_id value The S3/minio access key to use when using S3
-      proxy backend. [$BAZEL_REMOTE_S3_ACCESS_KEY_ID]
+      proxy backend. Applies to s3 auth method(s): access_key.
+      [$BAZEL_REMOTE_S3_ACCESS_KEY_ID]
 
    --s3.secret_access_key value The S3/minio secret access key to use when
-      using S3 proxy backend. [$BAZEL_REMOTE_S3_SECRET_ACCESS_KEY]
+      using S3 proxy backend. Applies to s3 auth method(s): access_key.
+      [$BAZEL_REMOTE_S3_SECRET_ACCESS_KEY]
+
+   --s3.aws_shared_credentials_file value Path to the AWS credentials file.
+      If not specified, the minio client will default to '~/.aws/credentials'.
+      Applies to s3 auth method(s): aws_credentials_file.
+      [$BAZEL_REMOTE_S3_AWS_SHARED_CREDENTIALS_FILE,
+      $AWS_SHARED_CREDENTIALS_FILE]
+
+   --s3.aws_profile value The aws credentials profile to use from within
+      s3.aws_shared_credentials_file. Applies to s3 auth method(s):
+      aws_credentials_file. (default: "default") [$BAZEL_REMOTE_S3_AWS_PROFILE,
+      $AWS_PROFILE]
 
    --s3.disable_ssl Whether to disable TLS/SSL when using the S3 proxy
       backend. (default: false, ie enable TLS/SSL)
       [$BAZEL_REMOTE_S3_DISABLE_SSL]
 
    --s3.iam_role_endpoint value Endpoint for using IAM security credentials.
       By default it will look for credentials in the standard locations for the
-      AWS platform. [$BAZEL_REMOTE_S3_IAM_ROLE_ENDPOINT]
+      AWS platform. Applies to s3 auth method(s): iam_role.
+      [$BAZEL_REMOTE_S3_IAM_ROLE_ENDPOINT]
 
    --s3.region value The AWS region. Required when not specifying S3/minio
       access keys. [$BAZEL_REMOTE_S3_REGION]
@@ -356,14 +374,24 @@ host: localhost
 #  endpoint: minio.example.com:9000
 #  bucket: test-bucket
 #  prefix: test-prefix
+#  disable_ssl: true
+#
+# Provide exactly one auth_method (access_key, iam_role, or credentials_file) and accompanying configuration.
+#
+# Access key authenticaiton:
+#  auth_method: access_key
 #  access_key_id: EXAMPLE_ACCESS_KEY
 #  secret_access_key: EXAMPLE_SECRET_KEY
-#  disable_ssl: true
 #
-# Provide either access_key_id/secret_access_key, or iam_role_endpoint/region.
-# iam_role_endpoint can also be left empty, and figured out automatically.
+# IAM Role authentication.
+#  auth_method: iam_role
 #  iam_role_endpoint: http://169.254.169.254
 #  region: us-east-1
+# 
+# AWS credentials file.
+#  auth_method: credentials_file
+#  aws_shared_credentials_file: path/to/aws/credentials
+#  aws_profile: my-profile
 #
 #http_proxy:
 #  url: https://remote-cache.com:8080/cache

cache/s3proxy/BUILD.bazel

@@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["s3proxy.go"],
+    srcs = [
+        "auth_methods.go",
+        "s3proxy.go",
+    ],
     importpath = "github.com/buchgr/bazel-remote/cache/s3proxy",
     visibility = ["//visibility:public"],
     deps = [

cache/s3proxy/auth_methods.go

@@ -0,0 +1,24 @@
+package s3proxy
+
+const (
+	AuthMethodIAMRole            = "iam_role"
+	AuthMethodAccessKey          = "access_key"
+	AuthMethodAWSCredentialsFile = "aws_credentials_file"
+)
+
+func GetAuthMethods() []string {
+	return []string{
+		AuthMethodIAMRole,
+		AuthMethodAccessKey,
+		AuthMethodAWSCredentialsFile,
+	}
+}
+
+func IsValidAuthMethod(authMethod string) bool {
+	for _, b := range GetAuthMethods() {
+		if authMethod == b {
+			return true
+		}
+	}
+	return false
+}

cache/s3proxy/minio-test-setup.md

@@ -26,6 +26,7 @@ production use.
     	--s3.endpoint 127.0.0.1:9000 \
     	--s3.bucket bazel-remote \
     	--s3.prefix files \
+    	--s3.auth_method access_key \
     	--s3.access_key_id minioadmin \
     	--s3.secret_access_key minioadmin \
     	--s3.disable_ssl

cache/s3proxy/s3proxy.go

@@ -54,10 +54,8 @@ func New(
 	Endpoint string,
 	Bucket string,
 	Prefix string,
-	AccessKeyID string,
-	SecretAccessKey string,
+	Credentials *credentials.Credentials,
 	DisableSSL bool,
-	IAMRoleEndpoint string,
 	Region string,
 
 	storageMode string, accessLogger cache.Logger,
@@ -68,38 +66,20 @@ func New(
 	var minioCore *minio.Core
 	var err error
 
-	if AccessKeyID != "" && SecretAccessKey != "" {
-		// Initialize minio client object.
-		opts := &minio.Options{
-			Creds:  credentials.NewStaticV4(AccessKeyID, SecretAccessKey, ""),
-			Secure: !DisableSSL,
-			Region: Region,
-		}
-		minioCore, err = minio.NewCore(Endpoint, opts)
-		if err != nil {
-			log.Fatalln(err)
-		}
-	} else {
-		// Initialize minio client object with IAM credentials
-		opts := &minio.Options{
-			// This config value may be empty.
-			Creds: credentials.NewIAM(IAMRoleEndpoint),
-
-			Region: Region,
-			Secure: !DisableSSL,
-		}
+	if Credentials == nil {
+		log.Fatalf("Failed to determine s3proxy credentials")
+	}
 
-		minioClient, err := minio.New(
-			Endpoint,
-			opts,
-		)
-		if err != nil {
-			log.Fatalln(err)
-		}
+	// Initialize minio client with credentials
+	opts := &minio.Options{
+		Creds: Credentials,
 
-		minioCore = &minio.Core{
-			Client: minioClient,
-		}
+		Region: Region,
+		Secure: !DisableSSL,
+	}
+	minioCore, err = minio.NewCore(Endpoint, opts)
+	if err != nil {
+		log.Fatalln(err)
 	}
 
 	if storageMode != "zstd" && storageMode != "uncompressed" {

config/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
         "config.go",
         "logger.go",
         "proxy.go",
+        "s3.go",
         "tls.go",
     ],
     importpath = "github.com/buchgr/bazel-remote/config",
@@ -15,6 +16,7 @@ go_library(
         "//cache/gcsproxy:go_default_library",
         "//cache/httpproxy:go_default_library",
         "//cache/s3proxy:go_default_library",
+        "@com_github_minio_minio_go_v7//pkg/credentials:go_default_library",
         "@com_github_urfave_cli_v2//:go_default_library",
         "@in_gopkg_yaml_v2//:go_default_library",
     ],

config/config.go

@@ -12,24 +12,12 @@ import (
 	"time"
 
 	"github.com/buchgr/bazel-remote/cache"
+	"github.com/buchgr/bazel-remote/cache/s3proxy"
 
 	"github.com/urfave/cli/v2"
 	yaml "gopkg.in/yaml.v2"
 )
 
-// S3CloudStorageConfig stores the configuration of an S3 API proxy backend.
-type S3CloudStorageConfig struct {
-	Endpoint        string `yaml:"endpoint"`
-	Bucket          string `yaml:"bucket"`
-	Prefix          string `yaml:"prefix"`
-	AccessKeyID     string `yaml:"access_key_id"`
-	SecretAccessKey string `yaml:"secret_access_key"`
-	DisableSSL      bool   `yaml:"disable_ssl"`
-	IAMRoleEndpoint string `yaml:"iam_role_endpoint"`
-	Region          string `yaml:"region"`
-	KeyVersion      *int   `yaml:"key_version"`
-}
-
 // GoogleCloudStorageConfig stores the configuration of a GCS proxy backend.
 type GoogleCloudStorageConfig struct {
 	Bucket                string `yaml:"bucket"`
@@ -269,8 +257,8 @@ func validateConfig(c *Config) error {
 	}
 
 	if c.S3CloudStorage != nil {
-		if c.S3CloudStorage.AccessKeyID != "" && c.S3CloudStorage.IAMRoleEndpoint != "" {
-			return errors.New("Expected either 's3.access_key_id' or 's3.iam_role_endpoint', found both")
+		if !s3proxy.IsValidAuthMethod(c.S3CloudStorage.AuthMethod) {
+			return fmt.Errorf("invalid s3.auth_method: %s", c.S3CloudStorage.AuthMethod)
 		}
 
 		if c.S3CloudStorage.KeyVersion != nil && *c.S3CloudStorage.KeyVersion != 2 {
@@ -335,14 +323,17 @@ func get(ctx *cli.Context) (*Config, error) {
 	var s3 *S3CloudStorageConfig
 	if ctx.String("s3.bucket") != "" {
 		s3 = &S3CloudStorageConfig{
-			Endpoint:        ctx.String("s3.endpoint"),
-			Bucket:          ctx.String("s3.bucket"),
-			Prefix:          ctx.String("s3.prefix"),
-			AccessKeyID:     ctx.String("s3.access_key_id"),
-			SecretAccessKey: ctx.String("s3.secret_access_key"),
-			DisableSSL:      ctx.Bool("s3.disable_ssl"),
-			IAMRoleEndpoint: ctx.String("s3.iam_role_endpoint"),
-			Region:          ctx.String("s3.region"),
+			Endpoint:                 ctx.String("s3.endpoint"),
+			Bucket:                   ctx.String("s3.bucket"),
+			Prefix:                   ctx.String("s3.prefix"),
+			AuthMethod:               ctx.String("s3.auth_method"),
+			AccessKeyID:              ctx.String("s3.access_key_id"),
+			SecretAccessKey:          ctx.String("s3.secret_access_key"),
+			DisableSSL:               ctx.Bool("s3.disable_ssl"),
+			IAMRoleEndpoint:          ctx.String("s3.iam_role_endpoint"),
+			Region:                   ctx.String("s3.region"),
+			AWSProfile:               ctx.String("s3.aws_profile"),
+			AWSSharedCredentialsFile: ctx.String("s3.aws_shared_credentials_file"),
 		}
 	}
 

config/config_test.go

@@ -179,6 +179,7 @@ s3_proxy:
   endpoint: minio.example.com:9000
   bucket: test-bucket
   prefix: test-prefix
+  auth_method: access_key
   access_key_id: EXAMPLE_ACCESS_KEY
   secret_access_key: EXAMPLE_SECRET_KEY
 `
@@ -197,6 +198,7 @@ s3_proxy:
 			Endpoint:        "minio.example.com:9000",
 			Bucket:          "test-bucket",
 			Prefix:          "test-prefix",
+			AuthMethod:      "access_key",
 			AccessKeyID:     "EXAMPLE_ACCESS_KEY",
 			SecretAccessKey: "EXAMPLE_SECRET_KEY",
 		},

config/proxy.go

@@ -40,14 +40,17 @@ func (c *Config) setProxy() error {
 	}
 
 	if c.S3CloudStorage != nil {
+		creds, err := c.S3CloudStorage.GetCredentials()
+		if err != nil {
+			return err
+		}
+
 		c.ProxyBackend = s3proxy.New(
 			c.S3CloudStorage.Endpoint,
 			c.S3CloudStorage.Bucket,
 			c.S3CloudStorage.Prefix,
-			c.S3CloudStorage.AccessKeyID,
-			c.S3CloudStorage.SecretAccessKey,
+			creds,
 			c.S3CloudStorage.DisableSSL,
-			c.S3CloudStorage.IAMRoleEndpoint,
 			c.S3CloudStorage.Region,
 			c.StorageMode, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads)
 		return nil

config/s3.go

@@ -0,0 +1,47 @@
+package config
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/buchgr/bazel-remote/cache/s3proxy"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// S3CloudStorageConfig stores the configuration of an S3 API proxy backend.
+type S3CloudStorageConfig struct {
+	Endpoint                 string `yaml:"endpoint"`
+	Bucket                   string `yaml:"bucket"`
+	Prefix                   string `yaml:"prefix"`
+	AuthMethod               string `yaml:"auth_method"`
+	AccessKeyID              string `yaml:"access_key_id"`
+	SecretAccessKey          string `yaml:"secret_access_key"`
+	DisableSSL               bool   `yaml:"disable_ssl"`
+	IAMRoleEndpoint          string `yaml:"iam_role_endpoint"`
+	Region                   string `yaml:"region"`
+	KeyVersion               *int   `yaml:"key_version"`
+	AWSProfile               string `yaml:"aws_profile"`
+	AWSSharedCredentialsFile string `yaml:"aws_shared_credentials_file"`
+}
+
+func (s3c S3CloudStorageConfig) GetCredentials() (*credentials.Credentials, error) {
+	if s3c.AuthMethod == s3proxy.AuthMethodAWSCredentialsFile {
+		log.Println("S3 Credentials: using AWS credentials file.")
+		return credentials.NewFileAWSCredentials(s3c.AWSSharedCredentialsFile, s3c.AWSProfile), nil
+	} else if s3c.AuthMethod == s3proxy.AuthMethodAccessKey {
+		if s3c.AccessKeyID == "" {
+			return nil, fmt.Errorf("missing s3.access_key_id for s3.auth_method = '%s'", s3proxy.AuthMethodAccessKey)
+		}
+		if s3c.SecretAccessKey == "" {
+			return nil, fmt.Errorf("missing s3.secret_access_key for s3.auth_method = '%s'", s3proxy.AuthMethodAccessKey)
+		}
+		log.Println("S3 Credentials: using access/secret access key.")
+		return credentials.NewStaticV4(s3c.AccessKeyID, s3c.SecretAccessKey, ""), nil
+	} else if s3c.AuthMethod == s3proxy.AuthMethodIAMRole {
+		// Fall back to getting credentials from IAM
+		log.Println("S3 Credentials: using IAM.")
+		return credentials.NewIAM(s3c.IAMRoleEndpoint), nil
+	}
+
+	return nil, fmt.Errorf("invalid s3.auth_method: %s", s3c.AuthMethod)
+}

utils/flags/BUILD.bazel

@@ -8,7 +8,10 @@ go_library(
     ],
     importpath = "github.com/buchgr/bazel-remote/utils/flags",
     visibility = ["//visibility:public"],
-    deps = ["@com_github_urfave_cli_v2//:go_default_library"],
+    deps = [
+        "//cache/s3proxy:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+    ],
 )
 
 go_test(