Merge branch 'master' into master

meltwater · Feb 2, 2021 · 186e697 · 186e697
2 parents 31dfe25 + 05a7985
commit 186e697
Show file tree

Hide file tree

Showing 8 changed files with 104 additions and 58 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -17,11 +17,15 @@ steps:
       - git fetch --tags
 
   - name: configure-buckets
-    image: minio/mc:RELEASE.2018-09-26T00-42-43Z
+    image: minio/mc:RELEASE.2020-10-03T02-54-56Z
     commands:
       - sleep 5
       - mc config host add minio http://minio:9000 AKIAIOSFODNN7EXAMPLE wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       - mc mb --region=eu-west-1 minio/drone-cache-bucket
+      - mc admin user add minio foo barbarbar
+      - "echo '{\"Version\": \"2012-10-17\", \"Statement\": [ { \"Action\": [ \"s3:GetObject\", \"s3:PutObject\", \"s3:DeleteObject\", \"s3:CreateBucket\", \"s3:DeleteBucket\" ], \"Effect\": \"Allow\", \"Resource\": [ \"arn:aws:s3:::s3-round-trip-with-role/*\", \"arn:aws:s3:::s3-round-trip-with-role\" ], \"Sid\": \"\" } ] }' >> /tmp/policy.json"
+      - mc admin policy add minio userpolicy /tmp/policy.json
+      - mc admin policy set minio userpolicy user=foo
 
   - name: build
     image: golang:1.14.4-alpine3.12
@@ -206,7 +210,7 @@ steps:
 
 services:
   - name: minio
-    image: minio/minio:RELEASE.2020-03-05T01-04-19Z
+    image: minio/minio:RELEASE.2020-11-06T23-17-07Z
     commands:
       - minio server /data
     environment:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Fixes [#132](https://github.com/meltwater/drone-cache/issues/132)
 - [#138](https://github.com/meltwater/drone-cache/pull/138) backend/gcs: Fix GCS to pass credentials correctly when `GCS_ENDPOINT` is not set.
 - [#135](https://github.com/meltwater/drone-cache/issues/135) backend/gcs: Fixed parsing of GCS JSON key.
-
+- [#142](https://github.com/meltwater/drone-cache/issues/142) backend/s3: Add option to assume AWS IAM role
 ### Added
 
 - [#102](https://github.com/meltwater/drone-cache/pull/102) Implement option to disable cache rebuild if it already exists in storage.

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # drone-cache
 
-[![Latest Release](https://img.shields.io/github/release/meltwater/drone-cache.svg?)](https://github.com/meltwater/drone-cache/releases/latest) [![Maintenance](https://img.shields.io/maintenance/yes/2020.svg)](https://github.com/meltwater/drone-cache/commits/master) ![GitHub](https://img.shields.io/github/license/meltwater/drone-cache) [![Gitter](https://badges.gitter.im/drone-cache/community.svg)](https://gitter.im/drone-cache/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![drone](https://cloud.drone.io/api/badges/meltwater/drone-cache/status.svg)](https://cloud.drone.io/meltwater/drone-cache) ![release](https://github.com/meltwater/drone-cache/workflows/release/badge.svg) ![snapshot](https://github.com/meltwater/drone-cache/workflows/snapshot/badge.svg)
+[![Latest Release](https://img.shields.io/github/release/meltwater/drone-cache.svg?)](https://github.com/meltwater/drone-cache/releases/latest) [![Maintenance](https://img.shields.io/maintenance/yes/2021.svg)](https://github.com/meltwater/drone-cache/commits/master) ![GitHub](https://img.shields.io/github/license/meltwater/drone-cache) [![drone](https://cloud.drone.io/api/badges/meltwater/drone-cache/status.svg)](https://cloud.drone.io/meltwater/drone-cache) ![release](https://github.com/meltwater/drone-cache/workflows/release/badge.svg) ![snapshot](https://github.com/meltwater/drone-cache/workflows/snapshot/badge.svg)
 
 [![Go Doc](https://godoc.org/github.com/meltwater/drone-cache?status.svg)](http://godoc.org/github.com/meltwater/drone-cache) [![Go Code reference](https://img.shields.io/badge/code%20reference-go.dev-darkblue.svg)](https://pkg.go.dev/github.com/meltwater/drone-cache?tab=subdirectories) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2713/badge)](https://bestpractices.coreinfrastructure.org/projects/2713) [![Go Report Card](https://goreportcard.com/badge/github.com/meltwater/drone-cache)](https://goreportcard.com/report/github.com/meltwater/drone-cache) [![codebeat badge](https://codebeat.co/badges/802c6149-ac2d-4514-8648-f618c63a8d9e)](https://codebeat.co/projects/github-com-meltwater-drone-cache-master)
 
@@ -24,6 +24,8 @@ If you want to learn more about the story behind `drone-cache`, you can read our
   * Other AWS API compatible stores:
     * [Minio](https://min.io/)
     * [Red Hat Ceph](https://www.redhat.com/en/technologies/storage/ceph)
+    * [IBM Object Store](https://www.ibm.com/cloud/object-storage)
+    * and many many others
 * [Azure Storage](https://azure.microsoft.com/en-us/services/storage/blobs/)
   * [Configuration](#)
   * [Example](#)
@@ -178,6 +180,7 @@ GLOBAL OPTIONS:
    --path-style                          AWS path style to use for bucket paths. (true for minio, false for aws) (default: false) [$PLUGIN_PATH_STYLE, $AWS_PLUGIN_PATH_STYLE]
    --acl value                           upload files with acl (private, public-read, ...) (default: "private") [$PLUGIN_ACL, $AWS_ACL]
    --encryption value                    server-side encryption algorithm, defaults to none. (AES256, aws:kms) [$PLUGIN_ENCRYPTION, $AWS_ENCRYPTION]
+   --role-arn value                      AWS IAM role ARN to assume [$PLUGIN_ASSUME_ROLE_ARN, $AWS_ASSUME_ROLE_ARN]
    --gcs.api-key value                   Google service account API key [$PLUGIN_API_KEY, $GCP_API_KEY]
    --gcs.json-key value                  Google service account JSON key [$PLUGIN_JSON_KEY, $GCS_CACHE_JSON_KEY]
    --gcs.acl value                       upload files with acl (private, public-read, ...) (default: "private") [$PLUGIN_GCS_ACL, $GCS_ACL]
@@ -242,30 +245,6 @@ Targets:
   help           	  Shows this help message
 ```
 
-```txt
-Usage:
-  make <target>
-
-Targets:
-  setup          	  Setups dev environment
-  drone-cache    	  Runs drone-cache target
-  build          	  Runs build target, always builds
-  clean          	  Cleans build resourcess
-  docs           	  Generates docs
-  generate       	  Generate documentation, website and yaml files,
-  vendor         	  Updates vendored copy of dependencies
-  compress       	  Creates compressed binary
-  container      	  Builds drone-cache docker image with latest tag
-  container-push 	  Pushes latest $(CONTAINER_REPO) image to repository
-  test           	  Runs tests
-  test-integration	  Runs integration tests
-  test-unit      	  Runs unit tests
-  lint           	  Runs golangci-lint analysis
-  fix            	  Runs golangci-lint fix
-  format         	  Runs gofmt
-  help           	  Shows this help message
-```
-
 ## Releases
 
 Release management handled by the CI pipeline. When you create a tag on `master` branch, CI handles the rest.

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '3'
 services:
   minio:
-    image: minio/minio:RELEASE.2020-03-05T01-04-19Z
+    image: minio/minio:RELEASE.2020-11-06T23-17-07Z
     environment:
       MINIO_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE
       MINIO_SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
@@ -25,8 +25,16 @@ services:
     - "10000:10000"
     command: azurite-blob --blobHost 0.0.0.0
   configure-buckets:
-    image: minio/mc:RELEASE.2020-02-20T23-49-54Z
+    image: minio/mc:RELEASE.2020-10-03T02-54-56Z
     entrypoint: sh
     depends_on:
       - minio
-    command: -c "sleep 5 && mc config host add minio http://minio:9000 AKIAIOSFODNN7EXAMPLE wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+    command:
+      - -c
+      - |
+        sleep 2 &&
+        mc config host add minio http://minio:9000 AKIAIOSFODNN7EXAMPLE wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY &&
+        mc admin user add minio foo barbarbar &&
+        echo '{"Version": "2012-10-17", "Statement": [{"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:CreateBucket", "s3:DeleteBucket"], "Effect": "Allow", "Resource": ["arn:aws:s3:::s3-round-trip-with-role/*", "arn:aws:s3:::s3-round-trip-with-role"], "Sid": ""}]}' > /tmp/policy.json &&
+        mc admin policy add minio userpolicy /tmp/policy.json &&
+        mc admin policy set minio userpolicy user=foo
diff --git a/main.go b/main.go
@@ -363,6 +363,12 @@ func main() {
 			Usage:   "server-side encryption algorithm, defaults to none. (AES256, aws:kms)",
 			EnvVars: []string{"PLUGIN_ENCRYPTION", "AWS_ENCRYPTION"},
 		},
+		&cli.StringFlag{
+			Name:    "role-arn",
+			Usage:   "AWS IAM role ARN to assume",
+			Value:   "",
+			EnvVars: []string{"PLUGIN_ASSUME_ROLE_ARN", "AWS_ASSUME_ROLE_ARN"},
+		},
 
 		// GCS specific Configs flags
 

diff --git a/storage/backend/s3/config.go b/storage/backend/s3/config.go
@@ -15,6 +15,7 @@ type Config struct {
 	Encryption string // if not "", enables server-side encryption. valid values are: AES256, aws:kms.
 	Endpoint   string
 	Key        string
+	RoleArn    string // if "", do not assume IAM role i.e. use the IAM user.
 
 	// us-east-1
 	// us-west-1

diff --git a/storage/backend/s3/s3.go b/storage/backend/s3/s3.go
@@ -9,9 +9,11 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 
@@ -44,6 +46,12 @@ func New(l log.Logger, c Config, debug bool) (*Backend, error) {
 		level.Warn(l).Log("msg", "aws key and/or Secret not provided (falling back to anonymous credentials)")
 	}
 
+	if c.RoleArn != "" {
+		conf.Credentials = credentials.NewStaticCredentials(c.Key, c.Secret, "")
+		crds := assumeRole(l, conf, c.RoleArn)
+		conf.Credentials = credentials.NewStaticCredentials(crds.AccessKeyID, crds.SecretAccessKey, crds.SessionToken)
+	}
+
 	level.Debug(l).Log("msg", "s3 backend", "config", fmt.Sprintf("%#v", c))
 
 	if debug {
@@ -138,3 +146,20 @@ func (b *Backend) Exists(ctx context.Context, p string) (bool, error) {
 	// Minio can return success status for without ETag, detect that here.
 	return *out.ETag != "", nil
 }
+
+func assumeRole(l log.Logger, c *aws.Config, roleArn string) credentials.Value {
+	client := sts.New(session.Must(session.NewSessionWithOptions(session.Options{})), c)
+
+	stsProvider := stscreds.AssumeRoleProvider{
+		Client:          client,
+		RoleARN:         roleArn,
+		RoleSessionName: "drone-cache",
+	}
+
+	role, err := stsProvider.Retrieve()
+	if err != nil {
+		level.Error(l).Log("msg", "s3 backend", "assume-role", err.Error())
+	}
+
+	return role
+}
diff --git a/storage/backend/s3/s3_test.go b/storage/backend/s3/s3_test.go
@@ -20,26 +20,58 @@ import (
 )
 
 const (
-	defaultEndpoint        = "127.0.0.1:9000"
-	defaultAccessKey       = "AKIAIOSFODNN7EXAMPLE"
-	defaultSecretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
-	defaultRegion          = "eu-west-1"
-	defaultACL             = "private"
+	defaultEndpoint            = "127.0.0.1:9000"
+	defaultAccessKey           = "AKIAIOSFODNN7EXAMPLE"
+	defaultSecretAccessKey     = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+	defaultRegion              = "eu-west-1"
+	defaultACL             	   = "private"
+	defaultUserAccessKey   	   = "foo"
+	defaultUserSecretAccessKey = "barbarbar"
 )
 
 var (
-	endpoint        = getEnv("TEST_S3_ENDPOINT", defaultEndpoint)
-	accessKey       = getEnv("TEST_S3_ACCESS_KEY", defaultAccessKey)
-	secretAccessKey = getEnv("TEST_S3_SECRET_KEY", defaultSecretAccessKey)
-	acl             = getEnv("TEST_S3_ACL", defaultACL)
+	endpoint        	= getEnv("TEST_S3_ENDPOINT", defaultEndpoint)
+	accessKey       	= getEnv("TEST_S3_ACCESS_KEY", defaultAccessKey)
+	secretAccessKey 	= getEnv("TEST_S3_SECRET_KEY", defaultSecretAccessKey)
+	acl             	= getEnv("TEST_S3_ACL", defaultACL)
+	userAccessKey       = getEnv("TEST_USER_S3_ACCESS_KEY", defaultUserAccessKey)
+	userSecretAccessKey = getEnv("TEST_USER_S3_SECRET_KEY", defaultUserSecretAccessKey)
 )
 
 func TestRoundTrip(t *testing.T) {
 	t.Parallel()
 
-	backend, cleanUp := setup(t)
+	backend, cleanUp := setup(t, Config{
+		ACL:       acl,
+		Bucket:    "s3-round-trip",
+		Endpoint:  endpoint,
+		Key:       accessKey,
+		PathStyle: true, // Should be true for minio and false for AWS.
+		Region:    defaultRegion,
+		Secret:    secretAccessKey,
+	})
+	t.Cleanup(cleanUp)
+	roundTrip(t, backend)
+}
+
+func TestRoundTripWithAssumeRole(t *testing.T) {
+	t.Parallel()
+
+	backend, cleanUp := setup(t, Config{
+		ACL:       acl,
+		Bucket:    "s3-round-trip-with-role",
+		Endpoint:  endpoint,
+		Key:       userAccessKey,
+		PathStyle: true, // Should be true for minio and false for AWS.
+		Region:    defaultRegion,
+		Secret:    userSecretAccessKey,
+		RoleArn:   "arn:aws:iam::account-id:role/TestRole",
+	})
 	t.Cleanup(cleanUp)
+	roundTrip(t, backend)
+}
 
+func roundTrip(t *testing.T, backend *Backend) {
 	content := "Hello world4"
 
 	// Test Put
@@ -62,44 +94,35 @@ func TestRoundTrip(t *testing.T) {
 
 // Helpers
 
-func setup(t *testing.T) (*Backend, func()) {
-	client := newClient()
-	bucket := "s3-round-trip"
+func setup(t *testing.T, config Config) (*Backend, func()) {
+	client := newClient(config)
 
 	_, err := client.CreateBucketWithContext(context.Background(), &s3.CreateBucketInput{
-		Bucket: aws.String(bucket),
+		Bucket: aws.String(config.Bucket),
 	})
 	test.Ok(t, err)
 
 	b, err := New(
 		log.NewNopLogger(),
-		Config{
-			ACL:       acl,
-			Bucket:    bucket,
-			Endpoint:  endpoint,
-			Key:       accessKey,
-			PathStyle: true, // Should be true for minio and false for AWS.
-			Region:    defaultRegion,
-			Secret:    secretAccessKey,
-		},
+		config,
 		false,
 	)
 	test.Ok(t, err)
 
 	return b, func() {
-		_, _ = client.DeleteBucket(&s3.DeleteBucketInput{
-			Bucket: aws.String(bucket),
+		_, err = client.DeleteBucket(&s3.DeleteBucketInput{
+			Bucket: aws.String(config.Bucket),
 		})
 	}
 }
 
-func newClient() *s3.S3 {
+func newClient(config Config) *s3.S3 {
 	conf := &aws.Config{
 		Region:           aws.String(defaultRegion),
 		Endpoint:         aws.String(endpoint),
 		DisableSSL:       aws.Bool(!strings.HasPrefix(endpoint, "https://")),
 		S3ForcePathStyle: aws.Bool(true),
-		Credentials:      credentials.NewStaticCredentials(accessKey, secretAccessKey, ""),
+		Credentials:      credentials.NewStaticCredentials(config.Key, config.Secret, ""),
 	}
 
 	return s3.New(session.Must(session.NewSessionWithOptions(session.Options{})), conf)