More efficient s3 paging

physical/common.go

@@ -0,0 +1,10 @@
+package physical
+
+func appendIfMissing(slice []string, i string) []string {
+	for _, ele := range slice {
+		if ele == i {
+			return slice
+		}
+	}
+	return append(slice, i)
+}

physical/common_test.go

@@ -0,0 +1,40 @@
+package physical
+
+import "testing"
+
+func TestAppendIfMissing(t *testing.T) {
+	keys := []string{}
+
+	keys = appendIfMissing(keys, "foo")
+
+	if len(keys) != 1 {
+		t.Fatalf("expected slice to be length of 1: %v", keys)
+	}
+	if keys[0] != "foo" {
+		t.Fatalf("expected slice to contain key 'foo': %v", keys)
+	}
+
+	keys = appendIfMissing(keys, "bar")
+
+	if len(keys) != 2 {
+		t.Fatalf("expected slice to be length of 2: %v", keys)
+	}
+	if keys[0] != "foo" {
+		t.Fatalf("expected slice to contain key 'foo': %v", keys)
+	}
+	if keys[1] != "bar" {
+		t.Fatalf("expected slice to contain key 'bar': %v", keys)
+	}
+
+	keys = appendIfMissing(keys, "foo")
+
+	if len(keys) != 2 {
+		t.Fatalf("expected slice to still be length of 2: %v", keys)
+	}
+	if keys[0] != "foo" {
+		t.Fatalf("expected slice to still contain key 'foo': %v", keys)
+	}
+	if keys[1] != "bar" {
+		t.Fatalf("expected slice to still contain key 'bar': %v", keys)
+	}
+}

physical/s3.go

@@ -205,24 +205,24 @@ func (s *S3Backend) List(prefix string) ([]string, error) {
 	defer s.permitPool.Release()
 
 	params := &s3.ListObjectsV2Input{
-		Bucket: aws.String(s.bucket),
-		Prefix: aws.String(prefix),
+		Bucket:    aws.String(s.bucket),
+		Prefix:    aws.String(prefix),
+		Delimiter: aws.String("/"),
 	}
 
 	keys := []string{}
 
 	err := s.client.ListObjectsV2Pages(params,
 		func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			// Add truncated 'folder' paths
+			for _, commonPrefix := range page.CommonPrefixes {
+				commonPrefix := strings.TrimPrefix(*commonPrefix.Prefix, prefix)
+				keys = append(keys, commonPrefix)
+			}
+			// Add objects only from the current 'folder'
 			for _, key := range page.Contents {
 				key := strings.TrimPrefix(*key.Key, prefix)
-
-				if i := strings.Index(key, "/"); i == -1 {
-					// Add objects only from the current 'folder'
-					keys = append(keys, key)
-				} else if i != -1 {
-					// Add truncated 'folder' paths
-					keys = appendIfMissing(keys, key[:i+1])
-				}
+				keys = append(keys, key)
 			}
 			return true
 		})
@@ -235,12 +235,3 @@ func (s *S3Backend) List(prefix string) ([]string, error) {
 
 	return keys, nil
 }
-
-func appendIfMissing(slice []string, i string) []string {
-	for _, ele := range slice {
-		if ele == i {
-			return slice
-		}
-	}
-	return append(slice, i)
-}

physical/s3_test.go

@@ -17,13 +17,15 @@ import (
 )
 
 func TestS3Backend(t *testing.T) {
-	if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
-		t.SkipNow()
-	}
+	creds := credentials.NewChainCredentials(
+		[]credentials.Provider{
+			&credentials.EnvProvider{},
+			&credentials.SharedCredentialsProvider{Filename: "", Profile: ""},
+		})
 
-	creds, err := credentials.NewEnvCredentials().Get()
+	credValue, err := creds.Get()
 	if err != nil {
-		t.Fatalf("err: %v", err)
+		t.SkipNow()
 	}
 
 	// If the variable is empty or doesn't exist, the default
@@ -36,7 +38,7 @@ func TestS3Backend(t *testing.T) {
 	}
 
 	s3conn := s3.New(session.New(&aws.Config{
-		Credentials: credentials.NewEnvCredentials(),
+		Credentials: creds,
 		Endpoint:    aws.String(endpoint),
 		Region:      aws.String(region),
 	}))
@@ -78,9 +80,9 @@ func TestS3Backend(t *testing.T) {
 	logger := logformat.NewVaultLogger(log.LevelTrace)
 
 	b, err := NewBackend("s3", logger, map[string]string{
-		"access_key":    creds.AccessKeyID,
-		"secret_key":    creds.SecretAccessKey,
-		"session_token": creds.SessionToken,
+		"access_key":    credValue.AccessKeyID,
+		"secret_key":    credValue.SecretAccessKey,
+		"session_token": credValue.SessionToken,
 		"bucket":        bucket,
 	})
 	if err != nil {