diff --git a/api/v1beta2/bucket_types.go b/api/v1beta2/bucket_types.go
index c9b748a54..90312f55e 100644
--- a/api/v1beta2/bucket_types.go
+++ b/api/v1beta2/bucket_types.go
@@ -23,6 +23,7 @@ import (
"github.com/fluxcd/pkg/apis/acl"
"github.com/fluxcd/pkg/apis/meta"
+
apiv1 "github.com/fluxcd/source-controller/api/v1"
)
@@ -73,6 +74,10 @@ type BucketSpec struct {
// +optional
Region string `json:"region,omitempty"`
+ // Prefix to use for server-side filtering of files in the Bucket.
+ // +optional
+ Prefix string `json:"prefix,omitempty"`
+
// SecretRef specifies the Secret containing authentication credentials
// for the Bucket.
// +optional
diff --git a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
index 57e644a88..2ef2fb603 100644
--- a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
+++ b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
@@ -331,6 +331,10 @@ spec:
to ensure efficient use of resources.
pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
type: string
+ prefix:
+ description: Prefix to use for server-side filtering of files in the
+ Bucket.
+ type: string
provider:
default: generic
description: Provider of the object storage bucket. Defaults to 'generic',
diff --git a/docs/api/v1beta2/source.md b/docs/api/v1beta2/source.md
index 3d58db692..60599e235 100644
--- a/docs/api/v1beta2/source.md
+++ b/docs/api/v1beta2/source.md
@@ -138,6 +138,18 @@ string
+prefix
+
+string
+
+ |
+
+(Optional)
+ Prefix to use for server-side filtering of files in the Bucket.
+ |
+
+
+
secretRef
@@ -1422,6 +1434,18 @@ string
|
+prefix
+
+string
+
+ |
+
+(Optional)
+ Prefix to use for server-side filtering of files in the Bucket.
+ |
+
+
+
secretRef
diff --git a/docs/spec/v1beta2/buckets.md b/docs/spec/v1beta2/buckets.md
index eb7eb8018..14d6a0d08 100644
--- a/docs/spec/v1beta2/buckets.md
+++ b/docs/spec/v1beta2/buckets.md
@@ -785,6 +785,15 @@ credentials for the object storage. For some `.spec.provider` implementations
the presence of the field is required, see [Provider](#provider) for more
details and examples.
+### Prefix
+
+`.spec.prefix` is an optional field to enable server-side filtering
+of files in the Bucket.
+
+**Note:** The server-side filtering works only with the `generic`, `aws`
+and `gcp` [provider](#provider) and is preferred over [`.spec.ignore`](#ignore)
+as a more efficient way of excluding files.
+
### Ignore
`.spec.ignore` is an optional field to specify rules in [the `.gitignore`
diff --git a/internal/controller/bucket_controller.go b/internal/controller/bucket_controller.go
index 29c3c5da2..c5c3267d2 100644
--- a/internal/controller/bucket_controller.go
+++ b/internal/controller/bucket_controller.go
@@ -145,7 +145,7 @@ type BucketProvider interface {
// bucket, calling visit for every item.
// If the underlying client or the visit callback returns an error,
// it returns early.
- VisitObjects(ctx context.Context, bucketName string, visit func(key, etag string) error) error
+ VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(key, etag string) error) error
// ObjectIsNotFound returns true if the given error indicates an object
// could not be found.
ObjectIsNotFound(error) bool
@@ -742,7 +742,7 @@ func fetchEtagIndex(ctx context.Context, provider BucketProvider, obj *bucketv1.
matcher := sourceignore.NewMatcher(ps)
// Build up index
- err = provider.VisitObjects(ctxTimeout, obj.Spec.BucketName, func(key, etag string) error {
+ err = provider.VisitObjects(ctxTimeout, obj.Spec.BucketName, obj.Spec.Prefix, func(key, etag string) error {
if strings.HasSuffix(key, "/") || key == sourceignore.IgnoreFile {
return nil
}
diff --git a/internal/controller/bucket_controller_fetch_test.go b/internal/controller/bucket_controller_fetch_test.go
index e8fb629d7..b31568ff8 100644
--- a/internal/controller/bucket_controller_fetch_test.go
+++ b/internal/controller/bucket_controller_fetch_test.go
@@ -69,7 +69,7 @@ func (m mockBucketClient) ObjectIsNotFound(e error) bool {
return e == errMockNotFound
}
-func (m mockBucketClient) VisitObjects(_ context.Context, _ string, f func(key, etag string) error) error {
+func (m mockBucketClient) VisitObjects(_ context.Context, _ string, _ string, f func(key, etag string) error) error {
for key, obj := range m.objects {
if err := f(key, obj.etag); err != nil {
return err
diff --git a/pkg/azure/blob.go b/pkg/azure/blob.go
index 89e85b4a2..940f429b7 100644
--- a/pkg/azure/blob.go
+++ b/pkg/azure/blob.go
@@ -265,7 +265,7 @@ func (c *BlobClient) FGetObject(ctx context.Context, bucketName, objectName, loc
// bucket, calling visit for every item.
// If the underlying client or the visit callback returns an error,
// it returns early.
-func (c *BlobClient) VisitObjects(ctx context.Context, bucketName string, visit func(path, etag string) error) error {
+func (c *BlobClient) VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(path, etag string) error) error {
items := c.NewListBlobsFlatPager(bucketName, nil)
for items.More() {
resp, err := items.NextPage(ctx)
diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go
index 419885cbb..77011fada 100644
--- a/pkg/gcp/gcp.go
+++ b/pkg/gcp/gcp.go
@@ -165,8 +165,10 @@ func (c *GCSClient) FGetObject(ctx context.Context, bucketName, objectName, loca
// bucket, calling visit for every item.
// If the underlying client or the visit callback returns an error,
// it returns early.
-func (c *GCSClient) VisitObjects(ctx context.Context, bucketName string, visit func(path, etag string) error) error {
- items := c.Client.Bucket(bucketName).Objects(ctx, nil)
+func (c *GCSClient) VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(path, etag string) error) error {
+ items := c.Client.Bucket(bucketName).Objects(ctx, &gcpstorage.Query{
+ Prefix: prefix,
+ })
for {
object, err := items.Next()
if err == IteratorDone {
diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go
index fb65bc1b9..53989aafe 100644
--- a/pkg/gcp/gcp_test.go
+++ b/pkg/gcp/gcp_test.go
@@ -170,7 +170,7 @@ func TestVisitObjects(t *testing.T) {
}
keys := []string{}
etags := []string{}
- err := gcpClient.VisitObjects(context.Background(), bucketName, func(key, etag string) error {
+ err := gcpClient.VisitObjects(context.Background(), bucketName, "", func(key, etag string) error {
keys = append(keys, key)
etags = append(etags, etag)
return nil
@@ -185,7 +185,7 @@ func TestVisitObjectsErr(t *testing.T) {
Client: client,
}
badBucketName := "bad-bucket"
- err := gcpClient.VisitObjects(context.Background(), badBucketName, func(key, etag string) error {
+ err := gcpClient.VisitObjects(context.Background(), badBucketName, "", func(key, etag string) error {
return nil
})
assert.Error(t, err, fmt.Sprintf("listing objects from bucket '%s' failed: storage: bucket doesn't exist", badBucketName))
@@ -196,7 +196,7 @@ func TestVisitObjectsCallbackErr(t *testing.T) {
Client: client,
}
mockErr := fmt.Errorf("mock")
- err := gcpClient.VisitObjects(context.Background(), bucketName, func(key, etag string) error {
+ err := gcpClient.VisitObjects(context.Background(), bucketName, "", func(key, etag string) error {
return mockErr
})
assert.Error(t, err, mockErr.Error())
diff --git a/pkg/minio/minio.go b/pkg/minio/minio.go
index deaa2f98f..7343f753e 100644
--- a/pkg/minio/minio.go
+++ b/pkg/minio/minio.go
@@ -105,9 +105,10 @@ func (c *MinioClient) FGetObject(ctx context.Context, bucketName, objectName, lo
// bucket, calling visit for every item.
// If the underlying client or the visit callback returns an error,
// it returns early.
-func (c *MinioClient) VisitObjects(ctx context.Context, bucketName string, visit func(key, etag string) error) error {
+func (c *MinioClient) VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(key, etag string) error) error {
for object := range c.Client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{
Recursive: true,
+ Prefix: prefix,
UseV1: s3utils.IsGoogleEndpoint(*c.Client.EndpointURL()),
}) {
if object.Err != nil {
diff --git a/pkg/minio/minio_test.go b/pkg/minio/minio_test.go
index 3e1598157..40eb3deee 100644
--- a/pkg/minio/minio_test.go
+++ b/pkg/minio/minio_test.go
@@ -36,6 +36,7 @@ import (
"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/sourceignore"
+
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
)
@@ -62,6 +63,7 @@ var (
var (
bucketName = "test-bucket-minio" + uuid.New().String()
+ prefix = ""
secret = corev1.Secret{
ObjectMeta: v1.ObjectMeta{
Name: "minio-secret",
@@ -228,7 +230,7 @@ func TestFGetObjectNotExists(t *testing.T) {
func TestVisitObjects(t *testing.T) {
keys := []string{}
etags := []string{}
- err := testMinioClient.VisitObjects(context.TODO(), bucketName, func(key, etag string) error {
+ err := testMinioClient.VisitObjects(context.TODO(), bucketName, prefix, func(key, etag string) error {
keys = append(keys, key)
etags = append(etags, etag)
return nil
@@ -241,7 +243,7 @@ func TestVisitObjects(t *testing.T) {
func TestVisitObjectsErr(t *testing.T) {
ctx := context.Background()
badBucketName := "bad-bucket"
- err := testMinioClient.VisitObjects(ctx, badBucketName, func(string, string) error {
+ err := testMinioClient.VisitObjects(ctx, badBucketName, prefix, func(string, string) error {
return nil
})
assert.Error(t, err, fmt.Sprintf("listing objects from bucket '%s' failed: The specified bucket does not exist", badBucketName))
@@ -249,7 +251,7 @@ func TestVisitObjectsErr(t *testing.T) {
func TestVisitObjectsCallbackErr(t *testing.T) {
mockErr := fmt.Errorf("mock")
- err := testMinioClient.VisitObjects(context.TODO(), bucketName, func(key, etag string) error {
+ err := testMinioClient.VisitObjects(context.TODO(), bucketName, prefix, func(key, etag string) error {
return mockErr
})
assert.Error(t, err, mockErr.Error())
|