chore: set limit on aws max objects

flanksource · Aug 27, 2024 · 48adc89 · 48adc89
1 parent b0d10be
commit 48adc89
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 14 deletions.
diff --git a/checks/folder.go b/checks/folder.go
@@ -6,9 +6,10 @@ import (
 	"os"
 	"strings"
 
+	"github.com/flanksource/artifacts"
+	artifactFS "github.com/flanksource/artifacts/fs"
 	"github.com/prometheus/client_golang/prometheus"
 
-	"github.com/flanksource/artifacts"
 	"github.com/flanksource/canary-checker/api/context"
 	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
@@ -111,18 +112,18 @@ func checkLocalFolder(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	return results
 }
 
-func genericFolderCheck(dirFS artifacts.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
+func genericFolderCheck(dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
 	return _genericFolderCheck(true, dirFS, path, recursive, filter)
 }
 
 // genericFolderCheckWithoutPrecheck is used for those filesystems that do not support fetching the stat of a directory.
 // Eg: s3, gcs.
 // It will not pre check whether the given path is a directory.
-func genericFolderCheckWithoutPrecheck(dirFS artifacts.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
+func genericFolderCheckWithoutPrecheck(dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
 	return _genericFolderCheck(false, dirFS, path, recursive, filter)
 }
 
-func _genericFolderCheck(supportsDirStat bool, dirFS artifacts.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
+func _genericFolderCheck(supportsDirStat bool, dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) {
 	result := FolderCheck{}
 	_filter, err := filter.New()
 	if err != nil {
@@ -170,7 +171,7 @@ func _genericFolderCheck(supportsDirStat bool, dirFS artifacts.Filesystem, path
 
 // getFolderContents walks the folder and returns all files.
 // Also supports recursively fetching contents
-func getFolderContents(dirFs artifacts.Filesystem, path string, filter *v1.FolderFilterContext) ([]fs.FileInfo, error) {
+func getFolderContents(dirFs artifactFS.Filesystem, path string, filter *v1.FolderFilterContext) ([]fs.FileInfo, error) {
 	files, err := dirFs.ReadDir(path)
 	if err != nil {
 		return nil, err

diff --git a/checks/folder_s3.go b/checks/folder_s3.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/flanksource/artifacts"
+	artifactFS "github.com/flanksource/artifacts/fs"
 	"github.com/flanksource/canary-checker/api/context"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
@@ -18,10 +19,6 @@ type S3 struct {
 	Bucket string
 }
 
-type Pager interface {
-	SetPageSize(maxList int)
-}
-
 func CheckS3Bucket(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 	result := pkg.Success(check, ctx.Canary)
 	var results pkg.Results
@@ -46,8 +43,8 @@ func CheckS3Bucket(ctx *context.Context, check v1.FolderCheck) pkg.Results {
 		return results.ErrorMessage(err)
 	}
 
-	if pagerFS, ok := fs.(Pager); ok {
-		pagerFS.SetPageSize(ctx.Properties().Int("s3.list.max-objects", 1000))
+	if limitFS, ok := fs.(artifactFS.ListItemLimiter); ok {
+		limitFS.SetMaxListItems(ctx.Properties().Int("s3.list.max-objects", 50_000))
 	}
 
 	folders, err := genericFolderCheckWithoutPrecheck(fs, check.Path, check.Recursive, check.Filter)

diff --git a/go.mod b/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/eko/gocache/lib/v4 v4.1.6
 	github.com/eko/gocache/store/bigcache/v4 v4.2.1
 	github.com/elastic/go-elasticsearch/v8 v8.13.1
-	github.com/flanksource/artifacts v1.0.13
+	github.com/flanksource/artifacts v1.0.14
 	github.com/flanksource/commons v1.28.0
 	github.com/flanksource/duty v1.0.606
 	github.com/flanksource/gomplate/v3 v3.24.24

diff --git a/go.sum b/go.sum
@@ -855,8 +855,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7TtkWXXJ+YRsxpKMDhxK0=
 github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw=
-github.com/flanksource/artifacts v1.0.13 h1:4M4GprCsxyHhMQtXAe5qicanALhjzUgC2dJ6jZPVrFU=
-github.com/flanksource/artifacts v1.0.13/go.mod h1:HyPcnBGjpn5+j0F2nMB0pffITzia4OX5REUVF3GUvhU=
+github.com/flanksource/artifacts v1.0.14 h1:Vv70bccsae0MwGaf/uSPp34J5V1/PyKfct9z5JYCTJU=
+github.com/flanksource/artifacts v1.0.14/go.mod h1:qHVCnQu5k50aWNJ5UhpcAKEl7pAzqUrFFKGSm147G70=
 github.com/flanksource/commons v1.28.0 h1:7wyTIU0Q1/wgckecRmIGmCS+zcFeAkId5qNzffR9Sho=
 github.com/flanksource/commons v1.28.0/go.mod h1:Q/9SkqJW5QwsVlynOYMVuVQzQtHSBGt1TJkkeX3GRjU=
 github.com/flanksource/duty v1.0.606 h1:7sTMS6hXlEOpYybck5vCIz4bjVBAZy0TKv/9LyucxvU=