Fix LIST benchmark (#160)

After SDK update recursive setting was missing. This would cause all operations to fail. * Use correct options for listing. * Fix object creation progress. * Fix missing error printing when skipped. Fixes #157
minio · Jan 12, 2021 · d3ee60c · d3ee60c
1 parent 0797d45
commit d3ee60c
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 13 deletions.
diff --git a/cli/analyze.go b/cli/analyze.go
@@ -290,16 +290,9 @@ func printAnalysis(ctx *cli.Context, o bench.Operations) {
 		return
 	}
 
-	for i, ops := range aggr.Operations {
+	for _, ops := range aggr.Operations {
 		typ := ops.Type
-		if i > 0 {
-			console.Println("\n-------------------")
-		}
-		fmt.Println("")
-		if ops.Skipped {
-			console.Println("Skipping", typ, "too few samples. Longer benchmark run required for reliable results.")
-			continue
-		}
+		console.Println("\n----------------------------------------")
 
 		opo := ops.ObjectsPerOperation
 		console.SetColor("Print", color.New(color.FgHiWhite))
@@ -327,11 +320,21 @@ func printAnalysis(ctx *cli.Context, o bench.Operations) {
 			console.SetColor("Print", color.New(color.FgHiRed))
 			console.Println("Errors:", ops.Errors)
 			if details {
+				console.SetColor("Print", color.New(color.FgWhite))
+				console.Println("First Errors:")
 				for _, err := range ops.FirstErrors {
-					console.Println(err)
+					console.Println(" *", err)
 				}
+				console.Println("")
 			}
 		}
+
+		if ops.Skipped {
+			console.SetColor("Print", color.New(color.FgHiWhite))
+			console.Println("Skipping", typ, "too few samples. Longer benchmark run required for reliable results.")
+			continue
+		}
+
 		if details {
 			printRequestAnalysis(ctx, ops, details)
 			console.SetColor("Print", color.New(color.FgHiWhite))

diff --git a/go.mod b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/klauspost/compress v1.11.3
 	github.com/minio/cli v1.22.0
 	github.com/minio/mc v0.0.0-20201106215650-63ce578c46e9
-	github.com/minio/md5-simd v1.1.0
+	github.com/minio/md5-simd v1.1.2-0.20210107194844-776275e0c9a7
 	github.com/minio/minio v0.0.0-20201102034248-d8e07f2c41c8
 	github.com/minio/minio-go/v7 v7.0.6
 	github.com/posener/complete v1.2.3

diff --git a/go.sum b/go.sum
@@ -202,6 +202,8 @@ github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo
 github.com/klauspost/cpuid v1.2.4/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
 github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/klauspost/cpuid/v2 v2.0.1 h1:lb04bBEJoAoV48eHs4Eq0UyhmJCkRSdIjQ3uS8WJRM4=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/klauspost/readahead v1.3.1/go.mod h1:AH9juHzNH7xqdqFHrMRSHeH2Ps+vFf+kblDqzPFiLJg=
 github.com/klauspost/reedsolomon v1.9.9/go.mod h1:O7yFFHiQwDR6b2t63KPUpccPtNdp5ADgh1gg4fd12wo=
@@ -245,6 +247,8 @@ github.com/minio/mc v0.0.0-20201106215650-63ce578c46e9 h1:QbmtnKTcl7jnpTFAf0btqo
 github.com/minio/mc v0.0.0-20201106215650-63ce578c46e9/go.mod h1:/Gah+B16Ns5rvIXB0XaRZPE7vKmQJvn8WLAUxiLhDbc=
 github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
+github.com/minio/md5-simd v1.1.2-0.20210107194844-776275e0c9a7 h1:uNbuDyqqb6/j017luX0eAd+E5TvjIYdKszRzk6v0qHM=
+github.com/minio/md5-simd v1.1.2-0.20210107194844-776275e0c9a7/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio v0.0.0-20201102034248-d8e07f2c41c8 h1:yCDFjUBUoE1uNr8dEqjHuwySdAtOyvVONchEVrPPOGQ=
 github.com/minio/minio v0.0.0-20201102034248-d8e07f2c41c8/go.mod h1:640kMkCwiyOX8dheptHYModdUw4HrnUFHKw3McqUXD8=
 github.com/minio/minio-go/v7 v7.0.6-0.20200929220449-755b5633803a/go.mod h1:CSt2ETZNs+bIIhWTse0mcZKZWMGrFU7Er7RR0TmkDYk=

diff --git a/pkg/bench/list.go b/pkg/bench/list.go
@@ -59,6 +59,7 @@ func (d *List) Prepare(ctx context.Context) error {
 	d.Collector = NewCollector()
 	d.objects = make([]generator.Objects, d.Concurrency)
 	var mu sync.Mutex
+	objsCreated := 0
 	var groupErr error
 	for i := 0; i < d.Concurrency; i++ {
 		go func(i int) {
@@ -123,7 +124,8 @@ func (d *List) Prepare(ctx context.Context) error {
 				mu.Lock()
 				obj.Reader = nil
 				d.objects[i] = append(d.objects[i], *obj)
-				d.prepareProgress(float64(len(d.objects)) / float64(objPerPrefix*d.Concurrency))
+				objsCreated++
+				d.prepareProgress(float64(objsCreated) / float64(objPerPrefix*d.Concurrency))
 				mu.Unlock()
 				rcv <- op
 			}
@@ -183,7 +185,7 @@ func (d *List) Start(ctx context.Context, wait chan struct{}) (Operations, error
 				op.Start = time.Now()
 
 				// List all objects with prefix
-				listCh := client.ListObjects(nonTerm, d.Bucket, minio.ListObjectsOptions{WithMetadata: true, Prefix: objs[0].Prefix})
+				listCh := client.ListObjects(nonTerm, d.Bucket, minio.ListObjectsOptions{WithMetadata: true, Prefix: objs[0].Prefix, Recursive: true})
 
 				// Wait for errCh to close.
 				for {