Skip to content

Commit

Permalink
Do concurrent cleanup (#85)
Browse files Browse the repository at this point in the history
Use one client per prefix.

- warp put --host=127.0.0.1:9001 -access-key=minio -secret-key=minio123 -duration=30s -obj.size=1K`. 

It seems to give a nice speedup.
  • Loading branch information
klauspost authored Mar 16, 2020
1 parent 03fe1e2 commit 06a9fd6
Showing 1 changed file with 17 additions and 13 deletions.
30 changes: 17 additions & 13 deletions pkg/bench/benchmark.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,20 +108,31 @@ func (c *Common) createEmptyBucket(ctx context.Context) error {
// deleteAllInBucket will delete all content in a bucket.
// If no prefixes are specified everything in bucket is deleted.
func (c *Common) deleteAllInBucket(ctx context.Context, prefixes ...string) {
doneCh := make(chan struct{})
defer close(doneCh)
cl, done := c.Client()
defer done()
if len(prefixes) == 0 {
prefixes = []string{""}
}
var wg sync.WaitGroup
remove := make(chan string, 1)
errCh := cl.RemoveObjectsWithContext(ctx, c.Bucket, remove)
wg.Add(len(prefixes))
for _, prefix := range prefixes {
go func(prefix string) {
defer wg.Done()

doneCh := make(chan struct{})
defer close(doneCh)
cl, done := c.Client()
defer done()
remove := make(chan string, 10)
errCh := cl.RemoveObjectsWithContext(ctx, c.Bucket, remove)
defer func() {
// Signal we are done
close(remove)
// Wait for deletes to finish
err := <-errCh
if err.Err != nil {
console.Error(err.Err)
}
}()

objects := cl.ListObjectsV2(c.Bucket, prefix, true, doneCh)
for {
select {
Expand All @@ -140,13 +151,6 @@ func (c *Common) deleteAllInBucket(ctx context.Context, prefixes ...string) {
}(prefix)
}
wg.Wait()
// Signal we are done
close(remove)
// Wait for deletes to finish
err := <-errCh
if err.Err != nil {
console.Error(err.Err)
}

}

Expand Down

0 comments on commit 06a9fd6

Please sign in to comment.