Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Avoid ListBuckets to keep Stat() ops on bucket simpler #2699

Merged
merged 1 commit into from
Feb 28, 2019
Merged

Avoid ListBuckets to keep Stat() ops on bucket simpler #2699

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ matrix:
env:
- ARCH=x86_64
- GO111MODULE=on
go: 1.11.4
go: 1.11.5
script:
- diff -au <(gofmt -d *.go) <(printf "")
- diff -au <(gofmt -d cmd) <(printf "")
Expand All @@ -28,7 +28,7 @@ matrix:
env:
- ARCH=x86_64
- GO111MODULE=on
go: 1.11.4
go: 1.11.5
script:
- go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\mc.exe
- for d in $(go list ./...); do go test -v -mod=vendor -race "$d"; done
Expand Down
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ checks:

getdeps:
@GO111MODULE=on
@echo "Installing golint" && go get -u golang.org/x/lint/golint
@echo "Installing golint" && go get golang.org/x/lint/golint
@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
Expand Down
89 changes: 34 additions & 55 deletions cmd/client-s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1012,15 +1012,11 @@ func (c *s3Client) Stat(isIncomplete, isFetchMeta bool, sse encrypt.ServerSide)
}

if object == "" {
exists, e := c.api.BucketExists(bucket)
if e != nil {
return nil, probe.NewError(e)
}
if !exists {
return nil, probe.NewError(BucketDoesNotExist{Bucket: bucket})
content, err := c.bucketStat(bucket)
if err != nil {
return nil, err.Trace(bucket)
}
bucketMetadata := c.bucketStat()
return &bucketMetadata, nil
return content, nil
}

// The following code tries to calculate if a given prefix/object does really exist
Expand Down Expand Up @@ -1474,18 +1470,18 @@ func (c *s3Client) listIncompleteRecursiveInRoutineDirOpt(contentCh chan *client
var allBuckets bool
// List all buckets if bucket and object are empty.
if bucket == "" && object == "" {
var err error
var e error
allBuckets = true
buckets, err = c.api.ListBuckets()
if err != nil {
contentCh <- &clientContent{URL: *c.targetURL, Err: probe.NewError(err)}
buckets, e = c.api.ListBuckets()
if e != nil {
contentCh <- &clientContent{Err: probe.NewError(e)}
return
}
} else if object == "" {
// Get bucket stat if object is empty.
content := c.bucketStat()
cContent = &content
if content.Err != nil {
contentCh <- cContent
content, err := c.bucketStat(bucket)
if err != nil {
contentCh <- &clientContent{Err: err.Trace(bucket)}
return
}
buckets = append(buckets, minio.BucketInfo{Name: bucket, CreationDate: content.Time})
Expand All @@ -1495,7 +1491,7 @@ func (c *s3Client) listIncompleteRecursiveInRoutineDirOpt(contentCh chan *client
content, perr := c.Stat(isIncomplete, false, nil)
cContent = content
if perr != nil {
contentCh <- &clientContent{URL: *c.targetURL, Err: perr}
contentCh <- &clientContent{Err: perr.Trace(bucket)}
return
}
buckets = append(buckets, minio.BucketInfo{Name: bucket, CreationDate: content.Time})
Expand Down Expand Up @@ -1553,21 +1549,15 @@ func (c *s3Client) objectInfo2ClientContent(bucket string, entry minio.ObjectInf
}

// Returns bucket stat info of current bucket.
func (c *s3Client) bucketStat() clientContent {
bucketName, _ := c.url2BucketAndObject()

buckets, err := c.api.ListBuckets()
if err != nil {
return clientContent{Err: probe.NewError(err)}
func (c *s3Client) bucketStat(bucket string) (*clientContent, *probe.Error) {
exists, e := c.api.BucketExists(bucket)
if e != nil {
return nil, probe.NewError(e)
}

for _, bucket := range buckets {
if bucket.Name == bucketName {
return clientContent{URL: *c.targetURL, Time: bucket.CreationDate, Type: os.ModeDir}
}
if !exists {
return nil, probe.NewError(BucketDoesNotExist{Bucket: bucket})
}

return clientContent{Err: probe.NewError(BucketDoesNotExist{Bucket: bucketName})}
return &clientContent{URL: *c.targetURL, Time: time.Unix(0, 0), Type: os.ModeDir}, nil
}

// Recursively lists objects.
Expand Down Expand Up @@ -1618,18 +1608,18 @@ func (c *s3Client) listRecursiveInRoutineDirOpt(contentCh chan *clientContent, d
var allBuckets bool
// List all buckets if bucket and object are empty.
if bucket == "" && object == "" {
var err error
var e error
allBuckets = true
buckets, err = c.api.ListBuckets()
if err != nil {
contentCh <- &clientContent{URL: *c.targetURL, Err: probe.NewError(err)}
buckets, e = c.api.ListBuckets()
if e != nil {
contentCh <- &clientContent{Err: probe.NewError(e)}
return
}
} else if object == "" {
// Get bucket stat if object is empty.
content := c.bucketStat()
cContent = &content
if content.Err != nil {
contentCh <- cContent
content, err := c.bucketStat(bucket)
if err != nil {
contentCh <- &clientContent{Err: err.Trace(bucket)}
return
}
buckets = append(buckets, minio.BucketInfo{Name: bucket, CreationDate: content.Time})
Expand All @@ -1640,7 +1630,7 @@ func (c *s3Client) listRecursiveInRoutineDirOpt(contentCh chan *clientContent, d
content, perr := c.Stat(isIncomplete, isFetchMeta, nil)
cContent = content
if perr != nil {
contentCh <- &clientContent{URL: *c.targetURL, Err: perr}
contentCh <- &clientContent{Err: perr.Trace(bucket)}
return
}
buckets = append(buckets, minio.BucketInfo{Name: bucket, CreationDate: content.Time})
Expand Down Expand Up @@ -1692,23 +1682,12 @@ func (c *s3Client) listInRoutine(contentCh chan *clientContent) {
contentCh <- content
}
case b != "" && !strings.HasSuffix(c.targetURL.Path, string(c.targetURL.Separator)) && o == "":
buckets, e := c.api.ListBuckets()
if e != nil {
contentCh <- &clientContent{
Err: probe.NewError(e),
}
}
for _, bucket := range buckets {
if bucket.Name == b {
content := &clientContent{}
content.URL = *c.targetURL
content.Size = 0
content.Time = bucket.CreationDate
content.Type = os.ModeDir
contentCh <- content
break
}
content, err := c.bucketStat(b)
if err != nil {
contentCh <- &clientContent{Err: err.Trace(b)}
return
}
contentCh <- content
default:
isRecursive := false
for object := range c.listObjectWrapper(b, o, isRecursive, nil) {
Expand Down
1 change: 1 addition & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.18.0 h1:IZl7mfBGfbhYx2p2rKRtYgDFw6SBz+kclmxYrCksPPA=
google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down