From 39e4235338442275a02b356baa51506385ce3230 Mon Sep 17 00:00:00 2001 From: Kirill Kouzoubov Date: Fri, 26 Nov 2021 20:17:40 +1100 Subject: [PATCH] command: add --request-payer flag allow access to requester pays buckets, issue #297 --- command/app.go | 5 ++++ command/cp.go | 3 ++ command/ls.go | 3 ++ storage/s3.go | 75 ++++++++++++++++++++++++++++------------------ storage/storage.go | 2 ++ 5 files changed, 59 insertions(+), 29 deletions(-) diff --git a/command/app.go b/command/app.go index 5a28165df..0bb7d806e 100644 --- a/command/app.go +++ b/command/app.go @@ -69,6 +69,10 @@ var app = &cli.App{ Name: "no-sign-request", Usage: "do not sign requests: credentials will not be loaded if --no-sign-request is provided", }, + &cli.StringFlag{ + Name: "request-payer", + Usage: "who pays for request (access requester pays buckets)", + }, }, Before: func(c *cli.Context) error { retryCount := c.Int("retry-count") @@ -139,6 +143,7 @@ func NewStorageOpts(c *cli.Context) storage.Options { NoVerifySSL: c.Bool("no-verify-ssl"), DryRun: c.Bool("dry-run"), NoSignRequest: c.Bool("no-sign-request"), + RequestPayer: c.String("request-payer"), } } diff --git a/command/cp.go b/command/cp.go index 0eb6addb3..f8203e066 100644 --- a/command/cp.go +++ b/command/cp.go @@ -94,6 +94,9 @@ Examples: 19. Copy all files from S3 bucket to another S3 bucket but exclude the ones starts with log > s5cmd {{.HelpName}} --exclude "log*" s3://bucket/* s3://destbucket + + 20. Download an S3 object from a requester pays bucket + > s5cmd --request-payer=requester {{.HelpName}} s3://bucket/prefix/object.gz . ` func NewCopyCommandFlags() []cli.Flag { diff --git a/command/ls.go b/command/ls.go index 2720f7eeb..7b0784612 100644 --- a/command/ls.go +++ b/command/ls.go @@ -42,6 +42,9 @@ Examples: 6. List all objects in a bucket but exclude the ones with prefix abc > s5cmd {{.HelpName}} --exclude "abc*" s3://bucket/* + + 7. List all object in a requester pays bucket + > s5cmd --request-payer=requester {{.HelpName}} s3://bucket/* ` func NewListCommand() *cli.Command { diff --git a/storage/s3.go b/storage/s3.go index 7ca292d1d..7d087e72a 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -53,11 +53,19 @@ var globalSessionCache = &SessionCache{ // S3 is a storage type which interacts with S3API, DownloaderAPI and // UploaderAPI. type S3 struct { - api s3iface.S3API - downloader s3manageriface.DownloaderAPI - uploader s3manageriface.UploaderAPI - endpointURL urlpkg.URL - dryRun bool + api s3iface.S3API + downloader s3manageriface.DownloaderAPI + uploader s3manageriface.UploaderAPI + endpointURL urlpkg.URL + dryRun bool + requestPayer string +} + +func (s *S3) RequestPayer() *string { + if s.requestPayer == "" { + return nil + } + return &s.requestPayer } func parseEndpoint(endpoint string) (urlpkg.URL, error) { @@ -90,19 +98,21 @@ func newS3Storage(ctx context.Context, opts Options) (*S3, error) { } return &S3{ - api: s3.New(awsSession), - downloader: s3manager.NewDownloader(awsSession), - uploader: s3manager.NewUploader(awsSession), - endpointURL: endpointURL, - dryRun: opts.DryRun, + api: s3.New(awsSession), + downloader: s3manager.NewDownloader(awsSession), + uploader: s3manager.NewUploader(awsSession), + endpointURL: endpointURL, + dryRun: opts.DryRun, + requestPayer: opts.RequestPayer, }, nil } // Stat retrieves metadata from S3 object without returning the object itself. func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error) { output, err := s.api.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(url.Bucket), - Key: aws.String(url.Path), + Bucket: aws.String(url.Bucket), + Key: aws.String(url.Path), + RequestPayer: s.RequestPayer(), }) if err != nil { if errHasCode(err, "NotFound") { @@ -133,8 +143,9 @@ func (s *S3) List(ctx context.Context, url *url.URL, _ bool) <-chan *Object { func (s *S3) listObjectsV2(ctx context.Context, url *url.URL) <-chan *Object { listInput := s3.ListObjectsV2Input{ - Bucket: aws.String(url.Bucket), - Prefix: aws.String(url.Prefix), + Bucket: aws.String(url.Bucket), + Prefix: aws.String(url.Prefix), + RequestPayer: s.RequestPayer(), } if url.Delimiter != "" { @@ -224,8 +235,9 @@ func (s *S3) listObjectsV2(ctx context.Context, url *url.URL) <-chan *Object { // ListObjectsV2 API. I'm looking at you GCS. func (s *S3) listObjects(ctx context.Context, url *url.URL) <-chan *Object { listInput := s3.ListObjectsInput{ - Bucket: aws.String(url.Bucket), - Prefix: aws.String(url.Prefix), + Bucket: aws.String(url.Bucket), + Prefix: aws.String(url.Prefix), + RequestPayer: s.RequestPayer(), } if url.Delimiter != "" { @@ -322,9 +334,10 @@ func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) err copySource := from.EscapedPath() input := &s3.CopyObjectInput{ - Bucket: aws.String(to.Bucket), - Key: aws.String(to.Path), - CopySource: aws.String(copySource), + Bucket: aws.String(to.Bucket), + Key: aws.String(to.Path), + CopySource: aws.String(copySource), + RequestPayer: s.RequestPayer(), } storageClass := metadata.StorageClass() @@ -367,8 +380,9 @@ func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) err // Read fetches the remote object and returns its contents as an io.ReadCloser. func (s *S3) Read(ctx context.Context, src *url.URL) (io.ReadCloser, error) { resp, err := s.api.GetObjectWithContext(ctx, &s3.GetObjectInput{ - Bucket: aws.String(src.Bucket), - Key: aws.String(src.Path), + Bucket: aws.String(src.Bucket), + Key: aws.String(src.Path), + RequestPayer: s.RequestPayer(), }) if err != nil { return nil, err @@ -391,8 +405,9 @@ func (s *S3) Get( } return s.downloader.DownloadWithContext(ctx, to, &s3.GetObjectInput{ - Bucket: aws.String(from.Bucket), - Key: aws.String(from.Path), + Bucket: aws.String(from.Bucket), + Key: aws.String(from.Path), + RequestPayer: s.RequestPayer(), }, func(u *s3manager.Downloader) { u.PartSize = partSize u.Concurrency = concurrency @@ -492,10 +507,11 @@ func (s *S3) Put( } input := &s3manager.UploadInput{ - Bucket: aws.String(to.Bucket), - Key: aws.String(to.Path), - Body: reader, - ContentType: aws.String(contentType), + Bucket: aws.String(to.Bucket), + Key: aws.String(to.Path), + Body: reader, + ContentType: aws.String(contentType), + RequestPayer: s.RequestPayer(), } storageClass := metadata.StorageClass() @@ -616,8 +632,9 @@ func (s *S3) doDelete(ctx context.Context, chunk chunk, resultch chan *Object) { bucket := chunk.Bucket o, err := s.api.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{ - Bucket: aws.String(bucket), - Delete: &s3.Delete{Objects: chunk.Keys}, + Bucket: aws.String(bucket), + Delete: &s3.Delete{Objects: chunk.Keys}, + RequestPayer: s.RequestPayer(), }) if err != nil { resultch <- &Object{Err: err} diff --git a/storage/storage.go b/storage/storage.go index c7d081c3c..329371ea8 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -53,6 +53,7 @@ func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, erro NoVerifySSL: opts.NoVerifySSL, DryRun: opts.DryRun, NoSignRequest: opts.NoSignRequest, + RequestPayer: opts.RequestPayer, bucket: url.Bucket, region: opts.region, } @@ -73,6 +74,7 @@ type Options struct { NoVerifySSL bool DryRun bool NoSignRequest bool + RequestPayer string bucket string region string }