command: add --request-payer flag

allow access to requester pays buckets, issue peak#297

command/app.go

@@ -69,6 +69,10 @@ var app = &cli.App{
 			Name:  "no-sign-request",
 			Usage: "do not sign requests: credentials will not be loaded if --no-sign-request is provided",
 		},
+		&cli.StringFlag{
+			Name:  "request-payer",
+			Usage: "who pays for request (access requester pays buckets)",
+		},
 	},
 	Before: func(c *cli.Context) error {
 		retryCount := c.Int("retry-count")
@@ -139,6 +143,7 @@ func NewStorageOpts(c *cli.Context) storage.Options {
 		NoVerifySSL:   c.Bool("no-verify-ssl"),
 		DryRun:        c.Bool("dry-run"),
 		NoSignRequest: c.Bool("no-sign-request"),
+		RequestPayer:  c.String("request-payer"),
 	}
 }
 

command/cp.go

@@ -94,6 +94,9 @@ Examples:
 
 	19. Copy all files from S3 bucket to another S3 bucket but exclude the ones starts with log
 		 > s5cmd {{.HelpName}} --exclude "log*" s3://bucket/* s3://destbucket
+
+	20. Download an S3 object from a requester pays bucket
+		 > s5cmd --request-payer=requester {{.HelpName}} s3://bucket/prefix/object.gz .
 `
 
 func NewCopyCommandFlags() []cli.Flag {

command/ls.go

@@ -42,6 +42,9 @@ Examples:
 
 	6. List all objects in a bucket but exclude the ones with prefix abc
 		 > s5cmd {{.HelpName}} --exclude "abc*" s3://bucket/*
+
+    7. List all object in a requester pays bucket
+	     > s5cmd --request-payer=requester {{.HelpName}} s3://bucket/*
 `
 
 func NewListCommand() *cli.Command {

storage/s3.go

@@ -53,11 +53,19 @@ var globalSessionCache = &SessionCache{
 // S3 is a storage type which interacts with S3API, DownloaderAPI and
 // UploaderAPI.
 type S3 struct {
-	api         s3iface.S3API
-	downloader  s3manageriface.DownloaderAPI
-	uploader    s3manageriface.UploaderAPI
-	endpointURL urlpkg.URL
-	dryRun      bool
+	api          s3iface.S3API
+	downloader   s3manageriface.DownloaderAPI
+	uploader     s3manageriface.UploaderAPI
+	endpointURL  urlpkg.URL
+	dryRun       bool
+	requestPayer string
+}
+
+func (s *S3) RequestPayer() *string {
+	if s.requestPayer == "" {
+		return nil
+	}
+	return &s.requestPayer
 }
 
 func parseEndpoint(endpoint string) (urlpkg.URL, error) {
@@ -90,19 +98,21 @@ func newS3Storage(ctx context.Context, opts Options) (*S3, error) {
 	}
 
 	return &S3{
-		api:         s3.New(awsSession),
-		downloader:  s3manager.NewDownloader(awsSession),
-		uploader:    s3manager.NewUploader(awsSession),
-		endpointURL: endpointURL,
-		dryRun:      opts.DryRun,
+		api:          s3.New(awsSession),
+		downloader:   s3manager.NewDownloader(awsSession),
+		uploader:     s3manager.NewUploader(awsSession),
+		endpointURL:  endpointURL,
+		dryRun:       opts.DryRun,
+		requestPayer: opts.RequestPayer,
 	}, nil
 }
 
 // Stat retrieves metadata from S3 object without returning the object itself.
 func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error) {
 	output, err := s.api.HeadObjectWithContext(ctx, &s3.HeadObjectInput{
-		Bucket: aws.String(url.Bucket),
-		Key:    aws.String(url.Path),
+		Bucket:       aws.String(url.Bucket),
+		Key:          aws.String(url.Path),
+		RequestPayer: s.RequestPayer(),
 	})
 	if err != nil {
 		if errHasCode(err, "NotFound") {
@@ -133,8 +143,9 @@ func (s *S3) List(ctx context.Context, url *url.URL, _ bool) <-chan *Object {
 
 func (s *S3) listObjectsV2(ctx context.Context, url *url.URL) <-chan *Object {
 	listInput := s3.ListObjectsV2Input{
-		Bucket: aws.String(url.Bucket),
-		Prefix: aws.String(url.Prefix),
+		Bucket:       aws.String(url.Bucket),
+		Prefix:       aws.String(url.Prefix),
+		RequestPayer: s.RequestPayer(),
 	}
 
 	if url.Delimiter != "" {
@@ -224,8 +235,9 @@ func (s *S3) listObjectsV2(ctx context.Context, url *url.URL) <-chan *Object {
 // ListObjectsV2 API. I'm looking at you GCS.
 func (s *S3) listObjects(ctx context.Context, url *url.URL) <-chan *Object {
 	listInput := s3.ListObjectsInput{
-		Bucket: aws.String(url.Bucket),
-		Prefix: aws.String(url.Prefix),
+		Bucket:       aws.String(url.Bucket),
+		Prefix:       aws.String(url.Prefix),
+		RequestPayer: s.RequestPayer(),
 	}
 
 	if url.Delimiter != "" {
@@ -322,9 +334,10 @@ func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) err
 	copySource := from.EscapedPath()
 
 	input := &s3.CopyObjectInput{
-		Bucket:     aws.String(to.Bucket),
-		Key:        aws.String(to.Path),
-		CopySource: aws.String(copySource),
+		Bucket:       aws.String(to.Bucket),
+		Key:          aws.String(to.Path),
+		CopySource:   aws.String(copySource),
+		RequestPayer: s.RequestPayer(),
 	}
 
 	storageClass := metadata.StorageClass()
@@ -367,8 +380,9 @@ func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) err
 // Read fetches the remote object and returns its contents as an io.ReadCloser.
 func (s *S3) Read(ctx context.Context, src *url.URL) (io.ReadCloser, error) {
 	resp, err := s.api.GetObjectWithContext(ctx, &s3.GetObjectInput{
-		Bucket: aws.String(src.Bucket),
-		Key:    aws.String(src.Path),
+		Bucket:       aws.String(src.Bucket),
+		Key:          aws.String(src.Path),
+		RequestPayer: s.RequestPayer(),
 	})
 	if err != nil {
 		return nil, err
@@ -391,8 +405,9 @@ func (s *S3) Get(
 	}
 
 	return s.downloader.DownloadWithContext(ctx, to, &s3.GetObjectInput{
-		Bucket: aws.String(from.Bucket),
-		Key:    aws.String(from.Path),
+		Bucket:       aws.String(from.Bucket),
+		Key:          aws.String(from.Path),
+		RequestPayer: s.RequestPayer(),
 	}, func(u *s3manager.Downloader) {
 		u.PartSize = partSize
 		u.Concurrency = concurrency
@@ -492,10 +507,11 @@ func (s *S3) Put(
 	}
 
 	input := &s3manager.UploadInput{
-		Bucket:      aws.String(to.Bucket),
-		Key:         aws.String(to.Path),
-		Body:        reader,
-		ContentType: aws.String(contentType),
+		Bucket:       aws.String(to.Bucket),
+		Key:          aws.String(to.Path),
+		Body:         reader,
+		ContentType:  aws.String(contentType),
+		RequestPayer: s.RequestPayer(),
 	}
 
 	storageClass := metadata.StorageClass()
@@ -616,8 +632,9 @@ func (s *S3) doDelete(ctx context.Context, chunk chunk, resultch chan *Object) {
 
 	bucket := chunk.Bucket
 	o, err := s.api.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{
-		Bucket: aws.String(bucket),
-		Delete: &s3.Delete{Objects: chunk.Keys},
+		Bucket:       aws.String(bucket),
+		Delete:       &s3.Delete{Objects: chunk.Keys},
+		RequestPayer: s.RequestPayer(),
 	})
 	if err != nil {
 		resultch <- &Object{Err: err}

storage/storage.go

@@ -53,6 +53,7 @@ func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, erro
 		NoVerifySSL:   opts.NoVerifySSL,
 		DryRun:        opts.DryRun,
 		NoSignRequest: opts.NoSignRequest,
+		RequestPayer:  opts.RequestPayer,
 		bucket:        url.Bucket,
 		region:        opts.region,
 	}
@@ -73,6 +74,7 @@ type Options struct {
 	NoVerifySSL   bool
 	DryRun        bool
 	NoSignRequest bool
+	RequestPayer  string
 	bucket        string
 	region        string
 }