From 5ac472b3f524699660eb0340fbce0908a8465999 Mon Sep 17 00:00:00 2001 From: Luc Talatinian <102624213+lucix-aws@users.noreply.github.com> Date: Tue, 28 Jan 2025 13:57:17 -0500 Subject: [PATCH] move transfer manager v2 integration tests to within module (#2987) move transfer manager v2 integration tests to within module --- .../api_op_PutObject_integ_test.go | 2 +- feature/s3/transfermanager/go.mod | 61 ++- feature/s3/transfermanager/go.sum | 22 +- .../s3/transfermanager/setup_integ_test.go | 440 ++++++++++++++++++ service/internal/integrationtest/go.mod | 2 - service/internal/integrationtest/go.sum | 6 +- .../s3transfermanager/setup_test.go | 259 ----------- 7 files changed, 494 insertions(+), 298 deletions(-) rename service/internal/integrationtest/s3transfermanager/api_test.go => feature/s3/transfermanager/api_op_PutObject_integ_test.go (95%) create mode 100644 feature/s3/transfermanager/setup_integ_test.go delete mode 100644 service/internal/integrationtest/s3transfermanager/setup_test.go diff --git a/service/internal/integrationtest/s3transfermanager/api_test.go b/feature/s3/transfermanager/api_op_PutObject_integ_test.go similarity index 95% rename from service/internal/integrationtest/s3transfermanager/api_test.go rename to feature/s3/transfermanager/api_op_PutObject_integ_test.go index 67b3de1881d..921e237e5dc 100644 --- a/service/internal/integrationtest/s3transfermanager/api_test.go +++ b/feature/s3/transfermanager/api_op_PutObject_integ_test.go @@ -1,7 +1,7 @@ //go:build integration // +build integration -package s3transfermanager +package transfermanager import ( "bytes" diff --git a/feature/s3/transfermanager/go.mod b/feature/s3/transfermanager/go.mod index ea0e3d4bbbd..26198a1a3b0 100644 --- a/feature/s3/transfermanager/go.mod +++ b/feature/s3/transfermanager/go.mod @@ -2,23 +2,60 @@ module github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager go 1.21 -toolchain go1.22.4 - require ( - github.com/aws/aws-sdk-go-v2 v1.32.3 + github.com/aws/aws-sdk-go-v2 v1.34.0 + github.com/aws/aws-sdk-go-v2/config v1.29.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1 - github.com/aws/smithy-go v1.22.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 + github.com/aws/smithy-go v1.22.2 ) require ( - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.55 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 // indirect ) replace github.com/aws/aws-sdk-go-v2 => ../../../ + +replace github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => ../../../aws/protocol/eventstream/ + +replace github.com/aws/aws-sdk-go-v2/config => ../../../config/ + +replace github.com/aws/aws-sdk-go-v2/credentials => ../../../credentials/ + +replace github.com/aws/aws-sdk-go-v2/feature/ec2/imds => ../../../feature/ec2/imds/ + +replace github.com/aws/aws-sdk-go-v2/internal/configsources => ../../../internal/configsources/ + +replace github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => ../../../internal/endpoints/v2/ + +replace github.com/aws/aws-sdk-go-v2/internal/ini => ../../../internal/ini/ + +replace github.com/aws/aws-sdk-go-v2/internal/v4a => ../../../internal/v4a/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../service/internal/accept-encoding/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/checksum => ../../../service/internal/checksum/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../service/internal/presigned-url/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../service/internal/s3shared/ + +replace github.com/aws/aws-sdk-go-v2/service/s3 => ../../../service/s3/ + +replace github.com/aws/aws-sdk-go-v2/service/sso => ../../../service/sso/ + +replace github.com/aws/aws-sdk-go-v2/service/ssooidc => ../../../service/ssooidc/ + +replace github.com/aws/aws-sdk-go-v2/service/sts => ../../../service/sts/ diff --git a/feature/s3/transfermanager/go.sum b/feature/s3/transfermanager/go.sum index b9beb150394..cad7e6ad46a 100644 --- a/feature/s3/transfermanager/go.sum +++ b/feature/s3/transfermanager/go.sum @@ -1,20 +1,2 @@ -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12 h1:DXFWyt7ymx/l1ygdyTTS0X923e+Q2wXIxConJzrgwc0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12/go.mod h1:mVOr/LbvaNySK1/BTy4cBOCjhCNY2raWBwK4v+WR5J4= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14 h1:oWccitSnByVU74rQRHac4gLfDqjB6Z1YQGOY/dXKedI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14/go.mod h1:8SaZBlQdCLrc/2U3CEO48rYj9uR8qRsPRkmzwNM52pM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12 h1:tzha+v1SCEBpXWEuw6B/+jm4h5z8hZbTpXz0zRZqTnw= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12/go.mod h1:n+nt2qjHGoseWeLHt1vEr6ZRCCxIN2KcNpJxBcYQSwI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1 h1:wsg9Z/vNnCmxWikfGIoOlnExtEU459cR+2d+iDJ8elo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1/go.mod h1:8rDw3mVwmvIWWX/+LWY3PPIMZuwnQdJMCt0iVFVT3qw= -github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= -github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= diff --git a/feature/s3/transfermanager/setup_integ_test.go b/feature/s3/transfermanager/setup_integ_test.go new file mode 100644 index 00000000000..d8efb29a220 --- /dev/null +++ b/feature/s3/transfermanager/setup_integ_test.go @@ -0,0 +1,440 @@ +//go:build integration +// +build integration + +package transfermanager + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/tls" + "flag" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go-v2/service/sts" +) + +var setupMetadata = struct { + AccountID string + Region string + Buckets struct { + Source struct { + Name string + ARN string + } + } +}{} + +// s3 client to use for integ testing +var s3Client *s3.Client + +// s3TransferManagerClient to use for integ testing +var s3TransferManagerClient *Client + +// sts client to use for integ testing +var stsClient *sts.Client + +// http client setting to use for integ testing +var httpClient *http.Client + +var region = "us-west-2" + +// large object buffer to test multipart upload +var largeObjectBuf []byte + +// TestMain executes at start of package tests +func TestMain(m *testing.M) { + flag.Parse() + flag.CommandLine.Visit(func(f *flag.Flag) { + if !(f.Name == "run" || f.Name == "test.run") { + return + } + value := f.Value.String() + if value == `NONE` { + os.Exit(0) + } + }) + + var result int + defer func() { + if r := recover(); r != nil { + fmt.Fprintln(os.Stderr, "S3 TransferManager integration tests panic,", r) + result = 1 + } + os.Exit(result) + }() + + var verifyTLS bool + var s3Endpoint string + + flag.StringVar(&s3Endpoint, "s3-endpoint", "", "integration endpoint for S3") + + flag.StringVar(&setupMetadata.AccountID, "account", "", "integration account id") + flag.BoolVar(&verifyTLS, "verify-tls", true, "verify server TLS certificate") + flag.Parse() + + httpClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: verifyTLS}, + }, + } + + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(region)) + if err != nil { + fmt.Fprintf(os.Stderr, "Error occurred while loading config with region %v, %v", region, err) + result = 1 + return + } + + // assign the http client + cfg.HTTPClient = httpClient + + // create a s3 client + s3cfg := cfg.Copy() + if len(s3Endpoint) != 0 { + s3cfg.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: s3Endpoint, + PartitionID: "aws", + SigningName: "s3", + SigningRegion: region, + SigningMethod: "s3v4", + }, nil + }) + } + + // build s3 client from config + s3Client = s3.NewFromConfig(s3cfg) + + // build s3 transfermanager client from config + s3TransferManagerClient = NewFromConfig(s3Client, s3cfg) + + // build sts client from config + stsClient = sts.NewFromConfig(cfg) + + // context + ctx := context.Background() + + setupMetadata.AccountID, err = getAccountID(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to get integration aws account id: %v\n", err) + result = 1 + return + } + + bucketCleanup, err := setupBuckets(ctx) + defer bucketCleanup() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to setup integration test buckets: %v\n", err) + result = 1 + return + } + + largeObjectBuf = make([]byte, 20*1024*1024) + _, err = rand.Read(largeObjectBuf) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to generate large object for multipart upload: %v\n", err) + result = 1 + return + } + + result = m.Run() +} + +// getAccountID retrieves account id +func getAccountID(ctx context.Context) (string, error) { + if len(setupMetadata.AccountID) != 0 { + return setupMetadata.AccountID, nil + } + identity, err := stsClient.GetCallerIdentity(ctx, nil) + if err != nil { + return "", fmt.Errorf("error fetching caller identity, %w", err) + } + return *identity.Account, nil +} + +// setupBuckets creates buckets needed for integration test +func setupBuckets(ctx context.Context) (func(), error) { + var cleanups []func() + + cleanup := func() { + for i := range cleanups { + cleanups[i]() + } + } + + bucketCreates := []struct { + name *string + arn *string + }{ + {name: &setupMetadata.Buckets.Source.Name, arn: &setupMetadata.Buckets.Source.ARN}, + } + + for _, bucket := range bucketCreates { + *bucket.name = GenerateBucketName() + + if err := SetupBucket(ctx, s3Client, *bucket.name); err != nil { + return cleanup, err + } + + // Compute ARN + bARN := arn.ARN{ + Partition: "aws", + Service: "s3", + Region: region, + AccountID: setupMetadata.AccountID, + Resource: fmt.Sprintf("bucket_name:%s", *bucket.name), + }.String() + + *bucket.arn = bARN + + bucketName := *bucket.name + cleanups = append(cleanups, func() { + if err := CleanupBucket(ctx, s3Client, bucketName); err != nil { + fmt.Fprintln(os.Stderr, err) + } + }) + } + + return cleanup, nil +} + +type putObjectTestData struct { + Body io.Reader + ExpectBody []byte + ExpectError string +} + +// UniqueID returns a unique UUID-like identifier for use in generating +// resources for integration tests. +// +// TODO: duped from service/internal/integrationtest, remove after beta. +func UniqueID() string { + uuid := make([]byte, 16) + io.ReadFull(rand.Reader, uuid) + return fmt.Sprintf("%x", uuid) +} + +func testPutObject(t *testing.T, bucket string, testData putObjectTestData, opts ...func(options *Options)) { + key := UniqueID() + + _, err := s3TransferManagerClient.PutObject(context.Background(), + &PutObjectInput{ + Bucket: bucket, + Key: key, + Body: testData.Body, + }, opts...) + if err != nil { + if len(testData.ExpectError) == 0 { + t.Fatalf("expect no error, got %v", err) + } + if e, a := testData.ExpectError, err.Error(); !strings.Contains(a, e) { + t.Fatalf("expect error to contain %v, got %v", e, a) + } + } else { + if e := testData.ExpectError; len(e) != 0 { + t.Fatalf("expect error: %v, got none", e) + } + } + + if len(testData.ExpectError) != 0 { + return + } + + resp, err := s3Client.GetObject(context.Background(), + &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + + b, _ := ioutil.ReadAll(resp.Body) + if e, a := testData.ExpectBody, b; !bytes.EqualFold(e, a) { + t.Errorf("expect %s, got %s", e, a) + } +} + +// TODO: duped from service/internal/integrationtest, remove after beta. +const expressAZID = "usw2-az3" + +// TODO: duped from service/internal/integrationtest, remove after beta. +const expressSuffix = "--usw2-az3--x-s3" + +// BucketPrefix is the root prefix of integration test buckets. +// +// TODO: duped from service/internal/integrationtest, remove after beta. +const BucketPrefix = "aws-sdk-go-v2-integration" + +// GenerateBucketName returns a unique bucket name. +// +// TODO: duped from service/internal/integrationtest, remove after beta. +func GenerateBucketName() string { + return fmt.Sprintf("%s-%s", + BucketPrefix, UniqueID()) +} + +// GenerateBucketName returns a unique express-formatted bucket name. +// +// TODO: duped from service/internal/integrationtest, remove after beta. +func GenerateExpressBucketName() string { + return fmt.Sprintf( + "%s-%s%s", + BucketPrefix, + UniqueID()[0:8], // express suffix adds length, regain that here + expressSuffix, + ) +} + +// SetupBucket returns a test bucket created for the integration tests. +// +// TODO: duped from service/internal/integrationtest, remove after beta. +func SetupBucket(ctx context.Context, svc *s3.Client, bucketName string) (err error) { + fmt.Println("Setup: Creating test bucket,", bucketName) + _, err = svc.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: &bucketName, + CreateBucketConfiguration: &types.CreateBucketConfiguration{ + LocationConstraint: "us-west-2", + }, + }) + if err != nil { + return fmt.Errorf("failed to create bucket %s, %v", bucketName, err) + } + + // TODO: change this to use waiter to wait until BucketExists instead of loop + // svc.WaitUntilBucketExists(HeadBucketInput) + + // HeadBucket to determine if bucket exists + var attempt = 0 + params := &s3.HeadBucketInput{ + Bucket: &bucketName, + } +pt: + _, err = svc.HeadBucket(ctx, params) + // increment an attempt + attempt++ + + // retry till 10 attempt + if err != nil { + if attempt < 10 { + goto pt + } + // fail if not succeed after 10 attempts + return fmt.Errorf("failed to determine if a bucket %s exists and you have permission to access it %v", bucketName, err) + } + + return nil +} + +// CleanupBucket deletes the contents of a S3 bucket, before deleting the bucket +// it self. +// TODO: list and delete methods should use paginators +// +// TODO: duped from service/internal/integrationtest, remove after beta. +func CleanupBucket(ctx context.Context, svc *s3.Client, bucketName string) (err error) { + var errs = make([]error, 0) + + fmt.Println("TearDown: Deleting objects from test bucket,", bucketName) + listObjectsResp, err := svc.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucketName, + }) + if err != nil { + return fmt.Errorf("failed to list objects, %w", err) + } + + for _, o := range listObjectsResp.Contents { + _, err := svc.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucketName, + Key: o.Key, + }) + if err != nil { + errs = append(errs, err) + } + } + if len(errs) != 0 { + return fmt.Errorf("failed to delete objects, %s", errs) + } + + fmt.Println("TearDown: Deleting partial uploads from test bucket,", bucketName) + multipartUploadResp, err := svc.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucketName, + }) + if err != nil { + return fmt.Errorf("failed to list multipart objects, %w", err) + } + + for _, u := range multipartUploadResp.Uploads { + _, err = svc.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucketName, + Key: u.Key, + UploadId: u.UploadId, + }) + if err != nil { + errs = append(errs, err) + } + } + if len(errs) != 0 { + return fmt.Errorf("failed to delete multipart upload objects, %s", errs) + } + + fmt.Println("TearDown: Deleting test bucket,", bucketName) + _, err = svc.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucketName, + }) + if err != nil { + return fmt.Errorf("failed to delete bucket, %s", bucketName) + } + + return nil +} + +// SetupExpressBucket returns an express bucket for testing. +// +// TODO: duped from service/internal/integrationtest, remove after beta. +func SetupExpressBucket(ctx context.Context, svc *s3.Client, bucketName string) error { + if !strings.HasSuffix(bucketName, expressSuffix) { + return fmt.Errorf("bucket name %s is missing required suffix %s", bucketName, expressSuffix) + } + + fmt.Println("Setup: Creating test express bucket,", bucketName) + _, err := svc.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: &bucketName, + CreateBucketConfiguration: &types.CreateBucketConfiguration{ + Location: &types.LocationInfo{ + Name: aws.String(expressAZID), + Type: types.LocationTypeAvailabilityZone, + }, + Bucket: &types.BucketInfo{ + DataRedundancy: types.DataRedundancySingleAvailabilityZone, + Type: types.BucketTypeDirectory, + }, + }, + }) + if err != nil { + return fmt.Errorf("create express bucket %s: %v", bucketName, err) + } + + w := s3.NewBucketExistsWaiter(svc) + err = w.Wait(ctx, &s3.HeadBucketInput{ + Bucket: &bucketName, + }, 10*time.Second) + if err != nil { + return fmt.Errorf("wait for express bucket %s: %v", bucketName, err) + } + + return nil +} diff --git a/service/internal/integrationtest/go.mod b/service/internal/integrationtest/go.mod index b3fd6dfbfe2..fc86f0e1da2 100644 --- a/service/internal/integrationtest/go.mod +++ b/service/internal/integrationtest/go.mod @@ -295,5 +295,3 @@ replace github.com/aws/aws-sdk-go-v2/service/wafregional => ../../../service/waf replace github.com/aws/aws-sdk-go-v2/service/wafv2 => ../../../service/wafv2/ replace github.com/aws/aws-sdk-go-v2/service/workspaces => ../../../service/workspaces/ - -replace github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager => ../../../feature/s3/transfermanager diff --git a/service/internal/integrationtest/go.sum b/service/internal/integrationtest/go.sum index 7c80ae776bc..609e6296b2c 100644 --- a/service/internal/integrationtest/go.sum +++ b/service/internal/integrationtest/go.sum @@ -1,8 +1,7 @@ github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -11,6 +10,5 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/service/internal/integrationtest/s3transfermanager/setup_test.go b/service/internal/integrationtest/s3transfermanager/setup_test.go deleted file mode 100644 index 6ac6a16f4a9..00000000000 --- a/service/internal/integrationtest/s3transfermanager/setup_test.go +++ /dev/null @@ -1,259 +0,0 @@ -//go:build integration -// +build integration - -package s3transfermanager - -import ( - "bytes" - "context" - "crypto/rand" - "crypto/tls" - "flag" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "strings" - "testing" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/arn" - tm "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" - "github.com/aws/aws-sdk-go-v2/service/internal/integrationtest" - "github.com/aws/aws-sdk-go-v2/service/internal/integrationtest/s3shared" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/sts" -) - -var setupMetadata = struct { - AccountID string - Region string - Buckets struct { - Source struct { - Name string - ARN string - } - } -}{} - -// s3 client to use for integ testing -var s3Client *s3.Client - -// s3TransferManagerClient to use for integ testing -var s3TransferManagerClient *tm.Client - -// sts client to use for integ testing -var stsClient *sts.Client - -// http client setting to use for integ testing -var httpClient *http.Client - -var region = "us-west-2" - -// large object buffer to test multipart upload -var largeObjectBuf []byte - -// TestMain executes at start of package tests -func TestMain(m *testing.M) { - flag.Parse() - flag.CommandLine.Visit(func(f *flag.Flag) { - if !(f.Name == "run" || f.Name == "test.run") { - return - } - value := f.Value.String() - if value == `NONE` { - os.Exit(0) - } - }) - - var result int - defer func() { - if r := recover(); r != nil { - fmt.Fprintln(os.Stderr, "S3 TransferManager integration tests panic,", r) - result = 1 - } - os.Exit(result) - }() - - var verifyTLS bool - var s3Endpoint string - - flag.StringVar(&s3Endpoint, "s3-endpoint", "", "integration endpoint for S3") - - flag.StringVar(&setupMetadata.AccountID, "account", "", "integration account id") - flag.BoolVar(&verifyTLS, "verify-tls", true, "verify server TLS certificate") - flag.Parse() - - httpClient = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: verifyTLS}, - }, - } - - cfg, err := integrationtest.LoadConfigWithDefaultRegion(region) - if err != nil { - fmt.Fprintf(os.Stderr, "Error occurred while loading config with region %v, %v", region, err) - result = 1 - return - } - - // assign the http client - cfg.HTTPClient = httpClient - - // create a s3 client - s3cfg := cfg.Copy() - if len(s3Endpoint) != 0 { - s3cfg.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { - return aws.Endpoint{ - URL: s3Endpoint, - PartitionID: "aws", - SigningName: "s3", - SigningRegion: region, - SigningMethod: "s3v4", - }, nil - }) - } - - // build s3 client from config - s3Client = s3.NewFromConfig(s3cfg) - - // build s3 transfermanager client from config - s3TransferManagerClient = tm.NewFromConfig(s3Client, s3cfg) - - // build sts client from config - stsClient = sts.NewFromConfig(cfg) - - // context - ctx := context.Background() - - setupMetadata.AccountID, err = getAccountID(ctx) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to get integration aws account id: %v\n", err) - result = 1 - return - } - - bucketCleanup, err := setupBuckets(ctx) - defer bucketCleanup() - if err != nil { - fmt.Fprintf(os.Stderr, "failed to setup integration test buckets: %v\n", err) - result = 1 - return - } - - largeObjectBuf = make([]byte, 20*1024*1024) - _, err = rand.Read(largeObjectBuf) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to generate large object for multipart upload: %v\n", err) - result = 1 - return - } - - result = m.Run() -} - -// getAccountID retrieves account id -func getAccountID(ctx context.Context) (string, error) { - if len(setupMetadata.AccountID) != 0 { - return setupMetadata.AccountID, nil - } - identity, err := stsClient.GetCallerIdentity(ctx, nil) - if err != nil { - return "", fmt.Errorf("error fetching caller identity, %w", err) - } - return *identity.Account, nil -} - -// setupBuckets creates buckets needed for integration test -func setupBuckets(ctx context.Context) (func(), error) { - var cleanups []func() - - cleanup := func() { - for i := range cleanups { - cleanups[i]() - } - } - - bucketCreates := []struct { - name *string - arn *string - }{ - {name: &setupMetadata.Buckets.Source.Name, arn: &setupMetadata.Buckets.Source.ARN}, - } - - for _, bucket := range bucketCreates { - *bucket.name = s3shared.GenerateBucketName() - - if err := s3shared.SetupBucket(ctx, s3Client, *bucket.name); err != nil { - return cleanup, err - } - - // Compute ARN - bARN := arn.ARN{ - Partition: "aws", - Service: "s3", - Region: region, - AccountID: setupMetadata.AccountID, - Resource: fmt.Sprintf("bucket_name:%s", *bucket.name), - }.String() - - *bucket.arn = bARN - - bucketName := *bucket.name - cleanups = append(cleanups, func() { - if err := s3shared.CleanupBucket(ctx, s3Client, bucketName); err != nil { - fmt.Fprintln(os.Stderr, err) - } - }) - } - - return cleanup, nil -} - -type putObjectTestData struct { - Body io.Reader - ExpectBody []byte - ExpectError string -} - -func testPutObject(t *testing.T, bucket string, testData putObjectTestData, opts ...func(options *tm.Options)) { - key := integrationtest.UniqueID() - - _, err := s3TransferManagerClient.PutObject(context.Background(), - &tm.PutObjectInput{ - Bucket: bucket, - Key: key, - Body: testData.Body, - }, opts...) - if err != nil { - if len(testData.ExpectError) == 0 { - t.Fatalf("expect no error, got %v", err) - } - if e, a := testData.ExpectError, err.Error(); !strings.Contains(a, e) { - t.Fatalf("expect error to contain %v, got %v", e, a) - } - } else { - if e := testData.ExpectError; len(e) != 0 { - t.Fatalf("expect error: %v, got none", e) - } - } - - if len(testData.ExpectError) != 0 { - return - } - - resp, err := s3Client.GetObject(context.Background(), - &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - - b, _ := ioutil.ReadAll(resp.Body) - if e, a := testData.ExpectBody, b; !bytes.EqualFold(e, a) { - t.Errorf("expect %s, got %s", e, a) - } -}