diff --git a/pkg/cli/rack.go b/pkg/cli/rack.go index fe4adc53b9..459b8693c4 100644 --- a/pkg/cli/rack.go +++ b/pkg/cli/rack.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/aws/aws-sdk-go/aws" - ss "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/convox/rack/pkg/helpers" "github.com/convox/rack/pkg/options" @@ -379,7 +378,7 @@ func RackSync(rack sdk.Interface, c *stdcli.Context) error { if c.String("name") != "" { rname = c.String("name") - s, err := ss.NewSession(&aws.Config{}) + s, err := helpers.NewSession() if err != nil { return err } diff --git a/pkg/helpers/aws.go b/pkg/helpers/aws.go index 7f01ac48ca..7c03098742 100644 --- a/pkg/helpers/aws.go +++ b/pkg/helpers/aws.go @@ -15,6 +15,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" @@ -23,6 +25,10 @@ import ( yaml "gopkg.in/yaml.v2" ) +const ( + MAX_RETRY = 10 +) + func AwsCredentialsLoad() error { if os.Getenv("AWS_ACCESS_KEY_ID") == "" { if err := exec.Command("which", "aws").Run(); err != nil { @@ -530,3 +536,18 @@ func writeLogEvents(w io.Writer, events []*cloudwatchlogs.FilteredLogEvent, opts return latest, nil } + +func NewSession() (*session.Session, error) { + s, err := session.NewSession(&aws.Config{ + MaxRetries: aws.Int(MAX_RETRY), + Retryer: client.DefaultRetryer{ + NumMaxRetries: MAX_RETRY, + MinRetryDelay: 1 * time.Second, + MaxRetryDelay: 5 * time.Second, + MinThrottleDelay: 10 * time.Second, + MaxThrottleDelay: 60 * time.Second, + }, + }) + + return s, err +} diff --git a/provider/aws/aws.go b/provider/aws/aws.go index 6d35155c57..e10db9e150 100644 --- a/provider/aws/aws.go +++ b/provider/aws/aws.go @@ -10,7 +10,6 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/acm" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/cloudformation" @@ -28,8 +27,10 @@ import ( "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sts" "github.com/convox/logger" + "github.com/convox/rack/pkg/helpers" "github.com/convox/rack/pkg/metrics" "github.com/convox/rack/pkg/structs" + "github.com/pkg/errors" ) const ( @@ -191,7 +192,12 @@ func (p *Provider) Initialize(opts structs.ProviderOptions) error { go p.Workers() } - p.CloudWatch = cloudwatch.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + return errors.WithStack(err) + } + + p.CloudWatch = cloudwatch.New(s, p.config()) return nil } @@ -264,63 +270,123 @@ func (p *Provider) logger(at string) *logger.Logger { } func (p *Provider) acm() *acm.ACM { - return acm.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return acm.New(s, p.config()) } func (p *Provider) autoscaling() *autoscaling.AutoScaling { - return autoscaling.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return autoscaling.New(s, p.config()) } func (p *Provider) cloudformation() *cloudformation.CloudFormation { - return cloudformation.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return cloudformation.New(s, p.config()) } func (p *Provider) cloudwatch() *cloudwatch.CloudWatch { - return cloudwatch.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return cloudwatch.New(s, p.config()) } func (p *Provider) cloudwatchlogs() *cloudwatchlogs.CloudWatchLogs { - return cloudwatchlogs.New(session.New(), p.config().WithLogLevel(aws.LogOff)) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return cloudwatchlogs.New(s, p.config().WithLogLevel(aws.LogOff)) } func (p *Provider) dynamodb() *dynamodb.DynamoDB { - return dynamodb.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return dynamodb.New(s, p.config()) } func (p *Provider) ec2() *ec2.EC2 { - return ec2.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return ec2.New(s, p.config()) } func (p *Provider) ecr() *ecr.ECR { - return ecr.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return ecr.New(s, p.config()) } func (p *Provider) ecs() *ecs.ECS { - return ecs.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return ecs.New(s, p.config()) } func (p *Provider) kms() *kms.KMS { - return kms.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return kms.New(s, p.config()) } func (p *Provider) iam() *iam.IAM { - return iam.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return iam.New(s, p.config()) } func (p *Provider) s3() *s3.S3 { - return s3.New(session.New(), p.config().WithS3ForcePathStyle(true)) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return s3.New(s, p.config().WithS3ForcePathStyle(true)) } func (p *Provider) sns() *sns.SNS { - return sns.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return sns.New(s, p.config()) } func (p *Provider) sqs() *sqs.SQS { - return sqs.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return sqs.New(s, p.config()) } func (p *Provider) sts() *sts.STS { - return sts.New(session.New(), p.config()) + s, err := helpers.NewSession() + if err != nil { + panic(errors.WithStack(err)) + } + return sts.New(s, p.config()) } // IsTest returns true when we're in test mode diff --git a/provider/aws/lambda/autoscale/handler.go b/provider/aws/lambda/autoscale/handler.go index 011605e0c1..1be392f68f 100644 --- a/provider/aws/lambda/autoscale/handler.go +++ b/provider/aws/lambda/autoscale/handler.go @@ -6,17 +6,20 @@ import ( "math" "os" "strconv" + "time" "github.com/aws/aws-lambda-go/lambda" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/ecs" ) const ( - haInstanceCountParam = "InstanceCount" - noHaInstanceCountParam = "NoHaInstanceCount" + HA_INSTANCE_COUNT_PARAM = "InstanceCount" + NO_HA_INSTANCE_COUNT_PARAM = "NoHaInstanceCount" + MAX_RETRY = 10 ) var ( @@ -55,9 +58,9 @@ func Handler(ctx context.Context) error { func autoscale(desired int64) error { stack := os.Getenv("STACK") ds := fmt.Sprintf("%d", desired) - icParam := haInstanceCountParam + icParam := HA_INSTANCE_COUNT_PARAM if !IsHA { - icParam = noHaInstanceCountParam + icParam = NO_HA_INSTANCE_COUNT_PARAM } debug("desired (ds) = %+v\n", ds) @@ -354,8 +357,20 @@ func min(ii ...int64) int64 { } func main() { - sessionConfig := &aws.Config{Region: aws.String(os.Getenv("REGION"))} - session, err := session.NewSession(sessionConfig) + session, err := session.NewSession( + &aws.Config{ + Region: aws.String(os.Getenv("REGION")), + MaxRetries: aws.Int(MAX_RETRY), + LogLevel: aws.LogLevel(aws.LogDebugWithRequestRetries), + Retryer: client.DefaultRetryer{ + NumMaxRetries: MAX_RETRY, + MinRetryDelay: 1 * time.Second, + MaxRetryDelay: 5 * time.Second, + MinThrottleDelay: 10 * time.Second, + MaxThrottleDelay: 60 * time.Second, + }, + }, + ) if err != nil { fmt.Printf("Error creating new session: %+v", err) return diff --git a/provider/aws/lambda/lifecycle/main.go b/provider/aws/lambda/lifecycle/main.go index 71e178bbd6..2e30cb37de 100644 --- a/provider/aws/lambda/lifecycle/main.go +++ b/provider/aws/lambda/lifecycle/main.go @@ -11,11 +11,16 @@ import ( "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-lambda-go/lambda" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ecs" ) +const ( + MAX_RETRY = 10 +) + var ( AutoScaling *autoscaling.AutoScaling ECS *ecs.ECS @@ -42,8 +47,20 @@ type Termination struct { } func main() { - sessionConfig := &aws.Config{Region: aws.String(os.Getenv("REGION"))} - session, err := session.NewSession(sessionConfig) + session, err := session.NewSession( + &aws.Config{ + Region: aws.String(os.Getenv("REGION")), + MaxRetries: aws.Int(MAX_RETRY), + LogLevel: aws.LogLevel(aws.LogDebugWithRequestRetries), + Retryer: client.DefaultRetryer{ + NumMaxRetries: MAX_RETRY, + MinRetryDelay: 1 * time.Second, + MaxRetryDelay: 5 * time.Second, + MinThrottleDelay: 10 * time.Second, + MaxThrottleDelay: 60 * time.Second, + }, + }, + ) if err != nil { fmt.Printf("Error creating new session: %+v", err) return diff --git a/provider/aws/system.go b/provider/aws/system.go index 37bc72ec42..311f574b91 100644 --- a/provider/aws/system.go +++ b/provider/aws/system.go @@ -16,7 +16,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/dynamodb" @@ -25,6 +24,7 @@ import ( "github.com/convox/rack/pkg/helpers" "github.com/convox/rack/pkg/structs" "github.com/fatih/color" + "github.com/pkg/errors" "golang.org/x/crypto/nacl/secretbox" "gopkg.in/cheggaaa/pb.v1" @@ -252,13 +252,17 @@ func (p *Provider) SystemInstall(w io.Writer, opts structs.SystemInstallOptions) var bar *pb.ProgressBar - cf := cloudformation.New(session.New(&aws.Config{})) + s, err := helpers.NewSession() + if err != nil { + return "", errors.WithStack(err) + } + cf := cloudformation.New(s) if !raw { fmt.Fprintf(w, "Preparing... ") } - err := helpers.CloudformationInstall(cf, name, template, params, tags, func(current, total int) { + err = helpers.CloudformationInstall(cf, name, template, params, tags, func(current, total int) { if raw { fmt.Fprintf(w, "{ \"stack\": %q, \"current\": %d, \"total\": %d }\n", name, current, total) return @@ -453,7 +457,11 @@ func (p *Provider) SystemUninstall(name string, w io.Writer, opts structs.System return err } - cf := cloudformation.New(session.New(&aws.Config{})) + s, err := helpers.NewSession() + if err != nil { + return errors.WithStack(err) + } + cf := cloudformation.New(s) dres, err := cf.DescribeStacks(&cloudformation.DescribeStacksInput{StackName: aws.String(name)}) if err != nil { @@ -620,7 +628,11 @@ func cloudformationProgress(stack, token string, template []byte, w io.Writer) e } } - cf := cloudformation.New(session.New(&aws.Config{})) + s, err := helpers.NewSession() + if err != nil { + return errors.WithStack(err) + } + cf := cloudformation.New(s) if w == nil { w = ioutil.Discard @@ -690,7 +702,11 @@ func cloudformationProgress(stack, token string, template []byte, w io.Writer) e } func rackDependencies(name string) ([]string, error) { - cf := cloudformation.New(session.New(&aws.Config{})) + s, err := helpers.NewSession() + if err != nil { + return nil, errors.WithStack(err) + } + cf := cloudformation.New(s) stacks := []string{} diff --git a/provider/kaws/kaws.go b/provider/kaws/kaws.go index 4c492ed92e..382fad1b93 100644 --- a/provider/kaws/kaws.go +++ b/provider/kaws/kaws.go @@ -5,7 +5,6 @@ import ( "fmt" "os" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" @@ -16,6 +15,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" + "github.com/convox/rack/pkg/helpers" "github.com/convox/rack/pkg/structs" "github.com/convox/rack/pkg/templater" "github.com/convox/rack/provider/k8s" @@ -151,7 +151,7 @@ func (p *Provider) applyOutputs(outputs map[string]string) { } func (p *Provider) initializeAwsServices() error { - s, err := session.NewSession() + s, err := helpers.NewSession() if err != nil { return err } diff --git a/provider/kaws/system.go b/provider/kaws/system.go index 25dfc08b3a..dc4ee54246 100644 --- a/provider/kaws/system.go +++ b/provider/kaws/system.go @@ -14,7 +14,6 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/convox/rack/pkg/helpers" "github.com/convox/rack/pkg/structs" @@ -79,7 +78,7 @@ func (p *Provider) SystemInstall(w io.Writer, opts structs.SystemInstallOptions) var bar *pb.ProgressBar - s, err := session.NewSession() + s, err := helpers.NewSession() if err != nil { return "", err } @@ -210,7 +209,7 @@ func (p *Provider) SystemTemplate(version string) ([]byte, error) { } func (p *Provider) SystemUninstall(name string, w io.Writer, opts structs.SystemUninstallOptions) error { - s, err := session.NewSession() + s, err := helpers.NewSession() if err != nil { return err }