Skip to content

Commit

Permalink
[#3578] Configure Retryer and Max Retry to avoid throttling error
Browse files Browse the repository at this point in the history
Use MaxRetry on aws.Config and set custom timers on DefaultRetryer.

Also, updates Go to 1.17 as `golang.org/x/sys` only allows 2 versions behind the latest one. Go 1.19 was released last month.
  • Loading branch information
Twsouza committed Sep 15, 2022
1 parent 732dc11 commit 62295e6
Show file tree
Hide file tree
Showing 8 changed files with 171 additions and 38 deletions.
3 changes: 1 addition & 2 deletions pkg/cli/rack.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"strings"

"github.com/aws/aws-sdk-go/aws"
ss "github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudformation"
"github.com/convox/rack/pkg/helpers"
"github.com/convox/rack/pkg/options"
Expand Down Expand Up @@ -379,7 +378,7 @@ func RackSync(rack sdk.Interface, c *stdcli.Context) error {

if c.String("name") != "" {
rname = c.String("name")
s, err := ss.NewSession(&aws.Config{})
s, err := helpers.NewSession()
if err != nil {
return err
}
Expand Down
21 changes: 21 additions & 0 deletions pkg/helpers/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudformation"
"github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
Expand All @@ -23,6 +25,10 @@ import (
yaml "gopkg.in/yaml.v2"
)

const (
MAX_RETRY = 10
)

func AwsCredentialsLoad() error {
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
if err := exec.Command("which", "aws").Run(); err != nil {
Expand Down Expand Up @@ -530,3 +536,18 @@ func writeLogEvents(w io.Writer, events []*cloudwatchlogs.FilteredLogEvent, opts

return latest, nil
}

func NewSession() (*session.Session, error) {
s, err := session.NewSession(&aws.Config{
MaxRetries: aws.Int(MAX_RETRY),
Retryer: client.DefaultRetryer{
NumMaxRetries: MAX_RETRY,
MinRetryDelay: 1 * time.Second,
MaxRetryDelay: 5 * time.Second,
MinThrottleDelay: 10 * time.Second,
MaxThrottleDelay: 60 * time.Second,
},
})

return s, err
}
100 changes: 83 additions & 17 deletions provider/aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/acm"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/aws/aws-sdk-go/service/cloudformation"
Expand All @@ -28,8 +27,10 @@ import (
"github.com/aws/aws-sdk-go/service/sqs"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/convox/logger"
"github.com/convox/rack/pkg/helpers"
"github.com/convox/rack/pkg/metrics"
"github.com/convox/rack/pkg/structs"
"github.com/pkg/errors"
)

const (
Expand Down Expand Up @@ -191,7 +192,12 @@ func (p *Provider) Initialize(opts structs.ProviderOptions) error {
go p.Workers()
}

p.CloudWatch = cloudwatch.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
return errors.WithStack(err)
}

p.CloudWatch = cloudwatch.New(s, p.config())

return nil
}
Expand Down Expand Up @@ -264,63 +270,123 @@ func (p *Provider) logger(at string) *logger.Logger {
}

func (p *Provider) acm() *acm.ACM {
return acm.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return acm.New(s, p.config())
}

func (p *Provider) autoscaling() *autoscaling.AutoScaling {
return autoscaling.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return autoscaling.New(s, p.config())
}

func (p *Provider) cloudformation() *cloudformation.CloudFormation {
return cloudformation.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return cloudformation.New(s, p.config())
}

func (p *Provider) cloudwatch() *cloudwatch.CloudWatch {
return cloudwatch.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return cloudwatch.New(s, p.config())
}

func (p *Provider) cloudwatchlogs() *cloudwatchlogs.CloudWatchLogs {
return cloudwatchlogs.New(session.New(), p.config().WithLogLevel(aws.LogOff))
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return cloudwatchlogs.New(s, p.config().WithLogLevel(aws.LogOff))
}

func (p *Provider) dynamodb() *dynamodb.DynamoDB {
return dynamodb.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return dynamodb.New(s, p.config())
}

func (p *Provider) ec2() *ec2.EC2 {
return ec2.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return ec2.New(s, p.config())
}

func (p *Provider) ecr() *ecr.ECR {
return ecr.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return ecr.New(s, p.config())
}

func (p *Provider) ecs() *ecs.ECS {
return ecs.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return ecs.New(s, p.config())
}

func (p *Provider) kms() *kms.KMS {
return kms.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return kms.New(s, p.config())
}

func (p *Provider) iam() *iam.IAM {
return iam.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return iam.New(s, p.config())
}

func (p *Provider) s3() *s3.S3 {
return s3.New(session.New(), p.config().WithS3ForcePathStyle(true))
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return s3.New(s, p.config().WithS3ForcePathStyle(true))
}

func (p *Provider) sns() *sns.SNS {
return sns.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return sns.New(s, p.config())
}

func (p *Provider) sqs() *sqs.SQS {
return sqs.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return sqs.New(s, p.config())
}

func (p *Provider) sts() *sts.STS {
return sts.New(session.New(), p.config())
s, err := helpers.NewSession()
if err != nil {
panic(errors.WithStack(err))
}
return sts.New(s, p.config())
}

// IsTest returns true when we're in test mode
Expand Down
27 changes: 21 additions & 6 deletions provider/aws/lambda/autoscale/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,20 @@ import (
"math"
"os"
"strconv"
"time"

"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudformation"
"github.com/aws/aws-sdk-go/service/ecs"
)

const (
haInstanceCountParam = "InstanceCount"
noHaInstanceCountParam = "NoHaInstanceCount"
HA_INSTANCE_COUNT_PARAM = "InstanceCount"
NO_HA_INSTANCE_COUNT_PARAM = "NoHaInstanceCount"
MAX_RETRY = 10
)

var (
Expand Down Expand Up @@ -55,9 +58,9 @@ func Handler(ctx context.Context) error {
func autoscale(desired int64) error {
stack := os.Getenv("STACK")
ds := fmt.Sprintf("%d", desired)
icParam := haInstanceCountParam
icParam := HA_INSTANCE_COUNT_PARAM
if !IsHA {
icParam = noHaInstanceCountParam
icParam = NO_HA_INSTANCE_COUNT_PARAM
}

debug("desired (ds) = %+v\n", ds)
Expand Down Expand Up @@ -354,8 +357,20 @@ func min(ii ...int64) int64 {
}

func main() {
sessionConfig := &aws.Config{Region: aws.String(os.Getenv("REGION"))}
session, err := session.NewSession(sessionConfig)
session, err := session.NewSession(
&aws.Config{
Region: aws.String(os.Getenv("REGION")),
MaxRetries: aws.Int(MAX_RETRY),
LogLevel: aws.LogLevel(aws.LogDebugWithRequestRetries),
Retryer: client.DefaultRetryer{
NumMaxRetries: MAX_RETRY,
MinRetryDelay: 1 * time.Second,
MaxRetryDelay: 5 * time.Second,
MinThrottleDelay: 10 * time.Second,
MaxThrottleDelay: 60 * time.Second,
},
},
)
if err != nil {
fmt.Printf("Error creating new session: %+v", err)
return
Expand Down
21 changes: 19 additions & 2 deletions provider/aws/lambda/lifecycle/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,16 @@ import (
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/aws/aws-sdk-go/service/ecs"
)

const (
MAX_RETRY = 10
)

var (
AutoScaling *autoscaling.AutoScaling
ECS *ecs.ECS
Expand All @@ -42,8 +47,20 @@ type Termination struct {
}

func main() {
sessionConfig := &aws.Config{Region: aws.String(os.Getenv("REGION"))}
session, err := session.NewSession(sessionConfig)
session, err := session.NewSession(
&aws.Config{
Region: aws.String(os.Getenv("REGION")),
MaxRetries: aws.Int(MAX_RETRY),
LogLevel: aws.LogLevel(aws.LogDebugWithRequestRetries),
Retryer: client.DefaultRetryer{
NumMaxRetries: MAX_RETRY,
MinRetryDelay: 1 * time.Second,
MaxRetryDelay: 5 * time.Second,
MinThrottleDelay: 10 * time.Second,
MaxThrottleDelay: 60 * time.Second,
},
},
)
if err != nil {
fmt.Printf("Error creating new session: %+v", err)
return
Expand Down
Loading

0 comments on commit 62295e6

Please sign in to comment.