From 26792a30451243e8d55c1182caf6d51c6e88f74a Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 8 Jan 2020 17:17:25 -0500 Subject: [PATCH] Add new config options for webhooks --- config/config.go | 22 ++++++++++++++-------- go.mod | 2 +- go.sum | 4 ++-- goflow/engine.go | 8 ++++++-- models/classifiers.go | 2 +- models/env.go | 2 +- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/config/config.go b/config/config.go index ec851f3fe..b574d8806 100644 --- a/config/config.go +++ b/config/config.go @@ -22,10 +22,13 @@ type Config struct { RetryPendingMessages bool `help:"whether to requeue pending messages older than five minutes to retry"` - WebhooksTimeout int `help:"the timeout in seconds for webhook calls from engine"` - SMTPServer string `help:"the smtp configuration for sending emails ex: smtp://user%40password@server:port/?from=foo%40gmail.com"` - MaxStepsPerSprint int `help:"the maximum number of steps allowed per engine sprint"` - MaxValueLength int `help:"the maximum size in characters for contact field values and run result values"` + WebhooksTimeout int `help:"the timeout in milliseconds for webhook calls from engine"` + WebhooksMaxRetries int `help:"the number of times to retry a failed webhook call"` + WebhooksInitialBackoff int `help:"the initial backoff in milliseconds when retrying a failed webhook call"` + WebhooksBackoffJitter float64 `help:"the amount of jitter to apply to backoff times"` + SMTPServer string `help:"the smtp configuration for sending emails ex: smtp://user%40password@server:port/?from=foo%40gmail.com"` + MaxStepsPerSprint int `help:"the maximum number of steps allowed per engine sprint"` + MaxValueLength int `help:"the maximum size in characters for contact field values and run result values"` LibratoUsername string `help:"the username that will be used to authenticate to Librato"` LibratoToken string `help:"the token that will be used to authenticate to Librato"` @@ -61,10 +64,13 @@ func NewMailroomConfig() *Config { LogLevel: "error", Version: "Dev", - WebhooksTimeout: 15, - SMTPServer: "", - MaxStepsPerSprint: 100, - MaxValueLength: 640, + WebhooksTimeout: 15000, + WebhooksMaxRetries: 2, + WebhooksInitialBackoff: 5000, + WebhooksBackoffJitter: 0.5, + SMTPServer: "", + MaxStepsPerSprint: 100, + MaxValueLength: 640, S3Endpoint: "https://s3.amazonaws.com", S3Region: "us-east-1", diff --git a/go.mod b/go.mod index d649a280e..5261cb2db 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 github.com/nyaruka/gocommon v1.1.1 - github.com/nyaruka/goflow v0.64.0 + github.com/nyaruka/goflow v0.64.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d github.com/nyaruka/null v1.2.0 diff --git a/go.sum b/go.sum index 7970e73f0..d8364f437 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= github.com/nyaruka/gocommon v1.1.1 h1:RnQ+kMzN1lA+W0NpkBDd0mGU3UqadJygR3SMpITMYTQ= github.com/nyaruka/gocommon v1.1.1/go.mod h1:QbdU2J9WBsqBmeZRuwndf2f6O7rD7mkC0bGn5UNnwjI= -github.com/nyaruka/goflow v0.64.0 h1:rKrm83laauEOlRkA5tPSq0rP4yPP87XNR6ZYvyzKkSA= -github.com/nyaruka/goflow v0.64.0/go.mod h1:fb6eGAXiTL2hjbzMpXwSIlNx0O4IsR9XBunXBN+8pWM= +github.com/nyaruka/goflow v0.64.1 h1:tAG4HDFh6h05SKvGZr0a8jimih65t9bDHfRjhfECd/c= +github.com/nyaruka/goflow v0.64.1/go.mod h1:fb6eGAXiTL2hjbzMpXwSIlNx0O4IsR9XBunXBN+8pWM= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc= diff --git a/goflow/engine.go b/goflow/engine.go index 7a79031ca..5fe8f8f94 100644 --- a/goflow/engine.go +++ b/goflow/engine.go @@ -101,10 +101,14 @@ func webhooksHTTP() (*http.Client, *httpx.RetryConfig) { webhooksHTTPClient = &http.Client{ Transport: t, - Timeout: time.Duration(config.Mailroom.WebhooksTimeout) * time.Second, + Timeout: time.Duration(config.Mailroom.WebhooksTimeout) * time.Millisecond, } - webhooksHTTPRetries = httpx.NewRetryDelays(3, 10) + webhooksHTTPRetries = httpx.NewExponentialRetries( + time.Duration(config.Mailroom.WebhooksInitialBackoff)*time.Millisecond, + config.Mailroom.WebhooksMaxRetries, + config.Mailroom.WebhooksBackoffJitter, + ) }) return webhooksHTTPClient, webhooksHTTPRetries } diff --git a/models/classifiers.go b/models/classifiers.go index 65b044080..a87306862 100644 --- a/models/classifiers.go +++ b/models/classifiers.go @@ -47,7 +47,7 @@ const ( // Register a classification service factory with the engine func init() { httpClient := &http.Client{Timeout: time.Duration(15 * time.Second)} - httpRetries := httpx.NewRetryDelays(3, 10) + httpRetries := httpx.NewFixedRetries(3, 10) goflow.RegisterClassificationServiceFactory( func(session flows.Session, classifier *flows.Classifier) (flows.ClassificationService, error) { diff --git a/models/env.go b/models/env.go index b5ef4d069..becf819d4 100644 --- a/models/env.go +++ b/models/env.go @@ -23,7 +23,7 @@ import ( func init() { // give airtime transfers an extra long timeout airtimeHTTPClient := &http.Client{Timeout: time.Duration(120 * time.Second)} - airtimeHTTPRetries := httpx.NewRetryDelays(5, 10) + airtimeHTTPRetries := httpx.NewFixedRetries(5, 10) goflow.RegisterEmailServiceFactory( func(session flows.Session) (flows.EmailService, error) {