Skip to content

Commit

Permalink
Engine: refactor task retry backoff logic (#123)
Browse files Browse the repository at this point in the history
* Engine: refactor task retry backoff logic
* Backoff README
  • Loading branch information
loopfz authored Apr 9, 2020
1 parent 6070417 commit 2572447
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 17 deletions.
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,11 @@ steps:
- `name`: a unique identifier
- `description`: a human readable sentence to convey the step's intent
- `dependencies`: a list of step names on which this step waits before running
- `retry_pattern`: (seconds|minutes|hours) define on what temporal order of magnitude the re-runs of this step should be spread
- `retry_pattern`: (`seconds`, `minutes`, `hours`) define on what temporal order of magnitude the re-runs of this step should be spread (default = `seconds`)

<p align="center">
<img src="./assets/img/utask_backoff.png" width="70%">
</p>

#### Action

Expand Down
Binary file added assets/img/utask_backoff.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
59 changes: 44 additions & 15 deletions engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ import (
"context"
"encoding/json"
"fmt"
"math"
"strings"
"time"

"github.com/cenkalti/backoff"
"github.com/ghodss/yaml"
expbk "github.com/jpillora/backoff"
"github.com/juju/errors"
"github.com/loopfz/gadgeto/zesty"
"github.com/ovh/configstore"
Expand Down Expand Up @@ -820,25 +820,54 @@ func minDuration(a, b time.Duration) time.Duration {
return b
}

// for retryCount: 1, 2, 3, 4, 5, 6......
// seconds: 10, 17, 21, 24, 26, 28.....
// minutes and hours: 1, 1, 7, 11, 14, 16.....
func computeDelay(d time.Duration, rc int) time.Duration {
// shorter two first retries for "minutes" and "hours" retry patterns
if d >= time.Minute {
if rc <= 2 {
return d
}
rc--
const maxMinutes = 30 * time.Minute

// 1m36s, 3m12s, 4m48s, 6m24s, 8m0s, 9m36s, 11m12s, 12m48s, 14m24s, 16m0s
// 17m36s, 19m12s, 20m48s, 22m24s, 24m0s, 25m36s, 27m12s, 28m48s, 30m0s
// 30m0s, 30m0s, ...
// cap 30m0s is reached at 20th retry
func computeDelayMinutesLinear(d time.Duration, rc int) time.Duration {
ret := time.Duration((float64(1.6) * float64(rc)) * float64(d))
if ret >= maxMinutes {
ret = maxMinutes
}
ret := time.Duration(float64(d) * math.Log(float64(rc)) * 10)
// baseline 10s for "seconds" retry pattern
if d < time.Minute {
ret += 10 * d
return ret
}

const maxHours = 3 * time.Hour

// 1h, 1h10, 1h20, 1h30, ... 3h, 3h, 3h, ...
// cap 3h is reached at 13th retry
func computeDelayHoursLinear(d time.Duration, rc int) time.Duration {
rc--
ret := d + (time.Duration(rc) * 10 * time.Minute)
if ret >= maxHours {
ret = maxHours
}
return ret
}

var expBackoff = expbk.Backoff{Min: 8 * time.Second, Max: 10 * time.Minute, Factor: 1.25}

// 10s, 12.5s, 15.625s, 19.53125s, 24.4140625s, 30.517578125s, 38.146972656s, 47.68371582s
// 59.604644775s, 1m14.505805969s, 1m33.132257461s, 1m56.415321826s, 2m25.519152283s
// 3m1.898940354s, 3m47.373675443s, 4m44.217094304s, 5m55.27136788s, 7m24.08920985s, 9m15.111512312s
// 10m0s, 10m0s, 10m0s, ...
// the 10m0s cap is reached at the 20th retry
func computeDelaySecondsExponential(d time.Duration, rc int) time.Duration {
return expBackoff.ForAttempt(float64(rc))
}

func computeDelay(d time.Duration, rc int) time.Duration {
switch d {
case time.Minute:
return computeDelayMinutesLinear(d, rc)
case time.Hour:
return computeDelayHoursLinear(d, rc)
}
return computeDelaySecondsExponential(time.Second, rc)
}

func resolutionStateSetter(res *resolution.Resolution, modifiedSteps map[string]bool) step.StateSetter {
return func(step, state, message string) {
if _, ok := res.Steps[step]; ok {
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ require (
github.com/huandu/xstrings v1.2.0 // indirect
github.com/imdario/mergo v0.3.7 // indirect
github.com/jinzhu/now v1.0.1 // indirect
github.com/jpillora/backoff v1.0.0
github.com/juju/errors v0.0.0-20190207033735-e65537c515d7
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 // indirect
Expand Down Expand Up @@ -57,7 +58,7 @@ require (
github.com/spf13/viper v1.4.0
github.com/stretchr/testify v1.4.0
github.com/tjarratt/babble v0.0.0-20191126185718-ccc47f626248 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
github.com/ugorji/go v1.1.7 // indirect
github.com/wI2L/fizz v0.0.0-20190425144348-6274bc96d962
github.com/ziutek/mymysql v1.5.4 // indirect
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
Expand Down

0 comments on commit 2572447

Please sign in to comment.