diff --git a/client/restarts.go b/client/restarts.go index bf3921dba8af..6da456e83acb 100644 --- a/client/restarts.go +++ b/client/restarts.go @@ -7,6 +7,9 @@ import ( "github.com/hashicorp/nomad/nomad/structs" ) +// jitter is the percent of jitter added to restart delays. +const jitter = 0.25 + func newRestartTracker(policy *structs.RestartPolicy) *RestartTracker { return &RestartTracker{ startTime: time.Now(), @@ -57,8 +60,8 @@ func (r *RestartTracker) shouldRestart(exitCode int) bool { // jitter returns the delay time plus a jitter. func (r *RestartTracker) jitter() time.Duration { d := r.policy.Delay.Nanoseconds() - j := r.rand.Int63n(d) / 4 // Up to 25% jitter - return time.Duration(d + j) + j := float64(r.rand.Int63n(d)) * jitter + return time.Duration(d + int64(j)) } // Returns a tracker that never restarts. diff --git a/client/restarts_test.go b/client/restarts_test.go index 04f2d362d930..79e4d2a56ff7 100644 --- a/client/restarts_test.go +++ b/client/restarts_test.go @@ -17,6 +17,13 @@ func testPolicy(success bool, mode string) *structs.RestartPolicy { } } +// withinJitter is a helper that returns whether the returned delay is within +// the jitter. +func withinJitter(expected, actual time.Duration) bool { + return float64((actual.Nanoseconds()-expected.Nanoseconds())/ + expected.Nanoseconds()) <= jitter +} + func TestClient_RestartTracker_ModeDelay(t *testing.T) { t.Parallel() p := testPolicy(true, structs.RestartPolicyModeDelay) @@ -26,8 +33,8 @@ func TestClient_RestartTracker_ModeDelay(t *testing.T) { if !actual { t.Fatalf("NextRestart() returned %v, want %v", actual, true) } - if when != p.Delay { - t.Fatalf("NextRestart() returned %v; want %v", when, p.Delay) + if !withinJitter(p.Delay, when) { + t.Fatalf("NextRestart() returned %v; want %v+jitter", when, p.Delay) } } @@ -52,8 +59,8 @@ func TestClient_RestartTracker_ModeFail(t *testing.T) { if !actual { t.Fatalf("NextRestart() returned %v, want %v", actual, true) } - if when != p.Delay { - t.Fatalf("NextRestart() returned %v; want %v", when, p.Delay) + if !withinJitter(p.Delay, when) { + t.Fatalf("NextRestart() returned %v; want %v+jitter", when, p.Delay) } }