From c2719d3bf5fa8cc39d5fc0509657d1904f4a43d1 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Tue, 1 Nov 2022 16:27:51 -0400 Subject: [PATCH] variables: limit rekey eval to half the nack timeout In order to limit how much the rekey job can monopolize a scheduler worker, we limit how long it can run to 1min before stopping work and emitting a new eval. But this exactly matches the default nack timeout, so it'll fail the eval rather than getting a chance to emit a new one. Set the timeout for the rekey eval to half the configured nack timeout. --- .changelog/15102.txt | 3 +++ nomad/core_sched.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changelog/15102.txt diff --git a/.changelog/15102.txt b/.changelog/15102.txt new file mode 100644 index 000000000000..6255a038bac7 --- /dev/null +++ b/.changelog/15102.txt @@ -0,0 +1,3 @@ +```release-note:bug +variables: Fixed a bug where a long-running rekey could hit the nack timeout +``` diff --git a/nomad/core_sched.go b/nomad/core_sched.go index 3936498537d3..f4e2aa48b3dc 100644 --- a/nomad/core_sched.go +++ b/nomad/core_sched.go @@ -1053,7 +1053,7 @@ func (c *CoreScheduler) rotateVariables(iter memdb.ResultIterator, eval *structs // // Instead, we'll rate limit RPC requests and have a timeout. If we still // haven't finished the set by the timeout, emit a new eval. - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), c.srv.GetConfig().EvalNackTimeout/2) defer cancel() limiter := rate.NewLimiter(rate.Limit(100), 100)