From 78aea890d3a97b7756327d1b8b6817aeec0b9c15 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 9 Feb 2023 10:46:49 -0500 Subject: [PATCH] eval broker: extend timeout on integration test Following the update to the go1.20 toolchain, the eval broker integration test started to become flaky. In order to isolate the problem from the test infrastructure and reduce test flakiness, update the test to use a longer timeout and to use `shoenig/test/wait` so we can have sensible reporting of the results rather than just a timeout error. --- nomad/eval_broker_test.go | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/nomad/eval_broker_test.go b/nomad/eval_broker_test.go index 897eee379f9c..add6f00b7060 100644 --- a/nomad/eval_broker_test.go +++ b/nomad/eval_broker_test.go @@ -9,14 +9,16 @@ import ( "time" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" + "github.com/shoenig/test" + "github.com/shoenig/test/must" + "github.com/shoenig/test/wait" + "github.com/stretchr/testify/require" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" - "github.com/shoenig/test" - "github.com/shoenig/test/must" - "github.com/stretchr/testify/require" ) var ( @@ -1567,16 +1569,27 @@ func TestEvalBroker_IntegrationTest(t *testing.T) { config := DefaultConfig() config.NumSchedulers = 4 - config.EvalReapCancelableInterval = time.Minute * 10 - require.NoError(t, srv.Reload(config)) + config.EvalReapCancelableInterval = time.Second + must.NoError(t, srv.Reload(config)) // assert that all but 2 evals were canceled and that the eval broker state // has been cleared - require.Eventually(t, func() bool { - got := getEvalStatuses() - return got[structs.EvalStatusComplete] == 2 && got[structs.EvalStatusCancelled] == 9 - }, 2*time.Second, time.Millisecond*100) + var got map[string]int + + must.Wait(t, wait.InitialSuccess( + wait.Timeout(5*time.Second), + wait.Gap(100*time.Millisecond), + wait.BoolFunc(func() bool { + got = getEvalStatuses() + return got[structs.EvalStatusComplete] == 2 && + got[structs.EvalStatusCancelled] == 9 + }), + ), + must.Func(func() string { + return fmt.Sprintf("expected map[complete:2 canceled:9] within timeout, got: %v with broker status=%#v", got, getStats()) + }), + ) must.Eq(t, BrokerStats{TotalReady: 0, TotalUnacked: 0, TotalPending: 0, TotalCancelable: 0}, getStats())