diff --git a/command/eval_delete.go b/command/eval_delete.go index 2a273b037fb3..027e94e9511e 100644 --- a/command/eval_delete.go +++ b/command/eval_delete.go @@ -17,11 +17,6 @@ type EvalDeleteCommand struct { filter string yes bool - // originalBrokerPaused indicates whether the broker was in a paused state - // before the command was run. This indicates what action, if any, should - // be taken before the command finishes. - originalBrokerPaused bool - // deleteByArg is set when the command is deleting an evaluation that has // been passed as an argument. This avoids need for confirmation. deleteByArg bool @@ -49,6 +44,11 @@ Usage: nomad eval delete [options] normal and outage scenarios, Nomads reconciliation and state management will handle evaluations as needed. + The eval broker is expected to be paused prior to running this command and + un-paused after. This can be done using the following two commands: + - nomad operator scheduler set-config -pause-eval-broker=true + - nomad operator scheduler set-config -pause-eval-broker=false + General Options: ` + generalOptionsUsage(usageOptsNoNamespace) + ` @@ -121,10 +121,19 @@ func (e *EvalDeleteCommand) Run(args []string) int { } e.client = client - // Pause the eval broker to ensure no more work can be dequeued and - // processed by the schedulers. - if err := e.pauseEvalBroker(); err != nil { - e.Ui.Error(fmt.Sprintf("Error pausing eval broker: %s", err)) + // Ensure the eval broker is paused. This check happens multiple times on + // the leader, but this check means we can provide quick and actionable + // feedback. + schedulerConfig, _, err := e.client.Operator().SchedulerGetConfiguration(nil) + if err != nil { + e.Ui.Error(fmt.Sprintf("Error querying scheduler configuration: %s", err)) + return 1 + } + + if !schedulerConfig.SchedulerConfig.PauseEvalBroker { + e.Ui.Error("Eval broker is not paused") + e.Ui.Output(`To delete evaluations you must first pause the eval broker by running "nomad operator scheduler set-config -pause-eval-broker=true"`) + e.Ui.Output(`After the deletion is complete, unpause the eval broker by running "nomad operator scheduler set-config -pause-eval-broker=false"`) return 1 } @@ -160,13 +169,6 @@ func (e *EvalDeleteCommand) Run(args []string) int { e.Ui.Output("No evaluations were deleted") } - // Always attempt to revert the state of the eval broker. In the event this - // succeeds, but the eval deletion failed, maintain the non-zero exit code - // so failures are clear. - if err := e.revertEvalBroker(); err != nil { - e.Ui.Error(fmt.Sprintf("Error reverting eval broker state: %s", err)) - exitCode = 1 - } return exitCode } @@ -189,68 +191,6 @@ func (e *EvalDeleteCommand) verifyArgsAndFlags(args []string) error { return nil } -// pauseEvalBroker pauses the eval broker if it is currently in an enabled -// state. Its previous state will be stored so the command can later revert any -// changes made. -func (e *EvalDeleteCommand) pauseEvalBroker() error { - - schedulerConfig, _, err := e.client.Operator().SchedulerGetConfiguration(nil) - if err != nil { - return err - } - - // Store the current and thus original eval broker paused state. - e.originalBrokerPaused = schedulerConfig.SchedulerConfig.PauseEvalBroker - - // It is possible the operator has already paused the broker via the operator - // API and CLI. If this is not the case, pause it now. - if !e.originalBrokerPaused { - schedulerConfig.SchedulerConfig.PauseEvalBroker = true - - newSchedulerConfig, _, err := e.client.Operator().SchedulerCASConfiguration(schedulerConfig.SchedulerConfig, nil) - if err != nil { - return err - } - - if !newSchedulerConfig.Updated { - return errors.New("failed to update scheduler config, please check server logs") - } - } - - return nil -} - -// revertEvalBroker reverts any initial changes made to the state of the eval -// broker. -func (e *EvalDeleteCommand) revertEvalBroker() error { - - // Read the configuration state first to ensure we are using the correct - // configuration and not overwriting changes. - schedulerConfig, _, err := e.client.Operator().SchedulerGetConfiguration(nil) - if err != nil { - return err - } - - // If the broker is paused, and it was paused before this command was - // invoked, there is nothing else to do. - if schedulerConfig.SchedulerConfig.PauseEvalBroker && e.originalBrokerPaused { - return nil - } - - // Modify the paused value and write this. - schedulerConfig.SchedulerConfig.PauseEvalBroker = false - - newSchedulerConfig, _, err := e.client.Operator().SchedulerCASConfiguration(schedulerConfig.SchedulerConfig, nil) - if err != nil { - return err - } - - if !newSchedulerConfig.Updated { - return errors.New("failed to update scheduler config, please check server logs") - } - return nil -} - // handleEvalArgDelete handles deletion and evaluation which was passed via // it's ID as a command argument. This is the simplest route to take and // doesn't require filtering or batching. diff --git a/command/eval_delete_test.go b/command/eval_delete_test.go index 90a50a098f30..fa50c3483ddd 100644 --- a/command/eval_delete_test.go +++ b/command/eval_delete_test.go @@ -37,16 +37,27 @@ func TestEvalDeleteCommand_Run(t *testing.T) { ui.ErrorWriter.Reset() ui.OutputWriter.Reset() - // Try deleting an eval by its ID that doesn't exist. + // Try running the command when the eval broker is not paused. require.Equal(t, 1, cmd.Run([]string{"-address=" + url, "fa3a8c37-eac3-00c7-3410-5ba3f7318fd8"})) - require.Contains(t, ui.ErrorWriter.String(), "404 (eval not found)") + require.Contains(t, ui.ErrorWriter.String(), "Eval broker is not paused") ui.ErrorWriter.Reset() ui.OutputWriter.Reset() - // Ensure the scheduler config broker is un-paused. + // Paused the eval broker, then try deleting with an eval that + // does not exist. schedulerConfig, _, err := client.Operator().SchedulerGetConfiguration(nil) require.NoError(t, err) require.False(t, schedulerConfig.SchedulerConfig.PauseEvalBroker) + + schedulerConfig.SchedulerConfig.PauseEvalBroker = true + _, _, err = client.Operator().SchedulerSetConfiguration(schedulerConfig.SchedulerConfig, nil) + require.NoError(t, err) + require.True(t, schedulerConfig.SchedulerConfig.PauseEvalBroker) + + require.Equal(t, 1, cmd.Run([]string{"-address=" + url, "fa3a8c37-eac3-00c7-3410-5ba3f7318fd8"})) + require.Contains(t, ui.ErrorWriter.String(), "eval not found") + ui.ErrorWriter.Reset() + ui.OutputWriter.Reset() }, name: "failures", }, diff --git a/website/content/docs/commands/eval/delete.mdx b/website/content/docs/commands/eval/delete.mdx index ae465fe826fd..42cf5125eb4f 100644 --- a/website/content/docs/commands/eval/delete.mdx +++ b/website/content/docs/commands/eval/delete.mdx @@ -2,7 +2,7 @@ layout: docs page_title: 'Commands: eval delete' description: | -The eval delete command is used to delete evaluations. + The eval delete command is used to delete evaluations. --- # Command: eval delete @@ -12,6 +12,11 @@ cautiously and only in outage situations where there is a large backlog of evaluations not being processed. During most normal and outage scenarios, Nomad's reconciliation and state management will handle evaluations as needed. +The eval broker is expected to be paused prior to running this command and +un-paused after. These actions can be performed by the +[`operator scheduler get-config`][scheduler_get_config] +and [`operator scheduler set-config`][scheduler_set_config] commands respectively. + ## Usage ```plaintext @@ -65,3 +70,6 @@ prompts: $ nomad eval delete -filter='Scheduler == "system" or Scheduler == "service"' -yes Successfuly deleted 23 evaluations ``` + +[scheduler_get_config]: /docs/commands/operator/scheduler-get-config +[scheduler_set_config]: /docs/commands/operator/scheduler-set-config diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index eb549a664c09..d8db9febe095 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -372,6 +372,10 @@ "title": "Overview", "path": "commands/eval" }, + { + "title": "delete", + "path": "commands/eval/delete" + }, { "title": "list", "path": "commands/eval/list"