diff --git a/CHANGELOG.md b/CHANGELOG.md index 16e5a54d42dd..c2a916970362 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## 0.8.4 (Unreleased) IMPROVEMENTS: + * cli: Add node drain monitoring with new `-monitor` flag on node drain + command [[GH-4260](https://github.com/hashicorp/nomad/issues/4260)] * cli: Add node drain details to node status [[GH-4247](https://github.com/hashicorp/nomad/issues/4247)] * command: Add -short option to init command that emits a minimal jobspec [[GH-4239](https://github.com/hashicorp/nomad/issues/4239)] diff --git a/command/node_drain.go b/command/node_drain.go index 198feb233ca2..7ab600f2aebb 100644 --- a/command/node_drain.go +++ b/command/node_drain.go @@ -113,7 +113,8 @@ func (c *NodeDrainCommand) Name() string { return "node-drain" } func (c *NodeDrainCommand) Run(args []string) int { var enable, disable, detach, force, - noDeadline, ignoreSystem, keepIneligible, self, autoYes bool + noDeadline, ignoreSystem, keepIneligible, + self, autoYes, monitor bool var deadline string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) @@ -128,14 +129,22 @@ func (c *NodeDrainCommand) Run(args []string) int { flags.BoolVar(&keepIneligible, "keep-ineligible", false, "Do not update the nodes scheduling eligibility") flags.BoolVar(&self, "self", false, "") flags.BoolVar(&autoYes, "yes", false, "Automatic yes to prompts.") + flags.BoolVar(&monitor, "monitor", false, "Monitor drain status.") if err := flags.Parse(args); err != nil { return 1 } + // Check that enable or disable is not set with monitor + if monitor && enable || monitor && disable { + c.Ui.Error("The -monitor flag cannot be used with the '-enable' or '-disable' flags") + c.Ui.Error(commandErrorText(c)) + return 1 + } + // Check that we got either enable or disable, but not both. - if (enable && disable) || (!enable && !disable) { - c.Ui.Error("Ethier the '-enable' or '-disable' flag must be set") + if (enable && disable) || (!monitor && !enable && !disable) { + c.Ui.Error("Ethier the '-enable' or '-disable' flag must be set, unless using '-monitor'") c.Ui.Error(commandErrorText(c)) return 1 } @@ -236,6 +245,13 @@ func (c *NodeDrainCommand) Run(args []string) int { return 1 } + // If monitoring the drain start the montior and return when done + if monitor { + c.Ui.Info(fmt.Sprintf("%s: Monitoring node %q: Ctrl-C to detach monitoring", formatTime(time.Now()), node.ID)) + c.monitorDrain(client, context.Background(), node, node.ModifyIndex-1, ignoreSystem) + return 0 + } + // Confirm drain if the node was a prefix match. if nodeID != node.ID && !autoYes { verb := "enable" @@ -290,20 +306,23 @@ func (c *NodeDrainCommand) Run(args []string) int { now := time.Now() c.Ui.Info(fmt.Sprintf("%s: Ctrl-C to stop monitoring: will not cancel the node drain", formatTime(now))) c.Ui.Output(fmt.Sprintf("%s: Node %q drain strategy set", formatTime(now), node.ID)) - outCh := client.Nodes().MonitorDrain(context.Background(), node.ID, meta.LastIndex, ignoreSystem) - for msg := range outCh { - switch msg.Level { - case api.MonitorMsgLevelInfo: - c.Ui.Info(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) - case api.MonitorMsgLevelWarn: - c.Ui.Warn(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) - case api.MonitorMsgLevelError: - c.Ui.Error(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) - default: - c.Ui.Output(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) - } - } + c.monitorDrain(client, context.Background(), node, meta.LastIndex, ignoreSystem) } - return 0 } + +func (c *NodeDrainCommand) monitorDrain(client *api.Client, ctx context.Context, node *api.Node, index uint64, ignoreSystem bool) { + outCh := client.Nodes().MonitorDrain(ctx, node.ID, index, ignoreSystem) + for msg := range outCh { + switch msg.Level { + case api.MonitorMsgLevelInfo: + c.Ui.Info(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) + case api.MonitorMsgLevelWarn: + c.Ui.Warn(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) + case api.MonitorMsgLevelError: + c.Ui.Error(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) + default: + c.Ui.Output(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) + } + } +} diff --git a/command/node_drain_test.go b/command/node_drain_test.go index 9ddc4482a232..c4e167894c92 100644 --- a/command/node_drain_test.go +++ b/command/node_drain_test.go @@ -237,6 +237,19 @@ func TestNodeDrainCommand_Monitor(t *testing.T) { if !strings.HasSuffix(out, expected) { t.Fatalf("expected output to end with:\n%s", expected) } + + // Test -monitor flag + outBuf.Reset() + args = []string{"-address=" + url, "-self", "-monitor", "-ignore-system"} + t.Logf("Running: %v", args) + if code := cmd.Run(args); code != 0 { + t.Fatalf("expected exit 0, got: %d\n%s", code, outBuf.String()) + } + + out = outBuf.String() + t.Logf("Output:\n%s", out) + + require.Contains(out, "marked all allocations for migration") } func TestNodeDrainCommand_Fails(t *testing.T) {