From d7b17a719c6efdfc4d947c13d3dad3db16726a97 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 7 May 2018 16:14:58 -0400 Subject: [PATCH 1/5] command: add '-monitor' flag to node drain --- CHANGELOG.md | 2 ++ command/node_drain.go | 53 ++++++++++++++++++++++++++------------ command/node_drain_test.go | 13 ++++++++++ 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64db8227437b..812e2bee3859 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ IMPROVEMENTS: isolate canary instances during deployments [[GH-4259](https://github.com/hashicorp/nomad/issues/4259)] * core: Emit Node events for drain and eligibility operations as well as for missed heartbeats [[GH-4284](https://github.com/hashicorp/nomad/issues/4284)], [[GH-4291](https://github.com/hashicorp/nomad/issues/4291)], [[GH-4292](https://github.com/hashicorp/nomad/issues/4292)] + * cli: Add node drain monitoring with new `-monitor` flag on node drain + command [[GH-4260](https://github.com/hashicorp/nomad/issues/4260)] * cli: Add node drain details to node status [[GH-4247](https://github.com/hashicorp/nomad/issues/4247)] * client: Avoid splitting log line across two files [[GH-4282](https://github.com/hashicorp/nomad/issues/4282)] * command: Add -short option to init command that emits a minimal diff --git a/command/node_drain.go b/command/node_drain.go index 198feb233ca2..7ab600f2aebb 100644 --- a/command/node_drain.go +++ b/command/node_drain.go @@ -113,7 +113,8 @@ func (c *NodeDrainCommand) Name() string { return "node-drain" } func (c *NodeDrainCommand) Run(args []string) int { var enable, disable, detach, force, - noDeadline, ignoreSystem, keepIneligible, self, autoYes bool + noDeadline, ignoreSystem, keepIneligible, + self, autoYes, monitor bool var deadline string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) @@ -128,14 +129,22 @@ func (c *NodeDrainCommand) Run(args []string) int { flags.BoolVar(&keepIneligible, "keep-ineligible", false, "Do not update the nodes scheduling eligibility") flags.BoolVar(&self, "self", false, "") flags.BoolVar(&autoYes, "yes", false, "Automatic yes to prompts.") + flags.BoolVar(&monitor, "monitor", false, "Monitor drain status.") if err := flags.Parse(args); err != nil { return 1 } + // Check that enable or disable is not set with monitor + if monitor && enable || monitor && disable { + c.Ui.Error("The -monitor flag cannot be used with the '-enable' or '-disable' flags") + c.Ui.Error(commandErrorText(c)) + return 1 + } + // Check that we got either enable or disable, but not both. - if (enable && disable) || (!enable && !disable) { - c.Ui.Error("Ethier the '-enable' or '-disable' flag must be set") + if (enable && disable) || (!monitor && !enable && !disable) { + c.Ui.Error("Ethier the '-enable' or '-disable' flag must be set, unless using '-monitor'") c.Ui.Error(commandErrorText(c)) return 1 } @@ -236,6 +245,13 @@ func (c *NodeDrainCommand) Run(args []string) int { return 1 } + // If monitoring the drain start the montior and return when done + if monitor { + c.Ui.Info(fmt.Sprintf("%s: Monitoring node %q: Ctrl-C to detach monitoring", formatTime(time.Now()), node.ID)) + c.monitorDrain(client, context.Background(), node, node.ModifyIndex-1, ignoreSystem) + return 0 + } + // Confirm drain if the node was a prefix match. if nodeID != node.ID && !autoYes { verb := "enable" @@ -290,20 +306,23 @@ func (c *NodeDrainCommand) Run(args []string) int { now := time.Now() c.Ui.Info(fmt.Sprintf("%s: Ctrl-C to stop monitoring: will not cancel the node drain", formatTime(now))) c.Ui.Output(fmt.Sprintf("%s: Node %q drain strategy set", formatTime(now), node.ID)) - outCh := client.Nodes().MonitorDrain(context.Background(), node.ID, meta.LastIndex, ignoreSystem) - for msg := range outCh { - switch msg.Level { - case api.MonitorMsgLevelInfo: - c.Ui.Info(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) - case api.MonitorMsgLevelWarn: - c.Ui.Warn(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) - case api.MonitorMsgLevelError: - c.Ui.Error(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) - default: - c.Ui.Output(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) - } - } + c.monitorDrain(client, context.Background(), node, meta.LastIndex, ignoreSystem) } - return 0 } + +func (c *NodeDrainCommand) monitorDrain(client *api.Client, ctx context.Context, node *api.Node, index uint64, ignoreSystem bool) { + outCh := client.Nodes().MonitorDrain(ctx, node.ID, index, ignoreSystem) + for msg := range outCh { + switch msg.Level { + case api.MonitorMsgLevelInfo: + c.Ui.Info(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) + case api.MonitorMsgLevelWarn: + c.Ui.Warn(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) + case api.MonitorMsgLevelError: + c.Ui.Error(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) + default: + c.Ui.Output(fmt.Sprintf("%s: %s", formatTime(time.Now()), msg)) + } + } +} diff --git a/command/node_drain_test.go b/command/node_drain_test.go index 9ddc4482a232..c4e167894c92 100644 --- a/command/node_drain_test.go +++ b/command/node_drain_test.go @@ -237,6 +237,19 @@ func TestNodeDrainCommand_Monitor(t *testing.T) { if !strings.HasSuffix(out, expected) { t.Fatalf("expected output to end with:\n%s", expected) } + + // Test -monitor flag + outBuf.Reset() + args = []string{"-address=" + url, "-self", "-monitor", "-ignore-system"} + t.Logf("Running: %v", args) + if code := cmd.Run(args); code != 0 { + t.Fatalf("expected exit 0, got: %d\n%s", code, outBuf.String()) + } + + out = outBuf.String() + t.Logf("Output:\n%s", out) + + require.Contains(out, "marked all allocations for migration") } func TestNodeDrainCommand_Fails(t *testing.T) { From 6287ac0009d255de5857346f90ffe67039a935e1 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Tue, 8 May 2018 16:21:29 -0400 Subject: [PATCH 2/5] command: use 0 as index for monitor request --- command/node_drain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/node_drain.go b/command/node_drain.go index 7ab600f2aebb..068ba9aeb149 100644 --- a/command/node_drain.go +++ b/command/node_drain.go @@ -248,7 +248,7 @@ func (c *NodeDrainCommand) Run(args []string) int { // If monitoring the drain start the montior and return when done if monitor { c.Ui.Info(fmt.Sprintf("%s: Monitoring node %q: Ctrl-C to detach monitoring", formatTime(time.Now()), node.ID)) - c.monitorDrain(client, context.Background(), node, node.ModifyIndex-1, ignoreSystem) + c.monitorDrain(client, context.Background(), node, 0, ignoreSystem) return 0 } From 10be0d535521de9d9664ff00eeb21580750755b6 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Wed, 9 May 2018 22:54:05 -0400 Subject: [PATCH 3/5] command: add docs for node drain -monitor flag --- command/node_drain.go | 5 ++++- website/source/docs/commands/node/drain.html.md.erb | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/command/node_drain.go b/command/node_drain.go index 068ba9aeb149..040b16ee19fa 100644 --- a/command/node_drain.go +++ b/command/node_drain.go @@ -49,6 +49,9 @@ Node Drain Options: -detach Return immediately instead of entering monitor mode. + -monitor + Enter monitor mode directly without modifying the drain status. + -force Force remove allocations off the node immediately. @@ -136,7 +139,7 @@ func (c *NodeDrainCommand) Run(args []string) int { } // Check that enable or disable is not set with monitor - if monitor && enable || monitor && disable { + if monitor && (enable || disable) { c.Ui.Error("The -monitor flag cannot be used with the '-enable' or '-disable' flags") c.Ui.Error(commandErrorText(c)) return 1 diff --git a/website/source/docs/commands/node/drain.html.md.erb b/website/source/docs/commands/node/drain.html.md.erb index d03ca3f0d6cf..1abbc4cfdcf9 100644 --- a/website/source/docs/commands/node/drain.html.md.erb +++ b/website/source/docs/commands/node/drain.html.md.erb @@ -57,6 +57,7 @@ operation is desired. node. Remaining allocations after the deadline are force removed from the node. Defaults to 1 hour. * `-detach`: Return immediately instead of entering monitor mode. +* `-monitor`: Enter monitor mode directly without modifying the drain status. * `-force`: Force remove allocations off the node immediately. * `-no-deadline`: No deadline allows the allocations to drain off the node without being force stopped after a certain deadline. @@ -114,6 +115,14 @@ $ nomad node drain -disable -keep-ineligible 4d2ba53b ... ``` +Enable drain mode and detach from monitoring, then reattach later: + +``` +$ nomad node drain -enable -detach -self +... +$ nomad node drain -self -monitor +... +``` [eligibility]: /docs/commands/node/eligibility.html [migrate]: /docs/job-specification/migrate.html From faeb612bd9f0fd7fbd45d0e45dc0219572637cba Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Tue, 22 May 2018 14:35:42 -0400 Subject: [PATCH 4/5] api: emit different monitor message if node's drain strategy is never set --- api/nodes.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/nodes.go b/api/nodes.go index cc1f4452b496..59d8a9f728de 100644 --- a/api/nodes.go +++ b/api/nodes.go @@ -193,6 +193,7 @@ func (n *Nodes) monitorDrainNode(ctx context.Context, nodeID string, index uint6 defer close(nodeCh) var lastStrategy *DrainStrategy + var strategyChanged bool q := QueryOptions{ AllowStale: true, WaitIndex: index, @@ -209,7 +210,12 @@ func (n *Nodes) monitorDrainNode(ctx context.Context, nodeID string, index uint6 } if node.DrainStrategy == nil { - msg := Messagef(MonitorMsgLevelInfo, "Node %q has marked all allocations for migration", nodeID) + var msg *MonitorMessage + if strategyChanged { + msg = Messagef(MonitorMsgLevelInfo, "Node %q has marked all allocations for migration", nodeID) + } else { + msg = Messagef(MonitorMsgLevelInfo, "No drain strategy set for node %s", nodeID) + } select { case nodeCh <- msg: case <-ctx.Done(): @@ -236,6 +242,7 @@ func (n *Nodes) monitorDrainNode(ctx context.Context, nodeID string, index uint6 } lastStrategy = node.DrainStrategy + strategyChanged = true // Drain still ongoing, update index and block for updates q.WaitIndex = meta.LastIndex From 0c62b9cd0fe528d55b43f6afd52ff1cc89d96b1b Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 24 May 2018 06:30:33 -0400 Subject: [PATCH 5/5] command: fix node drain monitor case --- command/node_drain_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/node_drain_test.go b/command/node_drain_test.go index c4e167894c92..33d714d1dcac 100644 --- a/command/node_drain_test.go +++ b/command/node_drain_test.go @@ -249,7 +249,7 @@ func TestNodeDrainCommand_Monitor(t *testing.T) { out = outBuf.String() t.Logf("Output:\n%s", out) - require.Contains(out, "marked all allocations for migration") + require.Contains(out, "No drain strategy set") } func TestNodeDrainCommand_Fails(t *testing.T) {