-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This adds a `nomad alloc stop` command that can be used to stop and force migrate an allocation to a different node. This is built on top of the AllocUpdateDesiredTransitionRequest and explicitly limits the scope of access to that transition to expose it under the alloc-lifecycle ACL. The API returns the follow up eval that can be used as part of monitoring in the CLI or parsed and used in an external tool.
- Loading branch information
1 parent
e613da5
commit 35a2c90
Showing
11 changed files
with
463 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,128 @@ | ||
package command | ||
|
||
import ( | ||
"fmt" | ||
"strings" | ||
) | ||
|
||
type AllocStopCommand struct { | ||
Meta | ||
} | ||
|
||
func (a *AllocStopCommand) Help() string { | ||
helpText := ` | ||
Usage: nomad alloc stop [options] <allocation> | ||
Alias: nomad stop | ||
stop an existing allocation. This command is used to signal a specific alloc | ||
to shut down. When the allocation has been shut down, it will then be | ||
rescheduled. An interactive monitoring session will display log lines as the | ||
allocation completes shutting down. It is safe to exit the monitor early with | ||
ctrl-c. | ||
General Options: | ||
` + generalOptionsUsage() + ` | ||
Stop Specific Options: | ||
-detach | ||
Return immediately instead of entering monitor mode. After the | ||
stop command is submitted, a new evaluation ID is printed to the | ||
screen, which can be used to examine the rescheduling evaluation using the | ||
eval-status command. | ||
-verbose | ||
Show full information. | ||
` | ||
return strings.TrimSpace(helpText) | ||
} | ||
|
||
func (c *AllocStopCommand) Name() string { return "alloc stop" } | ||
|
||
func (c *AllocStopCommand) Run(args []string) int { | ||
var detach, verbose bool | ||
|
||
flags := c.Meta.FlagSet(c.Name(), FlagSetClient) | ||
flags.Usage = func() { c.Ui.Output(c.Help()) } | ||
flags.BoolVar(&detach, "detach", false, "") | ||
flags.BoolVar(&verbose, "verbose", false, "") | ||
|
||
if err := flags.Parse(args); err != nil { | ||
return 1 | ||
} | ||
|
||
// Check that we got exactly one alloc | ||
args = flags.Args() | ||
if len(args) != 1 { | ||
c.Ui.Error("This command takes one argument: <alloc-id>") | ||
c.Ui.Error(commandErrorText(c)) | ||
return 1 | ||
} | ||
|
||
allocID := args[0] | ||
|
||
// Truncate the id unless full length is requested | ||
length := shortId | ||
if verbose { | ||
length = fullId | ||
} | ||
|
||
// Query the allocation info | ||
if len(allocID) == 1 { | ||
c.Ui.Error(fmt.Sprintf("Alloc ID must contain at least two characters.")) | ||
return 1 | ||
} | ||
|
||
allocID = sanitizeUUIDPrefix(allocID) | ||
|
||
// Get the HTTP client | ||
client, err := c.Meta.Client() | ||
if err != nil { | ||
c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) | ||
return 1 | ||
} | ||
|
||
allocs, _, err := client.Allocations().PrefixList(allocID) | ||
if err != nil { | ||
c.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err)) | ||
return 1 | ||
} | ||
|
||
if len(allocs) == 0 { | ||
c.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID)) | ||
return 1 | ||
} | ||
|
||
if len(allocs) > 1 { | ||
// Format the allocs | ||
out := formatAllocListStubs(allocs, verbose, length) | ||
c.Ui.Error(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", out)) | ||
return 1 | ||
} | ||
|
||
// Prefix lookup matched a single allocation | ||
alloc, _, err := client.Allocations().Info(allocs[0].ID, nil) | ||
if err != nil { | ||
c.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err)) | ||
return 1 | ||
} | ||
|
||
resp, err := client.Allocations().Stop(alloc, nil) | ||
if err != nil { | ||
c.Ui.Error(fmt.Sprintf("Error stopping allocation: %s", err)) | ||
return 1 | ||
} | ||
|
||
if detach { | ||
c.Ui.Output(resp.EvalID) | ||
return 0 | ||
} | ||
|
||
mon := newMonitor(c.Ui, client, length) | ||
return mon.monitor(resp.EvalID, false) | ||
} | ||
|
||
func (a *AllocStopCommand) Synopsis() string { | ||
return "Stop and reschedule a running allocation" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,112 @@ | ||
package command | ||
|
||
import ( | ||
"fmt" | ||
"testing" | ||
|
||
"github.com/hashicorp/nomad/api" | ||
"github.com/hashicorp/nomad/nomad/structs" | ||
"github.com/hashicorp/nomad/testutil" | ||
"github.com/mitchellh/cli" | ||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
func TestAllocStopCommand_Implements(t *testing.T) { | ||
t.Parallel() | ||
var _ cli.Command = &AllocStopCommand{} | ||
} | ||
|
||
func TestAllocStop_Fails(t *testing.T) { | ||
srv, _, url := testServer(t, false, nil) | ||
defer srv.Shutdown() | ||
|
||
require := require.New(t) | ||
ui := new(cli.MockUi) | ||
cmd := &AllocStopCommand{Meta: Meta{Ui: ui}} | ||
|
||
// Fails on misuse | ||
require.Equal(cmd.Run([]string{"some", "garbage", "args"}), 1, "Expected failure") | ||
require.Contains(ui.ErrorWriter.String(), commandErrorText(cmd), "Expected help output") | ||
ui.ErrorWriter.Reset() | ||
|
||
// Fails on connection failure | ||
require.Equal(cmd.Run([]string{"-address=nope", "foobar"}), 1, "expected failure") | ||
require.Contains(ui.ErrorWriter.String(), "Error querying allocation") | ||
ui.ErrorWriter.Reset() | ||
|
||
// Fails on missing alloc | ||
require.Equal(cmd.Run([]string{"-address=" + url, "26470238-5CF2-438F-8772-DC67CFB0705C"}), 1) | ||
require.Contains(ui.ErrorWriter.String(), "No allocation(s) with prefix or id") | ||
ui.ErrorWriter.Reset() | ||
|
||
// Fail on identifier with too few characters | ||
require.Equal(cmd.Run([]string{"-address=" + url, "2"}), 1) | ||
require.Contains(ui.ErrorWriter.String(), "must contain at least two characters") | ||
ui.ErrorWriter.Reset() | ||
|
||
// Identifiers with uneven length should produce a query result | ||
require.Equal(cmd.Run([]string{"-address=" + url, "123"}), 1) | ||
require.Contains(ui.ErrorWriter.String(), "No allocation(s) with prefix or id") | ||
ui.ErrorWriter.Reset() | ||
} | ||
|
||
func TestAllocStop_Run(t *testing.T) { | ||
srv, client, url := testServer(t, true, nil) | ||
defer srv.Shutdown() | ||
|
||
require := require.New(t) | ||
|
||
// Wait for a node to be ready | ||
testutil.WaitForResult(func() (bool, error) { | ||
nodes, _, err := client.Nodes().List(nil) | ||
if err != nil { | ||
return false, err | ||
} | ||
for _, node := range nodes { | ||
if _, ok := node.Drivers["mock_driver"]; ok && | ||
node.Status == structs.NodeStatusReady { | ||
return true, nil | ||
} | ||
} | ||
return false, fmt.Errorf("no ready nodes") | ||
}, func(err error) { | ||
t.Fatalf("err: %v", err) | ||
}) | ||
|
||
ui := new(cli.MockUi) | ||
cmd := &AllocStopCommand{Meta: Meta{Ui: ui}} | ||
|
||
jobID := "job1_sfx" | ||
job1 := testJob(jobID) | ||
resp, _, err := client.Jobs().Register(job1, nil) | ||
require.NoError(err) | ||
if code := waitForSuccess(ui, client, fullId, t, resp.EvalID); code != 0 { | ||
t.Fatalf("status code non zero saw %d", code) | ||
} | ||
// get an alloc id | ||
allocId1 := "" | ||
if allocs, _, err := client.Jobs().Allocations(jobID, false, nil); err == nil { | ||
if len(allocs) > 0 { | ||
allocId1 = allocs[0].ID | ||
} | ||
} | ||
require.NotEmpty(allocId1, "unable to find allocation") | ||
|
||
// Wait for alloc to be running | ||
testutil.WaitForResult(func() (bool, error) { | ||
alloc, _, err := client.Allocations().Info(allocId1, nil) | ||
if err != nil { | ||
return false, err | ||
} | ||
if alloc.ClientStatus == api.AllocClientStatusRunning { | ||
return true, nil | ||
} | ||
return false, fmt.Errorf("alloc is not running, is: %s", alloc.ClientStatus) | ||
}, func(err error) { | ||
t.Fatalf("err: %v", err) | ||
}) | ||
|
||
require.Equal(cmd.Run([]string{"-address=" + url, allocId1}), 0, "expected successful exit code") | ||
|
||
ui.OutputWriter.Reset() | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.