diff --git a/api/operator.go b/api/operator.go index d5bc5d061d56..cbc5e24a1900 100644 --- a/api/operator.go +++ b/api/operator.go @@ -304,3 +304,22 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro } return &reply, qm, nil } + +// Metrics returns a slice of bytes containing metrics, optionally formatted as either json or prometheus +func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { + if q == nil { + q = &QueryOptions{} + } + + metricsReader, err := op.c.rawQuery("/v1/metrics", q) + if err != nil { + return nil, err + } + + metricsBytes, err := ioutil.ReadAll(metricsReader) + if err != nil { + return nil, err + } + + return metricsBytes, nil +} diff --git a/command/commands.go b/command/commands.go index 3bb5831bff1b..67e61a37fc34 100644 --- a/command/commands.go +++ b/command/commands.go @@ -500,6 +500,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "operator metrics": func() (cli.Command, error) { + return &OperatorMetricsCommand{ + Meta: meta, + }, nil + }, "operator raft": func() (cli.Command, error) { return &OperatorRaftCommand{ Meta: meta, diff --git a/command/metrics.go b/command/metrics.go new file mode 100644 index 000000000000..5ceb39778669 --- /dev/null +++ b/command/metrics.go @@ -0,0 +1,101 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/posener/complete" +) + +type OperatorMetricsCommand struct { + Meta +} + +func (c *OperatorMetricsCommand) Help() string { + helpText := ` +Usage: nomad operator metrics [options] + +Get Nomad metrics +General Options: + + ` + generalOptionsUsage() + ` + +Metrics Specific Options + + -pretty + Pretty prints the JSON output + + -format + Specify output format (prometheus) +` + + return strings.TrimSpace(helpText) +} + +func (c *OperatorMetricsCommand) Synopsis() string { + return "Retrieve Nomad metrics" +} + +func (c *OperatorMetricsCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-pretty": complete.PredictAnything, + "-format": complete.PredictAnything, + }) +} + +func (c *OperatorMetricsCommand) Name() string { return "metrics" } + +func (c *OperatorMetricsCommand) Run(args []string) int { + var pretty bool + var format string + + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.BoolVar(&pretty, "pretty", false, "") + flags.StringVar(&format, "format", "", "") + + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing flags: %s", err)) + return 1 + } + + args = flags.Args() + if l := len(args); l != 0 { + c.Ui.Error("This command takes no arguments") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + params := map[string]string{} + + if pretty { + params["pretty"] = "1" + } + + if len(format) > 0 { + params["format"] = format + } + + query := &api.QueryOptions{ + Params: params, + } + + bs, err := client.Operator().Metrics(query) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error getting metrics: %v", err)) + return 1 + } + + resp := string(bs[:]) + c.Ui.Output(resp) + + return 0 +} diff --git a/command/metrics_test.go b/command/metrics_test.go new file mode 100644 index 000000000000..f909d4e6cca3 --- /dev/null +++ b/command/metrics_test.go @@ -0,0 +1,79 @@ +package command + +import ( + "testing" + + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +var _ cli.Command = &OperatorMetricsCommand{} + +func TestCommand_Metrics_Cases(t *testing.T) { + t.Parallel() + + srv, _, url := testServer(t, false, nil) + defer srv.Shutdown() + + ui := cli.NewMockUi() + cmd := &OperatorMetricsCommand{Meta: Meta{Ui: ui}} + + cases := []struct { + name string + args []string + expectedCode int + expectedOutput string + expectedError string + }{ + { + "pretty print json", + []string{"-address=" + url, "-pretty"}, + 0, + "{", + "", + }, + { + "prometheus format", + []string{"-address=" + url, "-format", "prometheus"}, + 0, + "# HELP", + "", + }, + { + "bad argument", + []string{"-address=" + url, "-foo", "bar"}, + 1, + "Usage: nomad operator metrics", + "flag provided but not defined: -foo", + }, + { + "bad address - no protocol", + []string{"-address=foo"}, + 1, + "", + "Error getting metrics: Get \"/v1/metrics\": unsupported protocol scheme", + }, + { + "bad address - fake host", + []string{"-address=http://foo"}, + 1, + "", + "no such host", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + code := cmd.Run(c.args) + out := ui.OutputWriter.String() + outerr := ui.ErrorWriter.String() + + require.Equalf(t, code, c.expectedCode, "expected exit code %d, got: %d: %s", c.expectedCode, code, outerr) + require.Contains(t, out, c.expectedOutput, "expected output \"%s\", got \"%s\"", c.expectedOutput, out) + require.Containsf(t, outerr, c.expectedError, "expected error \"%s\", got \"%s\"", c.expectedError, outerr) + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + }) + } +} diff --git a/command/operator_debug.go b/command/operator_debug.go index d8260f560de2..b240724e276f 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -200,6 +200,7 @@ func (c *OperatorDebugCommand) Run(args []string) int { flags.StringVar(&c.vault.tls.ClientKey, "vault-client-key", os.Getenv("VAULT_CLIENT_KEY"), "") if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing arguments: %q", err)) return 1 } @@ -575,6 +576,9 @@ func (c *OperatorDebugCommand) collectNomad(dir string, client *api.Client) erro vs, _, err := client.CSIVolumes().List(qo) c.writeJSON(dir, "volumes.json", vs, err) + metrics, err := client.Operator().Metrics(qo) + c.writeJSON(dir, "metrics.json", metrics, err) + return nil } diff --git a/command/operator_debug_test.go b/command/operator_debug_test.go index 3d18ab51d33a..d329deada8d6 100644 --- a/command/operator_debug_test.go +++ b/command/operator_debug_test.go @@ -32,12 +32,35 @@ func TestDebugUtils(t *testing.T) { require.Equal(t, "https://127.0.0.1:8500", e.addr("foo")) } +func TestDebugSuccesses(t *testing.T) { + t.Parallel() + srv, _, _ := testServer(t, false, nil) + defer srv.Shutdown() + + ui := cli.NewMockUi() + cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} + + // NOTE -- duration must be shorter than default 2m to prevent testify from timing out + + // Debug on the leader + code := cmd.Run([]string{"-duration", "250ms", "-server-id", "leader"}) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), "Starting debugger") + ui.OutputWriter.Reset() + + // Debug on all servers + code = cmd.Run([]string{"-duration", "250ms", "-server-id", "all"}) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), "Starting debugger") + ui.OutputWriter.Reset() +} + func TestDebugFails(t *testing.T) { t.Parallel() srv, _, _ := testServer(t, false, nil) defer srv.Shutdown() - ui := new(cli.MockUi) + ui := cli.NewMockUi() cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} // Fails incorrect args @@ -75,7 +98,7 @@ func TestDebugCapturedFiles(t *testing.T) { srv, _, url := testServer(t, false, nil) defer srv.Shutdown() - ui := new(cli.MockUi) + ui := cli.NewMockUi() cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} code := cmd.Run([]string{ @@ -111,8 +134,10 @@ func TestDebugCapturedFiles(t *testing.T) { // Multiple snapshots are collected, 00 is always created require.FileExists(t, filepath.Join(path, "nomad", "0000", "jobs.json")) require.FileExists(t, filepath.Join(path, "nomad", "0000", "nodes.json")) + require.FileExists(t, filepath.Join(path, "nomad", "0000", "metrics.json")) // Multiple snapshots are collected, 01 requires two intervals require.FileExists(t, filepath.Join(path, "nomad", "0001", "jobs.json")) require.FileExists(t, filepath.Join(path, "nomad", "0001", "nodes.json")) + require.FileExists(t, filepath.Join(path, "nomad", "0001", "metrics.json")) } diff --git a/vendor/github.com/hashicorp/nomad/api/operator.go b/vendor/github.com/hashicorp/nomad/api/operator.go index d5bc5d061d56..cbc5e24a1900 100644 --- a/vendor/github.com/hashicorp/nomad/api/operator.go +++ b/vendor/github.com/hashicorp/nomad/api/operator.go @@ -304,3 +304,22 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro } return &reply, qm, nil } + +// Metrics returns a slice of bytes containing metrics, optionally formatted as either json or prometheus +func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { + if q == nil { + q = &QueryOptions{} + } + + metricsReader, err := op.c.rawQuery("/v1/metrics", q) + if err != nil { + return nil, err + } + + metricsBytes, err := ioutil.ReadAll(metricsReader) + if err != nil { + return nil, err + } + + return metricsBytes, nil +}