Merge pull request #9034 from hashicorp/dmay-debug-metrics

Add metrics command / output to debug bundle

api/operator.go

@@ -304,3 +304,22 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro
 	}
 	return &reply, qm, nil
 }
+
+// Metrics returns a slice of bytes containing metrics, optionally formatted as either json or prometheus
+func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) {
+	if q == nil {
+		q = &QueryOptions{}
+	}
+
+	metricsReader, err := op.c.rawQuery("/v1/metrics", q)
+	if err != nil {
+		return nil, err
+	}
+
+	metricsBytes, err := ioutil.ReadAll(metricsReader)
+	if err != nil {
+		return nil, err
+	}
+
+	return metricsBytes, nil
+}

command/commands.go

@@ -500,6 +500,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"operator metrics": func() (cli.Command, error) {
+			return &OperatorMetricsCommand{
+				Meta: meta,
+			}, nil
+		},
 		"operator raft": func() (cli.Command, error) {
 			return &OperatorRaftCommand{
 				Meta: meta,

command/metrics.go

@@ -0,0 +1,101 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/posener/complete"
+)
+
+type OperatorMetricsCommand struct {
+	Meta
+}
+
+func (c *OperatorMetricsCommand) Help() string {
+	helpText := `
+Usage: nomad operator metrics [options]
+
+Get Nomad metrics
+General Options:
+
+  ` + generalOptionsUsage() + `
+
+Metrics Specific Options
+
+  -pretty
+    Pretty prints the JSON output
+
+  -format <format>
+    Specify output format (prometheus)
+`
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorMetricsCommand) Synopsis() string {
+	return "Retrieve Nomad metrics"
+}
+
+func (c *OperatorMetricsCommand) AutocompleteFlags() complete.Flags {
+	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
+		complete.Flags{
+			"-pretty": complete.PredictAnything,
+			"-format": complete.PredictAnything,
+		})
+}
+
+func (c *OperatorMetricsCommand) Name() string { return "metrics" }
+
+func (c *OperatorMetricsCommand) Run(args []string) int {
+	var pretty bool
+	var format string
+
+	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
+	flags.Usage = func() { c.Ui.Output(c.Help()) }
+	flags.BoolVar(&pretty, "pretty", false, "")
+	flags.StringVar(&format, "format", "", "")
+
+	if err := flags.Parse(args); err != nil {
+		c.Ui.Error(fmt.Sprintf("Error parsing flags: %s", err))
+		return 1
+	}
+
+	args = flags.Args()
+	if l := len(args); l != 0 {
+		c.Ui.Error("This command takes no arguments")
+		c.Ui.Error(commandErrorText(c))
+		return 1
+	}
+
+	client, err := c.Meta.Client()
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
+		return 1
+	}
+
+	params := map[string]string{}
+
+	if pretty {
+		params["pretty"] = "1"
+	}
+
+	if len(format) > 0 {
+		params["format"] = format
+	}
+
+	query := &api.QueryOptions{
+		Params: params,
+	}
+
+	bs, err := client.Operator().Metrics(query)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error getting metrics: %v", err))
+		return 1
+	}
+
+	resp := string(bs[:])
+	c.Ui.Output(resp)
+
+	return 0
+}

command/metrics_test.go

@@ -0,0 +1,79 @@
+package command
+
+import (
+	"testing"
+
+	"github.com/mitchellh/cli"
+	"github.com/stretchr/testify/require"
+)
+
+var _ cli.Command = &OperatorMetricsCommand{}
+
+func TestCommand_Metrics_Cases(t *testing.T) {
+	t.Parallel()
+
+	srv, _, url := testServer(t, false, nil)
+	defer srv.Shutdown()
+
+	ui := cli.NewMockUi()
+	cmd := &OperatorMetricsCommand{Meta: Meta{Ui: ui}}
+
+	cases := []struct {
+		name           string
+		args           []string
+		expectedCode   int
+		expectedOutput string
+		expectedError  string
+	}{
+		{
+			"pretty print json",
+			[]string{"-address=" + url, "-pretty"},
+			0,
+			"{",
+			"",
+		},
+		{
+			"prometheus format",
+			[]string{"-address=" + url, "-format", "prometheus"},
+			0,
+			"# HELP",
+			"",
+		},
+		{
+			"bad argument",
+			[]string{"-address=" + url, "-foo", "bar"},
+			1,
+			"Usage: nomad operator metrics",
+			"flag provided but not defined: -foo",
+		},
+		{
+			"bad address - no protocol",
+			[]string{"-address=foo"},
+			1,
+			"",
+			"Error getting metrics: Get \"/v1/metrics\": unsupported protocol scheme",
+		},
+		{
+			"bad address - fake host",
+			[]string{"-address=http://foo"},
+			1,
+			"",
+			"no such host",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			code := cmd.Run(c.args)
+			out := ui.OutputWriter.String()
+			outerr := ui.ErrorWriter.String()
+
+			require.Equalf(t, code, c.expectedCode, "expected exit code %d, got: %d: %s", c.expectedCode, code, outerr)
+			require.Contains(t, out, c.expectedOutput, "expected output \"%s\", got \"%s\"", c.expectedOutput, out)
+			require.Containsf(t, outerr, c.expectedError, "expected error \"%s\", got \"%s\"", c.expectedError, outerr)
+
+			ui.OutputWriter.Reset()
+			ui.ErrorWriter.Reset()
+		})
+	}
+}

command/operator_debug.go

@@ -200,6 +200,7 @@ func (c *OperatorDebugCommand) Run(args []string) int {
 	flags.StringVar(&c.vault.tls.ClientKey, "vault-client-key", os.Getenv("VAULT_CLIENT_KEY"), "")
 
 	if err := flags.Parse(args); err != nil {
+		c.Ui.Error(fmt.Sprintf("Error parsing arguments: %q", err))
 		return 1
 	}
 
@@ -575,6 +576,9 @@ func (c *OperatorDebugCommand) collectNomad(dir string, client *api.Client) erro
 	vs, _, err := client.CSIVolumes().List(qo)
 	c.writeJSON(dir, "volumes.json", vs, err)
 
+	metrics, err := client.Operator().Metrics(qo)
+	c.writeJSON(dir, "metrics.json", metrics, err)
+
 	return nil
 }
 

command/operator_debug_test.go

@@ -32,6 +32,29 @@ func TestDebugUtils(t *testing.T) {
 	require.Equal(t, "https://127.0.0.1:8500", e.addr("foo"))
 }
 
+func TestDebugSuccesses(t *testing.T) {
+	t.Parallel()
+	srv, _, _ := testServer(t, false, nil)
+	defer srv.Shutdown()
+
+	ui := cli.NewMockUi()
+	cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}}
+
+	// NOTE -- duration must be shorter than default 2m to prevent testify from timing out
+
+	// Debug on the leader
+	code := cmd.Run([]string{"-duration", "250ms", "-server-id", "leader"})
+	require.Equal(t, 0, code)
+	require.Contains(t, ui.OutputWriter.String(), "Starting debugger")
+	ui.OutputWriter.Reset()
+
+	// Debug on all servers
+	code = cmd.Run([]string{"-duration", "250ms", "-server-id", "all"})
+	require.Equal(t, 0, code)
+	require.Contains(t, ui.OutputWriter.String(), "Starting debugger")
+	ui.OutputWriter.Reset()
+}
+
 func TestDebugFails(t *testing.T) {
 	t.Parallel()
 	srv, _, _ := testServer(t, false, nil)
@@ -111,8 +134,10 @@ func TestDebugCapturedFiles(t *testing.T) {
 	// Multiple snapshots are collected, 00 is always created
 	require.FileExists(t, filepath.Join(path, "nomad", "0000", "jobs.json"))
 	require.FileExists(t, filepath.Join(path, "nomad", "0000", "nodes.json"))
+	require.FileExists(t, filepath.Join(path, "nomad", "0000", "metrics.json"))
 
 	// Multiple snapshots are collected, 01 requires two intervals
 	require.FileExists(t, filepath.Join(path, "nomad", "0001", "jobs.json"))
 	require.FileExists(t, filepath.Join(path, "nomad", "0001", "nodes.json"))
+	require.FileExists(t, filepath.Join(path, "nomad", "0001", "metrics.json"))
 }