diff --git a/.changelog/10808.txt b/.changelog/10808.txt new file mode 100644 index 000000000000..7bf40638057d --- /dev/null +++ b/.changelog/10808.txt @@ -0,0 +1,3 @@ +```release-note:improvement +cli: Added `nomad operator api` command to ease querying Nomad's HTTP API. +``` diff --git a/command/commands.go b/command/commands.go index 5282f2831747..58ff97077815 100644 --- a/command/commands.go +++ b/command/commands.go @@ -496,6 +496,12 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { }, nil }, + "operator api": func() (cli.Command, error) { + return &OperatorAPICommand{ + Meta: meta, + }, nil + }, + "operator autopilot": func() (cli.Command, error) { return &OperatorAutopilotCommand{ Meta: meta, diff --git a/command/operator_api.go b/command/operator_api.go new file mode 100644 index 000000000000..81f818362054 --- /dev/null +++ b/command/operator_api.go @@ -0,0 +1,449 @@ +package command + +import ( + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/hashicorp/go-cleanhttp" + "github.com/hashicorp/nomad/api" + "github.com/posener/complete" +) + +type OperatorAPICommand struct { + Meta + + verboseFlag bool + method string + body io.Reader +} + +func (*OperatorAPICommand) Help() string { + helpText := ` +Usage: nomad operator api [options] + + api is a utility command for accessing Nomad's HTTP API and is inspired by + the popular curl command line tool. Nomad's operator api command populates + Nomad's standard environment variables into their appropriate HTTP headers. + If the 'path' does not begin with "http" then $NOMAD_ADDR will be used. + + The 'path' can be in one of the following forms: + + /v1/allocations <- API Paths must start with a / + localhost:4646/v1/allocations <- Scheme will be inferred + https://localhost:4646/v1/allocations <- Scheme will be https:// + + Note that this command does not always match the popular curl program's + behavior. Instead Nomad's operator api command is optimized for common Nomad + HTTP API operations. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +Operator API Specific Options: + + -dryrun + Output equivalent curl command to stdout and exit. + HTTP Basic Auth will never be output. If the $NOMAD_HTTP_AUTH environment + variable is set, it will be referenced in the appropriate curl flag in the + output. + ACL tokens set via the $NOMAD_TOKEN environment variable will only be + referenced by environment variable as with HTTP Basic Auth above. However + if the -token flag is explicitly used, the token will also be included in + the output. + + -filter + Specifies an expression used to filter query results. + + -H
+ Adds an additional HTTP header to the request. May be specified more than + once. These headers take precedence over automatically set ones such as + X-Nomad-Token. + + -verbose + Output extra information to stderr similar to curl's --verbose flag. + + -X + HTTP method of request. If there is data piped to stdin, then the method + defaults to POST. Otherwise the method defaults to GET. +` + + return strings.TrimSpace(helpText) +} + +func (*OperatorAPICommand) Synopsis() string { + return "Query Nomad's HTTP API" +} + +func (c *OperatorAPICommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-dryrun": complete.PredictNothing, + }) +} + +func (c *OperatorAPICommand) AutocompleteArgs() complete.Predictor { + //TODO(schmichael) wouldn't it be cool to build path autocompletion off + // of our http mux? + return complete.PredictNothing +} + +func (*OperatorAPICommand) Name() string { return "operator api" } + +func (c *OperatorAPICommand) Run(args []string) int { + var dryrun bool + var filter string + headerFlags := newHeaderFlags() + + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.BoolVar(&dryrun, "dryrun", false, "") + flags.StringVar(&filter, "filter", "", "") + flags.BoolVar(&c.verboseFlag, "verbose", false, "") + flags.StringVar(&c.method, "X", "", "") + flags.Var(headerFlags, "H", "") + + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing flags: %v", err)) + return 1 + } + args = flags.Args() + + if len(args) < 1 { + c.Ui.Error("A path or URL is required") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + if n := len(args); n > 1 { + c.Ui.Error(fmt.Sprintf("operator api accepts exactly 1 argument, but %d arguments were found", n)) + c.Ui.Error(commandErrorText(c)) + return 1 + } + + // By default verbose func is a noop + verbose := func(string, ...interface{}) {} + if c.verboseFlag { + verbose = func(format string, a ...interface{}) { + // Use Warn instead of Info because Info goes to stdout + c.Ui.Warn(fmt.Sprintf(format, a...)) + } + } + + // Opportunistically read from stdin and POST unless method has been + // explicitly set. + stat, _ := os.Stdin.Stat() + if (stat.Mode() & os.ModeCharDevice) == 0 { + verbose("* Reading request body from stdin.") + c.body = os.Stdin + if c.method == "" { + c.method = "POST" + } + } else if c.method == "" { + c.method = "GET" + } + + config := c.clientConfig() + + // NewClient mutates or validates Config.Address, so call it to match + // the behavior of other commands. + _, err := api.NewClient(config) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %v", err)) + return 1 + } + + path, err := pathToURL(config, args[0]) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error turning path into URL: %v", err)) + return 1 + } + + // Set Filter query param + if filter != "" { + q := path.Query() + q.Set("filter", filter) + path.RawQuery = q.Encode() + } + + if dryrun { + out, err := c.apiToCurl(config, headerFlags.headers, path) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error creating curl command: %v", err)) + return 1 + } + c.Ui.Output(out) + return 0 + } + + // Re-implement a big chunk of api/api.go since we don't export it. + client := cleanhttp.DefaultClient() + transport := client.Transport.(*http.Transport) + transport.TLSHandshakeTimeout = 10 * time.Second + transport.TLSClientConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + } + + if err := api.ConfigureTLS(client, config.TLSConfig); err != nil { + c.Ui.Error(fmt.Sprintf("Error configuring TLS: %v", err)) + return 1 + } + + setQueryParams(config, path) + + verbose("> %s %s", c.method, path) + + req, err := http.NewRequest(c.method, path.String(), c.body) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error making request: %v", err)) + return 1 + } + + // Set headers from command line + req.Header = headerFlags.headers + + // Add token header if it doesn't already exist and is set + if req.Header.Get("X-Nomad-Token") == "" && config.SecretID != "" { + req.Header.Set("X-Nomad-Token", config.SecretID) + } + + // Configure HTTP basic authentication if set + if path.User != nil { + username := path.User.Username() + password, _ := path.User.Password() + req.SetBasicAuth(username, password) + } else if config.HttpAuth != nil { + req.SetBasicAuth(config.HttpAuth.Username, config.HttpAuth.Password) + } + + for k, vals := range req.Header { + for _, v := range vals { + verbose("> %s: %s", k, v) + } + } + + verbose("* Sending request and receiving response...") + + // Do the request! + resp, err := client.Do(req) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error performing request: %v", err)) + return 1 + } + defer resp.Body.Close() + + verbose("< %s %s", resp.Proto, resp.Status) + for k, vals := range resp.Header { + for _, v := range vals { + verbose("< %s: %s", k, v) + } + } + + n, err := io.Copy(os.Stdout, resp.Body) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error reading response after %d bytes: %v", n, err)) + return 1 + } + + if len(resp.Trailer) > 0 { + verbose("* Trailer Headers") + for k, vals := range resp.Trailer { + for _, v := range vals { + verbose("< %s: %s", k, v) + } + } + } + + return 0 +} + +// setQueryParams converts API configuration to query parameters. Updates path +// parameter in place. +func setQueryParams(config *api.Config, path *url.URL) { + queryParams := path.Query() + + // Prefer region explicitly set in path, otherwise fallback to config + // if one is set. + if queryParams.Get("region") == "" && config.Region != "" { + queryParams["region"] = []string{config.Region} + } + + // Prefer namespace explicitly set in path, otherwise fallback to + // config if one is set. + if queryParams.Get("namespace") == "" && config.Namespace != "" { + queryParams["namespace"] = []string{config.Namespace} + } + + // Re-encode query parameters + path.RawQuery = queryParams.Encode() +} + +// apiToCurl converts a Nomad HTTP API config and path to its corresponding +// curl command or returns an error. +func (c *OperatorAPICommand) apiToCurl(config *api.Config, headers http.Header, path *url.URL) (string, error) { + parts := []string{"curl"} + + if c.verboseFlag { + parts = append(parts, "--verbose") + } + + if c.method != "" { + parts = append(parts, "-X "+c.method) + } + + if c.body != nil { + parts = append(parts, "--data-binary @-") + } + + if config.TLSConfig != nil { + parts = tlsToCurl(parts, config.TLSConfig) + + // If a TLS server name is set we must alter the URL and use + // curl's --connect-to flag. + if v := config.TLSConfig.TLSServerName; v != "" { + pathHost, port, err := net.SplitHostPort(path.Host) + if err != nil { + return "", fmt.Errorf("error determining port: %v", err) + } + + // curl uses the url for SNI so override it with the + // configured server name + path.Host = net.JoinHostPort(v, port) + + // curl uses --connect-to to allow specifying a + // different connection address for the hostname in the + // path. The format is: + // logical-host:logical-port:actual-host:actual-port + // Ports will always match since only the hostname is + // overridden for SNI. + parts = append(parts, fmt.Sprintf(`--connect-to "%s:%s:%s:%s"`, + v, port, pathHost, port)) + } + } + + // Add headers + for k, vals := range headers { + for _, v := range vals { + parts = append(parts, fmt.Sprintf(`-H '%s: %s'`, k, v)) + } + } + + // Only write NOMAD_TOKEN to stdout if it was specified via -token. + // Otherwise output a static string that references the ACL token + // environment variable. + if headers.Get("X-Nomad-Token") == "" { + if c.Meta.token != "" { + parts = append(parts, fmt.Sprintf(`-H 'X-Nomad-Token: %s'`, c.Meta.token)) + } else if v := os.Getenv("NOMAD_TOKEN"); v != "" { + parts = append(parts, `-H "X-Nomad-Token: ${NOMAD_TOKEN}"`) + } + } + + // Never write http auth to stdout. Instead output a static string that + // references the HTTP auth environment variable. + if auth := os.Getenv("NOMAD_HTTP_AUTH"); auth != "" { + parts = append(parts, `-u "$NOMAD_HTTP_AUTH"`) + } + + setQueryParams(config, path) + + parts = append(parts, path.String()) + + return strings.Join(parts, " \\\n "), nil +} + +// tlsToCurl converts TLS configuration to their corresponding curl flags. +func tlsToCurl(parts []string, tlsConfig *api.TLSConfig) []string { + if v := tlsConfig.CACert; v != "" { + parts = append(parts, fmt.Sprintf(`--cacert "%s"`, v)) + } + + if v := tlsConfig.CAPath; v != "" { + parts = append(parts, fmt.Sprintf(`--capath "%s"`, v)) + } + + if v := tlsConfig.ClientCert; v != "" { + parts = append(parts, fmt.Sprintf(`--cert "%s"`, v)) + } + + if v := tlsConfig.ClientKey; v != "" { + parts = append(parts, fmt.Sprintf(`--key "%s"`, v)) + } + + // TLSServerName has already been configured as it may change the path. + + if tlsConfig.Insecure { + parts = append(parts, `--insecure`) + } + + return parts +} + +// pathToURL converts a curl path argumet to URL. Paths without a host are +// prefixed with $NOMAD_ADDR or http://127.0.0.1:4646. +func pathToURL(config *api.Config, path string) (*url.URL, error) { + // If the scheme is missing, add it + if !strings.HasPrefix(path, "http://") && !strings.HasPrefix(path, "https://") { + scheme := "http" + if config.TLSConfig != nil { + if config.TLSConfig.CACert != "" || + config.TLSConfig.CAPath != "" || + config.TLSConfig.ClientCert != "" || + config.TLSConfig.TLSServerName != "" || + config.TLSConfig.Insecure { + + // TLS configured, but scheme not set. Assume + // https. + scheme = "https" + } + } + + path = fmt.Sprintf("%s://%s", scheme, path) + } + + u, err := url.Parse(path) + if err != nil { + return nil, err + } + + // If URL.Scheme is empty, use defaults from client config + if u.Host == "" { + confURL, err := url.Parse(config.Address) + if err != nil { + return nil, fmt.Errorf("Unable to parse configured address: %v", err) + } + u.Host = confURL.Host + } + + return u, nil +} + +// headerFlags is a flag.Value implementation for collecting multiple -H flags. +type headerFlags struct { + headers http.Header +} + +func newHeaderFlags() *headerFlags { + return &headerFlags{ + headers: make(http.Header), + } +} + +func (*headerFlags) String() string { return "" } + +func (h *headerFlags) Set(v string) error { + parts := strings.SplitN(v, ":", 2) + if len(parts) != 2 { + return fmt.Errorf("Headers must be in the form 'Key: Value' but found: %q", v) + } + + h.headers.Add(parts[0], strings.TrimSpace(parts[1])) + return nil +} diff --git a/command/operator_api_test.go b/command/operator_api_test.go new file mode 100644 index 000000000000..937db63aa8e7 --- /dev/null +++ b/command/operator_api_test.go @@ -0,0 +1,103 @@ +package command + +import ( + "bytes" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +// TestOperatorAPICommand_Paths asserts that the op api command normalizes +// various path formats to the proper full address. +func TestOperatorAPICommand_Paths(t *testing.T) { + hits := make(chan *url.URL, 1) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits <- r.URL + })) + defer ts.Close() + + // Always expect the same URL to be hit + expected := "/v1/jobs" + + buf := bytes.NewBuffer(nil) + ui := &cli.BasicUi{ + ErrorWriter: buf, + Writer: buf, + } + cmd := &OperatorAPICommand{Meta: Meta{Ui: ui}} + + // Assert that absolute paths are appended to the configured address + exitCode := cmd.Run([]string{"-address=" + ts.URL, "/v1/jobs"}) + require.Zero(t, exitCode, buf.String()) + + select { + case hit := <-hits: + require.Equal(t, expected, hit.String()) + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for hit") + } + + buf.Reset() + + // Assert that full URLs are used as-is even if an invalid address is + // set. + exitCode = cmd.Run([]string{"-address=ftp://127.0.0.2:1", ts.URL + "/v1/jobs"}) + require.Zero(t, exitCode, buf.String()) + + select { + case hit := <-hits: + require.Equal(t, expected, hit.String()) + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for hit") + } + + buf.Reset() + + // Assert that URLs lacking a scheme are used even if an invalid + // address is set. + exitCode = cmd.Run([]string{"-address=ftp://127.0.0.2:1", ts.Listener.Addr().String() + "/v1/jobs"}) + require.Zero(t, exitCode, buf.String()) + + select { + case hit := <-hits: + require.Equal(t, expected, hit.String()) + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for hit") + } +} + +// TestOperatorAPICommand_Curl asserts that -dryrun outputs a valid curl +// command. +func TestOperatorAPICommand_Curl(t *testing.T) { + buf := bytes.NewBuffer(nil) + ui := &cli.BasicUi{ + ErrorWriter: buf, + Writer: buf, + } + cmd := &OperatorAPICommand{Meta: Meta{Ui: ui}} + + exitCode := cmd.Run([]string{ + "-dryrun", + "-address=http://127.0.0.1:1", + "-region=not even a valid region", + `-filter=this == "that" or this != "foo"`, + "-X", "POST", + "-token=acl-token", + "-H", "Some-Other-Header: ok", + "/url", + }) + require.Zero(t, exitCode, buf.String()) + + expected := `curl \ + -X POST \ + -H 'Some-Other-Header: ok' \ + -H 'X-Nomad-Token: acl-token' \ + http://127.0.0.1:1/url?filter=this+%3D%3D+%22that%22+or+this+%21%3D+%22foo%22®ion=not+even+a+valid+region +` + require.Equal(t, expected, buf.String()) +} diff --git a/website/content/docs/commands/operator/api.mdx b/website/content/docs/commands/operator/api.mdx new file mode 100644 index 000000000000..913831a86715 --- /dev/null +++ b/website/content/docs/commands/operator/api.mdx @@ -0,0 +1,95 @@ +--- +layout: docs +page_title: 'Commands: operator api' +description: | + operator api is a utility command for accessing Nomad's HTTP API similar to + the popular open source program curl. +--- + +# Command: operator api + +The `operator api` command allows easy access to Nomad's HTTP API similar to +the popular [curl] program. Nomad's `operator api` command reads [environment +variables][envvars] to dramatically ease HTTP API access compared to trying to +manually write the same command with the third party `curl` command. + +For example for the following environment: + +``` +NOMAD_TOKEN=d4434353-c797-19e4-a14d-4068241f86a4 +NOMAD_CACERT=$HOME/.nomad/ca.pem +NOMAD_CLIENT_CERT=$HOME/.nomad/cli.pem +NOMAD_CLIENT_KEY=$HOME/.nomad/client-key.pem +NOMAD_TLS_SERVER_NAME=client.global.nomad +NOMAD_ADDR=https://remote.client123.internal:4646 +``` + +Accessing Nomad's [`/v1/jobs`][jobs] HTTP endpoint with `nomad operator +api` would require: + +``` +nomad operator api /v1/jobs +``` + +Performing the same request using the external `curl` tool would require: + +``` +curl \ + --cacert "$HOME/.nomad/ca.pem" \ + --cert "$HOME/.nomad/client.pem" \ + --key "$HOME/.nomad/client-key.pem" \ + --connect-to "client.global.nomad:4646:remote.client123.internal:4646" \ + -H "X-Nomad-Token: ${NOMAD_TOKEN}" \ + https://client.global.nomad:4646/v1/jobs +``` + +## General Options + +@include 'general_options.mdx' + +## Operator API Options + +- `-dryrun`: output a curl command instead of performing the HTTP request + immediately. Note that you do *not* need the 3rd party `curl` command + installed to use `operator api`. The `curl` output from `-dryrun` is intended + for use in scripts or running in locations without a Nomad binary present. + +- `-filter`: Specifies an expression used to filter query results. + +- `-H`: Adds an additional HTTP header to the request. May be specified more + than once. These headers take precedence over automatically set ones such as + X-Nomad-Token. + +- `-verbose`: Output extra information to stderr similar to curl's --verbose + flag. + +- `-X`: HTTP method of request. If there is data piped to stdin, then the + method defaults to POST. Otherwise the method defaults to GET. + +## Examples + +```shell-session +$ nomad operator api -verbose /v1/agent/members?pretty +> GET http://127.0.0.1:4646/v1/agent/members?pretty= +* Sending request and receiving response... +< HTTP/1.1 200 OK +< Date: Wed, 02 Mar 2022 01:10:59 GMT +< Content-Type: application/json +< Vary: Accept-Encoding +{ + "Members": [ +... + + +$ nomad operator api -region eu-west -filter 'Status == "completed"' -dryrun /v1/evaluations +curl \ + -X GET \ + http://127.0.0.1:4646/v1/evaluations?filter=.Status+%3D%3D+%22completed%22®ion=eu-west +``` + + + + +[curl]: https://curl.se/ +[envvars]: /docs/commands#environment-variables +[jobs]: /api-docs/jobs diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index f6b3200d4b7d..6ba02c93bf1f 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -537,6 +537,10 @@ "title": "Overview", "path": "commands/operator" }, + { + "title": "api", + "path": "commands/operator/api" + }, { "title": "autopilot get-config", "path": "commands/operator/autopilot-get-config"