From 99cbdf2afccd829731cf29a9003fe87e4f80dd0a Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Wed, 23 Jun 2021 16:13:22 -0700 Subject: [PATCH 01/12] cli: add curl command Just a hackweek project at this point. --- command/commands.go | 5 + command/curl.go | 436 +++++++++++++++++++++++++ website/content/docs/commands/curl.mdx | 54 +++ website/data/docs-nav-data.json | 4 + 4 files changed, 499 insertions(+) create mode 100644 command/curl.go create mode 100644 website/content/docs/commands/curl.mdx diff --git a/command/commands.go b/command/commands.go index 287cdfaf1934..ae770ee571d3 100644 --- a/command/commands.go +++ b/command/commands.go @@ -214,6 +214,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "curl": func() (cli.Command, error) { + return &CurlCommand{ + Meta: meta, + }, nil + }, // operator debug was released in 0.12 as debug. This top-level alias preserves compatibility "debug": func() (cli.Command, error) { return &OperatorDebugCommand{ diff --git a/command/curl.go b/command/curl.go new file mode 100644 index 000000000000..8fe18a77e842 --- /dev/null +++ b/command/curl.go @@ -0,0 +1,436 @@ +package command + +import ( + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/hashicorp/go-cleanhttp" + "github.com/hashicorp/nomad/api" + "github.com/posener/complete" +) + +type CurlCommand struct { + Meta + + verboseFlag bool + method string + body io.Reader +} + +func (*CurlCommand) Help() string { + helpText := ` +Usage: nomad curl [options] + + curl is a utility command for accessing Nomad's HTTP API and is inspired by + the popular curl command line program. Nomad's curl command populates Nomad's + standard environment variables into their appropriate HTTP headers. If the + 'path' does not begin with "http" then $NOMAD_ADDR will be used. + + The 'path' can be in one of the following forms: + + /v1/allocations <- API Paths must start with a / + localhost:4646/v1/allocations <- Scheme will be inferred + https://localhost:4646/v1/allocations <- Scheme will be https:// + + Note that Nomad's curl does not always match the popular curl programs + behavior. Instead Nomad's curl is optimized for common Nomad HTTP API + operations. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +Curl Specific Options: + + -dryrun + Output curl command to stdout and exit. + HTTP Basic Auth will never be output. If the $NOMAD_HTTP_AUTH environment + variable is set, it will be referenced in the appropriate curl flag in the + output. + ACL tokens set via the $NOMAD_TOKEN environment variable will only be + referenced by environment variable as with HTTP Basic Auth above. However + if the -token flag is explicitly used, the token will also be included in + the output. + + -H
+ Adds an additional HTTP header to the request. May be specified more than + once. These headers take precedent over automatically ones such as + X-Nomad-Token. + + -verbose + Output extra information to stderr similar to curl's --verbose flag. + + -X + HTTP method of request. Defaults to GET. +` + + return strings.TrimSpace(helpText) +} + +func (*CurlCommand) Synopsis() string { + return "Query Nomad's HTTP API like curl" +} + +func (c *CurlCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-dryrun": complete.PredictNothing, + }) +} + +func (c *CurlCommand) AutocompleteArgs() complete.Predictor { + //TODO(schmichael) wouldn't it be cool to build path autocompletion off + // of our http mux? + return complete.PredictNothing +} + +func (*CurlCommand) Name() string { return "curl" } + +func (c *CurlCommand) Run(args []string) int { + var dryrun bool + headerFlags := newHeaderFlags() + + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.BoolVar(&dryrun, "dryrun", false, "") + flags.BoolVar(&c.verboseFlag, "verbose", false, "") + flags.StringVar(&c.method, "X", "", "") + flags.Var(headerFlags, "H", "") + + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing flags: %v", err)) + return 1 + } + args = flags.Args() + + if len(args) < 1 { + c.Ui.Error("A path or URL is required") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + if n := len(args); n > 1 { + c.Ui.Error(fmt.Sprintf("curl accepts exactly 1 argument, but %d arguments were found", n)) + c.Ui.Error(commandErrorText(c)) + return 1 + } + + // By default verbose func is a noop + verbose := func(string, ...interface{}) {} + if c.verboseFlag { + verbose = func(format string, a ...interface{}) { + // Use Warn instead of Info because Info goes to stdout + c.Ui.Warn(fmt.Sprintf(format, a...)) + } + } + + // Opportunistically read from stdin and POST unless method has been + // explicitly set. + stat, _ := os.Stdin.Stat() + if (stat.Mode() & os.ModeCharDevice) == 0 { + verbose("* Reading request body from stdin.") + c.body = os.Stdin + if c.method == "" { + c.method = "POST" + } + } else { + c.method = "GET" + } + + config := c.clientConfig() + + // NewClient mutates or validates Config.Address, so call it to match + // the behavior of other commands. + _, err := api.NewClient(config) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %v", err)) + return 1 + } + + path, err := pathToURL(config, args[0]) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error turning path into URL: %v", err)) + return 2 + } + + if dryrun { + out, err := c.apiToCurl(config, headerFlags.headers, path) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error creating curl command: %v", err)) + return 3 + } + c.Ui.Output(out) + return 0 + } + + // Re-implement a big chunk of api/api.go since we don't export it. + client := cleanhttp.DefaultClient() + transport := client.Transport.(*http.Transport) + transport.TLSHandshakeTimeout = 10 * time.Second + transport.TLSClientConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + } + + if err := api.ConfigureTLS(client, config.TLSConfig); err != nil { + c.Ui.Error(fmt.Sprintf("Error configuring TLS: %v", err)) + return 4 + } + + setQueryParams(config, path) + + verbose("> %s %s", c.method, path) + + req, err := http.NewRequest(c.method, path.String(), c.body) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error making request: %v", err)) + return 5 + } + + // Set headers from command line + req.Header = headerFlags.headers + + // Add token header if it doesn't already exist and is set + if req.Header.Get("X-Nomad-Token") == "" && config.SecretID != "" { + req.Header.Set("X-Nomad-Token", config.SecretID) + } + + // Configure HTTP basic authentication if set + if path.User != nil { + username := path.User.Username() + password, _ := path.User.Password() + req.SetBasicAuth(username, password) + } else if config.HttpAuth != nil { + req.SetBasicAuth(config.HttpAuth.Username, config.HttpAuth.Password) + } + + for k, vals := range req.Header { + for _, v := range vals { + verbose("> %s: %s", k, v) + } + } + + verbose("* Sending request and receiving response...") + + // Do the request! + resp, err := client.Do(req) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error performing request: %v", err)) + return 6 + } + defer resp.Body.Close() + + verbose("< %s %s", resp.Proto, resp.Status) + for k, vals := range resp.Header { + for _, v := range vals { + verbose("< %s: %s", k, v) + } + } + + n, err := io.Copy(os.Stdout, resp.Body) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error reading response after %d bytes: %v", n, err)) + return 7 + } + + if len(resp.Trailer) > 0 { + verbose("* Trailer Headers") + for k, vals := range resp.Trailer { + for _, v := range vals { + verbose("< %s: %s", k, v) + } + } + } + + return 0 +} + +// setQueryParams converts API configuration to query parameters. Updates path +// parameter in place. +func setQueryParams(config *api.Config, path *url.URL) { + queryParams := path.Query() + + // Prefer region explicitly set in path, otherwise fallback to config + // if one is set. + if queryParams.Get("region") == "" && config.Region != "" { + queryParams["region"] = []string{config.Region} + } + + // Prefer namespace explicitly set in path, otherwise fallback to + // config if one is set. + if queryParams.Get("namespace") == "" && config.Namespace != "" { + queryParams["namespace"] = []string{config.Namespace} + } + + // Re-encode query parameters + path.RawQuery = queryParams.Encode() +} + +// apiToCurl converts a Nomad HTTP API config and path to its corresponding +// curl command or returns an error. +func (c *CurlCommand) apiToCurl(config *api.Config, headers http.Header, path *url.URL) (string, error) { + parts := []string{"curl"} + + if c.verboseFlag { + parts = append(parts, "--verbose") + } + + if c.method != "" { + parts = append(parts, "-X "+c.method) + } + + if c.body != nil { + parts = append(parts, "--data-binary @-") + } + + if config.TLSConfig != nil { + parts = tlsToCurl(parts, config.TLSConfig) + + // If a TLS server name is set we must alter the URL and use + // curl's --connect-to flag. + if v := config.TLSConfig.TLSServerName; v != "" { + pathHost, port, err := net.SplitHostPort(path.Host) + if err != nil { + return "", fmt.Errorf("error determining port: %v", err) + } + + // curl uses the url for SNI so override it with the + // configured server name + path.Host = net.JoinHostPort(v, port) + + // curl uses --connect-to to allow specifying a + // different connection address for the hostname in the + // path. The format is: + // logical-host:logical-port:actual-host:actual-port + // Ports will always match since only the hostname is + // overridden for SNI. + parts = append(parts, fmt.Sprintf(`--connect-to "%s:%s:%s:%s"`, + v, port, pathHost, port)) + } + } + + // Add headers + for k, vals := range headers { + for _, v := range vals { + parts = append(parts, fmt.Sprintf(`-H '%s: %s'`, k, v)) + } + } + + // Only write NOMAD_TOKEN to stdout if it was specified via -token. + // Otherwise output a static string that references the ACL token + // environment variable. + if headers.Get("X-Nomad-Token") == "" { + if c.Meta.token != "" { + parts = append(parts, fmt.Sprintf(`-H "X-Nomad-Token: %s"`, c.Meta.token)) + } else if v := os.Getenv("NOMAD_TOKEN"); v != "" { + parts = append(parts, `-H "X-Nomad-Token: ${NOMAD_TOKEN}"`) + } + } + + // Never write http auth to stdout. Instead output a static string that + // references the HTTP auth environment variable. + if auth := os.Getenv("NOMAD_HTTP_AUTH"); auth != "" { + parts = append(parts, `-u "$NOMAD_HTTP_AUTH"`) + } + + setQueryParams(config, path) + + parts = append(parts, path.String()) + + return strings.Join(parts, " \\\n "), nil +} + +// tlsToCurl converts TLS configuration to their corresponding curl flags. +func tlsToCurl(parts []string, tlsConfig *api.TLSConfig) []string { + if v := tlsConfig.CACert; v != "" { + parts = append(parts, fmt.Sprintf(`--cacert "%s"`, v)) + } + + if v := tlsConfig.CAPath; v != "" { + parts = append(parts, fmt.Sprintf(`--capath "%s"`, v)) + } + + if v := tlsConfig.ClientCert; v != "" { + parts = append(parts, fmt.Sprintf(`--cert "%s"`, v)) + } + + if v := tlsConfig.ClientKey; v != "" { + parts = append(parts, fmt.Sprintf(`--key "%s"`, v)) + } + + // TLSServerName has already been configured as it may change the path. + + if tlsConfig.Insecure { + parts = append(parts, `--insecure`) + } + + return parts +} + +// pathToURL converts a curl path argumet to URL. Paths without a host are +// prefixed with $NOMAD_ADDR or http://127.0.0.1:4646. +func pathToURL(config *api.Config, path string) (*url.URL, error) { + // If the scheme is missing, add it + if !strings.HasPrefix(path, "http://") && !strings.HasPrefix(path, "https://") { + scheme := "http" + if config.TLSConfig != nil { + if config.TLSConfig.CACert != "" || + config.TLSConfig.CAPath != "" || + config.TLSConfig.ClientCert != "" || + config.TLSConfig.TLSServerName != "" || + config.TLSConfig.Insecure { + + // TLS configured, but scheme not set. Assume + // https. + scheme = "https" + } + } + + path = fmt.Sprintf("%s://%s", scheme, path) + } + + u, err := url.Parse(path) + if err != nil { + return nil, err + } + + // If URL.Scheme is empty, use defaults from client config + if u.Host == "" { + confURL, err := url.Parse(config.Address) + if err != nil { + return nil, fmt.Errorf("Unable to parse configured address: %v", err) + } + u.Host = confURL.Host + } + + return u, nil +} + +// headerFlags is a flag.Value implementation for collecting multiple -H flags. +type headerFlags struct { + headers http.Header +} + +func newHeaderFlags() *headerFlags { + return &headerFlags{ + headers: make(http.Header), + } +} + +func (*headerFlags) String() string { return "" } + +func (h *headerFlags) Set(v string) error { + parts := strings.SplitN(v, ":", 2) + if len(parts) != 2 { + return fmt.Errorf("Headers must be in the form 'Key: Value' but found: %q", v) + } + + h.headers.Add(parts[0], parts[1]) + return nil +} diff --git a/website/content/docs/commands/curl.mdx b/website/content/docs/commands/curl.mdx new file mode 100644 index 000000000000..fa5be8b01bef --- /dev/null +++ b/website/content/docs/commands/curl.mdx @@ -0,0 +1,54 @@ +--- +layout: docs +page_title: 'Commands: curl' +description: | + curl is a utility command for accessing Nomad's HTTP API similar to the + popular open source program of the same name. +--- + +# Command: curl + +The curl command allows easy access to Nomad's HTTP API similar to the popular +[curl] program. Nomad's curl command reads [environment variables][envvars] to +dramatically ease HTTP API access compared to trying to manually write the same +command with the third party `curl` command. + +For example for the following environment: + +``` +NOMAD_TOKEN=d4434353-c797-19e4-a14d-4068241f86a4 +NOMAD_CACERT=$HOME/.nomad/ca.pem +NOMAD_CLIENT_CERT=$HOME/.nomad/cli.pem +NOMAD_CLIENT_KEY=$HOME/.nomad/client-key.pem +NOMAD_TLS_SERVER_NAME=client.global.nomad +NOMAD_ADDR=https://remote.client123.internal:4646 +``` + +Accessing Nomad's [`/v1/metrics`][metrics] HTTP endpoint with `nomad curl` +would require: + +``` +nomad curl /v1/metrics +``` + +Performing the same request using the external `curl` tool would require: + +``` +curl \ + --cacert "$HOME/.nomad/ca.pem" \ + --cert "$HOME/.nomad/client.pem" \ + --key "$HOME/.nomad/client-key.pem" \ + --connect-to "client.global.nomad:4646:remote.client123.internal:4646" \ + -H "X-Nomad-Token: ${NOMAD_TOKEN}" \ + https://client.global.nomad:4646/v1/metrics +``` + +The `-dryrun` flag for `nomad curl` will output a curl command instead of +performing the HTTP request immediately. Note that you do *not* need the 3rd +party `curl` command installed to use `nomad curl`. The `curl` output from +`-dryrun` is intended for use in scripts or running in locations without a +Nomad binary present. + +[curl]: https://curl.se/ +[envvars]: /docs/commands#environment-variables +[metrics]: /api-docs/metrics#metrics-http-api diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 7a6b09ea850b..313afb421e30 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -328,6 +328,10 @@ } ] }, + { + "title": "curl", + "path": "commands/curl" + }, { "title": "deployment", "routes": [ From 6bc962fe03021337398e05caf32d9c7d9ed2bcb5 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 18 Feb 2022 16:47:39 -0800 Subject: [PATCH 02/12] rename `nomad curl` to `nomad operator api` --- command/commands.go | 11 ++-- command/{curl.go => operator_api.go} | 36 ++++++------- website/content/docs/commands/curl.mdx | 54 ------------------- .../content/docs/commands/operator/api.mdx | 54 +++++++++++++++++++ 4 files changed, 78 insertions(+), 77 deletions(-) rename command/{curl.go => operator_api.go} (90%) delete mode 100644 website/content/docs/commands/curl.mdx create mode 100644 website/content/docs/commands/operator/api.mdx diff --git a/command/commands.go b/command/commands.go index ae770ee571d3..63bb6d83f301 100644 --- a/command/commands.go +++ b/command/commands.go @@ -214,11 +214,6 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, - "curl": func() (cli.Command, error) { - return &CurlCommand{ - Meta: meta, - }, nil - }, // operator debug was released in 0.12 as debug. This top-level alias preserves compatibility "debug": func() (cli.Command, error) { return &OperatorDebugCommand{ @@ -501,6 +496,12 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { }, nil }, + "operator api": func() (cli.Command, error) { + return &OperatorAPICommand{ + Meta: meta, + }, nil + }, + "operator autopilot": func() (cli.Command, error) { return &OperatorAutopilotCommand{ Meta: meta, diff --git a/command/curl.go b/command/operator_api.go similarity index 90% rename from command/curl.go rename to command/operator_api.go index 8fe18a77e842..fe6d16d55fc0 100644 --- a/command/curl.go +++ b/command/operator_api.go @@ -16,7 +16,7 @@ import ( "github.com/posener/complete" ) -type CurlCommand struct { +type OperatorAPICommand struct { Meta verboseFlag bool @@ -24,14 +24,14 @@ type CurlCommand struct { body io.Reader } -func (*CurlCommand) Help() string { +func (*OperatorAPICommand) Help() string { helpText := ` -Usage: nomad curl [options] +Usage: nomad operator api [options] - curl is a utility command for accessing Nomad's HTTP API and is inspired by - the popular curl command line program. Nomad's curl command populates Nomad's - standard environment variables into their appropriate HTTP headers. If the - 'path' does not begin with "http" then $NOMAD_ADDR will be used. + api is a utility command for accessing Nomad's HTTP API and is inspired by + the popular curl command line program. Nomad's operator api command populates + Nomad's standard environment variables into their appropriate HTTP headers. + If the 'path' does not begin with "http" then $NOMAD_ADDR will be used. The 'path' can be in one of the following forms: @@ -39,15 +39,15 @@ Usage: nomad curl [options] localhost:4646/v1/allocations <- Scheme will be inferred https://localhost:4646/v1/allocations <- Scheme will be https:// - Note that Nomad's curl does not always match the popular curl programs - behavior. Instead Nomad's curl is optimized for common Nomad HTTP API - operations. + Note that this command does not always match the popular curl program's + behavior. Instead Nomad's operator api command is optimized for common Nomad + HTTP API operations. General Options: ` + generalOptionsUsage(usageOptsDefault) + ` -Curl Specific Options: +Operator API Specific Options: -dryrun Output curl command to stdout and exit. @@ -74,26 +74,26 @@ Curl Specific Options: return strings.TrimSpace(helpText) } -func (*CurlCommand) Synopsis() string { +func (*OperatorAPICommand) Synopsis() string { return "Query Nomad's HTTP API like curl" } -func (c *CurlCommand) AutocompleteFlags() complete.Flags { +func (c *OperatorAPICommand) AutocompleteFlags() complete.Flags { return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), complete.Flags{ "-dryrun": complete.PredictNothing, }) } -func (c *CurlCommand) AutocompleteArgs() complete.Predictor { +func (c *OperatorAPICommand) AutocompleteArgs() complete.Predictor { //TODO(schmichael) wouldn't it be cool to build path autocompletion off // of our http mux? return complete.PredictNothing } -func (*CurlCommand) Name() string { return "curl" } +func (*OperatorAPICommand) Name() string { return "operator api" } -func (c *CurlCommand) Run(args []string) int { +func (c *OperatorAPICommand) Run(args []string) int { var dryrun bool headerFlags := newHeaderFlags() @@ -117,7 +117,7 @@ func (c *CurlCommand) Run(args []string) int { } if n := len(args); n > 1 { - c.Ui.Error(fmt.Sprintf("curl accepts exactly 1 argument, but %d arguments were found", n)) + c.Ui.Error(fmt.Sprintf("operator api accepts exactly 1 argument, but %d arguments were found", n)) c.Ui.Error(commandErrorText(c)) return 1 } @@ -274,7 +274,7 @@ func setQueryParams(config *api.Config, path *url.URL) { // apiToCurl converts a Nomad HTTP API config and path to its corresponding // curl command or returns an error. -func (c *CurlCommand) apiToCurl(config *api.Config, headers http.Header, path *url.URL) (string, error) { +func (c *OperatorAPICommand) apiToCurl(config *api.Config, headers http.Header, path *url.URL) (string, error) { parts := []string{"curl"} if c.verboseFlag { diff --git a/website/content/docs/commands/curl.mdx b/website/content/docs/commands/curl.mdx deleted file mode 100644 index fa5be8b01bef..000000000000 --- a/website/content/docs/commands/curl.mdx +++ /dev/null @@ -1,54 +0,0 @@ ---- -layout: docs -page_title: 'Commands: curl' -description: | - curl is a utility command for accessing Nomad's HTTP API similar to the - popular open source program of the same name. ---- - -# Command: curl - -The curl command allows easy access to Nomad's HTTP API similar to the popular -[curl] program. Nomad's curl command reads [environment variables][envvars] to -dramatically ease HTTP API access compared to trying to manually write the same -command with the third party `curl` command. - -For example for the following environment: - -``` -NOMAD_TOKEN=d4434353-c797-19e4-a14d-4068241f86a4 -NOMAD_CACERT=$HOME/.nomad/ca.pem -NOMAD_CLIENT_CERT=$HOME/.nomad/cli.pem -NOMAD_CLIENT_KEY=$HOME/.nomad/client-key.pem -NOMAD_TLS_SERVER_NAME=client.global.nomad -NOMAD_ADDR=https://remote.client123.internal:4646 -``` - -Accessing Nomad's [`/v1/metrics`][metrics] HTTP endpoint with `nomad curl` -would require: - -``` -nomad curl /v1/metrics -``` - -Performing the same request using the external `curl` tool would require: - -``` -curl \ - --cacert "$HOME/.nomad/ca.pem" \ - --cert "$HOME/.nomad/client.pem" \ - --key "$HOME/.nomad/client-key.pem" \ - --connect-to "client.global.nomad:4646:remote.client123.internal:4646" \ - -H "X-Nomad-Token: ${NOMAD_TOKEN}" \ - https://client.global.nomad:4646/v1/metrics -``` - -The `-dryrun` flag for `nomad curl` will output a curl command instead of -performing the HTTP request immediately. Note that you do *not* need the 3rd -party `curl` command installed to use `nomad curl`. The `curl` output from -`-dryrun` is intended for use in scripts or running in locations without a -Nomad binary present. - -[curl]: https://curl.se/ -[envvars]: /docs/commands#environment-variables -[metrics]: /api-docs/metrics#metrics-http-api diff --git a/website/content/docs/commands/operator/api.mdx b/website/content/docs/commands/operator/api.mdx new file mode 100644 index 000000000000..6155b3115b8e --- /dev/null +++ b/website/content/docs/commands/operator/api.mdx @@ -0,0 +1,54 @@ +--- +layout: docs +page_title: 'Commands: operator api' +description: | + operator api is a utility command for accessing Nomad's HTTP API similar to + the popular open source program curl. +--- + +# Command: operator api + +The `operator api` command allows easy access to Nomad's HTTP API similar to +the popular [curl] program. Nomad's `operator api` command reads [environment +variables][envvars] to dramatically ease HTTP API access compared to trying to +manually write the same command with the third party `curl` command. + +For example for the following environment: + +``` +NOMAD_TOKEN=d4434353-c797-19e4-a14d-4068241f86a4 +NOMAD_CACERT=$HOME/.nomad/ca.pem +NOMAD_CLIENT_CERT=$HOME/.nomad/cli.pem +NOMAD_CLIENT_KEY=$HOME/.nomad/client-key.pem +NOMAD_TLS_SERVER_NAME=client.global.nomad +NOMAD_ADDR=https://remote.client123.internal:4646 +``` + +Accessing Nomad's [`/v1/jobs`][jobs] HTTP endpoint with `nomad operator +api` would require: + +``` +nomad operator api /v1/jobs +``` + +Performing the same request using the external `curl` tool would require: + +``` +curl \ + --cacert "$HOME/.nomad/ca.pem" \ + --cert "$HOME/.nomad/client.pem" \ + --key "$HOME/.nomad/client-key.pem" \ + --connect-to "client.global.nomad:4646:remote.client123.internal:4646" \ + -H "X-Nomad-Token: ${NOMAD_TOKEN}" \ + https://client.global.nomad:4646/v1/jobs +``` + +The `-dryrun` flag for `nomad operator api` will output a curl command instead +of performing the HTTP request immediately. Note that you do *not* need the 3rd +party `curl` command installed to use `operator api`. The `curl` output from +`-dryrun` is intended for use in scripts or running in locations without a +Nomad binary present. + +[curl]: https://curl.se/ +[envvars]: /docs/commands#environment-variables +[jobs]: /api-docs/jobs From bd6d7e0057c3ce7d58a7899e08456d687f9130bc Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 24 Feb 2022 15:52:16 -0800 Subject: [PATCH 03/12] cli: add filter support --- command/operator_api.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/command/operator_api.go b/command/operator_api.go index fe6d16d55fc0..dc9bb8200aa0 100644 --- a/command/operator_api.go +++ b/command/operator_api.go @@ -59,6 +59,9 @@ Operator API Specific Options: if the -token flag is explicitly used, the token will also be included in the output. + -filter + Specifies an expression used to filter query results. + -H
Adds an additional HTTP header to the request. May be specified more than once. These headers take precedent over automatically ones such as @@ -95,11 +98,13 @@ func (*OperatorAPICommand) Name() string { return "operator api" } func (c *OperatorAPICommand) Run(args []string) int { var dryrun bool + var filter string headerFlags := newHeaderFlags() flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } flags.BoolVar(&dryrun, "dryrun", false, "") + flags.StringVar(&filter, "filter", "", "") flags.BoolVar(&c.verboseFlag, "verbose", false, "") flags.StringVar(&c.method, "X", "", "") flags.Var(headerFlags, "H", "") @@ -160,6 +165,13 @@ func (c *OperatorAPICommand) Run(args []string) int { return 2 } + // Set Filter query param + if filter != "" { + q := path.Query() + q.Set("filter", filter) + path.RawQuery = q.Encode() + } + if dryrun { out, err := c.apiToCurl(config, headerFlags.headers, path) if err != nil { From a137c1dfe0f4f5136d8f0e207b192db7692eb5a1 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 24 Feb 2022 17:06:07 -0800 Subject: [PATCH 04/12] cli: add tests and minor fixes for op api Trimmed spaces around header values. Fixed method getting forced to GET. --- command/operator_api.go | 8 +-- command/operator_api_test.go | 103 +++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 command/operator_api_test.go diff --git a/command/operator_api.go b/command/operator_api.go index dc9bb8200aa0..8b28ec4c1e2b 100644 --- a/command/operator_api.go +++ b/command/operator_api.go @@ -106,7 +106,7 @@ func (c *OperatorAPICommand) Run(args []string) int { flags.BoolVar(&dryrun, "dryrun", false, "") flags.StringVar(&filter, "filter", "", "") flags.BoolVar(&c.verboseFlag, "verbose", false, "") - flags.StringVar(&c.method, "X", "", "") + flags.StringVar(&c.method, "X", "GET", "") flags.Var(headerFlags, "H", "") if err := flags.Parse(args); err != nil { @@ -145,8 +145,6 @@ func (c *OperatorAPICommand) Run(args []string) int { if c.method == "" { c.method = "POST" } - } else { - c.method = "GET" } config := c.clientConfig() @@ -339,7 +337,7 @@ func (c *OperatorAPICommand) apiToCurl(config *api.Config, headers http.Header, // environment variable. if headers.Get("X-Nomad-Token") == "" { if c.Meta.token != "" { - parts = append(parts, fmt.Sprintf(`-H "X-Nomad-Token: %s"`, c.Meta.token)) + parts = append(parts, fmt.Sprintf(`-H 'X-Nomad-Token: %s'`, c.Meta.token)) } else if v := os.Getenv("NOMAD_TOKEN"); v != "" { parts = append(parts, `-H "X-Nomad-Token: ${NOMAD_TOKEN}"`) } @@ -443,6 +441,6 @@ func (h *headerFlags) Set(v string) error { return fmt.Errorf("Headers must be in the form 'Key: Value' but found: %q", v) } - h.headers.Add(parts[0], parts[1]) + h.headers.Add(parts[0], strings.TrimSpace(parts[1])) return nil } diff --git a/command/operator_api_test.go b/command/operator_api_test.go new file mode 100644 index 000000000000..937db63aa8e7 --- /dev/null +++ b/command/operator_api_test.go @@ -0,0 +1,103 @@ +package command + +import ( + "bytes" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +// TestOperatorAPICommand_Paths asserts that the op api command normalizes +// various path formats to the proper full address. +func TestOperatorAPICommand_Paths(t *testing.T) { + hits := make(chan *url.URL, 1) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits <- r.URL + })) + defer ts.Close() + + // Always expect the same URL to be hit + expected := "/v1/jobs" + + buf := bytes.NewBuffer(nil) + ui := &cli.BasicUi{ + ErrorWriter: buf, + Writer: buf, + } + cmd := &OperatorAPICommand{Meta: Meta{Ui: ui}} + + // Assert that absolute paths are appended to the configured address + exitCode := cmd.Run([]string{"-address=" + ts.URL, "/v1/jobs"}) + require.Zero(t, exitCode, buf.String()) + + select { + case hit := <-hits: + require.Equal(t, expected, hit.String()) + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for hit") + } + + buf.Reset() + + // Assert that full URLs are used as-is even if an invalid address is + // set. + exitCode = cmd.Run([]string{"-address=ftp://127.0.0.2:1", ts.URL + "/v1/jobs"}) + require.Zero(t, exitCode, buf.String()) + + select { + case hit := <-hits: + require.Equal(t, expected, hit.String()) + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for hit") + } + + buf.Reset() + + // Assert that URLs lacking a scheme are used even if an invalid + // address is set. + exitCode = cmd.Run([]string{"-address=ftp://127.0.0.2:1", ts.Listener.Addr().String() + "/v1/jobs"}) + require.Zero(t, exitCode, buf.String()) + + select { + case hit := <-hits: + require.Equal(t, expected, hit.String()) + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for hit") + } +} + +// TestOperatorAPICommand_Curl asserts that -dryrun outputs a valid curl +// command. +func TestOperatorAPICommand_Curl(t *testing.T) { + buf := bytes.NewBuffer(nil) + ui := &cli.BasicUi{ + ErrorWriter: buf, + Writer: buf, + } + cmd := &OperatorAPICommand{Meta: Meta{Ui: ui}} + + exitCode := cmd.Run([]string{ + "-dryrun", + "-address=http://127.0.0.1:1", + "-region=not even a valid region", + `-filter=this == "that" or this != "foo"`, + "-X", "POST", + "-token=acl-token", + "-H", "Some-Other-Header: ok", + "/url", + }) + require.Zero(t, exitCode, buf.String()) + + expected := `curl \ + -X POST \ + -H 'Some-Other-Header: ok' \ + -H 'X-Nomad-Token: acl-token' \ + http://127.0.0.1:1/url?filter=this+%3D%3D+%22that%22+or+this+%21%3D+%22foo%22®ion=not+even+a+valid+region +` + require.Equal(t, expected, buf.String()) +} From 29461444c61530efe66ea0f245d0f12d61af1a20 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 24 Feb 2022 17:13:42 -0800 Subject: [PATCH 05/12] docs: add changelog for #10808 --- .changelog/10808.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/10808.txt diff --git a/.changelog/10808.txt b/.changelog/10808.txt new file mode 100644 index 000000000000..7bf40638057d --- /dev/null +++ b/.changelog/10808.txt @@ -0,0 +1,3 @@ +```release-note:improvement +cli: Added `nomad operator api` command to ease querying Nomad's HTTP API. +``` From 10b60d598337b0c5d8e4b3c99ac35d058a4da032 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 25 Feb 2022 16:21:14 -0800 Subject: [PATCH 06/12] docs: fix nav for op api --- website/data/docs-nav-data.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 313afb421e30..f16701bd2010 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -328,10 +328,6 @@ } ] }, - { - "title": "curl", - "path": "commands/curl" - }, { "title": "deployment", "routes": [ @@ -541,6 +537,10 @@ "title": "Overview", "path": "commands/operator" }, + { + "title": "api", + "path": "commands/operator/api" + }, { "title": "autopilot get-config", "path": "commands/operator/autopilot-get-config" From 08afbf476fecd626834243cc587eff0d8618b568 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 25 Feb 2022 16:23:31 -0800 Subject: [PATCH 07/12] cli: only return 1 on errors from op api We don't want people to expect stable error codes for errors, and I don't think these were useful for scripts anyway. --- command/operator_api.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/command/operator_api.go b/command/operator_api.go index 8b28ec4c1e2b..be932b67aa5d 100644 --- a/command/operator_api.go +++ b/command/operator_api.go @@ -160,7 +160,7 @@ func (c *OperatorAPICommand) Run(args []string) int { path, err := pathToURL(config, args[0]) if err != nil { c.Ui.Error(fmt.Sprintf("Error turning path into URL: %v", err)) - return 2 + return 1 } // Set Filter query param @@ -174,7 +174,7 @@ func (c *OperatorAPICommand) Run(args []string) int { out, err := c.apiToCurl(config, headerFlags.headers, path) if err != nil { c.Ui.Error(fmt.Sprintf("Error creating curl command: %v", err)) - return 3 + return 1 } c.Ui.Output(out) return 0 @@ -190,7 +190,7 @@ func (c *OperatorAPICommand) Run(args []string) int { if err := api.ConfigureTLS(client, config.TLSConfig); err != nil { c.Ui.Error(fmt.Sprintf("Error configuring TLS: %v", err)) - return 4 + return 1 } setQueryParams(config, path) @@ -200,7 +200,7 @@ func (c *OperatorAPICommand) Run(args []string) int { req, err := http.NewRequest(c.method, path.String(), c.body) if err != nil { c.Ui.Error(fmt.Sprintf("Error making request: %v", err)) - return 5 + return 1 } // Set headers from command line @@ -232,7 +232,7 @@ func (c *OperatorAPICommand) Run(args []string) int { resp, err := client.Do(req) if err != nil { c.Ui.Error(fmt.Sprintf("Error performing request: %v", err)) - return 6 + return 1 } defer resp.Body.Close() @@ -246,7 +246,7 @@ func (c *OperatorAPICommand) Run(args []string) int { n, err := io.Copy(os.Stdout, resp.Body) if err != nil { c.Ui.Error(fmt.Sprintf("Error reading response after %d bytes: %v", n, err)) - return 7 + return 1 } if len(resp.Trailer) > 0 { From 3b49cde589dcc04072bc7836e1ead3a8429ecbd8 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 25 Feb 2022 16:31:56 -0800 Subject: [PATCH 08/12] cli: fix op api typos Co-authored-by: Seth Hoenig --- command/operator_api.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/command/operator_api.go b/command/operator_api.go index be932b67aa5d..b21c34c08b47 100644 --- a/command/operator_api.go +++ b/command/operator_api.go @@ -29,7 +29,7 @@ func (*OperatorAPICommand) Help() string { Usage: nomad operator api [options] api is a utility command for accessing Nomad's HTTP API and is inspired by - the popular curl command line program. Nomad's operator api command populates + the popular curl command line tool. Nomad's operator api command populates Nomad's standard environment variables into their appropriate HTTP headers. If the 'path' does not begin with "http" then $NOMAD_ADDR will be used. @@ -50,7 +50,7 @@ General Options: Operator API Specific Options: -dryrun - Output curl command to stdout and exit. + Output equivalent curl command to stdout and exit. HTTP Basic Auth will never be output. If the $NOMAD_HTTP_AUTH environment variable is set, it will be referenced in the appropriate curl flag in the output. @@ -64,7 +64,7 @@ Operator API Specific Options: -H
Adds an additional HTTP header to the request. May be specified more than - once. These headers take precedent over automatically ones such as + once. These headers take precedence over automatically set ones such as X-Nomad-Token. -verbose @@ -78,7 +78,7 @@ Operator API Specific Options: } func (*OperatorAPICommand) Synopsis() string { - return "Query Nomad's HTTP API like curl" + return "Query Nomad's HTTP API" } func (c *OperatorAPICommand) AutocompleteFlags() complete.Flags { From ed95316bdf3e818543b79340e8466f03298774ea Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 1 Mar 2022 16:43:53 -0800 Subject: [PATCH 09/12] docs: add op api options --- .../content/docs/commands/operator/api.mdx | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/website/content/docs/commands/operator/api.mdx b/website/content/docs/commands/operator/api.mdx index 6155b3115b8e..0449894b18ef 100644 --- a/website/content/docs/commands/operator/api.mdx +++ b/website/content/docs/commands/operator/api.mdx @@ -43,11 +43,29 @@ curl \ https://client.global.nomad:4646/v1/jobs ``` -The `-dryrun` flag for `nomad operator api` will output a curl command instead -of performing the HTTP request immediately. Note that you do *not* need the 3rd -party `curl` command installed to use `operator api`. The `curl` output from -`-dryrun` is intended for use in scripts or running in locations without a -Nomad binary present. +## General Options + +@include 'general_options.mdx' + +## Operator API Options + +- `-dryrun`: output a curl command instead of performing the HTTP request + immediately. Note that you do *not* need the 3rd party `curl` command + installed to use `operator api`. The `curl` output from `-dryrun` is intended + for use in scripts or running in locations without a Nomad binary present. + +- `-filter`: Specifies an expression used to filter query results. + +- `-H`: Adds an additional HTTP header to the request. May be specified more + than once. These headers take precedence over automatically set ones such as + X-Nomad-Token. + +- `-verbose`: Output extra information to stderr similar to curl's --verbose + flag. + +- `-X`: HTTP method of request. If there is data piped to stdin, then the + method defaults to POST. Otherwise the method defaults to GET. + [curl]: https://curl.se/ [envvars]: /docs/commands#environment-variables From c23891b976c6c6ac1e9dfb10a7b73b295a751696 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 1 Mar 2022 16:44:15 -0800 Subject: [PATCH 10/12] cli: fix op api method handling --- command/operator_api.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/command/operator_api.go b/command/operator_api.go index b21c34c08b47..81f818362054 100644 --- a/command/operator_api.go +++ b/command/operator_api.go @@ -71,7 +71,8 @@ Operator API Specific Options: Output extra information to stderr similar to curl's --verbose flag. -X - HTTP method of request. Defaults to GET. + HTTP method of request. If there is data piped to stdin, then the method + defaults to POST. Otherwise the method defaults to GET. ` return strings.TrimSpace(helpText) @@ -106,7 +107,7 @@ func (c *OperatorAPICommand) Run(args []string) int { flags.BoolVar(&dryrun, "dryrun", false, "") flags.StringVar(&filter, "filter", "", "") flags.BoolVar(&c.verboseFlag, "verbose", false, "") - flags.StringVar(&c.method, "X", "GET", "") + flags.StringVar(&c.method, "X", "", "") flags.Var(headerFlags, "H", "") if err := flags.Parse(args); err != nil { @@ -145,6 +146,8 @@ func (c *OperatorAPICommand) Run(args []string) int { if c.method == "" { c.method = "POST" } + } else if c.method == "" { + c.method = "GET" } config := c.clientConfig() From a1000ee5b8e680643caae103cb1dd0ac40029f93 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 1 Mar 2022 17:12:58 -0800 Subject: [PATCH 11/12] docs: add op api examples --- .../content/docs/commands/operator/api.mdx | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/website/content/docs/commands/operator/api.mdx b/website/content/docs/commands/operator/api.mdx index 0449894b18ef..b05c6dc54f5a 100644 --- a/website/content/docs/commands/operator/api.mdx +++ b/website/content/docs/commands/operator/api.mdx @@ -66,6 +66,29 @@ curl \ - `-X`: HTTP method of request. If there is data piped to stdin, then the method defaults to POST. Otherwise the method defaults to GET. +## Examples + +```shell-session +$ nomad operator api -verbose /v1/agent/members?pretty +> GET http://127.0.0.1:4646/v1/agent/members?pretty= +* Sending request and receiving response... +< HTTP/1.1 200 OK +< Date: Wed, 02 Mar 2022 01:10:59 GMT +< Content-Type: application/json +< Vary: Accept-Encoding +{ + "Members": [ +... + + +$ nomad operator api -region eu-west -filter '.Status == "completed"' -dryrun /v1/evaluations +curl \ + -X GET \ + http://127.0.0.1:4646/v1/evaluations?filter=.Status+%3D%3D+%22completed%22®ion=eu-west +``` + + + [curl]: https://curl.se/ [envvars]: /docs/commands#environment-variables From 3020b4e8513f0dc3ae8d64856fac0e221052e79f Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 1 Mar 2022 17:15:26 -0800 Subject: [PATCH 12/12] docs: add op api examples --- website/content/docs/commands/operator/api.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/commands/operator/api.mdx b/website/content/docs/commands/operator/api.mdx index b05c6dc54f5a..913831a86715 100644 --- a/website/content/docs/commands/operator/api.mdx +++ b/website/content/docs/commands/operator/api.mdx @@ -81,7 +81,7 @@ $ nomad operator api -verbose /v1/agent/members?pretty ... -$ nomad operator api -region eu-west -filter '.Status == "completed"' -dryrun /v1/evaluations +$ nomad operator api -region eu-west -filter 'Status == "completed"' -dryrun /v1/evaluations curl \ -X GET \ http://127.0.0.1:4646/v1/evaluations?filter=.Status+%3D%3D+%22completed%22®ion=eu-west