diff --git a/command/agent/agent_endpoint.go b/command/agent/agent_endpoint.go index 23c5be215592..5456553ef83c 100644 --- a/command/agent/agent_endpoint.go +++ b/command/agent/agent_endpoint.go @@ -660,7 +660,7 @@ type healthResponseAgent struct { } // AgentHostRequest runs on servers and clients, and captures information about the host system to add -// to the nomad debug archive. +// to the nomad operator debug archive. func (s *HTTPServer) AgentHostRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { if req.Method != http.MethodGet { return nil, CodedError(405, ErrInvalidMethod) diff --git a/command/commands.go b/command/commands.go index dc0166bc8836..1d67fd3e1a11 100644 --- a/command/commands.go +++ b/command/commands.go @@ -201,8 +201,9 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + // operator debug was released in 0.12 as debug. This top-level alias preserves compatibility "debug": func() (cli.Command, error) { - return &DebugCommand{ + return &OperatorDebugCommand{ Meta: meta, }, nil }, @@ -484,6 +485,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "operator debug": func() (cli.Command, error) { + return &OperatorDebugCommand{ + Meta: meta, + }, nil + }, "operator keygen": func() (cli.Command, error) { return &OperatorKeygenCommand{ Meta: meta, diff --git a/command/debug.go b/command/operator_debug.go similarity index 90% rename from command/debug.go rename to command/operator_debug.go index 36901af427e8..73e5c8750183 100644 --- a/command/debug.go +++ b/command/operator_debug.go @@ -24,7 +24,7 @@ import ( "github.com/posener/complete" ) -type DebugCommand struct { +type OperatorDebugCommand struct { Meta timestamp string @@ -43,12 +43,12 @@ type DebugCommand struct { } const ( - userAgent = "nomad debug" + userAgent = "nomad operator debug" ) -func (c *DebugCommand) Help() string { +func (c *OperatorDebugCommand) Help() string { helpText := ` -Usage: nomad debug [options] +Usage: nomad operator debug [options] Build an archive containing Nomad cluster configuration and state, and Consul and Vault status. Include logs and pprof profiles for selected servers and client nodes. @@ -59,7 +59,7 @@ General Options: Debug Options: - -duration= + -duration= The duration of the log monitor command. Defaults to 2m. -interval= @@ -87,7 +87,7 @@ Debug Options: in the current directory. -consul-http-addr= - The address and port of the Consul HTTP agent. Can be specified by CONSUL_HTTP_ADDR + The address and port of the Consul HTTP agent. Overrides the CONSUL_HTTP_ADDR environment variable. -consul-token= Token used to query Consul. Overrides the CONSUL_HTTP_TOKEN environment @@ -140,11 +140,11 @@ Debug Options: return strings.TrimSpace(helpText) } -func (c *DebugCommand) Synopsis() string { +func (c *OperatorDebugCommand) Synopsis() string { return "Build a debug archive" } -func (c *DebugCommand) AutocompleteFlags() complete.Flags { +func (c *OperatorDebugCommand) AutocompleteFlags() complete.Flags { return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), complete.Flags{ "-duration": complete.PredictAnything, @@ -158,13 +158,13 @@ func (c *DebugCommand) AutocompleteFlags() complete.Flags { }) } -func (c *DebugCommand) AutocompleteArgs() complete.Predictor { +func (c *OperatorDebugCommand) AutocompleteArgs() complete.Predictor { return complete.PredictNothing } -func (c *DebugCommand) Name() string { return "debug" } +func (c *OperatorDebugCommand) Name() string { return "debug" } -func (c *DebugCommand) Run(args []string) int { +func (c *OperatorDebugCommand) Run(args []string) int { flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } @@ -317,7 +317,7 @@ func (c *DebugCommand) Run(args []string) int { } // collect collects data from our endpoints and writes the archive bundle -func (c *DebugCommand) collect(client *api.Client) error { +func (c *OperatorDebugCommand) collect(client *api.Client) error { // Version contains cluster meta information dir := "version" err := c.mkdir(dir) @@ -371,19 +371,19 @@ func (c *DebugCommand) collect(client *api.Client) error { } // path returns platform specific paths in the tmp root directory -func (c *DebugCommand) path(paths ...string) string { +func (c *OperatorDebugCommand) path(paths ...string) string { ps := []string{c.collectDir} ps = append(ps, paths...) return filepath.Join(ps...) } // mkdir creates directories in the tmp root directory -func (c *DebugCommand) mkdir(paths ...string) error { +func (c *OperatorDebugCommand) mkdir(paths ...string) error { return os.MkdirAll(c.path(paths...), 0755) } // startMonitors starts go routines for each node and client -func (c *DebugCommand) startMonitors(client *api.Client) { +func (c *OperatorDebugCommand) startMonitors(client *api.Client) { for _, id := range c.nodeIDs { go c.startMonitor("client", "node_id", id, client) } @@ -396,7 +396,7 @@ func (c *DebugCommand) startMonitors(client *api.Client) { // startMonitor starts one monitor api request, writing to a file. It blocks and should be // called in a go routine. Errors are ignored, we want to build the archive even if a node // is unavailable -func (c *DebugCommand) startMonitor(path, idKey, nodeID string, client *api.Client) { +func (c *OperatorDebugCommand) startMonitor(path, idKey, nodeID string, client *api.Client) { c.mkdir(path, nodeID) fh, err := os.Create(c.path(path, nodeID, "monitor.log")) if err != nil { @@ -432,7 +432,7 @@ func (c *DebugCommand) startMonitor(path, idKey, nodeID string, client *api.Clie } // collectAgentHosts calls collectAgentHost for each selected node -func (c *DebugCommand) collectAgentHosts(client *api.Client) { +func (c *OperatorDebugCommand) collectAgentHosts(client *api.Client) { for _, n := range c.nodeIDs { c.collectAgentHost("client", n, client) } @@ -444,7 +444,7 @@ func (c *DebugCommand) collectAgentHosts(client *api.Client) { } // collectAgentHost gets the agent host data -func (c *DebugCommand) collectAgentHost(path, id string, client *api.Client) { +func (c *OperatorDebugCommand) collectAgentHost(path, id string, client *api.Client) { var host *api.HostDataResponse var err error if path == "server" { @@ -459,7 +459,7 @@ func (c *DebugCommand) collectAgentHost(path, id string, client *api.Client) { } // collectPprofs captures the /agent/pprof for each listed node -func (c *DebugCommand) collectPprofs(client *api.Client) { +func (c *OperatorDebugCommand) collectPprofs(client *api.Client) { for _, n := range c.nodeIDs { c.collectPprof("client", n, client) } @@ -471,7 +471,7 @@ func (c *DebugCommand) collectPprofs(client *api.Client) { } // collectPprof captures pprof data for the node -func (c *DebugCommand) collectPprof(path, id string, client *api.Client) { +func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client) { opts := api.PprofOptions{Seconds: 1} if path == "server" { opts.ServerID = id @@ -503,7 +503,7 @@ func (c *DebugCommand) collectPprof(path, id string, client *api.Client) { // collectPeriodic runs for duration, capturing the cluster state every interval. It flushes and stops // the monitor requests -func (c *DebugCommand) collectPeriodic(client *api.Client) { +func (c *OperatorDebugCommand) collectPeriodic(client *api.Client) { // Not monitoring any logs, just capture the nomad context before exit if len(c.nodeIDs) == 0 && len(c.serverIDs) == 0 { dir := filepath.Join("nomad", "0000") @@ -539,7 +539,7 @@ func (c *DebugCommand) collectPeriodic(client *api.Client) { } // collectOperator captures some cluster meta information -func (c *DebugCommand) collectOperator(dir string, client *api.Client) { +func (c *OperatorDebugCommand) collectOperator(dir string, client *api.Client) { rc, err := client.Operator().RaftGetConfiguration(nil) c.writeJSON(dir, "operator-raft.json", rc, err) @@ -551,7 +551,7 @@ func (c *DebugCommand) collectOperator(dir string, client *api.Client) { } // collectNomad captures the nomad cluster state -func (c *DebugCommand) collectNomad(dir string, client *api.Client) error { +func (c *OperatorDebugCommand) collectNomad(dir string, client *api.Client) error { err := c.mkdir(dir) if err != nil { return err @@ -584,7 +584,7 @@ func (c *DebugCommand) collectNomad(dir string, client *api.Client) error { } // collectConsul calls the Consul API directly to collect data -func (c *DebugCommand) collectConsul(dir, consul string) error { +func (c *OperatorDebugCommand) collectConsul(dir, consul string) error { addr := c.consul.addr(consul) if addr == "" { return nil @@ -609,7 +609,7 @@ func (c *DebugCommand) collectConsul(dir, consul string) error { } // collectVault calls the Vault API directly to collect data -func (c *DebugCommand) collectVault(dir, vault string) error { +func (c *OperatorDebugCommand) collectVault(dir, vault string) error { addr := c.vault.addr(vault) if addr == "" { return nil @@ -628,7 +628,7 @@ func (c *DebugCommand) collectVault(dir, vault string) error { } // writeBytes writes a file to the archive, recording it in the manifest -func (c *DebugCommand) writeBytes(dir, file string, data []byte) error { +func (c *OperatorDebugCommand) writeBytes(dir, file string, data []byte) error { path := filepath.Join(dir, file) c.manifest = append(c.manifest, path) path = filepath.Join(c.collectDir, path) @@ -644,7 +644,7 @@ func (c *DebugCommand) writeBytes(dir, file string, data []byte) error { } // writeJSON writes JSON responses from the Nomad API calls to the archive -func (c *DebugCommand) writeJSON(dir, file string, data interface{}, err error) error { +func (c *OperatorDebugCommand) writeJSON(dir, file string, data interface{}, err error) error { if err != nil { return c.writeError(dir, file, err) } @@ -657,7 +657,7 @@ func (c *DebugCommand) writeJSON(dir, file string, data interface{}, err error) // writeError writes a JSON error object to capture errors in the debug bundle without // reporting -func (c *DebugCommand) writeError(dir, file string, err error) error { +func (c *OperatorDebugCommand) writeError(dir, file string, err error) error { bytes, err := json.Marshal(errorWrapper{Error: err.Error()}) if err != nil { return err @@ -670,7 +670,7 @@ type errorWrapper struct { } // writeBody is a helper that writes the body of an http.Response to the archive -func (c *DebugCommand) writeBody(dir, file string, resp *http.Response, err error) { +func (c *OperatorDebugCommand) writeBody(dir, file string, resp *http.Response, err error) { if err != nil { c.writeError(dir, file, err) return @@ -691,7 +691,7 @@ func (c *DebugCommand) writeBody(dir, file string, resp *http.Response, err erro } // writeManifest creates the index files -func (c *DebugCommand) writeManifest() error { +func (c *OperatorDebugCommand) writeManifest() error { // Write the JSON path := filepath.Join(c.collectDir, "index.json") jsonFh, err := os.Create(path) @@ -727,7 +727,7 @@ func (c *DebugCommand) writeManifest() error { } // trap captures signals, and closes stopCh -func (c *DebugCommand) trap() { +func (c *OperatorDebugCommand) trap() { sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGHUP, diff --git a/command/debug_test.go b/command/operator_debug_test.go similarity index 97% rename from command/debug_test.go rename to command/operator_debug_test.go index 9f16564197c0..3d18ab51d33a 100644 --- a/command/debug_test.go +++ b/command/operator_debug_test.go @@ -38,7 +38,7 @@ func TestDebugFails(t *testing.T) { defer srv.Shutdown() ui := new(cli.MockUi) - cmd := &DebugCommand{Meta: Meta{Ui: ui}} + cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} // Fails incorrect args code := cmd.Run([]string{"some", "bad", "args"}) @@ -76,7 +76,7 @@ func TestDebugCapturedFiles(t *testing.T) { defer srv.Shutdown() ui := new(cli.MockUi) - cmd := &DebugCommand{Meta: Meta{Ui: ui}} + cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} code := cmd.Run([]string{ "-address", url, diff --git a/main.go b/main.go index a7cf7fb8975a..cbc05e227ea5 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,7 @@ var ( "alloc-status", "check", "client-config", + "debug", "eval-status", "executor", "keygen", diff --git a/website/data/docs-navigation.js b/website/data/docs-navigation.js index 1fd33342aecc..637773f4961c 100644 --- a/website/data/docs-navigation.js +++ b/website/data/docs-navigation.js @@ -84,7 +84,6 @@ export default [ category: 'alloc', content: ['exec', 'fs', 'logs', 'restart', 'signal', 'status', 'stop'] }, - 'debug', { category: 'deployment', content: ['fail', 'list', 'pause', 'promote', 'resume', 'status', 'unblock'] @@ -127,6 +126,7 @@ export default [ content: [ 'autopilot-get-config', 'autopilot-set-config', + 'debug', 'keygen', 'keyring', 'raft-list-peers', diff --git a/website/pages/api-docs/agent.mdx b/website/pages/api-docs/agent.mdx index e9208ab22b3c..2daeb9a26e7b 100644 --- a/website/pages/api-docs/agent.mdx +++ b/website/pages/api-docs/agent.mdx @@ -501,7 +501,7 @@ $ curl \ This endpoint returns data about the agent's host environment from the perspective of the agent. It is included in the archive produced by -nomad debug. Known sensitive environment variables are shown as +nomad operator debug. Known sensitive environment variables are shown as ``, but the response may still contain sensitive information. diff --git a/website/pages/docs/commands/debug.mdx b/website/pages/docs/commands/debug.mdx deleted file mode 100644 index 0aee6b64a752..000000000000 --- a/website/pages/docs/commands/debug.mdx +++ /dev/null @@ -1,77 +0,0 @@ ---- -layout: docs -page_title: 'Commands: debug' -sidebar_title: debug -description: | - Build a debug archive. ---- - -# Command: debug - -The `debug` command builds an archive containing Nomad cluster -configuration and state information, Nomad server and client node -logs, and pprof profiles from the selected servers and client nodes. - -If no selection option is specified, the debug archive contains only -cluster meta information. - -## Usage - -```plaintext -nomad debug [options] -``` - -This command accepts comma separated `server-id` and `node-id` IDs for -monitoring and pprof profiling. If IDs are provided, the command will -monitor logs for the `duration`, saving a snapshot of Nomad state -every `interval`. Captured logs and configurations are subjected to -redaction, but may still contain sensitive information and the archive -contents should be reviewed before sharing. - -If an `output` path is provided, `debug` will create a timestamped -directory in that path instead of an archive. By default, the command -creates a compressed tar archive in the current directory. - -Consul and Vault status and version information are included if -configured. - -## General Options - -@include 'general_options.mdx' - -## Debug Options - -- `-duration=2m`: Set the duration of the log monitor command. - Defaults to `"2m"`. Logs will be captured from specified servers and - nodes at `log-level`. - -- `-interval=2m`: The interval between snapshots of the Nomad state. - If unspecified, only one snapshot is captured. - -- `-log-level=DEBUG`: The log level to monitor. Defaults to `DEBUG`. - -- `-node-id=n1,n2`: Comma separated list of Nomad client node ids, to - monitor for logs and include pprof data. Accepts id prefixes. - -- `-server-id=s1,s2`: Comma separated list of Nomad server names, or - the special server name "leader" to monitor for logs and include - pprof data. - -- `-output=path`: Path to the parent directory of the output - directory. Defaults to the current directory. If specified, no - archive is built. - -- `-consul-token`: Token used to query Consul. Defaults to `CONSUL_TOKEN` - -- `-vault-token`: Token used to query Vault. Defaults to `VAULT_TOKEN` - -## Output - -This command prints the name of the timestamped archive file produced. - -## Examples - -```shell-session -$ nomad debug -duration 20s -interval 5s -server-id leader -node-id 6e,dd -Created debug archive: nomad-debug-2020-06-10-145821Z.tar.gz -``` diff --git a/website/pages/docs/commands/operator/debug.mdx b/website/pages/docs/commands/operator/debug.mdx new file mode 100644 index 000000000000..05158f02fd0d --- /dev/null +++ b/website/pages/docs/commands/operator/debug.mdx @@ -0,0 +1,120 @@ +--- +layout: docs +page_title: 'Commands: operator debug' +sidebar_title: debug +description: | + Build an archive of debug data. +--- + +# Command: operator debug + +The `operator debug` command builds an archive containing Nomad cluster +configuration and state information, Nomad server and client node +logs, and pprof profiles from the selected servers and client nodes. + +If no selection option is specified, the debug archive contains only +cluster meta information. + +## Usage + +```plaintext +nomad operator debug [options] +``` + +This command accepts comma separated `server-id` and `node-id` IDs for +monitoring and pprof profiling. If IDs are provided, the command will +monitor logs for the `duration`, saving a snapshot of Nomad state +every `interval`. Captured logs and configurations are subjected to +redaction, but may still contain sensitive information and the archive +contents should be reviewed before sharing. + +If an `output` path is provided, `debug` will create a timestamped +directory in that path instead of an archive. By default, the command +creates a compressed tar archive in the current directory. + +Consul and Vault status and version information are included if +configured. + +## General Options + +@include 'general_options.mdx' + +## Debug Options + +- `-duration=2m`: Set the duration of the log monitor command. + Defaults to `"2m"`. Logs will be captured from specified servers and + nodes at `log-level`. + +- `-interval=2m`: The interval between snapshots of the Nomad state. + If unspecified, only one snapshot is captured. + +- `-log-level=DEBUG`: The log level to monitor. Defaults to `DEBUG`. + +- `-node-id=n1,n2`: Comma separated list of Nomad client node ids, to + monitor for logs and include pprof data. Accepts id prefixes. + +- `-server-id=s1,s2`: Comma separated list of Nomad server names, or + the special server name "leader" to monitor for logs and include + pprof data. + +- `-output=path`: Path to the parent directory of the output + directory. Defaults to the current directory. If specified, no + archive is built. + +- `-consul-http-addr=`: The address and port of the Consul HTTP + agent. Overrides the `CONSUL_HTTP_ADDR` environment variable. + +- `-consul-token=`: Token used to query Consul. Overrides the + `CONSUL_HTTP_TOKEN` environment variable and the Consul token + file. + +- `-consul-token-file=`: Path to the Consul token file. Overrides the `CONSUL_HTTP_TOKEN_FILE` + environment variable. + +- `-consul-client-cert=`: Path to the Consul client cert file. Overrides the `CONSUL_CLIENT_CERT` + environment variable. + +- `-consul-client-key=`: Path to the Consul client key file. Overrides the `CONSUL_CLIENT_KEY` + environment variable. + +- `-consul-ca-cert=`: Path to a CA file to use with Consul. Overrides the `CONSUL_CACERT` + environment variable and the Consul CA path. + +- `-consul-ca-path=`: Path to a directory of PEM encoded CA cert files to verify the Consul + certificate. Overrides the `CONSUL_CAPATH` environment variable. + +- `-vault-address=`: The address and port of the Vault HTTP agent. Overrides the `VAULT_ADDR` + environment variable. + +- `-vault-token=`: Token used to query Vault. Overrides the `VAULT_TOKEN` environment + variable. + +- `-vault-client-cert=`: Path to the Vault client cert file. Overrides the `VAULT_CLIENT_CERT` + environment variable. + +- `-vault-client-key=`: Path to the Vault client key file. Overrides the `VAULT_CLIENT_KEY` + environment variable. + +- `-vault-ca-cert=`: Path to a CA file to use with Vault. Overrides the `VAULT_CACERT` + environment variable and the Vault CA path. + +- `-vault-ca-path=`: Path to a directory of PEM encoded CA cert files to verify the Vault + certificate. Overrides the `VAULT_CAPATH` environment variable. + +## Output + +This command prints the name of the timestamped archive file produced. + +## Examples + +```shell-session +$ nomad operator debug -duration 20s -interval 5s -server-id leader -node-id 6e,dd +Starting debugger and capturing cluster data... + Interval: '5s' + Duration: '20s' + Capture interval 0000 + Capture interval 0001 + Capture interval 0002 + Capture interval 0003 +Created debug archive: nomad-debug-2020-07-20-205223Z.tar.gz +``` diff --git a/website/pages/docs/commands/operator/index.mdx b/website/pages/docs/commands/operator/index.mdx index 1705875dea9b..42aa7cb179bd 100644 --- a/website/pages/docs/commands/operator/index.mdx +++ b/website/pages/docs/commands/operator/index.mdx @@ -30,6 +30,8 @@ The following subcommands are available: - [`operator autopilot set-config`][set-config] - Modify the current Autopilot configuration + +- [`operator debug`][debug] - Build an archive of debug data - [`operator keygen`][keygen] - Generates a new encryption key