From 29b4b06511b792bc1f50e58c227d7365e4b9704b Mon Sep 17 00:00:00 2001 From: danishprakash Date: Sun, 9 Jan 2022 18:55:29 +0530 Subject: [PATCH 01/10] command/operator_debug: add pprof interval Signed-off-by: danishprakash --- command/operator_debug.go | 55 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/command/operator_debug.go b/command/operator_debug.go index 446ddc03edab..914732220450 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -38,6 +38,7 @@ type OperatorDebugCommand struct { collectDir string duration time.Duration interval time.Duration + pprofInterval time.Duration pprofDuration time.Duration logLevel string maxNodes int @@ -166,6 +167,10 @@ Debug Options: The interval between snapshots of the Nomad state. Set interval equal to duration to capture a single snapshot. Defaults to 30s. + -pprof-interval= + The interval between pprof collections. Set interval equal to + duration to capture a single snapshot. Defaults to 30s. + -log-level= The log level to monitor. Defaults to DEBUG. @@ -334,7 +339,7 @@ func (c *OperatorDebugCommand) Run(args []string) int { flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } - var duration, interval, output, pprofDuration, eventTopic string + var duration, interval, pprofInterval, output, pprofDuration, eventTopic string var eventIndex int64 var nodeIDs, serverIDs string var allowStale bool @@ -350,7 +355,8 @@ func (c *OperatorDebugCommand) Run(args []string) int { flags.StringVar(&serverIDs, "server-id", "all", "") flags.BoolVar(&allowStale, "stale", false, "") flags.StringVar(&output, "output", "", "") - flags.StringVar(&pprofDuration, "pprof-duration", "1s", "") + flags.StringVar(&pprofDuration, "pprof-duration", "1m", "") + flags.StringVar(&pprofInterval, "pprof-interval", "30s", "") flags.BoolVar(&c.verbose, "verbose", false, "") c.consul = &external{tls: &api.TLSConfig{}} @@ -423,6 +429,20 @@ func (c *OperatorDebugCommand) Run(args []string) int { } c.index = uint64(eventIndex) + // Parse the pprof capture interval + pi, err := time.ParseDuration(pprofInterval) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing pprof-interval: %s: %s", pprofInterval, err.Error())) + return 1 + } + c.pprofInterval = pi + + // Validate interval + if pi.Seconds() > pd.Seconds() { + c.Ui.Error(fmt.Sprintf("pprof-interval %s must be less than pprof-duration %s", pprofInterval, pprofDuration)) + return 1 + } + // Verify there are no extra arguments args = flags.Args() if l := len(args); l != 0 { @@ -595,6 +615,7 @@ func (c *OperatorDebugCommand) Run(args []string) int { } c.Ui.Output(fmt.Sprintf(" Interval: %s", interval)) c.Ui.Output(fmt.Sprintf(" Duration: %s", duration)) + c.Ui.Output(fmt.Sprintf(" Pprof Interval: %s", pprofInterval)) if c.pprofDuration.Seconds() != 1 { c.Ui.Output(fmt.Sprintf(" pprof Duration: %s", c.pprofDuration)) } @@ -663,7 +684,7 @@ func (c *OperatorDebugCommand) collect(client *api.Client) error { c.collectVault(clusterDir, vaultAddr) c.collectAgentHosts(client) - c.collectPprofs(client) + c.collectPeriodicPprofs(client) c.collectPeriodic(client) @@ -876,6 +897,34 @@ func (c *OperatorDebugCommand) collectAgentHost(path, id string, client *api.Cli c.writeJSON(path, "agent-host.json", host, err) } +func (c *OperatorDebugCommand) collectPeriodicPprofs(client *api.Client) { + duration := time.After(c.duration) + // Create a ticker to execute on every interval ticks + ticker := time.NewTicker(c.pprofInterval) + // Additionally, an out of loop execute to imitate first tick + c.collectPprofs(client) + + var pprofIntervalCount int + var name string + + for { + select { + case <-duration: + c.cancel() + return + + case <-ticker.C: + name = fmt.Sprintf("%04d", pprofIntervalCount) + c.Ui.Output(fmt.Sprintf(" Capture pprofInterval %s", name)) + c.collectPprofs(client) + pprofIntervalCount++ + + case <-c.ctx.Done(): + return + } + } +} + // collectPprofs captures the /agent/pprof for each listed node func (c *OperatorDebugCommand) collectPprofs(client *api.Client) { for _, n := range c.nodeIDs { From 657a04139adb4e654ed25ff58c9d9d84c4982663 Mon Sep 17 00:00:00 2001 From: danishprakash Date: Sat, 12 Feb 2022 16:48:52 +0530 Subject: [PATCH 02/10] command/operator_debug: update tests Signed-off-by: danishprakash --- command/operator_debug.go | 6 +++--- command/operator_debug_test.go | 20 +++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/command/operator_debug.go b/command/operator_debug.go index 914732220450..86e90218e936 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -169,7 +169,7 @@ Debug Options: -pprof-interval= The interval between pprof collections. Set interval equal to - duration to capture a single snapshot. Defaults to 30s. + duration to capture a single snapshot. Defaults to 250ms. -log-level= The log level to monitor. Defaults to DEBUG. @@ -355,8 +355,8 @@ func (c *OperatorDebugCommand) Run(args []string) int { flags.StringVar(&serverIDs, "server-id", "all", "") flags.BoolVar(&allowStale, "stale", false, "") flags.StringVar(&output, "output", "", "") - flags.StringVar(&pprofDuration, "pprof-duration", "1m", "") - flags.StringVar(&pprofInterval, "pprof-interval", "30s", "") + flags.StringVar(&pprofDuration, "pprof-duration", "1s", "") + flags.StringVar(&pprofInterval, "pprof-interval", "250ms", "") flags.BoolVar(&c.verbose, "verbose", false, "") c.consul = &external{tls: &api.TLSConfig{}} diff --git a/command/operator_debug_test.go b/command/operator_debug_test.go index afd09489f56c..e977e00b64f9 100644 --- a/command/operator_debug_test.go +++ b/command/operator_debug_test.go @@ -211,12 +211,12 @@ func TestDebug_MultiRegion(t *testing.T) { // Good { name: "no region - all servers, all clients", - args: []string{"-address", addrServer1, "-duration", "250ms", "-interval", "250ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "0"}, + args: []string{"-address", addrServer1, "-duration", "250ms", "-interval", "250ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "1s", "-pprof-interval", "250ms"}, expectedCode: 0, }, { name: "region1 - server1 address", - args: []string{"-address", addrServer1, "-region", region1, "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "0"}, + args: []string{"-address", addrServer1, "-region", region1, "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "1s", "-pprof-interval", "250ms"}, expectedCode: 0, expectedOutputs: []string{ "Region: " + region1 + "\n", @@ -227,7 +227,7 @@ func TestDebug_MultiRegion(t *testing.T) { }, { name: "region1 - client1 address", - args: []string{"-address", addrClient1, "-region", region1, "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "0"}, + args: []string{"-address", addrClient1, "-region", region1, "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "1s", "-pprof-interval", "250ms"}, expectedCode: 0, expectedOutputs: []string{ "Region: " + region1 + "\n", @@ -238,7 +238,7 @@ func TestDebug_MultiRegion(t *testing.T) { }, { name: "region2 - server2 address", - args: []string{"-address", addrServer2, "-region", region2, "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "0"}, + args: []string{"-address", addrServer2, "-region", region2, "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "1s", "-pprof-interval", "250ms"}, expectedCode: 0, expectedOutputs: []string{ "Region: " + region2 + "\n", @@ -249,7 +249,7 @@ func TestDebug_MultiRegion(t *testing.T) { }, { name: "region2 - client2 address", - args: []string{"-address", addrClient2, "-region", region2, "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "0"}, + args: []string{"-address", addrClient2, "-region", region2, "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "1s", "-pprof-interval", "250ms"}, expectedCode: 0, expectedOutputs: []string{ "Region: " + region2 + "\n", @@ -262,7 +262,7 @@ func TestDebug_MultiRegion(t *testing.T) { // Bad { name: "invalid region - all servers, all clients", - args: []string{"-address", addrServer1, "-region", "never", "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "0"}, + args: []string{"-address", addrServer1, "-region", "never", "-duration", "50ms", "-interval", "50ms", "-server-id", "all", "-node-id", "all", "-pprof-duration", "1s", "-pprof-interval", "250ms"}, expectedCode: 1, expectedError: "500 (No path to region)", }, @@ -347,6 +347,11 @@ func TestDebug_Failures(t *testing.T) { args: []string{"-pprof-duration", "baz"}, expectedCode: 1, }, + { + name: "Fails bad pprof interval", + args: []string{"-pprof-interval", "bar"}, + expectedCode: 1, + }, { name: "Fails bad address", args: []string{"-address", url + "bogus"}, @@ -482,7 +487,8 @@ func TestDebug_CapturedFiles(t *testing.T) { "-node-id", clientID, "-duration", duration.String(), "-interval", interval.String(), - "-pprof-duration", "0", + "-pprof-duration", "1s", + "-pprof-interval", "250ms", }) // Get capture directory From 9123886e86f6cf9ecf9d1867aebf6286b9995d77 Mon Sep 17 00:00:00 2001 From: danishprakash Date: Sat, 2 Apr 2022 14:17:07 +0530 Subject: [PATCH 03/10] command/operator_debug: pass pprofIntervalCount for filename Signed-off-by: danishprakash --- command/operator_debug.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/command/operator_debug.go b/command/operator_debug.go index 86e90218e936..e8a11825766f 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -901,12 +901,13 @@ func (c *OperatorDebugCommand) collectPeriodicPprofs(client *api.Client) { duration := time.After(c.duration) // Create a ticker to execute on every interval ticks ticker := time.NewTicker(c.pprofInterval) - // Additionally, an out of loop execute to imitate first tick - c.collectPprofs(client) var pprofIntervalCount int var name string + // Additionally, an out of loop execute to imitate first tick + c.collectPprofs(client, pprofIntervalCount) + for { select { case <-duration: @@ -916,7 +917,7 @@ func (c *OperatorDebugCommand) collectPeriodicPprofs(client *api.Client) { case <-ticker.C: name = fmt.Sprintf("%04d", pprofIntervalCount) c.Ui.Output(fmt.Sprintf(" Capture pprofInterval %s", name)) - c.collectPprofs(client) + c.collectPprofs(client, pprofIntervalCount) pprofIntervalCount++ case <-c.ctx.Done(): @@ -926,18 +927,18 @@ func (c *OperatorDebugCommand) collectPeriodicPprofs(client *api.Client) { } // collectPprofs captures the /agent/pprof for each listed node -func (c *OperatorDebugCommand) collectPprofs(client *api.Client) { +func (c *OperatorDebugCommand) collectPprofs(client *api.Client, interval int) { for _, n := range c.nodeIDs { - c.collectPprof(clientDir, n, client) + c.collectPprof(clientDir, n, client, interval) } for _, n := range c.serverIDs { - c.collectPprof(serverDir, n, client) + c.collectPprof(serverDir, n, client, interval) } } // collectPprof captures pprof data for the node -func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client) { +func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client, interval int) { pprofDurationSeconds := int(c.pprofDuration.Seconds()) opts := api.PprofOptions{Seconds: pprofDurationSeconds} if path == serverDir { @@ -960,7 +961,8 @@ func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client) return // only exit on 403 } } else { - err := c.writeBytes(path, "profile.prof", bs) + filename := fmt.Sprintf("profile_%d.prof", interval) + err := c.writeBytes(path, filename, bs) if err != nil { c.Ui.Error(err.Error()) } From 7401fe584a21cf943359a8189b38419219c39c44 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 4 Apr 2022 09:57:25 -0400 Subject: [PATCH 04/10] pad profile filename to match console output --- command/operator_debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/operator_debug.go b/command/operator_debug.go index e8a11825766f..cca8bb4bc9c7 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -961,7 +961,7 @@ func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client, return // only exit on 403 } } else { - filename := fmt.Sprintf("profile_%d.prof", interval) + filename := fmt.Sprintf("profile_%04d.prof", interval) err := c.writeBytes(path, filename, bs) if err != nil { c.Ui.Error(err.Error()) From bc4077ddb4237c14a96646855506a968ff0619c3 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 4 Apr 2022 09:57:33 -0400 Subject: [PATCH 05/10] docs and changelog entry --- .changelog/11938.txt | 3 +++ website/content/docs/commands/operator/debug.mdx | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changelog/11938.txt diff --git a/.changelog/11938.txt b/.changelog/11938.txt new file mode 100644 index 000000000000..fd86718bc1ce --- /dev/null +++ b/.changelog/11938.txt @@ -0,0 +1,3 @@ +```release-note:improvement +cli: added `-pprof-interval` to `nomad operator debug` command +``` diff --git a/website/content/docs/commands/operator/debug.mdx b/website/content/docs/commands/operator/debug.mdx index d6507ee72219..a583e2bfcc22 100644 --- a/website/content/docs/commands/operator/debug.mdx +++ b/website/content/docs/commands/operator/debug.mdx @@ -66,6 +66,10 @@ true. - `-pprof-duration=`: Duration for pprof collection. Defaults to 1s. +- `-pprof-interval=`: The interval between pprof + collections. Set interval equal to duration to capture a single + snapshot. Defaults to 250ms. + - `-server-id=,`: Comma separated list of Nomad server names to monitor for logs, API outputs, and pprof profiles. Accepts server names, "leader", or "all". Defaults to `all`. @@ -75,7 +79,7 @@ true. leadership, it may be necessary to get the configuration from a non-leader server. -- `-event-topic=:`: Enable event +- `-event-topic=:`: Enable event stream capture. Filter by comma delimited list of topic filters or "all". Defaults to "none" (disabled). Refer to the [Events API](/api-docs/events) for additional detail. From bd0d6b08b43696fabfa2e447a794a1a09a7fd5ad Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 4 Apr 2022 11:09:06 -0400 Subject: [PATCH 06/10] assert correct file name for Debug_CapturedFiles test --- command/operator_debug_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/operator_debug_test.go b/command/operator_debug_test.go index e977e00b64f9..057b35fd7593 100644 --- a/command/operator_debug_test.go +++ b/command/operator_debug_test.go @@ -441,7 +441,7 @@ func TestDebug_CapturedFiles(t *testing.T) { "goroutine-debug2.txt", "goroutine.prof", "heap.prof", - "profile.prof", + "profile_0000.prof", "threadcreate.prof", "trace.prof", } From 48d0d785c3e4a3e0daf9f7a7ab66c8bf47aafb81 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 4 Apr 2022 11:09:23 -0400 Subject: [PATCH 07/10] run periodic pprof collection concurrently --- command/operator_debug.go | 46 +++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/command/operator_debug.go b/command/operator_debug.go index cca8bb4bc9c7..c29bf539b135 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -167,10 +167,6 @@ Debug Options: The interval between snapshots of the Nomad state. Set interval equal to duration to capture a single snapshot. Defaults to 30s. - -pprof-interval= - The interval between pprof collections. Set interval equal to - duration to capture a single snapshot. Defaults to 250ms. - -log-level= The log level to monitor. Defaults to DEBUG. @@ -189,6 +185,10 @@ Debug Options: -pprof-duration= Duration for pprof collection. Defaults to 1s. + -pprof-interval= + The interval between pprof collections. Set interval equal to + duration to capture a single snapshot. Defaults to 250ms. + -server-id=, Comma separated list of Nomad server names to monitor for logs, API outputs, and pprof profiles. Accepts server names, "leader", or "all". @@ -406,6 +406,14 @@ func (c *OperatorDebugCommand) Run(args []string) int { return 1 } + // Parse the pprof capture interval + pi, err := time.ParseDuration(pprofInterval) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing pprof-interval: %s: %s", pprofInterval, err.Error())) + return 1 + } + c.pprofInterval = pi + // Parse the pprof capture duration pd, err := time.ParseDuration(pprofDuration) if err != nil { @@ -414,6 +422,12 @@ func (c *OperatorDebugCommand) Run(args []string) int { } c.pprofDuration = pd + // Validate pprof interval + if pi.Seconds() > pd.Seconds() { + c.Ui.Error(fmt.Sprintf("pprof-interval %s must be less than pprof-duration %s", pprofInterval, pprofDuration)) + return 1 + } + // Parse event stream topic filter t, err := topicsFromString(eventTopic) if err != nil { @@ -429,20 +443,6 @@ func (c *OperatorDebugCommand) Run(args []string) int { } c.index = uint64(eventIndex) - // Parse the pprof capture interval - pi, err := time.ParseDuration(pprofInterval) - if err != nil { - c.Ui.Error(fmt.Sprintf("Error parsing pprof-interval: %s: %s", pprofInterval, err.Error())) - return 1 - } - c.pprofInterval = pi - - // Validate interval - if pi.Seconds() > pd.Seconds() { - c.Ui.Error(fmt.Sprintf("pprof-interval %s must be less than pprof-duration %s", pprofInterval, pprofDuration)) - return 1 - } - // Verify there are no extra arguments args = flags.Args() if l := len(args); l != 0 { @@ -615,7 +615,7 @@ func (c *OperatorDebugCommand) Run(args []string) int { } c.Ui.Output(fmt.Sprintf(" Interval: %s", interval)) c.Ui.Output(fmt.Sprintf(" Duration: %s", duration)) - c.Ui.Output(fmt.Sprintf(" Pprof Interval: %s", pprofInterval)) + c.Ui.Output(fmt.Sprintf(" pprof Interval: %s", pprofInterval)) if c.pprofDuration.Seconds() != 1 { c.Ui.Output(fmt.Sprintf(" pprof Duration: %s", c.pprofDuration)) } @@ -684,7 +684,7 @@ func (c *OperatorDebugCommand) collect(client *api.Client) error { c.collectVault(clusterDir, vaultAddr) c.collectAgentHosts(client) - c.collectPeriodicPprofs(client) + go c.collectPeriodicPprofs(client) c.collectPeriodic(client) @@ -898,7 +898,7 @@ func (c *OperatorDebugCommand) collectAgentHost(path, id string, client *api.Cli } func (c *OperatorDebugCommand) collectPeriodicPprofs(client *api.Client) { - duration := time.After(c.duration) + duration := time.After(c.pprofDuration) // Create a ticker to execute on every interval ticks ticker := time.NewTicker(c.pprofInterval) @@ -948,10 +948,11 @@ func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client, } path = filepath.Join(path, id) + filename := fmt.Sprintf("profile_%04d.prof", interval) bs, err := client.Agent().CPUProfile(opts, c.queryOpts()) if err != nil { - c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof profile.prof, err: %v", path, err)) + c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof %s, err: %v", filename, path, err)) if structs.IsErrPermissionDenied(err) { // All Profiles require the same permissions, so we only need to see // one permission failure before we bail. @@ -961,7 +962,6 @@ func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client, return // only exit on 403 } } else { - filename := fmt.Sprintf("profile_%04d.prof", interval) err := c.writeBytes(path, filename, bs) if err != nil { c.Ui.Error(err.Error()) From 4d536bcd5178b541e0d1604b1ddbfef209793171 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 4 Apr 2022 12:12:21 -0400 Subject: [PATCH 08/10] periodic pprof termination should not end the whole collection --- command/operator_debug.go | 1 - 1 file changed, 1 deletion(-) diff --git a/command/operator_debug.go b/command/operator_debug.go index c29bf539b135..79da977c7fa6 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -911,7 +911,6 @@ func (c *OperatorDebugCommand) collectPeriodicPprofs(client *api.Client) { for { select { case <-duration: - c.cancel() return case <-ticker.C: From 0a250b146b2ef45d258f8a91db3a383161c17a16 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 4 Apr 2022 12:49:42 -0400 Subject: [PATCH 09/10] fixup for sigsegv test --- command/operator_debug_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/operator_debug_test.go b/command/operator_debug_test.go index 057b35fd7593..439026c70128 100644 --- a/command/operator_debug_test.go +++ b/command/operator_debug_test.go @@ -164,7 +164,7 @@ func TestDebug_ClientToServer(t *testing.T) { }, { name: "client1 address - verify no SIGSEGV panic", - args: []string{"-address", addrClient1, "-duration", "250ms", "-interval", "250ms", "-server-id", "all", "-node-id", "all"}, + args: []string{"-address", addrClient1, "-duration", "250ms", "-interval", "250ms", "-server-id", "all", "-node-id", "all", "-pprof-interval", "200ms", "-pprof-duration", "250ms"}, expectedCode: 0, expectedOutputs: []string{"Created debug archive"}, }, From 69a473f939768a01ce62482e9bbcf616917d5d83 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 4 Apr 2022 13:33:18 -0400 Subject: [PATCH 10/10] clean up test directories --- command/operator_debug_test.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/command/operator_debug_test.go b/command/operator_debug_test.go index 439026c70128..8d1026b490fb 100644 --- a/command/operator_debug_test.go +++ b/command/operator_debug_test.go @@ -384,9 +384,11 @@ func TestDebug_Bad_CSIPlugin_Names(t *testing.T) { // Setup mock UI ui := cli.NewMockUi() cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} + testDir := t.TempDir() + defer os.Remove(testDir) // Debug on the leader and all client nodes - code := cmd.Run([]string{"-address", url, "-duration", "250ms", "-interval", "250ms", "-server-id", "leader", "-node-id", "all", "-output", os.TempDir()}) + code := cmd.Run([]string{"-address", url, "-duration", "250ms", "-interval", "250ms", "-server-id", "leader", "-node-id", "all", "-output", testDir}) assert.Equal(t, 0, code) // Bad plugin name should be escaped before it reaches the sandbox test @@ -394,7 +396,6 @@ func TestDebug_Bad_CSIPlugin_Names(t *testing.T) { require.Contains(t, ui.OutputWriter.String(), "Starting debugger") path := cmd.collectDir - defer os.Remove(path) var pluginFiles []string for _, pluginName := range cases { @@ -475,6 +476,8 @@ func TestDebug_CapturedFiles(t *testing.T) { ui := cli.NewMockUi() cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} + testDir := t.TempDir() + defer os.Remove(testDir) duration := 2 * time.Second interval := 750 * time.Millisecond @@ -482,7 +485,7 @@ func TestDebug_CapturedFiles(t *testing.T) { code := cmd.Run([]string{ "-address", url, - "-output", os.TempDir(), + "-output", testDir, "-server-id", serverName, "-node-id", clientID, "-duration", duration.String(), @@ -491,10 +494,6 @@ func TestDebug_CapturedFiles(t *testing.T) { "-pprof-interval", "250ms", }) - // Get capture directory - path := cmd.collectDir - defer os.Remove(path) - // There should be no errors require.Empty(t, ui.ErrorWriter.String()) require.Equal(t, 0, code) @@ -535,11 +534,12 @@ func TestDebug_ExistingOutput(t *testing.T) { // Fails existing output format := "2006-01-02-150405Z" stamped := "nomad-debug-" + time.Now().UTC().Format(format) - path := filepath.Join(os.TempDir(), stamped) + tempDir := t.TempDir() + path := filepath.Join(tempDir, stamped) os.MkdirAll(path, 0755) - defer os.Remove(path) + defer os.Remove(tempDir) - code := cmd.Run([]string{"-output", os.TempDir(), "-duration", "50ms", "-interval", "50ms"}) + code := cmd.Run([]string{"-output", tempDir, "-duration", "50ms", "-interval", "50ms"}) require.Equal(t, 2, code) } @@ -642,12 +642,12 @@ func TestDebug_WriteBytes_Nil(t *testing.T) { ui := cli.NewMockUi() cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} - testDir = os.TempDir() + testDir = t.TempDir() + defer os.Remove(testDir) cmd.collectDir = testDir testFile = "test_nil.json" testPath = filepath.Join(testDir, testFile) - defer os.Remove(testPath) // Write nil file at top level of collect directory err := cmd.writeBytes("", testFile, testBytes) @@ -661,7 +661,7 @@ func TestDebug_WriteBytes_PathEscapesSandbox(t *testing.T) { var testDir, testFile string var testBytes []byte - testDir = os.TempDir() + testDir = t.TempDir() defer os.Remove(testDir) testFile = "testing.json" @@ -719,7 +719,7 @@ func TestDebug_CollectConsul(t *testing.T) { c.consul = ce // Setup capture directory - testDir := os.TempDir() + testDir := t.TempDir() defer os.Remove(testDir) c.collectDir = testDir @@ -762,7 +762,7 @@ func TestDebug_CollectVault(t *testing.T) { c.vault = ve // Set capture directory - testDir := os.TempDir() + testDir := t.TempDir() defer os.Remove(testDir) c.collectDir = testDir