debug: Improve namespace and region support

.changelog/11269.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Improve debug namespace and region support
+```

command/operator_debug.go

@@ -329,7 +329,7 @@ func (c *OperatorDebugCommand) Run(args []string) int {
 	nodeLookupFailCount := 0
 	nodeCaptureCount := 0
 
-	for _, id := range argNodes(nodeIDs) {
+	for _, id := range stringToSlice(nodeIDs) {
 		if id == "all" {
 			// Capture from all nodes using empty prefix filter
 			id = ""
@@ -382,15 +382,15 @@ func (c *OperatorDebugCommand) Run(args []string) int {
 		c.Ui.Error(fmt.Sprintf("Failed to retrieve server list; err: %v", err))
 		return 1
 	}
+
+	// Write complete list of server members to file
 	c.writeJSON("version", "members.json", members, err)
-	// We always write the error to the file, but don't range if no members found
-	if serverIDs == "all" && members != nil {
-		// Special case to capture from all servers
-		for _, member := range members.Members {
-			c.serverIDs = append(c.serverIDs, member.Name)
-		}
-	} else {
-		c.serverIDs = append(c.serverIDs, argNodes(serverIDs)...)
+
+	// Filter for servers matching criteria
+	c.serverIDs, err = parseMembers(members, serverIDs, c.region)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Failed to parse server list; err: %v", err))
+		return 1
 	}
 
 	serversFound := 0
@@ -412,6 +412,8 @@ func (c *OperatorDebugCommand) Run(args []string) int {
 	// Display general info about the capture
 	c.Ui.Output("Starting debugger...")
 	c.Ui.Output("")
+	c.Ui.Output(fmt.Sprintf("           Region: %s", c.region))
+	c.Ui.Output(fmt.Sprintf("        Namespace: %s", c.namespace))
 	c.Ui.Output(fmt.Sprintf("          Servers: (%d/%d) %v", serverCaptureCount, serversFound, c.serverIDs))
 	c.Ui.Output(fmt.Sprintf("          Clients: (%d/%d) %v", nodeCaptureCount, nodesFound, c.nodeIDs))
 	if nodeCaptureCount > 0 && nodeCaptureCount == c.maxNodes {
@@ -468,6 +470,13 @@ func (c *OperatorDebugCommand) collect(client *api.Client) error {
 	self, err := client.Agent().Self()
 	c.writeJSON(dir, "agent-self.json", self, err)
 
+	var qo *api.QueryOptions
+	namespaces, _, err := client.Namespaces().List(qo)
+	c.writeJSON(dir, "namespaces.json", namespaces, err)
+
+	regions, err := client.Regions().List()
+	c.writeJSON(dir, "regions.json", regions, err)
+
 	// Fetch data directly from consul and vault. Ignore errors
 	var consul, vault string
 
@@ -1055,8 +1064,44 @@ func TarCZF(archive string, src, target string) error {
 	})
 }
 
-// argNodes splits node ids from the command line by ","
-func argNodes(input string) []string {
+// parseMembers returns a slice of server member names matching the search criteria
+func parseMembers(serverMembers *api.ServerMembers, serverIDs string, region string) (membersFound []string, err error) {
+	prefixes := stringToSlice(serverIDs)
+
+	if serverMembers.Members == nil {
+		return nil, fmt.Errorf("Failed to parse server members, members==nil")
+	}
+
+	for _, member := range serverMembers.Members {
+		// If region is provided it must match exactly
+		if region != "" && member.Tags["region"] != region {
+			continue
+		}
+
+		// Always include "all"
+		if serverIDs == "all" {
+			membersFound = append(membersFound, member.Name)
+			continue
+		}
+
+		// Special case passthrough as literally "leader"
+		if serverIDs == "leader" {
+			membersFound = append(membersFound, "leader")
+			continue
+		}
+
+		// Include member if name matches any prefix
+		if helper.SliceStringContainsPrefix(prefixes, member.Name) {
+			membersFound = append(membersFound, member.Name)
+		}
+	}
+
+	return membersFound, nil
+}
+
+// stringToSlice splits CSV string into slice, trims whitespace, and prunes
+// empty values
+func stringToSlice(input string) []string {
 	ns := strings.Split(input, ",")
 	var out []string
 	for _, n := range ns {

command/operator_debug_test.go

@@ -73,13 +73,15 @@ func TestDebug_NodeClass(t *testing.T) {
 	}
 
 	// Start client 1
-	client1 := agent.NewTestAgent(t, "client1", agentConfFunc1)
-	defer client1.Shutdown()
+	agent1 := agent.NewTestAgent(t, "client1", agentConfFunc1)
+	defer agent1.Shutdown()
 
 	// Wait for client1 to connect
-	client1NodeID := client1.Agent.Client().NodeID()
-	testutil.WaitForClient(t, srv.Agent.RPC, client1NodeID)
-	t.Logf("[TEST] Client1 ready, id: %s", client1NodeID)
+	client1 := agent1.Agent.Client()
+	client1NodeID := client1.NodeID()
+	client1Region := client1.Region()
+	testutil.WaitForClient(t, srv.Agent.RPC, client1NodeID, client1Region)
+	t.Logf("[TEST] Client1 ready, id: %s, region: %s", client1NodeID, client1Region)
 
 	// Setup client 2 (nodeclass = clientb)
 	agentConfFunc2 := func(c *agent.Config) {
@@ -91,13 +93,15 @@ func TestDebug_NodeClass(t *testing.T) {
 	}
 
 	// Start client 2
-	client2 := agent.NewTestAgent(t, "client2", agentConfFunc2)
-	defer client2.Shutdown()
+	agent2 := agent.NewTestAgent(t, "client2", agentConfFunc2)
+	defer agent2.Shutdown()
 
 	// Wait for client2 to connect
-	client2NodeID := client2.Agent.Client().NodeID()
-	testutil.WaitForClient(t, srv.Agent.RPC, client2NodeID)
-	t.Logf("[TEST] Client2 ready, id: %s", client2NodeID)
+	client2 := agent2.Agent.Client()
+	client2NodeID := client2.NodeID()
+	client2Region := client2.Region()
+	testutil.WaitForClient(t, srv.Agent.RPC, client2NodeID, client2Region)
+	t.Logf("[TEST] Client2 ready, id: %s, region: %s", client2NodeID, client2Region)
 
 	// Setup client 3 (nodeclass = clienta)
 	agentConfFunc3 := func(c *agent.Config) {
@@ -107,13 +111,15 @@ func TestDebug_NodeClass(t *testing.T) {
 	}
 
 	// Start client 3
-	client3 := agent.NewTestAgent(t, "client3", agentConfFunc3)
-	defer client3.Shutdown()
+	agent3 := agent.NewTestAgent(t, "client3", agentConfFunc3)
+	defer agent3.Shutdown()
 
 	// Wait for client3 to connect
-	client3NodeID := client3.Agent.Client().NodeID()
-	testutil.WaitForClient(t, srv.Agent.RPC, client3NodeID)
-	t.Logf("[TEST] Client3 ready, id: %s", client3NodeID)
+	client3 := agent3.Agent.Client()
+	client3NodeID := client3.NodeID()
+	client3Region := client3.Region()
+	testutil.WaitForClient(t, srv.Agent.RPC, client3NodeID, client3Region)
+	t.Logf("[TEST] Client3 ready, id: %s, region: %s", client3NodeID, client3Region)
 
 	// Setup test cases
 	cases := testCases{
@@ -169,17 +175,19 @@ func TestDebug_ClientToServer(t *testing.T) {
 	}
 
 	// Start client 1
-	client1 := agent.NewTestAgent(t, "client1", agentConfFunc1)
-	defer client1.Shutdown()
+	agent1 := agent.NewTestAgent(t, "client1", agentConfFunc1)
+	defer agent1.Shutdown()
 
 	// Wait for client 1 to connect
-	client1NodeID := client1.Agent.Client().NodeID()
-	testutil.WaitForClient(t, srv.Agent.RPC, client1NodeID)
-	t.Logf("[TEST] Client1 ready, id: %s", client1NodeID)
+	client1 := agent1.Agent.Client()
+	client1NodeID := client1.NodeID()
+	client1Region := client1.Region()
+	testutil.WaitForClient(t, srv.Agent.RPC, client1NodeID, client1Region)
+	t.Logf("[TEST] Client1 ready, id: %s, region: %s", client1NodeID, client1Region)
 
 	// Get API addresses
 	addrServer := srv.HTTPAddr()
-	addrClient1 := client1.HTTPAddr()
+	addrClient1 := agent1.HTTPAddr()
 
 	t.Logf("[TEST] testAgent api address: %s", url)
 	t.Logf("[TEST] Server    api address: %s", addrServer)
@@ -210,6 +218,88 @@ func TestDebug_ClientToServer(t *testing.T) {
 	runTestCases(t, cases)
 }
 
+func TestDebug_ClientToServer_Region(t *testing.T) {
+	agentConfFunc := func(c *agent.Config) {
+		c.Region = "testregion"
+	}
+
+	// Start test server and API client
+	srv, _, url := testServer(t, false, agentConfFunc)
+	defer srv.Shutdown()
+
+	// Wait for leadership to establish
+	testutil.WaitForLeader(t, srv.Agent.RPC)
+
+	// Retrieve server RPC address to join client
+	srvRPCAddr := srv.GetConfig().AdvertiseAddrs.RPC
+	t.Logf("[TEST] Leader started, srv.GetConfig().AdvertiseAddrs.RPC: %s", srvRPCAddr)
+
+	// Setup client 1 (nodeclass = clienta)
+	agentConfFunc1 := func(c *agent.Config) {
+		c.Region = "testregion"
+		c.Server.Enabled = false
+		c.Client.NodeClass = "clienta"
+		c.Client.Enabled = true
+		c.Client.Servers = []string{srvRPCAddr}
+	}
+
+	// Start client 1
+	agent1 := agent.NewTestAgent(t, "client1", agentConfFunc1)
+	defer agent1.Shutdown()
+
+	// Wait for client 1 to connect
+	client1NodeID := agent1.Agent.Client().NodeID()
+	client1Region := agent1.Agent.Client().Region()
+	testutil.WaitForClient(t, srv.Agent.RPC, client1NodeID, client1Region)
+	t.Logf("[TEST] Client1 ready, id: %s, region: %s", client1NodeID, client1Region)
+
+	// Get API addresses
+	addrServer := srv.HTTPAddr()
+	addrClient1 := agent1.HTTPAddr()
+
+	t.Logf("[TEST] testAgent api address: %s", url)
+	t.Logf("[TEST] Server    api address: %s", addrServer)
+	t.Logf("[TEST] Client1   api address: %s", addrClient1)
+
+	// Setup test cases
+	var cases = testCases{
+		// Good
+		{
+			name:         "testAgent api server",
+			args:         []string{"-address", url, "-region", "testregion", "-duration", "250ms", "-interval", "250ms", "-server-id", "all", "-node-id", "all"},
+			expectedCode: 0,
+			expectedOutputs: []string{
+				"Region: testregion\n",
+				"Servers: (1/1)",
+				"Clients: (1/1)",
+				"Created debug archive",
+			},
+		},
+		{
+			name:            "server address",
+			args:            []string{"-address", addrServer, "-region", "testregion", "-duration", "250ms", "-interval", "250ms", "-server-id", "all", "-node-id", "all"},
+			expectedCode:    0,
+			expectedOutputs: []string{"Created debug archive"},
+		},
+		{
+			name:            "client1 address - verify no SIGSEGV panic",
+			args:            []string{"-address", addrClient1, "-region", "testregion", "-duration", "250ms", "-interval", "250ms", "-server-id", "all", "-node-id", "all"},
+			expectedCode:    0,
+			expectedOutputs: []string{"Created debug archive"},
+		},
+
+		// Bad
+		{
+			name:          "invalid region - all servers, all clients",
+			args:          []string{"-address", url, "-region", "never", "-duration", "250ms", "-interval", "250ms", "-server-id", "all", "-node-id", "all"},
+			expectedCode:  1,
+			expectedError: "500 (No path to region)",
+		},
+	}
+
+	runTestCases(t, cases)
+}
+
 func TestDebug_SingleServer(t *testing.T) {
 	srv, _, url := testServer(t, false, nil)
 	defer srv.Shutdown()
@@ -432,15 +522,26 @@ func TestDebug_Fail_Pprof(t *testing.T) {
 	require.Contains(t, ui.OutputWriter.String(), "Created debug archive")   // Archive should be generated anyway
 }
 
-func TestDebug_Utils(t *testing.T) {
+func TestDebug_StringToSlice(t *testing.T) {
 	t.Parallel()
 
-	xs := argNodes("foo, bar")
-	require.Equal(t, []string{"foo", "bar"}, xs)
+	cases := []struct {
+		input    string
+		expected []string
+	}{
+		{input: ",,", expected: []string(nil)},
+		{input: "", expected: []string(nil)},
+		{input: "foo, bar", expected: []string{"foo", "bar"}},
+	}
+	for _, tc := range cases {
+		out := stringToSlice(tc.input)
+		require.Equal(t, tc.expected, out)
+		require.Equal(t, true, helper.CompareSliceSetString(tc.expected, out))
+	}
+}
 
-	xs = argNodes("")
-	require.Len(t, xs, 0)
-	require.Empty(t, xs)
+func TestDebug_External(t *testing.T) {
+	t.Parallel()
 
 	// address calculation honors CONSUL_HTTP_SSL
 	// ssl: true - Correct alignment

helper/funcs.go

@@ -197,6 +197,16 @@ func SliceStringContains(list []string, item string) bool {
 	return false
 }
 
+// SliceStringContainsPrefix returns true if any string in list matches prefix
+func SliceStringContainsPrefix(list []string, prefix string) bool {
+	for _, s := range list {
+		if strings.HasPrefix(s, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
 func SliceSetDisjoint(first, second []string) (bool, []string) {
 	contained := make(map[string]struct{}, len(first))
 	for _, k := range first {

nomad/client_agent_endpoint_test.go

@@ -548,7 +548,7 @@ func TestAgentProfile_RemoteClient(t *testing.T) {
 	})
 	defer cleanupC()
 
-	testutil.WaitForClient(t, s2.RPC, c.NodeID())
+	testutil.WaitForClient(t, s2.RPC, c.NodeID(), c.Region())
 	testutil.WaitForResult(func() (bool, error) {
 		nodes := s2.connectedNodes()
 		return len(nodes) == 1, nil

testutil/wait.go

@@ -112,12 +112,15 @@ func WaitForLeader(t testing.TB, rpc rpcFn) {
 }
 
 // WaitForClient blocks until the client can be found
-func WaitForClient(t testing.TB, rpc rpcFn, nodeID string) {
+func WaitForClient(t testing.TB, rpc rpcFn, nodeID string, region string) {
 	t.Helper()
+	if region == "" {
+		region = "global"
+	}
 	WaitForResult(func() (bool, error) {
 		req := structs.NodeSpecificRequest{
 			NodeID:       nodeID,
-			QueryOptions: structs.QueryOptions{Region: "global"},
+			QueryOptions: structs.QueryOptions{Region: region},
 		}
 		var out structs.SingleNodeResponse