Add walkSecretsTree helper function

changelog/20464.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Add listSecretsRecursive helper function, which produces a recursive list of secrets rooted at the given path
+```

command/kv_helpers.go

@@ -4,10 +4,12 @@
 package command
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
-	"path"
+	paths "path"
+	"sort"
 	"strings"
 
 	"github.com/hashicorp/go-secure-stdlib/strutil"
@@ -128,7 +130,7 @@ func isKVv2(path string, client *api.Client) (string, bool, error) {
 
 func addPrefixToKVPath(p, mountPath, apiPrefix string) string {
 	if p == mountPath || p == strings.TrimSuffix(mountPath, "/") {
-		return path.Join(mountPath, apiPrefix)
+		return paths.Join(mountPath, apiPrefix)
 	}
 
 	tp := strings.TrimPrefix(p, mountPath)
@@ -148,7 +150,7 @@ func addPrefixToKVPath(p, mountPath, apiPrefix string) string {
 		tp = strings.TrimPrefix(tp, mountPath)
 	}
 
-	return path.Join(mountPath, apiPrefix, tp)
+	return paths.Join(mountPath, apiPrefix, tp)
 }
 
 func getHeaderForMap(header string, data map[string]interface{}) string {
@@ -197,3 +199,67 @@ func padEqualSigns(header string, totalLen int) string {
 
 	return fmt.Sprintf("%s %s %s", strings.Repeat("=", equalSigns/2), header, strings.Repeat("=", equalSigns/2))
 }
+
+// listSecretsRecursive dfs-traverses the secrets tree rooted at the given path
+// and returns a list of the leaf paths. Note: for kv-v2, a "metadata" path is
+// expected and "metadata" paths will be returned. If includeDirectories is
+// specified, the output will include non-leaf nodes with "/" suffixes.
+func listSecretsRecursive(ctx context.Context, client *api.Client, path string, includeDirectories bool) ([]string, error) {
+	var descendants []string
+
+	resp, err := client.Logical().ListWithContext(ctx, path)
+	if err != nil {
+		return nil, fmt.Errorf("could not list %q path: %w", path, err)
+	}
+
+	if resp == nil || resp.Data == nil {
+		return nil, fmt.Errorf("no value found at %q: %w", path, err)
+	}
+
+	keysRaw, ok := resp.Data["keys"]
+	if !ok {
+		return nil, fmt.Errorf("unexpected list response at %q", path)
+	}
+
+	keysRawSlice, ok := keysRaw.([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("unexpected list response type %T at %q", keysRaw, path)
+	}
+
+	keys := make([]string, 0, len(keysRawSlice))
+
+	for _, keyRaw := range keysRawSlice {
+		key, ok := keyRaw.(string)
+		if !ok {
+			return nil, fmt.Errorf("unexpected key type %T at %q", keyRaw, path)
+		}
+		keys = append(keys, key)
+	}
+
+	// sort the keys for a deterministic output
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		// the keys are relative to the current path: combine them
+		child := paths.Join(path, key)
+
+		if strings.HasSuffix(key, "/") {
+			// if requested, include the directory suffixed with "/"
+			if includeDirectories {
+				descendants = append(descendants, child+"/")
+			}
+
+			// this is not a leaf node: we need to go deeper...
+			d, err := listSecretsRecursive(ctx, client, child, includeDirectories)
+			if err != nil {
+				return nil, err
+			}
+			descendants = append(descendants, d...)
+		} else {
+			// this is a leaf node: add it to the list
+			descendants = append(descendants, child)
+		}
+	}
+
+	return descendants, nil
+}

command/kv_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -1523,6 +1524,211 @@ func TestPadEqualSigns(t *testing.T) {
 	}
 }
 
+// TestListSecretsRecursive tests the listSecretsRecursive helper function
+func TestListSecretsRecursive(t *testing.T) {
+	// test setup
+	client, closer := testVaultServer(t)
+	defer closer()
+
+	// enable kv-v1 backend
+	if err := client.Sys().Mount("kv-v1/", &api.MountInput{
+		Type: "kv-v1",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	// enable kv-v2 backend
+	if err := client.Sys().Mount("kv-v2/", &api.MountInput{
+		Type: "kv-v2",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	ctx, cancelContextFunc := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancelContextFunc()
+
+	// populate secrets
+	for _, path := range []string{
+		"foo",
+		"app-1/foo",
+		"app-1/bar",
+		"app-1/nested/x/y/z",
+		"app-1/nested/x/y",
+		"app-1/nested/bar",
+	} {
+		if err := client.KVv1("kv-v1").Put(ctx, path, map[string]interface{}{
+			"password": "Hashi123",
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		if _, err := client.KVv2("kv-v2").Put(ctx, path, map[string]interface{}{
+			"password": "Hashi123",
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	cases := []struct {
+		name               string
+		path               string
+		includeDirectories bool
+		expected           []string
+		expectedError      bool
+	}{
+		{
+			name:               "kv-v1-simple",
+			path:               "kv-v1/app-1/nested/x/y",
+			includeDirectories: false,
+			expected:           []string{"kv-v1/app-1/nested/x/y/z"},
+			expectedError:      false,
+		},
+		{
+			name:               "kv-v2-simple",
+			path:               "kv-v2/metadata/app-1/nested/x/y",
+			includeDirectories: false,
+			expected:           []string{"kv-v2/metadata/app-1/nested/x/y/z"},
+			expectedError:      false,
+		},
+		{
+			name:               "kv-v1-nested",
+			path:               "kv-v1/app-1/nested/",
+			includeDirectories: false,
+			expected: []string{
+				"kv-v1/app-1/nested/bar",
+				"kv-v1/app-1/nested/x/y",
+				"kv-v1/app-1/nested/x/y/z",
+			},
+			expectedError: false,
+		},
+		{
+			name:               "kv-v2-nested",
+			path:               "kv-v2/metadata/app-1/nested/",
+			includeDirectories: false,
+			expected: []string{
+				"kv-v2/metadata/app-1/nested/bar",
+				"kv-v2/metadata/app-1/nested/x/y",
+				"kv-v2/metadata/app-1/nested/x/y/z",
+			},
+			expectedError: false,
+		},
+		{
+			name:               "kv-v1-all",
+			path:               "kv-v1",
+			includeDirectories: false,
+			expected: []string{
+				"kv-v1/app-1/bar",
+				"kv-v1/app-1/foo",
+				"kv-v1/app-1/nested/bar",
+				"kv-v1/app-1/nested/x/y",
+				"kv-v1/app-1/nested/x/y/z",
+				"kv-v1/foo",
+			},
+			expectedError: false,
+		},
+		{
+			name:               "kv-v2-all",
+			path:               "kv-v2/metadata",
+			includeDirectories: false,
+			expected: []string{
+				"kv-v2/metadata/app-1/bar",
+				"kv-v2/metadata/app-1/foo",
+				"kv-v2/metadata/app-1/nested/bar",
+				"kv-v2/metadata/app-1/nested/x/y",
+				"kv-v2/metadata/app-1/nested/x/y/z",
+				"kv-v2/metadata/foo",
+			},
+			expectedError: false,
+		},
+		{
+			name:               "kv-v1-all-include-directories",
+			path:               "kv-v1",
+			includeDirectories: true,
+			expected: []string{
+				"kv-v1/app-1/",
+				"kv-v1/app-1/bar",
+				"kv-v1/app-1/foo",
+				"kv-v1/app-1/nested/",
+				"kv-v1/app-1/nested/bar",
+				"kv-v1/app-1/nested/x/",
+				"kv-v1/app-1/nested/x/y",
+				"kv-v1/app-1/nested/x/y/",
+				"kv-v1/app-1/nested/x/y/z",
+				"kv-v1/foo",
+			},
+			expectedError: false,
+		},
+		{
+			name:               "kv-v2-all-include-directories",
+			path:               "kv-v2/metadata",
+			includeDirectories: true,
+			expected: []string{
+				"kv-v2/metadata/app-1/",
+				"kv-v2/metadata/app-1/bar",
+				"kv-v2/metadata/app-1/foo",
+				"kv-v2/metadata/app-1/nested/",
+				"kv-v2/metadata/app-1/nested/bar",
+				"kv-v2/metadata/app-1/nested/x/",
+				"kv-v2/metadata/app-1/nested/x/y",
+				"kv-v2/metadata/app-1/nested/x/y/",
+				"kv-v2/metadata/app-1/nested/x/y/z",
+				"kv-v2/metadata/foo",
+			},
+			expectedError: false,
+		},
+		{
+			name:               "kv-v1-not-found",
+			path:               "kv-v1/does/not/exist",
+			includeDirectories: false,
+			expected:           nil,
+			expectedError:      true,
+		},
+		{
+			name:               "kv-v2-not-found",
+			path:               "kv-v2/metadata/does/not/exist",
+			includeDirectories: false,
+			expected:           nil,
+			expectedError:      true,
+		},
+		{
+			name:               "kv-v1-not-listable-leaf-node",
+			path:               "kv-v1/foo",
+			includeDirectories: false,
+			expected:           nil,
+			expectedError:      true,
+		},
+		{
+			name:               "kv-v2-not-listable-leaf-node",
+			path:               "kv-v2/metadata/foo",
+			includeDirectories: false,
+			expected:           nil,
+			expectedError:      true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual, err := listSecretsRecursive(ctx, client, tc.path, tc.includeDirectories)
+
+			if tc.expectedError {
+				if err == nil {
+					t.Fatal("an error was expected but the test succeeded")
+				}
+			} else {
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if !reflect.DeepEqual(tc.expected, actual) {
+					t.Fatalf("unexpected list output; want: %v, got: %v", tc.expected, actual)
+				}
+			}
+		})
+	}
+}
+
 func createTokenForPolicy(t *testing.T, client *api.Client, policy string) (*api.SecretAuth, error) {
 	t.Helper()