Refactor /inventory code to expose some helpers

Makefile

@@ -164,7 +164,7 @@ ui: node_modules $(shell find ui -type f) ## Build the UI
 
 node_modules: ## Install node modules
 	rm -rf .parcel-cache
-	yarn config set network-timeout 600000 && yarn --pure-lockfile
+	yarn config set network-timeout 600000 && yarn --frozen-lockfile
 
 ui-lint: ## Run linter against the UI
 	yarn lint

core/server/inventory.go

@@ -14,8 +14,10 @@ import (
 	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
 	"github.com/fluxcd/pkg/ssa"
+	"github.com/go-logr/logr"
 	"github.com/weaveworks/weave-gitops/core/server/types"
 	pb "github.com/weaveworks/weave-gitops/pkg/api/core"
+	"github.com/weaveworks/weave-gitops/pkg/health"
 	"github.com/weaveworks/weave-gitops/pkg/server/auth"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,6 +28,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+// ObjectWithChildren is a recursive data structure containing a tree of Unstructured
+// values.
+type ObjectWithChildren struct {
+	Object   *unstructured.Unstructured
+	Children []*ObjectWithChildren
+}
+
 func (cs *coreServer) GetInventory(ctx context.Context, msg *pb.GetInventoryRequest) (*pb.GetInventoryResponse, error) {
 	clustersClient, err := cs.clustersManager.GetImpersonatedClient(ctx, auth.Principal(ctx))
 	if err != nil {
@@ -37,16 +46,16 @@ func (cs *coreServer) GetInventory(ctx context.Context, msg *pb.GetInventoryRequ
 		return nil, fmt.Errorf("error getting scoped client for cluster=%s: %w", msg.ClusterName, err)
 	}
 
-	var entries []*unstructured.Unstructured
+	var inventoryRefs []*unstructured.Unstructured
 
 	switch msg.Kind {
 	case kustomizev1.KustomizationKind:
-		entries, err = cs.getKustomizationInventory(ctx, client, msg.Name, msg.Namespace)
+		inventoryRefs, err = cs.getKustomizationInventory(ctx, client, msg.Name, msg.Namespace)
 		if err != nil {
 			return nil, fmt.Errorf("failed getting kustomization inventory: %w", err)
 		}
 	case helmv2.HelmReleaseKind:
-		entries, err = cs.getHelmReleaseInventory(ctx, client, msg.Name, msg.Namespace)
+		inventoryRefs, err = cs.getHelmReleaseInventory(ctx, client, msg.Name, msg.Namespace)
 		if err != nil {
 			return nil, fmt.Errorf("failed getting helm Release inventory: %w", err)
 		}
@@ -55,44 +64,58 @@ func (cs *coreServer) GetInventory(ctx context.Context, msg *pb.GetInventoryRequ
 		if err != nil {
 			return nil, err
 		}
-		entries, err = getFluxLikeInventory(ctx, client, msg.Name, msg.Namespace, *gvk)
+		inventoryRefs, err = getFluxLikeInventory(ctx, client, msg.Name, msg.Namespace, *gvk)
 		if err != nil {
 			return nil, fmt.Errorf("failed getting flux like inventory: %w", err)
 		}
 	}
 
-	resources := cs.getInventoryResources(ctx, msg.ClusterName, client, entries, msg.Namespace, msg.WithChildren)
+	objsWithChildren, err := GetObjectsWithChildren(ctx, inventoryRefs, client, msg.WithChildren, cs.logger)
+	if err != nil {
+		return nil, fmt.Errorf("failed getting objects with children: %w", err)
+	}
+
+	entries := []*pb.InventoryEntry{}
+	clusterUserNamespaces := cs.clustersManager.GetUserNamespaces(auth.Principal(ctx))
+	for _, oc := range objsWithChildren {
+		entry, err := unstructuredToInventoryEntry(msg.ClusterName, *oc, clusterUserNamespaces, cs.healthChecker)
+		if err != nil {
+			return nil, fmt.Errorf("failed converting inventory entry: %w", err)
+		}
+		entries = append(entries, entry)
+	}
 
 	return &pb.GetInventoryResponse{
-		Entries: resources,
+		Entries: entries,
 	}, nil
 }
 
 func (cs *coreServer) getKustomizationInventory(ctx context.Context, k8sClient client.Client, name, namespace string) ([]*unstructured.Unstructured, error) {
-	kust := &kustomizev1.Kustomization{
+	ks := &kustomizev1.Kustomization{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: namespace,
 		},
 	}
 
-	if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(kust), kust); err != nil {
+	if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(ks), ks); err != nil {
 		return nil, fmt.Errorf("failed to get kustomization: %w", err)
 	}
 
-	if kust.Status.Inventory == nil {
+	if ks.Status.Inventory == nil {
 		return nil, nil
 	}
 
-	if kust.Status.Inventory.Entries == nil {
+	if ks.Status.Inventory.Entries == nil {
 		return nil, nil
 	}
 
 	objects := []*unstructured.Unstructured{}
-	for _, e := range kust.Status.Inventory.Entries {
-		obj, err := resourceRefToUnstructured(e)
+	for _, ref := range ks.Status.Inventory.Entries {
+		obj, err := ResourceRefToUnstructured(ref.ID, ref.Version)
 		if err != nil {
-			return nil, fmt.Errorf("failed converting inventory entry: %w", err)
+			cs.logger.Error(err, "failed converting inventory entry", "entry", ref)
+			return nil, err
 		}
 		objects = append(objects, &obj)
 	}
@@ -120,40 +143,6 @@ func (cs *coreServer) getHelmReleaseInventory(ctx context.Context, k8sClient cli
 	return objects, nil
 }
 
-func (cs *coreServer) getInventoryResources(ctx context.Context, clusterName string, k8sClient client.Client, objects []*unstructured.Unstructured, namespace string, withChildren bool) []*pb.InventoryEntry {
-	result := []*pb.InventoryEntry{}
-	resultMu := sync.Mutex{}
-
-	wg := sync.WaitGroup{}
-
-	for _, o := range objects {
-		wg.Add(1)
-
-		go func(obj unstructured.Unstructured) {
-			defer wg.Done()
-
-			if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(&obj), &obj); err != nil {
-				cs.logger.Error(err, "failed to get object", "entry", obj)
-				return
-			}
-
-			entry, err := cs.unstructuredToInventoryEntry(ctx, clusterName, k8sClient, obj, namespace, withChildren)
-			if err != nil {
-				cs.logger.Error(err, "failed converting inventory entry", "entry", obj)
-				return
-			}
-
-			resultMu.Lock()
-			result = append(result, entry)
-			resultMu.Unlock()
-		}(*o)
-	}
-
-	wg.Wait()
-
-	return result
-}
-
 // Returns the list of resources applied in the helm chart.
 func getHelmReleaseObjects(ctx context.Context, k8sClient client.Client, helmRelease *helmv2.HelmRelease) ([]*unstructured.Unstructured, error) {
 	storageNamespace := helmRelease.GetStorageNamespace()
@@ -222,36 +211,34 @@ func getHelmReleaseObjects(ctx context.Context, k8sClient client.Client, helmRel
 	return objects, nil
 }
 
-func (cs *coreServer) unstructuredToInventoryEntry(ctx context.Context, clusterName string, k8sClient client.Client, unstructuredObj unstructured.Unstructured, ns string, withChildren bool) (*pb.InventoryEntry, error) {
-	var err error
-
+func unstructuredToInventoryEntry(clusterName string, objWithChildren ObjectWithChildren, clusterUserNamespaces map[string][]v1.Namespace, healthChecker health.HealthChecker) (*pb.InventoryEntry, error) {
+	unstructuredObj := *objWithChildren.Object
 	if unstructuredObj.GetKind() == "Secret" {
+		var err error
 		unstructuredObj, err = sanitizeUnstructuredSecret(unstructuredObj)
 		if err != nil {
 			return nil, fmt.Errorf("error sanitizing secrets: %w", err)
 		}
 	}
-
-	children := []*pb.InventoryEntry{}
-
-	if withChildren {
-		children, err = cs.getChildren(ctx, clusterName, k8sClient, unstructuredObj, ns)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	bytes, err := unstructuredObj.MarshalJSON()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to marshal unstructured object: %w", err)
 	}
 
-	clusterUserNss := cs.clustersManager.GetUserNamespaces(auth.Principal(ctx))
-	tenant := GetTenant(unstructuredObj.GetNamespace(), clusterName, clusterUserNss)
+	tenant := GetTenant(unstructuredObj.GetNamespace(), clusterName, clusterUserNamespaces)
 
-	health, err := cs.healthChecker.Check(unstructuredObj)
+	health, err := healthChecker.Check(unstructuredObj)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to check health: %w", err)
+	}
+
+	children := []*pb.InventoryEntry{}
+	for _, c := range objWithChildren.Children {
+		child, err := unstructuredToInventoryEntry(clusterName, *c, clusterUserNamespaces, healthChecker)
+		if err != nil {
+			return nil, fmt.Errorf("failed converting child inventory entry: %w", err)
+		}
+		children = append(children, child)
 	}
 
 	entry := &pb.InventoryEntry{
@@ -268,7 +255,53 @@ func (cs *coreServer) unstructuredToInventoryEntry(ctx context.Context, clusterN
 	return entry, nil
 }
 
-func (cs *coreServer) getChildren(ctx context.Context, clusterName string, k8sClient client.Client, parentObj unstructured.Unstructured, ns string) ([]*pb.InventoryEntry, error) {
+// GetObjectsWithChildren returns objects with their children populated if withChildren is true.
+// Objects are retrieved in parallel.
+// Children are retrieved recusively, e.g. Deployment -> ReplicaSet -> Pod
+func GetObjectsWithChildren(ctx context.Context, objects []*unstructured.Unstructured, k8sClient client.Client, withChildren bool, logger logr.Logger) ([]*ObjectWithChildren, error) {
+	result := []*ObjectWithChildren{}
+	resultMu := sync.Mutex{}
+
+	wg := sync.WaitGroup{}
+
+	for _, o := range objects {
+		wg.Add(1)
+
+		go func(obj unstructured.Unstructured) {
+			defer wg.Done()
+
+			if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(&obj), &obj); err != nil {
+				logger.Error(err, "failed to get object", "entry", obj)
+				return
+			}
+
+			children := []*ObjectWithChildren{}
+			if withChildren {
+				var err error
+				children, err = getChildren(ctx, k8sClient, obj)
+				if err != nil {
+					logger.Error(err, "failed getting children", "entry", obj)
+					return
+				}
+			}
+
+			entry := &ObjectWithChildren{
+				Object:   &obj,
+				Children: children,
+			}
+
+			resultMu.Lock()
+			result = append(result, entry)
+			resultMu.Unlock()
+		}(*o)
+	}
+
+	wg.Wait()
+
+	return result, nil
+}
+
+func getChildren(ctx context.Context, k8sClient client.Client, parentObj unstructured.Unstructured) ([]*ObjectWithChildren, error) {
 	listResult := unstructured.UnstructuredList{}
 
 	switch parentObj.GetObjectKind().GroupVersionKind().Kind {
@@ -285,10 +318,10 @@ func (cs *coreServer) getChildren(ctx context.Context, clusterName string, k8sCl
 			Kind:    "Pod",
 		})
 	default:
-		return []*pb.InventoryEntry{}, nil
+		return []*ObjectWithChildren{}, nil
 	}
 
-	if err := k8sClient.List(ctx, &listResult, client.InNamespace(ns)); err != nil {
+	if err := k8sClient.List(ctx, &listResult, client.InNamespace(parentObj.GetNamespace())); err != nil {
 		return nil, fmt.Errorf("could not get unstructured object: %s", err)
 	}
 
@@ -308,39 +341,46 @@ func (cs *coreServer) getChildren(ctx context.Context, clusterName string, k8sCl
 		}
 	}
 
-	children := []*pb.InventoryEntry{}
+	children := []*ObjectWithChildren{}
 
 	for _, c := range unstructuredChildren {
-		entry, err := cs.unstructuredToInventoryEntry(ctx, clusterName, k8sClient, c, ns, true)
+		var err error
+		children, err = getChildren(ctx, k8sClient, c)
 		if err != nil {
 			return nil, err
 		}
 
+		entry := &ObjectWithChildren{
+			Object:   &c,
+			Children: children,
+		}
 		children = append(children, entry)
 	}
 
 	return children, nil
 }
 
-func resourceRefToUnstructured(entry kustomizev1.ResourceRef) (unstructured.Unstructured, error) {
+// ResourceRefToUnstructured converts a flux like resource entry pair of (id, version) into a unstructured object
+func ResourceRefToUnstructured(id, version string) (unstructured.Unstructured, error) {
 	u := unstructured.Unstructured{}
 
-	objMetadata, err := object.ParseObjMetadata(entry.ID)
+	objMetadata, err := object.ParseObjMetadata(id)
 	if err != nil {
 		return u, err
 	}
 
 	u.SetGroupVersionKind(schema.GroupVersionKind{
 		Group:   objMetadata.GroupKind.Group,
 		Kind:    objMetadata.GroupKind.Kind,
-		Version: entry.Version,
+		Version: version,
 	})
 	u.SetName(objMetadata.Name)
 	u.SetNamespace(objMetadata.Namespace)
 
 	return u, nil
 }
 
+// sanitizeUnstructuredSecret redacts the data field of a Secret object
 func sanitizeUnstructuredSecret(obj unstructured.Unstructured) (unstructured.Unstructured, error) {
 	redactedUnstructured := unstructured.Unstructured{}
 	s := &v1.Secret{}
@@ -396,8 +436,8 @@ func parseInventoryFromUnstructured(obj *unstructured.Unstructured) ([]*unstruct
 	}
 
 	objects := []*unstructured.Unstructured{}
-	for _, entry := range resourceInventory.Entries {
-		u, err := resourceRefToUnstructured(entry)
+	for _, ref := range resourceInventory.Entries {
+		u, err := ResourceRefToUnstructured(ref.ID, ref.Version)
 		if err != nil {
 			return nil, fmt.Errorf("error converting resource ref to unstructured: %w", err)
 		}

core/server/inventory_internal_test.go

@@ -218,3 +218,42 @@ func TestParseInventoryFromUnstructured(t *testing.T) {
 		})
 	}
 }
+
+func TestSanitizeUnstructuredSecret(t *testing.T) {
+	unstructuredSecret := unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "v1",
+			"kind":       "Secret",
+			"metadata": map[string]interface{}{
+				"name":      "my-secret",
+				"namespace": "my-namespace",
+			},
+			"type": "Opaque",
+			"data": map[string]interface{}{
+				"key": "dGVzdA==",
+			},
+		},
+	}
+
+	expected := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "v1",
+			"kind":       "Secret",
+			"metadata": map[string]interface{}{
+				"name":              "my-secret",
+				"namespace":         "my-namespace",
+				"creationTimestamp": nil,
+			},
+			"type": "Opaque",
+			"data": map[string]interface{}{
+				"redacted": nil,
+			},
+		},
+	}
+
+	secret, err := sanitizeUnstructuredSecret(unstructuredSecret)
+
+	g := NewGomegaWithT(t)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(&secret).To(Equal(expected))
+}