Cluster values full templating

e2e/assets/single-cluster/helm-cluster-values.yaml

@@ -0,0 +1,9 @@
+kind: GitRepo
+apiVersion: fleet.cattle.io/v1alpha1
+metadata:
+  name: helm-cluster-values
+spec:
+  repo: https://github.com/rancher/fleet-examples
+  branch: test-cluster-values
+  paths:
+  - single-cluster/helm-cluster-values

e2e/single-cluster/helm_verify_test.go

@@ -4,6 +4,10 @@ import (
 	"github.com/rancher/fleet/e2e/testenv"
 	"github.com/rancher/fleet/e2e/testenv/kubectl"
 
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	yaml "sigs.k8s.io/yaml"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -22,9 +26,15 @@ var _ = Describe("Helm Chart Values", func() {
 		Expect(err).ToNot(HaveOccurred(), out)
 	})
 
+	AfterEach(func() {
+		out, err := k.Delete("-f", testenv.AssetPath(asset))
+		Expect(err).ToNot(HaveOccurred(), out)
+
+	})
+
 	When("helm chart validates inputs", func() {
 		BeforeEach(func() {
-			asset = "helm-verify.yaml"
+			asset = "single-cluster/helm-verify.yaml"
 		})
 
 		It("replaces cluster label in values and valuesFiles", func() {
@@ -40,4 +50,52 @@ var _ = Describe("Helm Chart Values", func() {
 			))
 		})
 	})
+
+	When("fleet.yaml has templated values", func() {
+		BeforeEach(func() {
+			asset = "single-cluster/helm-cluster-values.yaml"
+		})
+
+		It("generates values and applies them", func() {
+			Eventually(func() string {
+				out, _ := k.Namespace("default").Get("configmaps")
+
+				return out
+			}).Should(ContainSubstring("test-template-values"))
+			out, _ := k.Namespace("default").Get("configmap", "test-template-values", "-o", "jsonpath={.data}")
+
+			var t map[string]interface{}
+			err := yaml.Unmarshal([]byte(out), &t)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(t).To(HaveKeyWithValue("name", "name-local"))
+			Expect(t).To(HaveKeyWithValue("namespace", ContainSubstring("fleet-local")))
+			Expect(t).To(HaveKey("annotations"))
+			Expect(t["annotations"]).To(Equal(`{"app":"fleet","more":"data"}`))
+			Expect(t).To(HaveKeyWithValue("image", "rancher/mirrored-library-busybox:1.34.1"))
+			Expect(t).To(HaveKeyWithValue("imagePullPolicy", "IfNotPresent"))
+			Expect(t).To(HaveKeyWithValue("clusterValues", "{}"))
+			Expect(t).To(HaveKeyWithValue("global", "null"))
+
+			Eventually(func() string {
+				out, _ := k.Namespace("default").Get("deployments")
+
+				return out
+			}).Should(ContainSubstring("test-template-values"))
+			out, _ = k.Namespace("default").Get("deployments", "test-template-values", "-ojson")
+			d := appsv1.Deployment{}
+			err = yaml.Unmarshal([]byte(out), &d)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(*d.Spec.Replicas).To(Equal(int32(1)))
+			Expect(d.Spec.Template.Spec.Containers[0].Image).To(Equal("rancher/mirrored-library-busybox:1.34.1"))
+			Expect(d.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullIfNotPresent))
+			Expect(d.Spec.Template.Annotations).To(SatisfyAll(
+				HaveKeyWithValue("app", "fleet"),
+				HaveKeyWithValue("more", "data"),
+			))
+			Expect(d.Spec.Template.Labels).To(HaveKeyWithValue("name", "local"))
+			Expect(d.Spec.Template.Labels).To(HaveKeyWithValue("policy", ""))
+			Expect(d.Spec.Template.Spec.HostAliases[0].Hostnames).To(ContainElements("one", "two", "three"))
+		})
+	})
 })

pkg/target/target.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	kyaml "sigs.k8s.io/yaml"
 
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 	"github.com/rancher/fleet/pkg/bundlematcher"
@@ -338,8 +339,8 @@ func preprocessHelmValues(opts *fleet.BundleDeploymentOptions, cluster *fleet.Cl
 		values := map[string]interface{}{
 			"ClusterNamespace":   cluster.Namespace,
 			"ClusterName":        cluster.Name,
-			"ClusterLabels":      clusterLabels,
-			"ClusterAnnotations": clusterAnnotations,
+			"ClusterLabels":      toDict(clusterLabels),
+			"ClusterAnnotations": toDict(clusterAnnotations),
 			"ClusterValues":      templateValues,
 		}
 
@@ -354,6 +355,15 @@ func preprocessHelmValues(opts *fleet.BundleDeploymentOptions, cluster *fleet.Cl
 
 }
 
+// sprig dictionary functions like "default" and "hasKey" expect map[string]interface{}
+func toDict(values map[string]string) map[string]interface{} {
+	dict := make(map[string]interface{}, len(values))
+	for k, v := range values {
+		dict[k] = v
+	}
+	return dict
+}
+
 // foldInDeployments adds the existing bundledeployments to the targets.
 func (m *Manager) foldInDeployments(bundle *fleet.Bundle, targets []*Target) error {
 	bundleDeployments, err := m.bundleDeploymentCache.List("", labels.SelectorFromSet(deploymentLabelsForSelector(bundle)))
@@ -595,73 +605,35 @@ func tplFuncMap() template.FuncMap {
 	return f
 }
 
-func processTemplateValues(valuesMap map[string]interface{}, templateContext map[string]interface{}) (map[string]interface{}, error) {
-	tplFn := template.New("values").Funcs(tplFuncMap()).Option("missingkey=error")
-	recursionDepth := 0
-	tplResult, err := templateSubstitutions(valuesMap, templateContext, tplFn, recursionDepth)
+func processTemplateValues(helmValues map[string]interface{}, templateContext map[string]interface{}) (map[string]interface{}, error) {
+	data, err := kyaml.Marshal(helmValues)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to marshal helm values section into a template: %w", err)
 	}
-	compiledYaml, ok := tplResult.(map[string]interface{})
-	if !ok {
-		return nil, fmt.Errorf("templated result was expected to be map[string]interface{}, got %T", tplResult)
-	}
-
-	return compiledYaml, nil
-}
 
-func templateSubstitutions(src interface{}, templateContext map[string]interface{}, tplFn *template.Template, recursionDepth int) (interface{}, error) {
-	if recursionDepth > maxTemplateRecursionDepth {
-		return nil, fmt.Errorf("maximum recursion depth of %v exceeded for current templating operation, too many nested values", maxTemplateRecursionDepth)
+	// fleet.yaml must be valid yaml, however '{}[]' are YAML control
+	// characters and will be interpreted as JSON data structures. This
+	// causes issues when parsing the fleet.yaml so we change the delims
+	// for templating to '${ }'
+	tmpl := template.New("values").Funcs(tplFuncMap()).Option("missingkey=error").Delims("${", "}")
+	tmpl, err = tmpl.Parse(string(data))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse helm values template: %w", err)
 	}
 
-	switch tplVal := src.(type) {
-	case string:
-		tpl, err := tplFn.Parse(tplVal)
-		if err != nil {
-			return nil, err
-		}
+	var b bytes.Buffer
+	err = tmpl.Execute(&b, templateContext)
+	if err != nil {
+		return nil, fmt.Errorf("failed to render helm values template: %w", err)
+	}
 
-		var tplBytes bytes.Buffer
-		defer func() {
-			if r := recover(); r != nil {
-				err = fmt.Errorf("failed to process template substitution for string '%s': [%v]", tplVal, err)
-			}
-		}()
-		err = tpl.Execute(&tplBytes, templateContext)
-		if err != nil {
-			return nil, fmt.Errorf("failed to process template substitution for string '%s': [%v]", tplVal, err)
-		}
-		return tplBytes.String(), nil
-	case map[string]interface{}:
-		newMap := make(map[string]interface{})
-		for key, val := range tplVal {
-			processedKey, err := templateSubstitutions(key, templateContext, tplFn, recursionDepth+1)
-			if err != nil {
-				return nil, err
-			}
-			keyAsString, ok := processedKey.(string)
-			if !ok {
-				return nil, fmt.Errorf("expected a string to be returned, but instead got [%T]", processedKey)
-			}
-			if newMap[keyAsString], err = templateSubstitutions(val, templateContext, tplFn, recursionDepth+1); err != nil {
-				return nil, err
-			}
-		}
-		return newMap, nil
-	case []interface{}:
-		newSlice := make([]interface{}, len(tplVal))
-		for i, v := range tplVal {
-			newVal, err := templateSubstitutions(v, templateContext, tplFn, recursionDepth+1)
-			if err != nil {
-				return nil, err
-			}
-			newSlice[i] = newVal
-		}
-		return newSlice, nil
-	default:
-		return tplVal, nil
+	var renderedValues map[string]interface{}
+	err = kyaml.Unmarshal(b.Bytes(), &renderedValues)
+	if err != nil {
+		return nil, fmt.Errorf("failed to interpret rendered template as helm values: %#v, %v", renderedValues, err)
 	}
+
+	return renderedValues, nil
 }
 
 func processLabelValues(valuesMap map[string]interface{}, clusterLabels map[string]string, recursionDepth int) error {