feat: allow chart deploy overrides

- This feature allows chart deployment overrides.
- #2133

Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com>

adr/0023-chart-override.md

@@ -0,0 +1,28 @@
+# 23. Simplifying Helm Chart Value Overrides
+
+Date: 2024-03-25
+
+## Status
+
+Proposed
+
+
+## Context
+
+The process of deploying applications with Helm charts in Kubernetes environments often necessitates the customization of chart values to align with specific operational or environmental requirements. The current method for customizing these values—either through manual edits or `###`. A more streamlined approach would greatly enhance the deployment experience by offering both flexibility and reliability.
+
+## Decision
+
+To address this issue, we propose the introduction of a feature designed to simplify the process of overriding chart values at the time of deployment. This feature would allow users to easily specify overrides for any chart values directly via command-line arguments, eliminating the need to alter the chart's default values file or manage multiple command-line arguments for each override.
+
+Key aspects of the proposed implementation include:
+- Use existing `--set`  flags to specify overrides for chart values.
+- The ability to list all overrides in a structured and easily understandable format within the ZarfConfig file.
+- Ensuring that during deployment, these specified overrides take precedence over the chart's default values, thus facilitating customized deployments without necessitating permanent modifications to the chart.
+
+## Consequences
+
+Adopting this feature would lead to several key improvements:
+- **Streamlined Configuration Process**: Centralizing overrides in a single, unified file significantly simplifies the management of configuration settings, aligning more closely with standard Helm practices and reducing the reliance on extensive command-line arguments.
+
+Ultimately, this feature is aimed at enhancing the deployment workflow by offering a straightforward and efficient means of customizing Helm chart deployments via command-line inputs.

examples/helm-charts/zarf.yaml

@@ -16,6 +16,13 @@ components:
         localPath: chart
         valuesFiles:
           - values.yaml
+        # Variables are used to override the default values in the chart
+        # This can be overridden by the user at deployment time with the `--set` flag
+        variables:
+          - name: replicaCount
+            description: "Override the number of pod replicas"
+            path: replicaCount
+            value: 2
 
       - name: podinfo-oci
         version: 6.4.0
@@ -77,3 +84,4 @@ components:
                 name: cool-release-name-podinfo
                 namespace: podinfo-from-repo
                 condition: available
+

src/pkg/packager/deploy.go

@@ -547,24 +547,89 @@ func (p *Packager) pushReposToRepository(reposPath string, repos []string) error
 	return nil
 }
 
+// generateValuesOverrides creates a map containing overrides for chart values based on the given inputs.
+// It prioritizes the overrides in this order:
+// 1. Variables defined in the ZarfComponent, including both Set Variables and Chart Variables.
+// 2. Deployment options specified through DeployOpts.
+// This function is useful for customizing the deployment of a Helm chart by programmatically setting chart values.
+//
+// Returns:
+// - A map containing the final set of value overrides for the chart, where keys are the variable names.
+func generateValuesOverrides(chartVariables []types.ZarfChartVariable,
+	setVariableMap map[string]*types.ZarfSetVariable,
+	deployOpts types.ZarfDeployOptions,
+	componentName, chartName string) (map[string]any, error) {
+
+	valuesOverrides := make(map[string]any)
+	chartOverrides := make(map[string]any)
+
+	for _, variable := range chartVariables {
+		if setVar, ok := setVariableMap[variable.Name]; ok && setVar != nil {
+			// Use the variable's path as a key to ensure unique entries for variables with the same name but different paths.
+			if err := mergePathAndValueIntoMap(chartOverrides, variable.Path, setVar.Value); err != nil {
+				return nil, fmt.Errorf("unable to merge path and value into map: %w", err)
+			}
+		}
+	}
+
+	// Apply any direct overrides specified in the deployment options for this component and chart
+	if componentOverrides, ok := deployOpts.ValuesOverridesMap[componentName]; ok {
+		if chartSpecificOverrides, ok := componentOverrides[chartName]; ok {
+			for k, v := range chartSpecificOverrides {
+				// Directly apply overrides to valuesOverrides to avoid overwriting by path.
+				valuesOverrides[k] = v
+			}
+		}
+	}
+
+	// Merge chartOverrides into valuesOverrides to ensure all overrides are applied.
+	// This corrects the logic to ensure that chartOverrides and valuesOverrides are merged correctly.
+	return helpers.MergeMapRecursive(valuesOverrides, chartOverrides), nil
+}
+
+// mergePathAndValueIntoMap takes a path in dot notation as a string and a value (also as a string for simplicity),
+// then merges this into the provided map. The value can be any type.
+func mergePathAndValueIntoMap(m map[string]any, path string, value any) error {
+	pathParts := strings.Split(path, ".")
+	current := m
+	for i, part := range pathParts {
+		if i == len(pathParts)-1 {
+			// Set the value at the last key in the path.
+			current[part] = value
+		} else {
+			if _, exists := current[part]; !exists {
+				// If the part does not exist, create a new map for it.
+				current[part] = make(map[string]any)
+			}
+
+			nextMap, ok := current[part].(map[string]any)
+			if !ok {
+				return fmt.Errorf("conflict at '%s', expected map but got %T", strings.Join(pathParts[:i+1], "."), current[part])
+			}
+			current = nextMap
+		}
+	}
+	return nil
+}
+
 // Install all Helm charts and raw k8s manifests into the k8s cluster.
 func (p *Packager) installChartAndManifests(componentPaths *layout.ComponentPaths, component types.ZarfComponent) (installedCharts []types.InstalledChart, err error) {
 	for _, chart := range component.Charts {
-
 		// zarf magic for the value file
 		for idx := range chart.ValuesFiles {
 			chartValueName := helm.StandardValuesName(componentPaths.Values, chart, idx)
 			if err := p.valueTemplate.Apply(component, chartValueName, false); err != nil {
 				return installedCharts, err
 			}
 		}
-
-		// TODO (@WSTARR): Currently this logic is library-only and is untested while it is in an experimental state - it may eventually get added as shorthand in Zarf Variables though
-		var valuesOverrides map[string]any
-		if componentChartValuesOverrides, ok := p.cfg.DeployOpts.ValuesOverridesMap[component.Name]; ok {
-			if chartValuesOverrides, ok := componentChartValuesOverrides[chart.Name]; ok {
-				valuesOverrides = chartValuesOverrides
-			}
+		// this is to get the overrides for the chart from the commandline and
+		// the chart variables set in the ZarfComponent and as well as DeployOpts set from the Library user.
+		// The order of precedence is as follows:
+		// 1. Set Variables and Chart Variables from the ZarfComponent
+		// 2. DeployOpts
+		valuesOverrides, err := generateValuesOverrides(chart.Variables, p.cfg.SetVariableMap, p.cfg.DeployOpts, component.Name, chart.Name)
+		if err != nil {
+			return installedCharts, err
 		}
 
 		helmCfg := helm.New(

src/pkg/packager/deploy_test.go

@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2021-Present The Zarf Authors
+
+package packager
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/defenseunicorns/zarf/src/types"
+)
+
+func TestGenerateValuesOverrides(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		chartVariables []types.ZarfChartVariable
+		setVariableMap map[string]*types.ZarfSetVariable
+		deployOpts     types.ZarfDeployOptions
+		componentName  string
+		chartName      string
+		want           map[string]any
+	}{
+		{
+			name:           "Empty inputs",
+			chartVariables: []types.ZarfChartVariable{},
+			setVariableMap: map[string]*types.ZarfSetVariable{},
+			deployOpts:     types.ZarfDeployOptions{},
+			componentName:  "",
+			chartName:      "",
+			want:           map[string]any{},
+		},
+		{
+			name:           "Single variable",
+			chartVariables: []types.ZarfChartVariable{{Name: "testVar", Path: "testVar"}},
+			setVariableMap: map[string]*types.ZarfSetVariable{"testVar": {Value: "testValue"}},
+			deployOpts:     types.ZarfDeployOptions{},
+			componentName:  "testComponent",
+			chartName:      "testChart",
+			want:           map[string]any{"testVar": "testValue"},
+		},
+		{
+			name:           "Non-matching setVariable",
+			chartVariables: []types.ZarfChartVariable{{Name: "expectedVar", Path: "path.to.expectedVar"}},
+			setVariableMap: map[string]*types.ZarfSetVariable{"unexpectedVar": {Value: "unexpectedValue"}},
+			deployOpts:     types.ZarfDeployOptions{},
+			componentName:  "testComponent",
+			chartName:      "testChart",
+			want:           map[string]any{},
+		},
+		{
+			name: "Nested 3 level setVariableMap",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "level1.level2.level3Var", Path: "level1.level2.level3Var"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"level1.level2.level3Var": {Value: "nestedValue"},
+			},
+			deployOpts:    types.ZarfDeployOptions{},
+			componentName: "nestedComponent",
+			chartName:     "nestedChart",
+			want: map[string]any{
+				"level1": map[string]any{
+					"level2": map[string]any{
+						"level3Var": "nestedValue",
+					},
+				},
+			},
+		},
+		{
+			name: "Multiple variables with nested and non-nested paths",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "nestedVar.level2", Path: "nestedVar.level2"},
+				{Name: "simpleVar", Path: "simpleVar"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"nestedVar.level2": {Value: "nestedValue"},
+				"simpleVar":        {Value: "simpleValue"},
+			},
+			deployOpts:    types.ZarfDeployOptions{},
+			componentName: "mixedComponent",
+			chartName:     "mixedChart",
+			want: map[string]any{
+				"nestedVar": map[string]any{
+					"level2": "nestedValue",
+				},
+				"simpleVar": "simpleValue",
+			},
+		},
+		{
+			name: "Values override test",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "overrideVar", Path: "path"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"path": {Value: "overrideValue"},
+			},
+			deployOpts: types.ZarfDeployOptions{
+				ValuesOverridesMap: map[string]map[string]map[string]any{
+					"testComponent": {
+						"testChart": {
+							"path": "deployOverrideValue",
+						},
+					},
+				},
+			},
+			componentName: "testComponent",
+			chartName:     "testChart",
+			want: map[string]any{
+				"path": "deployOverrideValue",
+			},
+		},
+		{
+			name: "Missing variable in setVariableMap but present in ValuesOverridesMap",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "missingVar", Path: "missingVarPath"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{},
+			deployOpts: types.ZarfDeployOptions{
+				ValuesOverridesMap: map[string]map[string]map[string]any{
+					"testComponent": {
+						"testChart": {
+							"missingVarPath": "overrideValue",
+						},
+					},
+				},
+			},
+			componentName: "testComponent",
+			chartName:     "testChart",
+			want: map[string]any{
+				"missingVarPath": "overrideValue",
+			},
+		},
+		{
+			name:           "Non-existent component or chart",
+			chartVariables: []types.ZarfChartVariable{{Name: "someVar", Path: "someVar"}},
+			setVariableMap: map[string]*types.ZarfSetVariable{"someVar": {Value: "value"}},
+			deployOpts: types.ZarfDeployOptions{
+				ValuesOverridesMap: map[string]map[string]map[string]any{
+					"nonExistentComponent": {
+						"nonExistentChart": {
+							"someVar": "overrideValue",
+						},
+					},
+				},
+			},
+			componentName: "actualComponent",
+			chartName:     "actualChart",
+			want:          map[string]any{"someVar": "value"},
+		},
+		{
+			name:           "Variable in setVariableMap but not in chartVariables",
+			chartVariables: []types.ZarfChartVariable{},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"orphanVar": {Value: "orphanValue"},
+			},
+			deployOpts:    types.ZarfDeployOptions{},
+			componentName: "orphanComponent",
+			chartName:     "orphanChart",
+			want:          map[string]any{},
+		},
+		{
+			name: "Empty ValuesOverridesMap with non-empty setVariableMap and chartVariables",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "var1", Path: "path.to.var1"},
+				{Name: "var2", Path: "path.to.var2"},
+				{Name: "var3", Path: "path.to3.var3"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"var1": {Value: "value1"},
+				"var2": {Value: "value2"},
+				"var3": {Value: "value3"},
+			},
+			deployOpts: types.ZarfDeployOptions{
+				ValuesOverridesMap: map[string]map[string]map[string]any{},
+			},
+			componentName: "componentWithVars",
+			chartName:     "chartWithVars",
+			want: map[string]any{
+				"path": map[string]any{
+					"to": map[string]any{
+						"var1": "value1",
+						"var2": "value2",
+					},
+					"to3": map[string]any{
+						"var3": "value3",
+					},
+				},
+			},
+		},
+		{
+			name:           "Empty chartVariables and non-empty setVariableMap",
+			chartVariables: []types.ZarfChartVariable{},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"var1": {Value: "value1"},
+				"var2": {Value: "value2"},
+			},
+			deployOpts:    types.ZarfDeployOptions{},
+			componentName: "componentWithVars",
+			chartName:     "chartWithVars",
+			want:          map[string]any{},
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			got, _ := generateValuesOverrides(tt.chartVariables, tt.setVariableMap, tt.deployOpts, tt.componentName, tt.chartName)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("%s: generateValuesOverrides() got = %v, want %v", tt.name, got, tt.want)
+			}
+		})
+	}
+}

src/types/component.go

@@ -105,16 +105,24 @@ type ZarfFile struct {
 
 // ZarfChart defines a helm chart to be deployed.
 type ZarfChart struct {
-	Name        string   `json:"name" jsonschema:"description=The name of the chart within Zarf; note that this must be unique and does not need to be the same as the name in the chart repo"`
-	Version     string   `json:"version,omitempty" jsonschema:"description=The version of the chart to deploy; for git-based charts this is also the tag of the git repo by default (when not using the '@' syntax for 'repos')"`
-	URL         string   `json:"url,omitempty" jsonschema:"example=OCI registry: oci://ghcr.io/stefanprodan/charts/podinfo,example=helm chart repo: https://stefanprodan.github.io/podinfo,example=git repo: https://github.com/stefanprodan/podinfo (note the '@' syntax for 'repos' is supported here too)" jsonschema_description:"The URL of the OCI registry, chart repository, or git repo where the helm chart is stored"`
-	RepoName    string   `json:"repoName,omitempty" jsonschema:"description=The name of a chart within a Helm repository (defaults to the Zarf name of the chart)"`
-	GitPath     string   `json:"gitPath,omitempty" jsonschema:"description=(git repo only) The sub directory to the chart within a git repo,example=charts/your-chart"`
-	LocalPath   string   `json:"localPath,omitempty" jsonschema:"description=The path to a local chart's folder or .tgz archive"`
-	Namespace   string   `json:"namespace" jsonschema:"description=The namespace to deploy the chart to"`
-	ReleaseName string   `json:"releaseName,omitempty" jsonschema:"description=The name of the Helm release to create (defaults to the Zarf name of the chart)"`
-	NoWait      bool     `json:"noWait,omitempty" jsonschema:"description=Whether to not wait for chart resources to be ready before continuing"`
-	ValuesFiles []string `json:"valuesFiles,omitempty" jsonschema:"description=List of local values file paths or remote URLs to include in the package; these will be merged together when deployed"`
+	Name        string              `json:"name" jsonschema:"description=The name of the chart within Zarf; note that this must be unique and does not need to be the same as the name in the chart repo"`
+	Version     string              `json:"version,omitempty" jsonschema:"description=The version of the chart to deploy; for git-based charts this is also the tag of the git repo by default (when not using the '@' syntax for 'repos')"`
+	URL         string              `json:"url,omitempty" jsonschema:"example=OCI registry: oci://ghcr.io/stefanprodan/charts/podinfo,example=helm chart repo: https://stefanprodan.github.io/podinfo,example=git repo: https://github.com/stefanprodan/podinfo (note the '@' syntax for 'repos' is supported here too)" jsonschema_description:"The URL of the OCI registry, chart repository, or git repo where the helm chart is stored"`
+	RepoName    string              `json:"repoName,omitempty" jsonschema:"description=The name of a chart within a Helm repository (defaults to the Zarf name of the chart)"`
+	GitPath     string              `json:"gitPath,omitempty" jsonschema:"description=(git repo only) The sub directory to the chart within a git repo,example=charts/your-chart"`
+	LocalPath   string              `json:"localPath,omitempty" jsonschema:"description=The path to a local chart's folder or .tgz archive"`
+	Namespace   string              `json:"namespace" jsonschema:"description=The namespace to deploy the chart to"`
+	ReleaseName string              `json:"releaseName,omitempty" jsonschema:"description=The name of the Helm release to create (defaults to the Zarf name of the chart)"`
+	NoWait      bool                `json:"noWait,omitempty" jsonschema:"description=Whether to not wait for chart resources to be ready before continuing"`
+	ValuesFiles []string            `json:"valuesFiles,omitempty" jsonschema:"description=List of local values file paths or remote URLs to include in the package; these will be merged together when deployed"`
+	Variables   []ZarfChartVariable `json:"variables,omitempty" jsonschema:"description=List of variables to set in the Helm chart"`
+}
+
+// ZarfChartVariable represents a variable that can be set for a Helm chart overrides.
+type ZarfChartVariable struct {
+	Name        string `json:"name" jsonschema:"description=The name of the variable"`
+	Description string `json:"description" jsonschema:"description=A brief description of what the variable controls"`
+	Path        string `json:"path" jsonschema:"description=The path within the Helm chart values where this variable applies"`
 }
 
 // ZarfManifest defines raw manifests Zarf will deploy as a helm chart.