Merge branch 'main' into docs-next

.github/pull_request_template.md

@@ -8,12 +8,6 @@ Fixes #
 <!-- or -->
 Relates to #
 
-## Type of change
-
-- [ ] Bug fix (non-breaking change which fixes an issue)
-- [ ] New feature (non-breaking change which adds functionality)
-- [ ] Other (security config, docs update, etc)
-
 ## Checklist before merging
 
 - [ ] Test, docs, adr added or updated as needed

adr/0023-chart-override.md

@@ -0,0 +1,65 @@
+# 23. Simplifying Helm Chart Value Overrides
+
+Date: 2024-03-25
+
+## Status
+
+Accepted
+
+
+## Context
+
+The process of deploying applications with Helm charts in Kubernetes environments often necessitates the customization of chart values to align with specific operational or environmental requirements. The current method for customizing these values—either through manual edits or `###ZARF_VAR_XYZ###`. A more streamlined approach would greatly enhance the deployment experience by offering both flexibility and reliability.
+
+## Decision
+
+To address this issue, we propose the introduction of a feature designed to simplify the process of overriding chart values at the time of deployment. This feature would allow users to easily specify overrides for any chart values directly via command-line arguments, eliminating the need to alter the chart's default values file or manage multiple command-line arguments for each override.
+
+Key aspects of the proposed implementation include:
+- Use existing `--set`  flags to specify overrides for chart values.
+- The ability to list all overrides in a structured and easily understandable format within `zarf.yaml`.
+- Ensuring that during deployment, these specified overrides take precedence over the chart's default values, thus facilitating customized deployments without necessitating permanent modifications to the chart.
+
+## Consequences
+
+Adopting this feature would lead to several key improvements:
+- **Streamlined Configuration Process**: Allowing helm values overrides in the zarf package schema simplifies the user experience by reducing the reliance on extensive custom `###ZARF_VAR_XYZ###` templating and aligning more closely with standard Helm practices
+
+Ultimately, this feature is aimed at enhancing the deployment workflow by offering a straightforward and efficient means of customizing Helm chart deployments via command-line inputs.
+
+## Example Configuration
+
+Below is an example of how the `zarf.yaml` configuration file might be structured to utilize the new override feature for Helm chart values:
+
+```yaml
+kind: ZarfPackageConfig
+metadata:
+  name: helm-charts
+  description: Example showcasing multiple ways to deploy helm charts
+  version: 0.0.1
+
+components:
+  - name: demo-helm-charts
+    required: true
+    charts:
+      - name: podinfo-local
+        version: 6.4.0
+        namespace: podinfo-from-local-chart
+        localPath: chart
+        valuesFiles:
+          - values.yaml
+        variables:
+          - name: REPLICA_COUNT
+            description: "Override the number of pod replicas"
+            path: replicaCount
+```
+This configuration allows for the specification of default values and descriptions for variables that can be overridden at deployment time. The variables section under each chart specifies the variables that can be overridden, along with a path that indicates where in the values file the variable is located.
+
+### Command Line Example
+
+To override the `REPLICA_COUNT` variable at deployment time, the following command can be used:
+
+```bash
+zarf package deploy zarf-package-helm-charts-arm64-0.0.1.tar.zst --set REPLICA_COUNT=5
+```
+This command demonstrates how users can easily customize their Helm chart deployments by specifying overrides for chart values directly via command-line arguments, in line with the proposed feature.

examples/helm-charts/zarf.yaml

@@ -16,6 +16,12 @@ components:
         localPath: chart
         valuesFiles:
           - values.yaml
+        # Variables are used to override the default values in the chart
+        # This can be overridden by the user at deployment time with the `--set` flag
+        variables:
+          - name: REPLICA_COUNT
+            description: "Override the number of pod replicas"
+            path: replicaCount
 
       - name: podinfo-oci
         version: 6.4.0

src/pkg/packager/deploy.go

@@ -547,24 +547,61 @@ func (p *Packager) pushReposToRepository(reposPath string, repos []string) error
 	return nil
 }
 
+// generateValuesOverrides creates a map containing overrides for chart values based on the given inputs.
+// It prioritizes the overrides in this order:
+// 1. Variables defined in the ZarfComponent, including both Set Variables and Chart Variables.
+// 2. Deployment options specified through DeployOpts.
+// This function is useful for customizing the deployment of a Helm chart by programmatically setting chart values.
+//
+// Returns:
+// - A map containing the final set of value overrides for the chart, where keys are the variable names.
+func generateValuesOverrides(chartVariables []types.ZarfChartVariable,
+	setVariableMap map[string]*types.ZarfSetVariable,
+	deployOpts types.ZarfDeployOptions,
+	componentName, chartName string) (map[string]any, error) {
+
+	valuesOverrides := make(map[string]any)
+	chartOverrides := make(map[string]any)
+
+	for _, variable := range chartVariables {
+		if setVar, ok := setVariableMap[variable.Name]; ok && setVar != nil {
+			// Use the variable's path as a key to ensure unique entries for variables with the same name but different paths.
+			if err := helpers.MergePathAndValueIntoMap(chartOverrides, variable.Path, setVar.Value); err != nil {
+				return nil, fmt.Errorf("unable to merge path and value into map: %w", err)
+			}
+		}
+	}
+
+	// Apply any direct overrides specified in the deployment options for this component and chart
+	if componentOverrides, ok := deployOpts.ValuesOverridesMap[componentName]; ok {
+		if chartSpecificOverrides, ok := componentOverrides[chartName]; ok {
+			valuesOverrides = chartSpecificOverrides
+		}
+	}
+
+	// Merge chartOverrides into valuesOverrides to ensure all overrides are applied.
+	// This corrects the logic to ensure that chartOverrides and valuesOverrides are merged correctly.
+	return helpers.MergeMapRecursive(valuesOverrides, chartOverrides), nil
+}
+
 // Install all Helm charts and raw k8s manifests into the k8s cluster.
 func (p *Packager) installChartAndManifests(componentPaths *layout.ComponentPaths, component types.ZarfComponent) (installedCharts []types.InstalledChart, err error) {
 	for _, chart := range component.Charts {
-
 		// zarf magic for the value file
 		for idx := range chart.ValuesFiles {
 			chartValueName := helm.StandardValuesName(componentPaths.Values, chart, idx)
 			if err := p.valueTemplate.Apply(component, chartValueName, false); err != nil {
 				return installedCharts, err
 			}
 		}
-
-		// TODO (@WSTARR): Currently this logic is library-only and is untested while it is in an experimental state - it may eventually get added as shorthand in Zarf Variables though
-		var valuesOverrides map[string]any
-		if componentChartValuesOverrides, ok := p.cfg.DeployOpts.ValuesOverridesMap[component.Name]; ok {
-			if chartValuesOverrides, ok := componentChartValuesOverrides[chart.Name]; ok {
-				valuesOverrides = chartValuesOverrides
-			}
+		// this is to get the overrides for the chart from the commandline and
+		// the chart variables set in the ZarfComponent and as well as DeployOpts set from the Library user.
+		// The order of precedence is as follows:
+		// 1. Set Variables and Chart Variables from the ZarfComponent
+		// 2. DeployOpts
+		valuesOverrides, err := generateValuesOverrides(chart.Variables, p.cfg.SetVariableMap, p.cfg.DeployOpts, component.Name, chart.Name)
+		if err != nil {
+			return installedCharts, err
 		}
 
 		helmCfg := helm.New(

src/pkg/packager/deploy_test.go

@@ -0,0 +1,218 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2021-Present The Zarf Authors
+
+package packager
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/defenseunicorns/zarf/src/types"
+)
+
+func TestGenerateValuesOverrides(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		chartVariables []types.ZarfChartVariable
+		setVariableMap map[string]*types.ZarfSetVariable
+		deployOpts     types.ZarfDeployOptions
+		componentName  string
+		chartName      string
+		want           map[string]any
+	}{
+		{
+			name:           "Empty inputs",
+			chartVariables: []types.ZarfChartVariable{},
+			setVariableMap: map[string]*types.ZarfSetVariable{},
+			deployOpts:     types.ZarfDeployOptions{},
+			componentName:  "",
+			chartName:      "",
+			want:           map[string]any{},
+		},
+		{
+			name:           "Single variable",
+			chartVariables: []types.ZarfChartVariable{{Name: "testVar", Path: "testVar"}},
+			setVariableMap: map[string]*types.ZarfSetVariable{"testVar": {Value: "testValue"}},
+			deployOpts:     types.ZarfDeployOptions{},
+			componentName:  "testComponent",
+			chartName:      "testChart",
+			want:           map[string]any{"testVar": "testValue"},
+		},
+		{
+			name:           "Non-matching setVariable",
+			chartVariables: []types.ZarfChartVariable{{Name: "expectedVar", Path: "path.to.expectedVar"}},
+			setVariableMap: map[string]*types.ZarfSetVariable{"unexpectedVar": {Value: "unexpectedValue"}},
+			deployOpts:     types.ZarfDeployOptions{},
+			componentName:  "testComponent",
+			chartName:      "testChart",
+			want:           map[string]any{},
+		},
+		{
+			name: "Nested 3 level setVariableMap",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "level1.level2.level3Var", Path: "level1.level2.level3Var"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"level1.level2.level3Var": {Value: "nestedValue"},
+			},
+			deployOpts:    types.ZarfDeployOptions{},
+			componentName: "nestedComponent",
+			chartName:     "nestedChart",
+			want: map[string]any{
+				"level1": map[string]any{
+					"level2": map[string]any{
+						"level3Var": "nestedValue",
+					},
+				},
+			},
+		},
+		{
+			name: "Multiple variables with nested and non-nested paths, distinct values",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "NESTED_VAR_LEVEL2", Path: "nestedVar.level2"},
+				{Name: "simpleVar", Path: "simpleVar"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"NESTED_VAR_LEVEL2": {Value: "distinctNestedValue"},
+				"simpleVar":         {Value: "distinctSimpleValue"},
+			},
+			deployOpts:    types.ZarfDeployOptions{},
+			componentName: "mixedComponent",
+			chartName:     "mixedChart",
+			want: map[string]any{
+				"nestedVar": map[string]any{
+					"level2": "distinctNestedValue",
+				},
+				"simpleVar": "distinctSimpleValue",
+			},
+		},
+		{
+			name: "Values override test",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "overrideVar", Path: "path"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"path": {Value: "overrideValue"},
+			},
+			deployOpts: types.ZarfDeployOptions{
+				ValuesOverridesMap: map[string]map[string]map[string]any{
+					"testComponent": {
+						"testChart": {
+							"path": "deployOverrideValue",
+						},
+					},
+				},
+			},
+			componentName: "testComponent",
+			chartName:     "testChart",
+			want: map[string]any{
+				"path": "deployOverrideValue",
+			},
+		},
+		{
+			name: "Missing variable in setVariableMap but present in ValuesOverridesMap",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "missingVar", Path: "missingVarPath"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{},
+			deployOpts: types.ZarfDeployOptions{
+				ValuesOverridesMap: map[string]map[string]map[string]any{
+					"testComponent": {
+						"testChart": {
+							"missingVarPath": "overrideValue",
+						},
+					},
+				},
+			},
+			componentName: "testComponent",
+			chartName:     "testChart",
+			want: map[string]any{
+				"missingVarPath": "overrideValue",
+			},
+		},
+		{
+			name:           "Non-existent component or chart",
+			chartVariables: []types.ZarfChartVariable{{Name: "someVar", Path: "someVar"}},
+			setVariableMap: map[string]*types.ZarfSetVariable{"someVar": {Value: "value"}},
+			deployOpts: types.ZarfDeployOptions{
+				ValuesOverridesMap: map[string]map[string]map[string]any{
+					"nonExistentComponent": {
+						"nonExistentChart": {
+							"someVar": "overrideValue",
+						},
+					},
+				},
+			},
+			componentName: "actualComponent",
+			chartName:     "actualChart",
+			want:          map[string]any{"someVar": "value"},
+		},
+		{
+			name:           "Variable in setVariableMap but not in chartVariables",
+			chartVariables: []types.ZarfChartVariable{},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"orphanVar": {Value: "orphanValue"},
+			},
+			deployOpts:    types.ZarfDeployOptions{},
+			componentName: "orphanComponent",
+			chartName:     "orphanChart",
+			want:          map[string]any{},
+		},
+		{
+			name: "Empty ValuesOverridesMap with non-empty setVariableMap and chartVariables",
+			chartVariables: []types.ZarfChartVariable{
+				{Name: "var1", Path: "path.to.var1"},
+				{Name: "var2", Path: "path.to.var2"},
+				{Name: "var3", Path: "path.to3.var3"},
+			},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"var1": {Value: "value1"},
+				"var2": {Value: "value2"},
+				"var3": {Value: "value3"},
+			},
+			deployOpts: types.ZarfDeployOptions{
+				ValuesOverridesMap: map[string]map[string]map[string]any{},
+			},
+			componentName: "componentWithVars",
+			chartName:     "chartWithVars",
+			want: map[string]any{
+				"path": map[string]any{
+					"to": map[string]any{
+						"var1": "value1",
+						"var2": "value2",
+					},
+					"to3": map[string]any{
+						"var3": "value3",
+					},
+				},
+			},
+		},
+		{
+			name:           "Empty chartVariables and non-empty setVariableMap",
+			chartVariables: []types.ZarfChartVariable{},
+			setVariableMap: map[string]*types.ZarfSetVariable{
+				"var1": {Value: "value1"},
+				"var2": {Value: "value2"},
+			},
+			deployOpts:    types.ZarfDeployOptions{},
+			componentName: "componentWithVars",
+			chartName:     "chartWithVars",
+			want:          map[string]any{},
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := generateValuesOverrides(tt.chartVariables, tt.setVariableMap, tt.deployOpts, tt.componentName, tt.chartName)
+			if err != nil {
+				t.Errorf("%s: generateValuesOverrides() error = %v", tt.name, err)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("%s: generateValuesOverrides() got = %v, want %v", tt.name, got, tt.want)
+			}
+		})
+	}
+}