From 3cab72e09c6d70fcc0bae66996dc1bf7b32de772 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 3 Apr 2024 09:43:12 -0600 Subject: [PATCH 01/23] chore: refactor variables into separate package --- src/cmd/tools/zarf.go | 3 +- src/internal/packager/helm/chart.go | 6 - src/internal/packager/helm/common.go | 33 ++- src/internal/packager/helm/post-render.go | 21 +- src/internal/packager/helm/repo.go | 2 +- src/internal/packager/helm/zarf.go | 15 +- src/internal/packager/images/push.go | 2 +- src/internal/packager/template/template.go | 247 ++++-------------- src/internal/packager/template/yaml.go | 27 -- src/internal/packager/validate/validate.go | 5 +- src/pkg/interactive/prompt.go | 4 +- src/pkg/message/message.go | 20 +- src/pkg/packager/actions/actions.go | 31 +-- src/pkg/packager/common.go | 10 +- src/pkg/packager/composer/list.go | 15 +- src/pkg/packager/composer/list_test.go | 81 +++--- src/pkg/packager/creator/normal.go | 8 +- src/pkg/packager/creator/template.go | 5 +- src/pkg/packager/deploy.go | 98 ++++--- .../deprecated/pluralize-set-variable.go | 3 +- src/pkg/packager/dev.go | 5 +- src/pkg/packager/interactive.go | 5 +- src/pkg/packager/mirror.go | 2 +- src/pkg/packager/prepare.go | 50 ++-- src/pkg/packager/remove.go | 10 +- src/pkg/packager/variables/variables.go | 54 ---- src/pkg/variables/common.go | 29 ++ src/pkg/variables/templates.go | 152 +++++++++++ src/pkg/variables/types.go | 48 ++++ src/pkg/variables/variables.go | 78 ++++++ src/types/component.go | 32 +-- src/types/package.go | 36 +-- src/types/packager.go | 30 --- 33 files changed, 596 insertions(+), 571 deletions(-) delete mode 100644 src/internal/packager/template/yaml.go delete mode 100644 src/pkg/packager/variables/variables.go create mode 100644 src/pkg/variables/common.go create mode 100644 src/pkg/variables/templates.go create mode 100644 src/pkg/variables/types.go create mode 100644 src/pkg/variables/variables.go diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index 54f5b18067..363fe87c63 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/internal/packager/helm" + "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" @@ -138,7 +139,7 @@ var updateCredsCmd = &cobra.Command{ } // Update Zarf 'init' component Helm releases if present - h := helm.NewClusterOnly(&types.PackagerConfig{State: newState}, c) + h := helm.NewClusterOnly(&types.PackagerConfig{}, template.GetZarfVariableConfig(), newState, c) if slices.Contains(args, message.RegistryKey) && newState.RegistryInfo.InternalRegistry { err = h.UpdateZarfRegistryValues() diff --git a/src/internal/packager/helm/chart.go b/src/internal/packager/helm/chart.go index 0e10c0ef99..72b80d65f9 100644 --- a/src/internal/packager/helm/chart.go +++ b/src/internal/packager/helm/chart.go @@ -44,12 +44,6 @@ func (h *Helm) InstallOrUpgradeChart() (types.ConnectStrings, string, error) { h.chart.ReleaseName = h.chart.Name } - // Do not wait for the chart to be ready if data injections are present. - if len(h.component.DataInjections) > 0 { - spinner.Updatef("Data injections detected, not waiting for chart to be ready") - h.chart.NoWait = true - } - // Setup K8s connection. err := h.createActionConfig(h.chart.Namespace, spinner) if err != nil { diff --git a/src/internal/packager/helm/common.go b/src/internal/packager/helm/common.go index 335fbf0f6c..46a6cdccbb 100644 --- a/src/internal/packager/helm/common.go +++ b/src/internal/packager/helm/common.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/variables" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -29,19 +30,20 @@ type Helm struct { chartPath string valuesPath string - cfg *types.PackagerConfig - component types.ZarfComponent - cluster *cluster.Cluster - timeout time.Duration - retries int + cfg *types.PackagerConfig + cluster *cluster.Cluster + timeout time.Duration + retries int kubeVersion string chartOverride *chart.Chart valuesOverrides map[string]any - settings *cli.EnvSettings - actionConfig *action.Configuration + settings *cli.EnvSettings + actionConfig *action.Configuration + variableConfig *variables.VariableConfig + state *types.ZarfState } // Modifier is a function that modifies the Helm config. @@ -64,12 +66,14 @@ func New(chart types.ZarfChart, chartPath string, valuesPath string, mods ...Mod } // NewClusterOnly returns a new Helm config struct geared toward interacting with the cluster (not packages) -func NewClusterOnly(cfg *types.PackagerConfig, cluster *cluster.Cluster) *Helm { +func NewClusterOnly(cfg *types.PackagerConfig, variableConfig *variables.VariableConfig, state *types.ZarfState, cluster *cluster.Cluster) *Helm { return &Helm{ - cfg: cfg, - cluster: cluster, - timeout: config.ZarfDefaultTimeout, - retries: config.ZarfDefaultRetries, + cfg: cfg, + variableConfig: variableConfig, + state: state, + cluster: cluster, + timeout: config.ZarfDefaultTimeout, + retries: config.ZarfDefaultRetries, } } @@ -133,10 +137,11 @@ func NewFromZarfManifest(manifest types.ZarfManifest, manifestPath, packageName, } // WithDeployInfo adds the necessary information to deploy a given chart -func WithDeployInfo(component types.ZarfComponent, cfg *types.PackagerConfig, cluster *cluster.Cluster, valuesOverrides map[string]any, timeout time.Duration, retries int) Modifier { +func WithDeployInfo(cfg *types.PackagerConfig, variableConfig *variables.VariableConfig, state *types.ZarfState, cluster *cluster.Cluster, valuesOverrides map[string]any, timeout time.Duration, retries int) Modifier { return func(h *Helm) { - h.component = component h.cfg = cfg + h.variableConfig = variableConfig + h.state = state h.cluster = cluster h.valuesOverrides = valuesOverrides h.timeout = timeout diff --git a/src/internal/packager/helm/post-render.go b/src/internal/packager/helm/post-render.go index ed3dec5d9f..24cdae1395 100644 --- a/src/internal/packager/helm/post-render.go +++ b/src/internal/packager/helm/post-render.go @@ -13,7 +13,6 @@ import ( "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -29,22 +28,11 @@ type renderer struct { *Helm connectStrings types.ConnectStrings namespaces map[string]*corev1.Namespace - values template.Values } func (h *Helm) newRenderer() (*renderer, error) { message.Debugf("helm.NewRenderer()") - valueTemplate, err := template.Generate(h.cfg) - if err != nil { - return nil, err - } - - // TODO (@austinabro321) this should be cleaned up after https://github.com/defenseunicorns/zarf/pull/2276 gets merged - if h.cfg.State == nil { - valueTemplate.SetState(&types.ZarfState{}) - } - namespaces := make(map[string]*corev1.Namespace) if h.cluster != nil { namespaces[h.chart.Namespace] = h.cluster.NewZarfManagedNamespace(h.chart.Namespace) @@ -54,7 +42,6 @@ func (h *Helm) newRenderer() (*renderer, error) { Helm: h, connectStrings: make(types.ConnectStrings), namespaces: namespaces, - values: *valueTemplate, }, nil } @@ -71,7 +58,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { } // Run the template engine against the chart output - if _, err := template.ProcessYamlFilesInPath(tempDir, r.component, r.values); err != nil { + if _, err := r.variableConfig.ProcessYamlFilesInPath(tempDir); err != nil { return nil, fmt.Errorf("error templating the helm chart: %w", err) } @@ -143,12 +130,12 @@ func (r *renderer) adoptAndUpdateNamespaces() error { } // If the package is marked as YOLO and the state is empty, skip the secret creation for this namespace - if r.cfg.Pkg.Metadata.YOLO && r.cfg.State.Distro == "YOLO" { + if r.cfg.Pkg.Metadata.YOLO && r.state.Distro == "YOLO" { continue } // Create the secret - validRegistrySecret := c.GenerateRegistryPullCreds(name, config.ZarfImagePullSecretName, r.cfg.State.RegistryInfo) + validRegistrySecret := c.GenerateRegistryPullCreds(name, config.ZarfImagePullSecretName, r.state.RegistryInfo) // Try to get a valid existing secret currentRegistrySecret, _ := c.GetSecret(name, config.ZarfImagePullSecretName) @@ -159,7 +146,7 @@ func (r *renderer) adoptAndUpdateNamespaces() error { } // Generate the git server secret - gitServerSecret := c.GenerateGitPullCreds(name, config.ZarfGitServerSecretName, r.cfg.State.GitServer) + gitServerSecret := c.GenerateGitPullCreds(name, config.ZarfGitServerSecretName, r.state.GitServer) // Create or update the zarf git server secret if _, err := c.CreateOrUpdateSecret(gitServerSecret); err != nil { diff --git a/src/internal/packager/helm/repo.go b/src/internal/packager/helm/repo.go index d91349a5c6..97582a64b9 100644 --- a/src/internal/packager/helm/repo.go +++ b/src/internal/packager/helm/repo.go @@ -376,7 +376,7 @@ func (h *Helm) listAvailableChartsAndVersions(pull *action.Pull) error { versions += entry.Version + separator } - versions = message.Truncate(versions, 75, false) + versions = helpers.Truncate(versions, 75, false) chartData = append(chartData, []string{name, versions}) } diff --git a/src/internal/packager/helm/zarf.go b/src/internal/packager/helm/zarf.go index b8d42c3f84..d6087090b5 100644 --- a/src/internal/packager/helm/zarf.go +++ b/src/internal/packager/helm/zarf.go @@ -7,23 +7,25 @@ package helm import ( "fmt" + "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/variables" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/action" ) // UpdateZarfRegistryValues updates the Zarf registry deployment with the new state values func (h *Helm) UpdateZarfRegistryValues() error { - pushUser, err := utils.GetHtpasswdString(h.cfg.State.RegistryInfo.PushUsername, h.cfg.State.RegistryInfo.PushPassword) + pushUser, err := utils.GetHtpasswdString(h.state.RegistryInfo.PushUsername, h.state.RegistryInfo.PushPassword) if err != nil { return fmt.Errorf("error generating htpasswd string: %w", err) } - pullUser, err := utils.GetHtpasswdString(h.cfg.State.RegistryInfo.PullUsername, h.cfg.State.RegistryInfo.PullPassword) + pullUser, err := utils.GetHtpasswdString(h.state.RegistryInfo.PullUsername, h.state.RegistryInfo.PullPassword) if err != nil { return fmt.Errorf("error generating htpasswd string: %w", err) } @@ -90,10 +92,7 @@ func (h *Helm) UpdateZarfAgentValues() error { Namespace: "zarf", ReleaseName: release.Name, } - h.component = types.ZarfComponent{ - Name: "zarf-agent", - } - h.cfg.Pkg.Constants = []types.ZarfPackageConstant{ + h.variableConfig.Constants = []variables.Constant{ { Name: "AGENT_IMAGE", Value: currentAgentImage.Path, @@ -103,6 +102,10 @@ func (h *Helm) UpdateZarfAgentValues() error { Value: currentAgentImage.Tag, }, } + h.variableConfig.ApplicationTemplates, err = template.GetZarfTemplates("zarf-agent", h.state) + if err != nil { + return fmt.Errorf("error setting up the templates: %w", err) + } err := h.UpdateReleaseValues(map[string]interface{}{}) if err != nil { diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index 5daff0a14f..900b925651 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -91,7 +91,7 @@ func (i *ImageConfig) PushToZarfRegistry() error { } for refInfo, img := range refInfoToImage { - refTruncated := message.Truncate(refInfo.Reference, 55, true) + refTruncated := helpers.Truncate(refInfo.Reference, 55, true) progressBar.UpdateTitle(fmt.Sprintf("Pushing %s", refTruncated)) // If this is not a no checksum image push it for use with the Zarf agent diff --git a/src/internal/packager/template/template.go b/src/internal/packager/template/template.go index 19eed3e93c..8fd66cc916 100644 --- a/src/internal/packager/template/template.go +++ b/src/internal/packager/template/template.go @@ -5,11 +5,8 @@ package template import ( - "bufio" "encoding/base64" "fmt" - "os" - "regexp" "strings" "github.com/defenseunicorns/zarf/src/types" @@ -18,83 +15,34 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/variables" ) -// TextTemplate represents a value to be templated into a text file. -type TextTemplate struct { - Sensitive bool - AutoIndent bool - Type types.VariableType - Value string -} - -// Values contains the values to be used in the template. -type Values struct { - config *types.PackagerConfig - htpasswd string -} - -// Generate returns a Values struct with the values to be used in the template. -func Generate(cfg *types.PackagerConfig) (*Values, error) { - message.Debug("template.Generate()") - var generated Values - - if cfg == nil { - return nil, fmt.Errorf("config is nil") - } - - generated.config = cfg - - if cfg.State == nil { - return &generated, nil - } - - regInfo := cfg.State.RegistryInfo - - // Only calculate this for internal registries to allow longer external passwords - if regInfo.InternalRegistry { - pushUser, err := utils.GetHtpasswdString(regInfo.PushUsername, regInfo.PushPassword) - if err != nil { - return nil, fmt.Errorf("error generating htpasswd string: %w", err) - } - - pullUser, err := utils.GetHtpasswdString(regInfo.PullUsername, regInfo.PullPassword) - if err != nil { - return nil, fmt.Errorf("error generating htpasswd string: %w", err) - } - - generated.htpasswd = fmt.Sprintf("%s\\n%s", pushUser, pullUser) - } - - return &generated, nil -} - -// Ready returns true if the Values struct is ready to be used in the template. -func (values *Values) Ready() bool { - return values.config.State != nil -} +const ( + depMarkerOld = "DATA_INJECTON_MARKER" + depMarkerNew = "DATA_INJECTION_MARKER" +) -// SetState sets the state -func (values *Values) SetState(state *types.ZarfState) { - values.config.State = state +// GetZarfVariableConfig gets a variable configuration specific to Zarf +func GetZarfVariableConfig() *variables.VariableConfig { + return variables.New( + "zarf", + deprecatedKeys(), + make(variables.SetVariableMap), + make([]variables.Constant, 0), + message.Warnf) } -// GetVariables returns the variables to be used in the template. -func (values *Values) GetVariables(component types.ZarfComponent) (templateMap map[string]*TextTemplate, deprecations map[string]string) { - templateMap = make(map[string]*TextTemplate) +// GetZarfTemplates returns the template keys and values to be used for templating. +func GetZarfTemplates(componentName string, state *types.ZarfState) (templateMap map[string]*variables.TextTemplate, err error) { + templateMap = make(map[string]*variables.TextTemplate) - depMarkerOld := "DATA_INJECTON_MARKER" - depMarkerNew := "DATA_INJECTION_MARKER" - deprecations = map[string]string{ - fmt.Sprintf("###ZARF_%s###", depMarkerOld): fmt.Sprintf("###ZARF_%s###", depMarkerNew), - } - - if values.config.State != nil { - regInfo := values.config.State.RegistryInfo - gitInfo := values.config.State.GitServer + if state != nil { + regInfo := state.RegistryInfo + gitInfo := state.GitServer builtinMap := map[string]string{ - "STORAGE_CLASS": values.config.State.StorageClass, + "STORAGE_CLASS": state.StorageClass, // Registry info "REGISTRY": regInfo.Address, @@ -109,34 +57,35 @@ func (values *Values) GetVariables(component types.ZarfComponent) (templateMap m "GIT_AUTH_PULL": gitInfo.PullPassword, } - // Include the data injection marker template if the component has data injections - if len(component.DataInjections) > 0 { - // Preserve existing misspelling for backwards compatibility - builtinMap[depMarkerOld] = config.GetDataInjectionMarker() - builtinMap[depMarkerNew] = config.GetDataInjectionMarker() - } + // Preserve existing misspelling for backwards compatibility + builtinMap[depMarkerOld] = config.GetDataInjectionMarker() + builtinMap[depMarkerNew] = config.GetDataInjectionMarker() // Don't template component-specific variables for every component - switch component.Name { + switch componentName { case "zarf-agent": - agentTLS := values.config.State.AgentTLS + agentTLS := state.AgentTLS builtinMap["AGENT_CRT"] = base64.StdEncoding.EncodeToString(agentTLS.Cert) builtinMap["AGENT_KEY"] = base64.StdEncoding.EncodeToString(agentTLS.Key) builtinMap["AGENT_CA"] = base64.StdEncoding.EncodeToString(agentTLS.CA) case "zarf-seed-registry", "zarf-registry": builtinMap["SEED_REGISTRY"] = fmt.Sprintf("%s:%s", helpers.IPV4Localhost, config.ZarfSeedPort) - builtinMap["HTPASSWD"] = values.htpasswd + htpasswd, err := generateHtpasswd(®Info) + if err != nil { + return templateMap, err + } + builtinMap["HTPASSWD"] = htpasswd builtinMap["REGISTRY_SECRET"] = regInfo.Secret case "logging": - builtinMap["LOGGING_AUTH"] = values.config.State.LoggingSecret + builtinMap["LOGGING_AUTH"] = state.LoggingSecret } // Iterate over any custom variables and add them to the mappings for templating for key, value := range builtinMap { // Builtin keys are always uppercase in the format ###ZARF_KEY### - templateMap[strings.ToUpper(fmt.Sprintf("###ZARF_%s###", key))] = &TextTemplate{ + templateMap[strings.ToUpper(fmt.Sprintf("###ZARF_%s###", key))] = &variables.TextTemplate{ Value: value, } @@ -149,131 +98,39 @@ func (values *Values) GetVariables(component types.ZarfComponent) (templateMap m } } - for key, variable := range values.config.SetVariableMap { - // Variable keys are always uppercase in the format ###ZARF_VAR_KEY### - templateMap[strings.ToUpper(fmt.Sprintf("###ZARF_VAR_%s###", key))] = &TextTemplate{ - Value: variable.Value, - Sensitive: variable.Sensitive, - AutoIndent: variable.AutoIndent, - Type: variable.Type, - } - } - - for _, constant := range values.config.Pkg.Constants { - // Constant keys are always uppercase in the format ###ZARF_CONST_KEY### - templateMap[strings.ToUpper(fmt.Sprintf("###ZARF_CONST_%s###", constant.Name))] = &TextTemplate{ - Value: constant.Value, - AutoIndent: constant.AutoIndent, - } - } - debugPrintTemplateMap(templateMap) - message.Debugf("deprecations = %#v", deprecations) - return templateMap, deprecations + return templateMap, nil } -// Apply renders the template and writes the result to the given path. -func (values *Values) Apply(component types.ZarfComponent, path string, ignoreReady bool) error { - // If Apply() is called before all values are loaded, fail unless ignoreReady is true - if !values.Ready() && !ignoreReady { - return fmt.Errorf("template.Apply() called before template.Generate()") +// deprecatedKeys returns a map of template keys that are deprecated +func deprecatedKeys() map[string]string { + return map[string]string{ + fmt.Sprintf("###ZARF_%s###", depMarkerOld): fmt.Sprintf("###ZARF_%s###", depMarkerNew), } - - templateMap, deprecations := values.GetVariables(component) - err := ReplaceTextTemplate(path, templateMap, deprecations, "###ZARF_[A-Z0-9_]+###") - - return err } -// ReplaceTextTemplate loads a file from a given path, replaces text in it and writes it back in place. -func ReplaceTextTemplate(path string, mappings map[string]*TextTemplate, deprecations map[string]string, templateRegex string) error { - textFile, err := os.Open(path) - if err != nil { - return err - } - - // This regex takes a line and parses the text before and after a discovered template: https://regex101.com/r/ilUxAz/1 - regexTemplateLine := regexp.MustCompile(fmt.Sprintf("(?P.*?)(?P