From 4ffb43a042979c063df986151ec6efe14bcf8239 Mon Sep 17 00:00:00 2001 From: Yusuke KUOKA Date: Tue, 4 Jun 2019 13:23:22 +0900 Subject: [PATCH] feat: state values This adds `values` to state files as proposed in #640. ```yaml values: - key1: val1 - defaults.yaml environments: default: - values: - environments/default.yaml production: - values: - environments/production.yaml ``` `{{ .Valuese.key1 }}` evaluates to `val1` if and only if it is not overrode via the production or the default env, or command-line args. Resolves #640 --- pkg/app/app_test.go | 124 +++++++++++++++++++++++++ pkg/app/desired_state_file_loader.go | 1 + pkg/app/two_pass_renderer.go | 40 ++++++-- pkg/environment/environment.go | 28 ++++-- pkg/maputil/maputil_test.go | 61 ++++++++++++ pkg/state/create.go | 37 ++++++-- pkg/state/environment_values_loader.go | 2 +- pkg/state/release.go | 8 ++ pkg/state/state.go | 5 +- pkg/state/state_exec_tmpl.go | 43 ++++++++- pkg/state/types.go | 8 +- 11 files changed, 329 insertions(+), 28 deletions(-) create mode 100644 pkg/maputil/maputil_test.go diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index db4a5b23..6e55bcf4 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -900,6 +900,130 @@ bar: "bar1" } } +func TestVisitDesiredStatesWithReleasesFiltered_StateValueOverrides(t *testing.T) { + envTmplExpr := "{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \"b\" | first | first | pluck \"c\" | first }}" + relTmplExpr := "\"{{`{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \\\"b\\\" | first | first | pluck \\\"c\\\" | first }}`}}\"" + + testcases := []struct { + expr, env, expected string + }{ + { + expr: envTmplExpr, + env: "default", + expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C", + }, + { + expr: envTmplExpr, + env: "production", + expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C", + }, + { + expr: relTmplExpr, + env: "default", + expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C", + }, + { + expr: relTmplExpr, + env: "production", + expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C", + }, + } + for i := range testcases { + testcase := testcases[i] + t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { + files := map[string]string{ + "/path/to/helmfile.yaml": fmt.Sprintf(` +# The top-level "values" are "base" values has inherited to state values with the lowest priority. +# The lowest priority results in environment-specific values to override values defined in the base. +values: +- values.yaml + +environments: + default: + values: + - default.yaml + production: + values: + - production.yaml +--- +releases: +- name: %s + chart: %s + namespace: %s +`, testcase.expr, testcase.expr, testcase.expr), + "/path/to/values.yaml": ` +foo: foo +bar: bar +baz: baz +hoge: hoge +fuga: fuga + +a: [] +`, + "/path/to/default.yaml": ` +bar: "bar_default" +baz: "baz_default" + +a: +- b: [] +`, + "/path/to/production.yaml": ` +bar: "bar_production" +baz: "baz_production" + +a: +- b: [] +`, + "/path/to/overrides.yaml": ` +baz: baz_override +hoge: hoge_override + +a: +- b: + - c: C +`, + } + + actual := []state.ReleaseSpec{} + + collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error { + for _, r := range st.Releases { + actual = append(actual, r) + } + return []error{} + } + app := appWithFs(&App{ + KubeContext: "default", + Logger: helmexec.NewLogger(os.Stderr, "debug"), + Reverse: false, + Namespace: "", + Selectors: []string{}, + Env: testcase.env, + ValuesFiles: []string{"overrides.yaml"}, + Set: map[string]interface{}{"hoge": "hoge_set", "fuga": "fuga_set"}, + }, files) + err := app.VisitDesiredStatesWithReleasesFiltered( + "helmfile.yaml", collectReleases, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(actual) != 1 { + t.Errorf("unexpected number of processed releases: expected=1, got=%d", len(actual)) + } + if actual[0].Name != testcase.expected { + t.Errorf("unexpected name: expected=%s, got=%s", testcase.expected, actual[0].Name) + } + if actual[0].Chart != testcase.expected { + t.Errorf("unexpected chart: expected=%s, got=%s", testcase.expected, actual[0].Chart) + } + if actual[0].Namespace != testcase.expected { + t.Errorf("unexpected namespace: expected=%s, got=%s", testcase.expected, actual[0].Namespace) + } + }) + } +} + func TestLoadDesiredStateFromYaml_DuplicateReleaseName(t *testing.T) { yamlFile := "example/path/to/yaml/file" yamlContent := []byte(`releases: diff --git a/pkg/app/desired_state_file_loader.go b/pkg/app/desired_state_file_loader.go index 19820c34..51af19e5 100644 --- a/pkg/app/desired_state_file_loader.go +++ b/pkg/app/desired_state_file_loader.go @@ -83,6 +83,7 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) { return ld.loadFileWithOverrides(inheritedEnv, nil, baseDir, file, evaluateBases) } + func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) { var f string if filepath.IsAbs(file) { diff --git a/pkg/app/two_pass_renderer.go b/pkg/app/two_pass_renderer.go index 51d6b061..17ab39ae 100644 --- a/pkg/app/two_pass_renderer.go +++ b/pkg/app/two_pass_renderer.go @@ -19,8 +19,12 @@ func prependLineNumbers(text string) string { return buf.String() } -func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) *environment.Environment { - tmplData := state.EnvironmentTemplateData{Environment: *firstPassEnv, Namespace: r.namespace} +func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) (*environment.Environment, *state.HelmState) { + tmplData := state.EnvironmentTemplateData{ + Environment: *firstPassEnv, + Namespace: r.namespace, + Values: map[string]interface{}{}, + } firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData) // parse as much as we can, tolerate errors, this is a preparse @@ -29,7 +33,7 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content))) if yamlBuf == nil { // we have a template syntax error, let the second parse report r.logger.Debugf("template syntax error: %v", err) - return firstPassEnv + return firstPassEnv, nil } } yamlData := yamlBuf.String() @@ -57,7 +61,8 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ if prestate != nil { firstPassEnv = &prestate.Env } - return firstPassEnv + + return firstPassEnv, prestate } type RenderOpts struct { @@ -88,13 +93,18 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en r.logger.Debugf("first-pass uses: %v", initEnv) } - renderedEnv := r.renderEnvironment(initEnv, baseDir, filename, content) + renderedEnv, prestate := r.renderPrestate(initEnv, baseDir, filename, content) if r.logger != nil { r.logger.Debugf("first-pass produced: %v", renderedEnv) } - finalEnv, err := renderedEnv.Merge(overrode) + finalEnv, err := inherited.Merge(renderedEnv) + if err != nil { + return nil, err + } + + finalEnv, err = finalEnv.Merge(overrode) if err != nil { return nil, err } @@ -103,7 +113,23 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, *finalEnv) } - tmplData := state.EnvironmentTemplateData{Environment: *finalEnv, Namespace: r.namespace} + vals := map[string]interface{}{} + if prestate != nil { + prestate.Env = *finalEnv + vals, err = prestate.Values() + if err != nil { + return nil, err + } + } + if prestate != nil { + r.logger.Debugf("vals:\n%v\ndefaultVals:%v", vals, prestate.DefaultValues) + } + + tmplData := state.EnvironmentTemplateData{ + Environment: *finalEnv, + Namespace: r.namespace, + Values: vals, + } secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData) yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content) if err != nil { diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index 69bc59d7..f125b2e9 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -7,28 +7,44 @@ import ( ) type Environment struct { - Name string - Values map[string]interface{} + Name string + Values map[string]interface{} + Defaults map[string]interface{} } var EmptyEnvironment Environment func (e Environment) DeepCopy() Environment { - bytes, err := yaml.Marshal(e.Values) + valuesBytes, err := yaml.Marshal(e.Values) if err != nil { panic(err) } var values map[string]interface{} - if err := yaml.Unmarshal(bytes, &values); err != nil { + if err := yaml.Unmarshal(valuesBytes, &values); err != nil { panic(err) } values, err = maputil.CastKeysToStrings(values) if err != nil { panic(err) } + + defaultsBytes, err := yaml.Marshal(e.Defaults) + if err != nil { + panic(err) + } + var defaults map[string]interface{} + if err := yaml.Unmarshal(defaultsBytes, &defaults); err != nil { + panic(err) + } + defaults, err = maputil.CastKeysToStrings(defaults) + if err != nil { + panic(err) + } + return Environment{ - Name: e.Name, - Values: values, + Name: e.Name, + Values: values, + Defaults: defaults, } } diff --git a/pkg/maputil/maputil_test.go b/pkg/maputil/maputil_test.go new file mode 100644 index 00000000..dd43b9c5 --- /dev/null +++ b/pkg/maputil/maputil_test.go @@ -0,0 +1,61 @@ +package maputil + +import "testing" + +func TestMapUtil_StrKeys(t *testing.T) { + m := map[string]interface{}{ + "a": []interface{}{ + map[string]interface{}{ + "b": []interface{}{ + map[string]interface{}{ + "c": "C", + }, + }, + }, + }, + } + + r, err := CastKeysToStrings(m) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + a := r["a"].([]interface{}) + a0 := a[0].(map[string]interface{}) + b := a0["b"].([]interface{}) + b0 := b[0].(map[string]interface{}) + c := b0["c"] + + if c != "C" { + t.Errorf("unexpected c: expected=C, got=%s", c) + } +} + +func TestMapUtil_IFKeys(t *testing.T) { + m := map[interface{}]interface{}{ + "a": []interface{}{ + map[interface{}]interface{}{ + "b": []interface{}{ + map[interface{}]interface{}{ + "c": "C", + }, + }, + }, + }, + } + + r, err := CastKeysToStrings(m) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + a := r["a"].([]interface{}) + a0 := a[0].(map[string]interface{}) + b := a0["b"].([]interface{}) + b0 := b[0].(map[string]interface{}) + c := b0["c"] + + if c != "C" { + t.Errorf("unexpected c: expected=C, got=%s", c) + } +} diff --git a/pkg/state/create.go b/pkg/state/create.go index 9c4d42fa..7c3acf3d 100644 --- a/pkg/state/create.go +++ b/pkg/state/create.go @@ -114,6 +114,12 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi if err != nil { return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err} } + + e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues) + if err != nil { + return nil, err + } + state.Env = *e return &state, nil @@ -137,7 +143,12 @@ func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envNam return nil, err } - return c.LoadEnvValues(state, envName, envValues) + state, err = c.LoadEnvValues(state, envName, envValues) + if err != nil { + return nil, err + } + + return state, nil } func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmState, baseDir string) (*HelmState, error) { @@ -164,13 +175,8 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, envVals := map[string]interface{}{} envSpec, ok := st.Environments[name] if ok { - envValues := append([]interface{}{}, envSpec.Values...) - ld := &EnvironmentValuesLoader{ - storage: st.storage(), - readFile: st.readFile, - } var err error - envVals, err = ld.LoadEnvironmentValues(envSpec.MissingFileHandler, envValues) + envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values) if err != nil { return nil, err } @@ -237,3 +243,20 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, return newEnv, nil } + +func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) { + envVals := map[string]interface{}{} + + valuesEntries := append([]interface{}{}, entries...) + ld := &EnvironmentValuesLoader{ + storage: st.storage(), + readFile: st.readFile, + } + var err error + envVals, err = ld.LoadEnvironmentValues(missingFileHandler, valuesEntries) + if err != nil { + return nil, err + } + + return envVals, nil +} diff --git a/pkg/state/environment_values_loader.go b/pkg/state/environment_values_loader.go index c02507f0..3e81bd1c 100644 --- a/pkg/state/environment_values_loader.go +++ b/pkg/state/environment_values_loader.go @@ -39,7 +39,7 @@ func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *str } for _, envvalFullPath := range resolved { - tmplData := EnvironmentTemplateData{Environment: environment.EmptyEnvironment, Namespace: ""} + tmplData := EnvironmentTemplateData{environment.EmptyEnvironment, "", map[string]interface{}{}} r := tmpl.NewFileRenderer(ld.readFile, filepath.Dir(envvalFullPath), tmplData) bytes, err := r.RenderToBytes(envvalFullPath) if err != nil { diff --git a/pkg/state/release.go b/pkg/state/release.go index 6c9c2b61..f3ee680f 100644 --- a/pkg/state/release.go +++ b/pkg/state/release.go @@ -15,6 +15,14 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R return nil, fmt.Errorf("failed executing template expressions in release \"%s\": %v", r.Name, err) } + { + ts := result.Name + result.Name, err = renderer.RenderTemplateContentToString([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".name = \"%s\": %v", r.Name, ts, err) + } + } + { ts := result.Chart result.Chart, err = renderer.RenderTemplateContentToString([]byte(ts)) diff --git a/pkg/state/state.go b/pkg/state/state.go index 62524bb8..8c2cf0ed 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -27,6 +27,9 @@ type HelmState struct { basePath string FilePath string + // DefaultValues is the default values to be overrode by environment values and command-line overrides + DefaultValues []interface{} `yaml:"values"` + Environments map[string]EnvironmentSpec `yaml:"environments"` Bases []string `yaml:"bases"` @@ -1243,7 +1246,7 @@ func (st *HelmState) flagsForLint(helm helmexec.Interface, release *ReleaseSpec, } func (st *HelmState) RenderValuesFileToBytes(path string) ([]byte, error) { - r := tmpl.NewFileRenderer(st.readFile, filepath.Dir(path), st.envTemplateData()) + r := tmpl.NewFileRenderer(st.readFile, filepath.Dir(path), st.valuesFileTemplateData()) return r.RenderToBytes(path) } diff --git a/pkg/state/state_exec_tmpl.go b/pkg/state/state_exec_tmpl.go index 38164199..34a3573c 100644 --- a/pkg/state/state_exec_tmpl.go +++ b/pkg/state/state_exec_tmpl.go @@ -2,23 +2,58 @@ package state import ( "fmt" + "github.com/imdario/mergo" + "github.com/roboll/helmfile/pkg/maputil" "github.com/roboll/helmfile/pkg/tmpl" ) -func (st *HelmState) envTemplateData() EnvironmentTemplateData { +func (st *HelmState) Values() (map[string]interface{}, error) { + vals := map[string]interface{}{} + + if err := mergo.Merge(&vals, st.Env.Defaults, mergo.WithOverride); err != nil { + return nil, err + } + if err := mergo.Merge(&vals, st.Env.Values, mergo.WithOverride); err != nil { + return nil, err + } + + vals, err := maputil.CastKeysToStrings(vals) + if err != nil { + return nil, err + } + + return vals, nil +} + +func (st *HelmState) mustLoadVals() map[string]interface{} { + vals, err := st.Values() + if err != nil { + panic(err) + } + return vals +} + +func (st *HelmState) valuesFileTemplateData() EnvironmentTemplateData { return EnvironmentTemplateData{ - st.Env, - st.Namespace, + Environment: st.Env, + Namespace: st.Namespace, + Values: st.mustLoadVals(), } } func (st *HelmState) ExecuteTemplates() (*HelmState, error) { r := *st + vals, err := st.Values() + if err != nil { + return nil, err + } + for i, rt := range st.Releases { - tmplData := ReleaseTemplateData{ + tmplData := releaseTemplateData{ Environment: st.Env, Release: rt, + Values: vals, } renderer := tmpl.NewFileRenderer(st.readFile, st.basePath, tmplData) r, err := rt.ExecuteTemplateExpressions(renderer) diff --git a/pkg/state/types.go b/pkg/state/types.go index e998a7ec..fed10683 100644 --- a/pkg/state/types.go +++ b/pkg/state/types.go @@ -15,12 +15,16 @@ type EnvironmentTemplateData struct { Environment environment.Environment // Namespace is accessible as `.Namespace` from any non-values template executed by the renderer Namespace string + // Values is accessible as `.Values` and it contains default state values overrode by environment values and override values. + Values map[string]interface{} } -// ReleaseTemplateData provides variables accessible while executing golang text/template expressions in releases of a helmfile YAML file -type ReleaseTemplateData struct { +// releaseTemplateData provides variables accessible while executing golang text/template expressions in releases of a helmfile YAML file +type releaseTemplateData struct { // Environment is accessible as `.Environment` from any template expression executed by the renderer Environment environment.Environment // Release is accessible as `.Release` from any template expression executed by the renderer Release ReleaseSpec + // Values is accessible as `.Values` and it contains default state values overrode by environment values and override values. + Values map[string]interface{} }