Skip to content

Commit

Permalink
feat: state values (#647)
Browse files Browse the repository at this point in the history
This adds `values` to state files as proposed in #640.

```yaml
values:
- key1: val1
- defaults.yaml

environments:
  default:
  - values:
    - environments/default.yaml
  production:
  - values:
    - environments/production.yaml

```

`{{ .Valuese.key1 }}` evaluates to `val1` if and only if it is not overrode via the production or the default env, or command-line args.

Resolves #640
  • Loading branch information
mumoshu authored Jun 4, 2019
1 parent 2d2b3e4 commit 3710f62
Show file tree
Hide file tree
Showing 11 changed files with 329 additions and 28 deletions.
124 changes: 124 additions & 0 deletions pkg/app/app_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -900,6 +900,130 @@ bar: "bar1"
}
}

func TestVisitDesiredStatesWithReleasesFiltered_StateValueOverrides(t *testing.T) {
envTmplExpr := "{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \"b\" | first | first | pluck \"c\" | first }}"
relTmplExpr := "\"{{`{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \\\"b\\\" | first | first | pluck \\\"c\\\" | first }}`}}\""

testcases := []struct {
expr, env, expected string
}{
{
expr: envTmplExpr,
env: "default",
expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C",
},
{
expr: envTmplExpr,
env: "production",
expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C",
},
{
expr: relTmplExpr,
env: "default",
expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C",
},
{
expr: relTmplExpr,
env: "production",
expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C",
},
}
for i := range testcases {
testcase := testcases[i]
t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
files := map[string]string{
"/path/to/helmfile.yaml": fmt.Sprintf(`
# The top-level "values" are "base" values has inherited to state values with the lowest priority.
# The lowest priority results in environment-specific values to override values defined in the base.
values:
- values.yaml
environments:
default:
values:
- default.yaml
production:
values:
- production.yaml
---
releases:
- name: %s
chart: %s
namespace: %s
`, testcase.expr, testcase.expr, testcase.expr),
"/path/to/values.yaml": `
foo: foo
bar: bar
baz: baz
hoge: hoge
fuga: fuga
a: []
`,
"/path/to/default.yaml": `
bar: "bar_default"
baz: "baz_default"
a:
- b: []
`,
"/path/to/production.yaml": `
bar: "bar_production"
baz: "baz_production"
a:
- b: []
`,
"/path/to/overrides.yaml": `
baz: baz_override
hoge: hoge_override
a:
- b:
- c: C
`,
}

actual := []state.ReleaseSpec{}

collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error {
for _, r := range st.Releases {
actual = append(actual, r)
}
return []error{}
}
app := appWithFs(&App{
KubeContext: "default",
Logger: helmexec.NewLogger(os.Stderr, "debug"),
Reverse: false,
Namespace: "",
Selectors: []string{},
Env: testcase.env,
ValuesFiles: []string{"overrides.yaml"},
Set: map[string]interface{}{"hoge": "hoge_set", "fuga": "fuga_set"},
}, files)
err := app.VisitDesiredStatesWithReleasesFiltered(
"helmfile.yaml", collectReleases,
)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(actual) != 1 {
t.Errorf("unexpected number of processed releases: expected=1, got=%d", len(actual))
}
if actual[0].Name != testcase.expected {
t.Errorf("unexpected name: expected=%s, got=%s", testcase.expected, actual[0].Name)
}
if actual[0].Chart != testcase.expected {
t.Errorf("unexpected chart: expected=%s, got=%s", testcase.expected, actual[0].Chart)
}
if actual[0].Namespace != testcase.expected {
t.Errorf("unexpected namespace: expected=%s, got=%s", testcase.expected, actual[0].Namespace)
}
})
}
}

func TestLoadDesiredStateFromYaml_DuplicateReleaseName(t *testing.T) {
yamlFile := "example/path/to/yaml/file"
yamlContent := []byte(`releases:
Expand Down
1 change: 1 addition & 0 deletions pkg/app/desired_state_file_loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e
func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
return ld.loadFileWithOverrides(inheritedEnv, nil, baseDir, file, evaluateBases)
}

func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
var f string
if filepath.IsAbs(file) {
Expand Down
40 changes: 33 additions & 7 deletions pkg/app/two_pass_renderer.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ func prependLineNumbers(text string) string {
return buf.String()
}

func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) *environment.Environment {
tmplData := state.EnvironmentTemplateData{Environment: *firstPassEnv, Namespace: r.namespace}
func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) (*environment.Environment, *state.HelmState) {
tmplData := state.EnvironmentTemplateData{
Environment: *firstPassEnv,
Namespace: r.namespace,
Values: map[string]interface{}{},
}
firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData)

// parse as much as we can, tolerate errors, this is a preparse
Expand All @@ -29,7 +33,7 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ
r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
if yamlBuf == nil { // we have a template syntax error, let the second parse report
r.logger.Debugf("template syntax error: %v", err)
return firstPassEnv
return firstPassEnv, nil
}
}
yamlData := yamlBuf.String()
Expand Down Expand Up @@ -57,7 +61,8 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ
if prestate != nil {
firstPassEnv = &prestate.Env
}
return firstPassEnv

return firstPassEnv, prestate
}

type RenderOpts struct {
Expand Down Expand Up @@ -88,13 +93,18 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en
r.logger.Debugf("first-pass uses: %v", initEnv)
}

renderedEnv := r.renderEnvironment(initEnv, baseDir, filename, content)
renderedEnv, prestate := r.renderPrestate(initEnv, baseDir, filename, content)

if r.logger != nil {
r.logger.Debugf("first-pass produced: %v", renderedEnv)
}

finalEnv, err := renderedEnv.Merge(overrode)
finalEnv, err := inherited.Merge(renderedEnv)
if err != nil {
return nil, err
}

finalEnv, err = finalEnv.Merge(overrode)
if err != nil {
return nil, err
}
Expand All @@ -103,7 +113,23 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en
r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, *finalEnv)
}

tmplData := state.EnvironmentTemplateData{Environment: *finalEnv, Namespace: r.namespace}
vals := map[string]interface{}{}
if prestate != nil {
prestate.Env = *finalEnv
vals, err = prestate.Values()
if err != nil {
return nil, err
}
}
if prestate != nil {
r.logger.Debugf("vals:\n%v\ndefaultVals:%v", vals, prestate.DefaultValues)
}

tmplData := state.EnvironmentTemplateData{
Environment: *finalEnv,
Namespace: r.namespace,
Values: vals,
}
secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData)
yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
if err != nil {
Expand Down
28 changes: 22 additions & 6 deletions pkg/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,44 @@ import (
)

type Environment struct {
Name string
Values map[string]interface{}
Name string
Values map[string]interface{}
Defaults map[string]interface{}
}

var EmptyEnvironment Environment

func (e Environment) DeepCopy() Environment {
bytes, err := yaml.Marshal(e.Values)
valuesBytes, err := yaml.Marshal(e.Values)
if err != nil {
panic(err)
}
var values map[string]interface{}
if err := yaml.Unmarshal(bytes, &values); err != nil {
if err := yaml.Unmarshal(valuesBytes, &values); err != nil {
panic(err)
}
values, err = maputil.CastKeysToStrings(values)
if err != nil {
panic(err)
}

defaultsBytes, err := yaml.Marshal(e.Defaults)
if err != nil {
panic(err)
}
var defaults map[string]interface{}
if err := yaml.Unmarshal(defaultsBytes, &defaults); err != nil {
panic(err)
}
defaults, err = maputil.CastKeysToStrings(defaults)
if err != nil {
panic(err)
}

return Environment{
Name: e.Name,
Values: values,
Name: e.Name,
Values: values,
Defaults: defaults,
}
}

Expand Down
61 changes: 61 additions & 0 deletions pkg/maputil/maputil_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package maputil

import "testing"

func TestMapUtil_StrKeys(t *testing.T) {
m := map[string]interface{}{
"a": []interface{}{
map[string]interface{}{
"b": []interface{}{
map[string]interface{}{
"c": "C",
},
},
},
},
}

r, err := CastKeysToStrings(m)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

a := r["a"].([]interface{})
a0 := a[0].(map[string]interface{})
b := a0["b"].([]interface{})
b0 := b[0].(map[string]interface{})
c := b0["c"]

if c != "C" {
t.Errorf("unexpected c: expected=C, got=%s", c)
}
}

func TestMapUtil_IFKeys(t *testing.T) {
m := map[interface{}]interface{}{
"a": []interface{}{
map[interface{}]interface{}{
"b": []interface{}{
map[interface{}]interface{}{
"c": "C",
},
},
},
},
}

r, err := CastKeysToStrings(m)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

a := r["a"].([]interface{})
a0 := a[0].(map[string]interface{})
b := a0["b"].([]interface{})
b0 := b[0].(map[string]interface{})
c := b0["c"]

if c != "C" {
t.Errorf("unexpected c: expected=C, got=%s", c)
}
}
37 changes: 30 additions & 7 deletions pkg/state/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,12 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi
if err != nil {
return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err}
}

e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues)
if err != nil {
return nil, err
}

state.Env = *e

return &state, nil
Expand All @@ -137,7 +143,12 @@ func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envNam
return nil, err
}

return c.LoadEnvValues(state, envName, envValues)
state, err = c.LoadEnvValues(state, envName, envValues)
if err != nil {
return nil, err
}

return state, nil
}

func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmState, baseDir string) (*HelmState, error) {
Expand All @@ -164,13 +175,8 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
envVals := map[string]interface{}{}
envSpec, ok := st.Environments[name]
if ok {
envValues := append([]interface{}{}, envSpec.Values...)
ld := &EnvironmentValuesLoader{
storage: st.storage(),
readFile: st.readFile,
}
var err error
envVals, err = ld.LoadEnvironmentValues(envSpec.MissingFileHandler, envValues)
envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -237,3 +243,20 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,

return newEnv, nil
}

func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) {
envVals := map[string]interface{}{}

valuesEntries := append([]interface{}{}, entries...)
ld := &EnvironmentValuesLoader{
storage: st.storage(),
readFile: st.readFile,
}
var err error
envVals, err = ld.LoadEnvironmentValues(missingFileHandler, valuesEntries)
if err != nil {
return nil, err
}

return envVals, nil
}
Loading

0 comments on commit 3710f62

Please sign in to comment.