diff --git a/README.md b/README.md index b8596ece..e915f685 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ repositories: context: kube-context # kube-context (--kube-context) #default values to set for args along with dedicated keys that can be set by contributers, cli args take precedence overe these -helmDefaults: +helmDefaults: tillerNamespace: tiller-namespace #dedicated default key for tiller-namespace kubeContext: kube-context #dedicated default key for kube-context # additional and global args passed to helm @@ -396,7 +396,7 @@ releases: # snip {{ end }} - name: myapp - # snip + # snip ``` ## Environment Values @@ -424,6 +424,7 @@ releases: ```yaml domain: prod.example.com +releaseName: prod ``` `values.yaml.gotmpl` @@ -435,6 +436,38 @@ domain: {{ .Environment.Values.domain | default "dev.example.com" }} `helmfile sync` installs `myapp` with the value `domain=dev.example.com`, whereas `helmfile --environment production sync` installs the app with the value `domain=production.example.com`. +For even more flexibility, you can now use values declared in the `environments:` section in other parts of your helmfiles: + +consider: +`default.yaml` + +```yaml +domain: dev.example.com +releaseName: dev +``` + +```yaml +environments: + default: + values: + - default.yaml + production: + values: + - production.yaml # template directives with potential side-effects like `exec` and `readFile` will NOT be executed + - other.yaml.gotmpl # `exec` and `readFile` will be honoured + +releases: +- name: myapp-{{ .Environment.Values.releaseName }} # release name will be one of `dev` or `prod` depending on selected environment + values: + - values.yaml.gotmpl + +{{ if eq (.Environment.Values.releaseName "prod" ) }} +# this release would be installed only if selected environment is `production` +- name: production-specific-release + ... +{{ end }} +``` + ## Environment Secrets Environment Secrets are encrypted versions of `Environment Values`. diff --git a/main.go b/main.go index 3cf93a5b..438aeb58 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "fmt" "log" "os" @@ -588,6 +589,67 @@ func (e *noMatchingHelmfileError) Error() string { ) } +func prependLineNumbers(text string) string { + buf := bytes.NewBufferString("") + lines := strings.Split(text, "\n") + for i, line := range lines { + buf.WriteString(fmt.Sprintf("%2d: %s\n", i, line)) + } + return buf.String() +} + +type twoPassRenderer struct { + reader func(string) ([]byte, error) + env string + filename string + logger *zap.SugaredLogger + abs func(string) (string, error) +} + +func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environment { + firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)} + firstPassRenderer := tmpl.NewFirstPassRenderer(firstPassEnv) + + // parse as much as we can, tolerate errors, this is a preparse + yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content) + if err != nil && logger != nil { + r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content))) + } + c := state.NewCreator(r.logger, r.reader, r.abs) + c.Strict = false + // create preliminary state, as we may have an environment. Tolerate errors. + prestate, err := c.CreateFromYaml(yamlBuf.Bytes(), r.filename, r.env) + if err != nil && r.logger != nil { + switch err.(type) { + case *state.StateLoadError: + r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err) + } + r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String())) + } + if prestate != nil { + firstPassEnv = prestate.Env + } + return firstPassEnv +} + +func (r *twoPassRenderer) renderTemplate(content []byte) (*bytes.Buffer, error) { + // try a first pass render. This will always succeed, but can produce a limited env + firstPassEnv := r.renderEnvironment(content) + + secondPassRenderer := tmpl.NewFileRenderer(r.reader, "", firstPassEnv) + yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content) + if err != nil { + if r.logger != nil { + r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content))) + } + return nil, err + } + if r.logger != nil { + r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String())) + } + return yamlBuf, nil +} + func (a *app) FindAndIterateOverDesiredStates(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error, namespace string, selectors []string, env string) error { desiredStateFiles, err := a.findDesiredStateFiles(fileOrDir) if err != nil { @@ -596,10 +658,23 @@ func (a *app) FindAndIterateOverDesiredStates(fileOrDir string, converge func(*s noMatchInHelmfiles := true for _, f := range desiredStateFiles { a.logger.Debugf("Processing %s", f) - yamlBuf, err := tmpl.NewFileRenderer(a.readFile, "", environment.Environment{Name: env, Values: map[string]interface{}(nil)}).RenderTemplateFileToBuffer(f) + + content, err := a.readFile(f) if err != nil { return err } + // render template, in two runs + r := &twoPassRenderer{ + reader: a.readFile, + env: env, + filename: f, + logger: a.logger, + abs: a.abs, + } + yamlBuf, err := r.renderTemplate(content) + if err != nil { + return fmt.Errorf("error during %s parsing: %v", f, err) + } st, noMatchInThisHelmfile, err := a.loadDesiredStateFromYaml( yamlBuf.Bytes(), @@ -757,7 +832,7 @@ func (a *app) loadDesiredStateFromYaml(yaml []byte, file string, namespace strin releaseNameCounts := map[string]int{} for _, r := range st.Releases { - releaseNameCounts[r.Name] += 1 + releaseNameCounts[r.Name]++ } for name, c := range releaseNameCounts { if c > 1 { diff --git a/main_test.go b/main_test.go index 66cebbc2..284ed90c 100644 --- a/main_test.go +++ b/main_test.go @@ -1,9 +1,14 @@ package main import ( + "fmt" "io/ioutil" "path/filepath" + "strings" "testing" + + "github.com/roboll/helmfile/state" + "gopkg.in/yaml.v2" ) // See https://github.com/roboll/helmfile/issues/193 @@ -35,3 +40,195 @@ func TestReadFromYaml_DuplicateReleaseName(t *testing.T) { t.Errorf("unexpected error happened: %v", err) } } + +func makeRenderer(readFile func(string) ([]byte, error), env string) *twoPassRenderer { + return &twoPassRenderer{ + reader: readFile, + env: env, + filename: "", + logger: logger, + abs: filepath.Abs, + } +} + +func TestReadFromYaml_MakeEnvironmentHasNoSideEffects(t *testing.T) { + + yamlContent := []byte(` +environments: + staging: + values: + - default/values.yaml + production: + +releases: +- name: {{ readFile "other/default/values.yaml" }} + chart: mychart1 +`) + + fileReaderCalls := 0 + // make a reader that returns a simulated context + fileReader := func(filename string) ([]byte, error) { + expectedFilename := filepath.Clean("default/values.yaml") + if !strings.HasSuffix(filename, expectedFilename) { + return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename) + } + fileReaderCalls++ + if fileReaderCalls == 2 { + return []byte("SecondPass"), nil + } + return []byte(""), nil + } + + r := makeRenderer(fileReader, "staging") + yamlBuf, err := r.renderTemplate(yamlContent) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + var state state.HelmState + err = yaml.Unmarshal(yamlBuf.Bytes(), &state) + + if fileReaderCalls > 2 { + t.Error("reader should be called only twice") + } + + if state.Releases[0].Name != "SecondPass" { + t.Errorf("release name should have ben set as SecondPass") + } +} + +func TestReadFromYaml_RenderTemplate(t *testing.T) { + + defaultValuesYalm := []byte(` +releaseName: "hello" +conditionalReleaseTag: "yes" +`) + + yamlContent := []byte(` +environments: + staging: + values: + - default/values.yaml + production: + +releases: +- name: {{ .Environment.Values.releaseName }} + chart: mychart1 + +{{ if (eq .Environment.Values.conditionalReleaseTag "yes") }} +- name: conditionalRelease +{{ end }} + +`) + + // make a reader that returns a simulated context + fileReader := func(filename string) ([]byte, error) { + expectedFilename := filepath.Clean("default/values.yaml") + if !strings.HasSuffix(filename, expectedFilename) { + return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename) + } + return defaultValuesYalm, nil + } + + r := makeRenderer(fileReader, "staging") + // test the double rendering + yamlBuf, err := r.renderTemplate(yamlContent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var state state.HelmState + err = yaml.Unmarshal(yamlBuf.Bytes(), &state) + + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if len(state.Releases) != 2 { + t.Fatal("there should be 2 releases") + } + + if state.Releases[0].Name != "hello" { + t.Errorf("release name should be hello") + } + + if state.Releases[1].Name != "conditionalRelease" { + t.Error("conditional release should have been present") + } +} + +func TestReadFromYaml_RenderTemplateWithValuesReferenceError(t *testing.T) { + defaultValuesYalm := []byte("") + + yamlContent := []byte(` +environments: + staging: + values: + - default/values.yaml + production: + +{{ if (eq .Environment.Values.releaseName "a") }} # line 8 +releases: +- name: a + chart: mychart1 +{{ end }} +`) + + // make a reader that returns a simulated context + fileReader := func(filename string) ([]byte, error) { + return defaultValuesYalm, nil + } + + r := makeRenderer(fileReader, "staging") + // test the double rendering + _, err := r.renderTemplate(yamlContent) + + if !strings.Contains(err.Error(), "stringTemplate:8") { + t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err) + } +} + +// This test shows that a gotmpl reference will get rendered correctly +// even if the pre-render disables the readFile and exec functions. +// This does not apply to .gotmpl files, which is a nice side-effect. +func TestReadFromYaml_RenderTemplateWithGotmpl(t *testing.T) { + + defaultValuesYalmGotmpl := []byte(` +releaseName: {{ readFile "nonIgnoredFile" }} +`) + + yamlContent := []byte(` +environments: + staging: + values: + - values.yaml.gotmpl + production: + +{{ if (eq .Environment.Values.releaseName "release-a") }} # line 8 +releases: +- name: a + chart: mychart1 +{{ end }} +`) + + fileReader := func(filename string) ([]byte, error) { + if strings.HasSuffix(filename, "nonIgnoredFile") { + return []byte("release-a"), nil + } + return defaultValuesYalmGotmpl, nil + } + + r := makeRenderer(fileReader, "staging") + rendered, _ := r.renderTemplate(yamlContent) + + var state state.HelmState + yaml.Unmarshal(rendered.Bytes(), &state) + + if len(state.Releases) != 1 { + t.Fatal("there should be 1 release") + } + + if state.Releases[0].Name != "a" { + t.Fatal("release should have been declared") + } +} diff --git a/state/create.go b/state/create.go index b63687a8..6b525f78 100644 --- a/state/create.go +++ b/state/create.go @@ -35,6 +35,7 @@ func createFromYaml(content []byte, file string, env string, logger *zap.Sugared logger, ioutil.ReadFile, filepath.Abs, + true, } return c.CreateFromYaml(content, file, env) } @@ -43,6 +44,8 @@ type creator struct { logger *zap.SugaredLogger readFile func(string) ([]byte, error) abs func(string) (string, error) + + Strict bool } func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), abs func(string) (string, error)) *creator { @@ -50,6 +53,7 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error) logger: logger, readFile: readFile, abs: abs, + Strict: true, } } @@ -62,7 +66,11 @@ func (c *creator) CreateFromYaml(content []byte, file string, env string) (*Helm } state.basePath = basePath - if err := yaml.UnmarshalStrict(content, &state); err != nil { + unmarshal := yaml.UnmarshalStrict + if !c.Strict { + unmarshal = yaml.Unmarshal + } + if err := unmarshal(content, &state); err != nil { return nil, &StateLoadError{fmt.Sprintf("failed to read %s", file), err} } state.FilePath = file @@ -81,7 +89,7 @@ func (c *creator) CreateFromYaml(content []byte, file string, env string) (*Helm if err != nil { return nil, &StateLoadError{fmt.Sprintf("failed to read %s", file), err} } - state.env = *e + state.Env = *e state.readFile = c.readFile diff --git a/state/create_test.go b/state/create_test.go index cdca4ae0..7389c96c 100644 --- a/state/create_test.go +++ b/state/create_test.go @@ -103,7 +103,7 @@ bar: {{ readFile "bar.txt" }} t.Errorf("unexpected error: %v", err) } - actual := state.env.Values + actual := state.Env.Values if !reflect.DeepEqual(actual, expected) { t.Errorf("unexpected environment values: expected=%v, actual=%v", expected, actual) } diff --git a/state/state.go b/state/state.go index 84466bc0..ce7c5e88 100644 --- a/state/state.go +++ b/state/state.go @@ -37,7 +37,7 @@ type HelmState struct { Repositories []RepositorySpec `yaml:"repositories"` Releases []ReleaseSpec `yaml:"releases"` - env environment.Environment + Env environment.Environment logger *zap.SugaredLogger @@ -876,7 +876,7 @@ func (state *HelmState) flagsForLint(helm helmexec.Interface, release *ReleaseSp } func (state *HelmState) RenderValuesFileToBytes(path string) ([]byte, error) { - r := valuesfile.NewRenderer(state.readFile, state.basePath, state.env) + r := valuesfile.NewRenderer(state.readFile, state.basePath, state.Env) return r.RenderToBytes(path) } diff --git a/tmpl/context.go b/tmpl/context.go index 180ef649..6c119415 100644 --- a/tmpl/context.go +++ b/tmpl/context.go @@ -1,6 +1,7 @@ package tmpl type Context struct { - basePath string - readFile func(string) ([]byte, error) + preRender bool + basePath string + readFile func(string) ([]byte, error) } diff --git a/tmpl/file.go b/tmpl/file.go index 29e556d5..ebef172e 100644 --- a/tmpl/file.go +++ b/tmpl/file.go @@ -2,6 +2,8 @@ package tmpl import ( "bytes" + "io/ioutil" + "github.com/roboll/helmfile/environment" ) @@ -33,11 +35,29 @@ func NewFileRenderer(readFile func(filename string) ([]byte, error), basePath st } } +func NewFirstPassRenderer(env environment.Environment) *templateFileRenderer { + return &templateFileRenderer{ + ReadFile: ioutil.ReadFile, + Context: &Context{ + preRender: true, + basePath: "", + readFile: ioutil.ReadFile, + }, + Data: TemplateData{ + Environment: env, + }, + } +} + func (r *templateFileRenderer) RenderTemplateFileToBuffer(file string) (*bytes.Buffer, error) { content, err := r.ReadFile(file) if err != nil { return nil, err } + return r.RenderTemplateContentToBuffer(content) +} + +func (r *templateFileRenderer) RenderTemplateContentToBuffer(content []byte) (*bytes.Buffer, error) { return r.Context.RenderTemplateToBuffer(string(content), r.Data) } diff --git a/tmpl/funcs.go b/tmpl/funcs.go index 746e0fbb..5cfb48cd 100644 --- a/tmpl/funcs.go +++ b/tmpl/funcs.go @@ -15,7 +15,7 @@ import ( type Values = map[string]interface{} func (c *Context) createFuncMap() template.FuncMap { - return template.FuncMap{ + funcMap := template.FuncMap{ "exec": c.Exec, "readFile": c.ReadFile, "toYaml": ToYaml, @@ -23,6 +23,17 @@ func (c *Context) createFuncMap() template.FuncMap { "setValueAtPath": SetValueAtPath, "requiredEnv": RequiredEnv, } + if c.preRender { + // disable potential side-effect template calls + funcMap["exec"] = func(string, []interface{}, ...string) (string, error) { + return "", nil + } + funcMap["readFile"] = func(string) (string, error) { + return "", nil + } + } + + return funcMap } func (c *Context) Exec(command string, args []interface{}, inputs ...string) (string, error) { diff --git a/tmpl/tmpl.go b/tmpl/tmpl.go index d91a7ad9..70445bf6 100644 --- a/tmpl/tmpl.go +++ b/tmpl/tmpl.go @@ -11,7 +11,13 @@ func (c *Context) stringTemplate() *template.Template { for name, f := range c.createFuncMap() { funcMap[name] = f } - return template.New("stringTemplate").Funcs(funcMap) + tmpl := template.New("stringTemplate").Funcs(funcMap) + if c.preRender { + tmpl.Option("missingkey=zero") + } else { + tmpl.Option("missingkey=error") + } + return tmpl } func (c *Context) RenderTemplateToBuffer(s string, data ...interface{}) (*bytes.Buffer, error) { @@ -28,7 +34,7 @@ func (c *Context) RenderTemplateToBuffer(s string, data ...interface{}) (*bytes. var execErr = t.Execute(&tplString, d) if execErr != nil { - return nil, execErr + return &tplString, execErr } return &tplString, nil