feat: double render the helmfile (#308)

This allows using the environment values defined in the environments: section of helmfile.yaml to be used from other sections of the file.

This works by having two template renderers, the first-pass and the second-pass renderer.

The first-pass render renders a helmfile.yaml template with replacing template functions has side-effects with noop. So, use only funcs that don't have side-effects to compose your environment values.

Then the second-pass renderer renders the same helmfile.yaml template, but with the environment values loaded by the first-pass renderer.

The implementation uses a buffer instead of re-reading the file twice.

Resolves #297

README.md

@@ -40,7 +40,7 @@ repositories:
 context: kube-context					 # kube-context (--kube-context)
 
 #default values to set for args along with dedicated keys that can be set by contributers, cli args take precedence overe these
-helmDefaults:           
+helmDefaults:
   tillerNamespace: tiller-namespace  #dedicated default key for tiller-namespace
   kubeContext: kube-context          #dedicated default key for kube-context
   # additional and global args passed to helm
@@ -396,7 +396,7 @@ releases:
   # snip
 {{ end }}
 - name: myapp
-  # snip  
+  # snip
 ```
 
 ## Environment Values
@@ -424,6 +424,7 @@ releases:
 
 ```yaml
 domain: prod.example.com
+releaseName: prod
 ```
 
 `values.yaml.gotmpl`
@@ -435,6 +436,38 @@ domain: {{ .Environment.Values.domain | default "dev.example.com" }}
 `helmfile sync` installs `myapp` with the value `domain=dev.example.com`,
 whereas `helmfile --environment production sync` installs the app with the value `domain=production.example.com`.
 
+For even more flexibility, you can now use values declared in the `environments:` section in other parts of your helmfiles:
+
+consider:
+`default.yaml`
+
+```yaml
+domain: dev.example.com
+releaseName: dev
+```
+
+```yaml
+environments:
+  default:
+    values:
+    - default.yaml
+  production:
+    values:
+    - production.yaml #  template directives with potential side-effects like `exec` and `readFile` will NOT be executed
+    - other.yaml.gotmpl  #  `exec` and `readFile` will be honoured
+
+releases:
+- name: myapp-{{ .Environment.Values.releaseName }} # release name will be one of `dev` or `prod` depending on selected environment
+  values:
+  - values.yaml.gotmpl
+
+{{ if eq (.Environment.Values.releaseName "prod" ) }}
+# this release would be installed only if selected environment is `production`
+- name: production-specific-release
+  ...
+{{ end }}
+```
+
 ## Environment Secrets
 
 Environment Secrets are encrypted versions of `Environment Values`.

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"log"
 	"os"
@@ -588,6 +589,67 @@ func (e *noMatchingHelmfileError) Error() string {
 	)
 }
 
+func prependLineNumbers(text string) string {
+	buf := bytes.NewBufferString("")
+	lines := strings.Split(text, "\n")
+	for i, line := range lines {
+		buf.WriteString(fmt.Sprintf("%2d: %s\n", i, line))
+	}
+	return buf.String()
+}
+
+type twoPassRenderer struct {
+	reader   func(string) ([]byte, error)
+	env      string
+	filename string
+	logger   *zap.SugaredLogger
+	abs      func(string) (string, error)
+}
+
+func (r *twoPassRenderer) renderEnvironment(content []byte) environment.Environment {
+	firstPassEnv := environment.Environment{Name: r.env, Values: map[string]interface{}(nil)}
+	firstPassRenderer := tmpl.NewFirstPassRenderer(firstPassEnv)
+
+	// parse as much as we can, tolerate errors, this is a preparse
+	yamlBuf, err := firstPassRenderer.RenderTemplateContentToBuffer(content)
+	if err != nil && logger != nil {
+		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
+	}
+	c := state.NewCreator(r.logger, r.reader, r.abs)
+	c.Strict = false
+	// create preliminary state, as we may have an environment. Tolerate errors.
+	prestate, err := c.CreateFromYaml(yamlBuf.Bytes(), r.filename, r.env)
+	if err != nil && r.logger != nil {
+		switch err.(type) {
+		case *state.StateLoadError:
+			r.logger.Infof("could not deduce `environment:` block, configuring only .Environment.Name. error: %v", err)
+		}
+		r.logger.Debugf("error in first-pass rendering: result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
+	}
+	if prestate != nil {
+		firstPassEnv = prestate.Env
+	}
+	return firstPassEnv
+}
+
+func (r *twoPassRenderer) renderTemplate(content []byte) (*bytes.Buffer, error) {
+	// try a first pass render. This will always succeed, but can produce a limited env
+	firstPassEnv := r.renderEnvironment(content)
+
+	secondPassRenderer := tmpl.NewFileRenderer(r.reader, "", firstPassEnv)
+	yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
+	if err != nil {
+		if r.logger != nil {
+			r.logger.Debugf("second-pass rendering failed, input of \"%s\":\n%s", r.filename, prependLineNumbers(string(content)))
+		}
+		return nil, err
+	}
+	if r.logger != nil {
+		r.logger.Debugf("second-pass rendering result of \"%s\":\n%s", r.filename, prependLineNumbers(yamlBuf.String()))
+	}
+	return yamlBuf, nil
+}
+
 func (a *app) FindAndIterateOverDesiredStates(fileOrDir string, converge func(*state.HelmState, helmexec.Interface) []error, namespace string, selectors []string, env string) error {
 	desiredStateFiles, err := a.findDesiredStateFiles(fileOrDir)
 	if err != nil {
@@ -596,10 +658,23 @@ func (a *app) FindAndIterateOverDesiredStates(fileOrDir string, converge func(*s
 	noMatchInHelmfiles := true
 	for _, f := range desiredStateFiles {
 		a.logger.Debugf("Processing %s", f)
-		yamlBuf, err := tmpl.NewFileRenderer(a.readFile, "", environment.Environment{Name: env, Values: map[string]interface{}(nil)}).RenderTemplateFileToBuffer(f)
+
+		content, err := a.readFile(f)
 		if err != nil {
 			return err
 		}
+		// render template, in two runs
+		r := &twoPassRenderer{
+			reader:   a.readFile,
+			env:      env,
+			filename: f,
+			logger:   a.logger,
+			abs:      a.abs,
+		}
+		yamlBuf, err := r.renderTemplate(content)
+		if err != nil {
+			return fmt.Errorf("error during %s parsing: %v", f, err)
+		}
 
 		st, noMatchInThisHelmfile, err := a.loadDesiredStateFromYaml(
 			yamlBuf.Bytes(),
@@ -757,7 +832,7 @@ func (a *app) loadDesiredStateFromYaml(yaml []byte, file string, namespace strin
 
 	releaseNameCounts := map[string]int{}
 	for _, r := range st.Releases {
-		releaseNameCounts[r.Name] += 1
+		releaseNameCounts[r.Name]++
 	}
 	for name, c := range releaseNameCounts {
 		if c > 1 {

main_test.go

@@ -1,9 +1,14 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
+	"strings"
 	"testing"
+
+	"github.com/roboll/helmfile/state"
+	"gopkg.in/yaml.v2"
 )
 
 // See https://github.com/roboll/helmfile/issues/193
@@ -35,3 +40,195 @@ func TestReadFromYaml_DuplicateReleaseName(t *testing.T) {
 		t.Errorf("unexpected error happened: %v", err)
 	}
 }
+
+func makeRenderer(readFile func(string) ([]byte, error), env string) *twoPassRenderer {
+	return &twoPassRenderer{
+		reader:   readFile,
+		env:      env,
+		filename: "",
+		logger:   logger,
+		abs:      filepath.Abs,
+	}
+}
+
+func TestReadFromYaml_MakeEnvironmentHasNoSideEffects(t *testing.T) {
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - default/values.yaml
+  production:
+
+releases:
+- name: {{ readFile "other/default/values.yaml" }}
+  chart: mychart1
+`)
+
+	fileReaderCalls := 0
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		expectedFilename := filepath.Clean("default/values.yaml")
+		if !strings.HasSuffix(filename, expectedFilename) {
+			return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename)
+		}
+		fileReaderCalls++
+		if fileReaderCalls == 2 {
+			return []byte("SecondPass"), nil
+		}
+		return []byte(""), nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	yamlBuf, err := r.renderTemplate(yamlContent)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	var state state.HelmState
+	err = yaml.Unmarshal(yamlBuf.Bytes(), &state)
+
+	if fileReaderCalls > 2 {
+		t.Error("reader should be called only twice")
+	}
+
+	if state.Releases[0].Name != "SecondPass" {
+		t.Errorf("release name should have ben set as SecondPass")
+	}
+}
+
+func TestReadFromYaml_RenderTemplate(t *testing.T) {
+
+	defaultValuesYalm := []byte(`
+releaseName: "hello"
+conditionalReleaseTag: "yes"
+`)
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - default/values.yaml
+  production:
+
+releases:
+- name: {{ .Environment.Values.releaseName }}
+  chart: mychart1
+
+{{ if (eq .Environment.Values.conditionalReleaseTag "yes") }}
+- name: conditionalRelease
+{{ end }}
+
+`)
+
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		expectedFilename := filepath.Clean("default/values.yaml")
+		if !strings.HasSuffix(filename, expectedFilename) {
+			return nil, fmt.Errorf("unexpected filename: expected=%s, actual=%s", expectedFilename, filename)
+		}
+		return defaultValuesYalm, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	// test the double rendering
+	yamlBuf, err := r.renderTemplate(yamlContent)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var state state.HelmState
+	err = yaml.Unmarshal(yamlBuf.Bytes(), &state)
+
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	if len(state.Releases) != 2 {
+		t.Fatal("there should be 2 releases")
+	}
+
+	if state.Releases[0].Name != "hello" {
+		t.Errorf("release name should be hello")
+	}
+
+	if state.Releases[1].Name != "conditionalRelease" {
+		t.Error("conditional release should have been present")
+	}
+}
+
+func TestReadFromYaml_RenderTemplateWithValuesReferenceError(t *testing.T) {
+	defaultValuesYalm := []byte("")
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - default/values.yaml
+  production:
+
+{{ if (eq .Environment.Values.releaseName "a") }} # line 8
+releases:
+- name: a
+	chart: mychart1
+{{ end }}
+`)
+
+	// make a reader that returns a simulated context
+	fileReader := func(filename string) ([]byte, error) {
+		return defaultValuesYalm, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	// test the double rendering
+	_, err := r.renderTemplate(yamlContent)
+
+	if !strings.Contains(err.Error(), "stringTemplate:8") {
+		t.Fatalf("error should contain a stringTemplate error (reference to unknow key) %v", err)
+	}
+}
+
+// This test shows that a gotmpl reference will get rendered correctly
+// even if the pre-render disables the readFile and exec functions.
+// This does not apply to .gotmpl files, which is a nice side-effect.
+func TestReadFromYaml_RenderTemplateWithGotmpl(t *testing.T) {
+
+	defaultValuesYalmGotmpl := []byte(`
+releaseName: {{ readFile "nonIgnoredFile" }}
+`)
+
+	yamlContent := []byte(`
+environments:
+  staging:
+    values:
+    - values.yaml.gotmpl
+  production:
+
+{{ if (eq .Environment.Values.releaseName "release-a") }} # line 8
+releases:
+- name: a
+  chart: mychart1
+{{ end }}
+`)
+
+	fileReader := func(filename string) ([]byte, error) {
+		if strings.HasSuffix(filename, "nonIgnoredFile") {
+			return []byte("release-a"), nil
+		}
+		return defaultValuesYalmGotmpl, nil
+	}
+
+	r := makeRenderer(fileReader, "staging")
+	rendered, _ := r.renderTemplate(yamlContent)
+
+	var state state.HelmState
+	yaml.Unmarshal(rendered.Bytes(), &state)
+
+	if len(state.Releases) != 1 {
+		t.Fatal("there should be 1 release")
+	}
+
+	if state.Releases[0].Name != "a" {
+		t.Fatal("release should have been declared")
+	}
+}