Merge pull request #587 from roboll/layering-enhancements

feat: helmfile.yaml layering enhancements

The current  [Layering](https://github.com/roboll/helmfile/blob/master/docs/writing-helmfile.md#layering) system didn't work as documented, as it relies on helmfile to template each "part" of your helmfile.yaml THEN merge them one by one.

The reality was that helmfile template all the parts of your helmfile.yaml at once, and then merge those YAML documents. In #388 (comment), @sruon was making a GREAT point that we may need to change helmfile to render templates earlier - that is to evaluate a template per each helmfile.yaml part separated by `---`. Sorry I missed my expertise to follow your great idea last year @sruon  😭 

Anyways, this, in combination with the wrong documentation, has made so many people confused. To finally overcome this situation, here's a fairly large PR that introduces the 2 enhancements:

- `bases:` for easier layering without go template expressions, especially `{{ readFunc "path/to/file" }}`s. This is the first commit of this PR.
- `helmfile.yaml` is splited by the separator `---` at first. Each part is then rendered as a go template(double-render applies as before). Finally, All the results are merged in the order of occurence. I assume this as an enhanced version of @sruon's work. This is the second commit of this PR.

Resolves #388
Resolve #584
Resolves #585 (`HELMFILE_EXPERIMENTA=true -f helmfile.yaml helmfile` disables the whole-file templating, treating the helmfile.yaml as a regular YAML file as the file ext. denotes. Use `helmfile.yaml.gotmpl` or `helmfile.gotmpl` to enable)
Fixes #568 (Use `bases` or `readFile` rather than not importing implicitly with `helmfile.d`

docs/writing-helmfile.md

@@ -100,10 +100,10 @@ Use Layering to extract the common parts into a dedicated *library helmfile*s, s
 Let's assume that your `helmfile.yaml` looks like:
 
 ```
-{ readFile "commons.yaml" }}
----
-{{ readFile "environments.yaml" }}
----
+bases:
+- commons.yaml
+- environments.yaml
+
 releases:
 - name: myapp
   chart: mychart
@@ -125,23 +125,26 @@ environments:
   production:
 ```
 
-At run time, template expressions in your `helmfile.yaml` are executed:
+At run time, `bases` in your `helmfile.yaml` are evaluated to produce:
 
 ```yaml
+# commons.yaml
 releases:
 - name: metricbaet
   chart: stable/metricbeat
 ---
+# environments.yaml
 environments:
   development:
   production:
 ---
+# helmfile.yaml
 releases:
 - name: myapp
   chart: mychart
 ```
 
-Resulting YAML documents are merged in the order of occurrence,
+Finally the resulting YAML documents are merged in the order of occurrence,
 so that your `helmfile.yaml` becomes:
 
 ```yaml
@@ -159,3 +162,5 @@ releases:
 Great!
 
 Now, repeat the above steps for each your `helmfile.yaml`, so that all your helmfiles becomes DRY.
+
+Please also see [the discussion in the issue 388](https://github.com/roboll/helmfile/issues/388#issuecomment-491710348) for more advanced layering examples.

pkg/app/app.go

@@ -7,14 +7,14 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"syscall"
 
 	"github.com/roboll/helmfile/helmexec"
 	"github.com/roboll/helmfile/state"
 	"go.uber.org/zap"
 
 	"path/filepath"
 	"sort"
-	"syscall"
 )
 
 type App struct {
@@ -111,34 +111,42 @@ func (a *App) visitStateFiles(fileOrDir string, do func(string) error) error {
 	return nil
 }
 
+func (a *App) loadDesiredStateFromYaml(file string) (*state.HelmState, error) {
+	ld := &desiredStateLoader{
+		readFile:  a.readFile,
+		env:       a.Env,
+		namespace: a.Namespace,
+		logger:    a.Logger,
+		abs:       a.abs,
+
+		Reverse:     a.Reverse,
+		KubeContext: a.KubeContext,
+		glob:        a.glob,
+	}
+	return ld.Load(file)
+}
+
 func (a *App) VisitDesiredStates(fileOrDir string, selector []string, converge func(*state.HelmState, helmexec.Interface) (bool, []error)) error {
 	noMatchInHelmfiles := true
 
 	err := a.visitStateFiles(fileOrDir, func(f string) error {
-		content, err := a.readFile(f)
-		if err != nil {
-			return err
-		}
-		// render template, in two runs
-		r := &twoPassRenderer{
-			reader:    a.readFile,
-			env:       a.Env,
-			namespace: a.Namespace,
-			filename:  f,
-			logger:    a.Logger,
-			abs:       a.abs,
-		}
-		yamlBuf, err := r.renderTemplate(content)
-		if err != nil {
-			return fmt.Errorf("error during %s parsing: %v", f, err)
-		}
-
-		st, err := a.loadDesiredStateFromYaml(
-			yamlBuf.Bytes(),
-			f,
-			a.Namespace,
-			a.Env,
-		)
+		st, err := a.loadDesiredStateFromYaml(f)
+
+		sigs := make(chan os.Signal, 1)
+		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+		go func() {
+			sig := <-sigs
+
+			errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)}
+			_ = context{a, st}.clean(errs)
+			// See http://tldp.org/LDP/abs/html/exitcodes.html
+			switch sig {
+			case syscall.SIGINT:
+				os.Exit(130)
+			case syscall.SIGTERM:
+				os.Exit(143)
+			}
+		}()
 
 		ctx := context{a, st}
 
@@ -313,78 +321,6 @@ func directoryExistsAt(path string) bool {
 	return err == nil && fileInfo.Mode().IsDir()
 }
 
-func (a *App) loadDesiredStateFromYaml(yaml []byte, file string, namespace string, env string) (*state.HelmState, error) {
-	c := state.NewCreator(a.Logger, a.readFile, a.abs)
-	st, err := c.CreateFromYaml(yaml, file, env)
-	if err != nil {
-		return nil, err
-	}
-
-	helmfiles := []state.SubHelmfileSpec{}
-	for _, hf := range st.Helmfiles {
-		globPattern := hf.Path
-		var absPathPattern string
-		if filepath.IsAbs(globPattern) {
-			absPathPattern = globPattern
-		} else {
-			absPathPattern = st.JoinBase(globPattern)
-		}
-		matches, err := a.glob(absPathPattern)
-		if err != nil {
-			return nil, fmt.Errorf("failed processing %s: %v", globPattern, err)
-		}
-		sort.Strings(matches)
-		for _, match := range matches {
-			newHelmfile := hf
-			newHelmfile.Path = match
-			helmfiles = append(helmfiles, newHelmfile)
-		}
-
-	}
-	st.Helmfiles = helmfiles
-
-	if a.Reverse {
-		rev := func(i, j int) bool {
-			return j < i
-		}
-		sort.Slice(st.Releases, rev)
-		sort.Slice(st.Helmfiles, rev)
-	}
-
-	if a.KubeContext != "" {
-		if st.HelmDefaults.KubeContext != "" {
-			log.Printf("err: Cannot use option --kube-context and set attribute helmDefaults.kubeContext.")
-			os.Exit(1)
-		}
-		st.HelmDefaults.KubeContext = a.KubeContext
-	}
-	if namespace != "" {
-		if st.Namespace != "" {
-			log.Printf("err: Cannot use option --namespace and set attribute namespace.")
-			os.Exit(1)
-		}
-		st.Namespace = namespace
-	}
-
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-	go func() {
-		sig := <-sigs
-
-		errs := []error{fmt.Errorf("Received [%s] to shutdown ", sig)}
-		_ = context{a, st}.clean(errs)
-		// See http://tldp.org/LDP/abs/html/exitcodes.html
-		switch sig {
-		case syscall.SIGINT:
-			os.Exit(130)
-		case syscall.SIGTERM:
-			os.Exit(143)
-		}
-	}()
-
-	return st, nil
-}
-
 type Error struct {
 	msg string