(re)enable map key override for plain YAML

- introduce tree walker/visitor for AST.

pkg/cmd/template/cmd_test.go

@@ -528,10 +528,31 @@ func TestPlainYAMLNoTemplateProcessing(t *testing.T) {
 		yamlTplData := []byte(`
 #@ load("funcs/funcs.lib.yml", "yamlfunc")
 annotation: 5 #@ 1 + 2
-text_template: (@= "string" @)`)
+text_template: (@= "string" @)
+versions:
+- &version
+  name: v1alpha1
+  served: true
+- << : *version
+  name: v1beta1
+- << : *version
+  name: v1
+  storage: true
+- << : *version
+`)
 
 		expectedYAMLTplData := `annotation: 5
 text_template: (@= "string" @)
+versions:
+- name: v1alpha1
+  served: true
+- served: true
+  name: v1beta1
+- served: true
+  name: v1
+  storage: true
+- name: v1alpha1
+  served: true
 `
 
 		filesToProcess := []*files.File{

pkg/workspace/template_loader.go

@@ -159,13 +159,20 @@ func (l *TemplateLoader) EvalYAML(libraryCtx LibraryExecutionContext, file *file
 		return nil, nil, err
 	}
 
-	l.ui.Debugf("### ast\n")
-	docSet.Print(l.ui.DebugWriter())
+	l.ui.Debugf("### ast ")
 
 	if !file.IsTemplate() && !file.IsLibrary() || !yamltemplate.HasTemplating(docSet) {
+		docSet.OverrideMapKeys()
+
+		l.ui.Debugf("(plain)\n")
+		docSet.Print(l.ui.DebugWriter())
+
 		return nil, docSet, nil
 	}
 
+	l.ui.Debugf("(templated)\n")
+	docSet.Print(l.ui.DebugWriter())
+
 	tplOpts := yamltemplate.TemplateOpts{
 		IgnoreUnknownComments:   l.opts.IgnoreUnknownComments,
 		ImplicitMapKeyOverrides: l.opts.ImplicitMapKeyOverrides,

pkg/yamlmeta/document_set.go

@@ -60,3 +60,8 @@ func (ds *DocumentSet) AsBytesWithPrinter(printerFunc func(io.Writer) DocumentPr
 
 	return buf.Bytes(), nil
 }
+
+// OverrideMapKeys within any contained Map, where there is more than one MapItem with the same key, delete all but the last.
+func (ds *DocumentSet) OverrideMapKeys() {
+	_ = Walk(ds, &overrideMapKeys{})
+}

pkg/yamlmeta/map_key_overrides.go

@@ -0,0 +1,51 @@
+// Copyright 2020 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package yamlmeta
+
+import "sort"
+
+type overrideMapKeys struct{}
+
+// Visit if `node` is a Map, among its MapItem's that have duplicate keys, removes all but the last.
+// This visitor never returns an non-nil error.
+func (r *overrideMapKeys) visit(node Node) error {
+	mapNode, isMap := node.(*Map)
+	if !isMap {
+		return nil
+	}
+
+	keyToIdxs := r.collectIndexesByKey(mapNode)
+	idxsToRemove := r.selectRedundantIdxs(keyToIdxs)
+	for _, idx := range idxsToRemove {
+		mapNode.Items = append(mapNode.Items[:idx], mapNode.Items[idx+1:]...)
+	}
+	return nil
+}
+
+// collectIndexesByKey iterates over `mapNode's MapItem's and produces a mapping keys to the index of MapItem within
+// this map. The values in this mapping are in ascending order.
+func (r *overrideMapKeys) collectIndexesByKey(mapNode *Map) map[interface{}][]int {
+	keyToIndexes := make(map[interface{}][]int)
+	for idx, item := range mapNode.Items {
+		idxs := keyToIndexes[item.Key]
+		idxs = append(idxs, idx)
+		keyToIndexes[item.Key] = idxs
+	}
+	return keyToIndexes
+}
+
+// selectRedundantIdxs given a mapping from key to list of indexes of MapItem's that have that key, produces a single
+// list of indexes that refer to all _but_ the last MapItem.
+// Values of `keyToIdxs` are assumed to be in ascending order.
+func (r *overrideMapKeys) selectRedundantIdxs(keyToIdxs map[interface{}][]int) []int {
+	var idxsToRemove []int
+	for _, idxs := range keyToIdxs {
+		if len(idxs) == 1 {
+			continue
+		}
+		idxsToRemove = append(idxsToRemove, idxs[:len(idxs)-1]...)
+		sort.Sort(sort.Reverse(sort.IntSlice(idxsToRemove)))
+	}
+	return idxsToRemove
+}

pkg/yamlmeta/map_key_overrides_test.go

@@ -0,0 +1,98 @@
+// Copyright 2020 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package yamlmeta_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/k14s/ytt/pkg/yamlmeta"
+)
+
+func TestMapKeyOverridePlainYAML(t *testing.T) {
+	t.Run("when no maps have duplicate keys, is a no op", func(t *testing.T) {
+		docSet := &yamlmeta.DocumentSet{
+			Items: []*yamlmeta.Document{{
+				Value: &yamlmeta.Map{
+					Items: []*yamlmeta.MapItem{
+						{Key: "foo", Value: 1},
+						{Key: "bar", Value: 2},
+					},
+				},
+			}},
+		}
+		expectedDocSet := docSet.DeepCopyAsNode()
+		docSet.OverrideMapKeys()
+
+		printer := yamlmeta.NewPrinterWithOpts(os.Stdout, yamlmeta.PrinterOpts{ExcludeRefs: true})
+
+		result := printer.PrintStr(docSet)
+		expected := printer.PrintStr(expectedDocSet)
+		assertEqual(t, result, expected)
+	})
+	t.Run("when there are duplicates, last map item overrides", func(t *testing.T) {
+		docSet := &yamlmeta.DocumentSet{
+			Items: []*yamlmeta.Document{{
+				Value: &yamlmeta.Map{
+					Items: []*yamlmeta.MapItem{
+						{Key: "foo", Value: 1},
+						{Key: "foo", Value: 2},
+						{Key: "foo", Value: 3},
+						{Key: "foo", Value: 4},
+					},
+				},
+			}},
+		}
+		expectedDocSet := &yamlmeta.DocumentSet{
+			Items: []*yamlmeta.Document{{
+				Value: &yamlmeta.Map{
+					Items: []*yamlmeta.MapItem{
+						{Key: "foo", Value: 4},
+					},
+				},
+			}},
+		}
+		docSet.OverrideMapKeys()
+
+		printer := yamlmeta.NewPrinterWithOpts(os.Stdout, yamlmeta.PrinterOpts{ExcludeRefs: true})
+
+		result := printer.PrintStr(docSet)
+		expected := printer.PrintStr(expectedDocSet)
+		assertEqual(t, result, expected)
+	})
+	t.Run("when there are multiple keys with duplicates, last map item for each key overrides the others", func(t *testing.T) {
+		docSet := &yamlmeta.DocumentSet{
+			Items: []*yamlmeta.Document{{
+				Value: &yamlmeta.Map{
+					Items: []*yamlmeta.MapItem{
+						{Key: "foo", Value: 1},
+						{Key: "bar", Value: 2},
+						{Key: "ree", Value: 3},
+						{Key: "ree", Value: 4},
+						{Key: "foo", Value: 5},
+						{Key: "bar", Value: 6},
+					},
+				},
+			}},
+		}
+		expectedDocSet := &yamlmeta.DocumentSet{
+			Items: []*yamlmeta.Document{{
+				Value: &yamlmeta.Map{
+					Items: []*yamlmeta.MapItem{
+						{Key: "ree", Value: 4},
+						{Key: "foo", Value: 5},
+						{Key: "bar", Value: 6},
+					},
+				},
+			}},
+		}
+		docSet.OverrideMapKeys()
+
+		printer := yamlmeta.NewPrinterWithOpts(os.Stdout, yamlmeta.PrinterOpts{ExcludeRefs: true})
+
+		result := printer.PrintStr(docSet)
+		expected := printer.PrintStr(expectedDocSet)
+		assertEqual(t, result, expected)
+	})
+}

pkg/yamlmeta/walk.go

@@ -0,0 +1,29 @@
+// Copyright 2020 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package yamlmeta
+
+// Visitor performs an operation on the given Node while traversing the AST.
+// Typically defines the action taken during a Walk().
+type Visitor interface {
+	visit(Node) error
+}
+
+// Walk traverses the tree starting at `n`, recursively, depth-first, invoking `v` on each node.
+// if `v` returns non-nil error, the traversal is aborted.
+func Walk(n Node, v Visitor) error {
+	err := v.visit(n)
+	if err != nil {
+		return err
+	}
+
+	for _, c := range n.GetValues() {
+		if cn, ok := c.(Node); ok {
+			err := Walk(cn, v)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}