Pipeline: extend functionality with Export and Env operations

Signed-off-by: Richard Kosegi <richard.kosegi@gmail.com>

pipeline/env_op.go

@@ -0,0 +1,89 @@
+/*
+Copyright 2024 Richard Kosegi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pipeline
+
+import (
+	"fmt"
+	"github.com/rkosegi/yaml-toolkit/dom"
+	"github.com/rkosegi/yaml-toolkit/utils"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// TODO : move these predicates to common place, maybe utils package
+var (
+	MatchAny = func() StringPredicateFn {
+		return func(s string) bool {
+			return true
+		}
+	}
+	MatchNone = func() StringPredicateFn {
+		return func(s string) bool {
+			return false
+		}
+	}
+	MatchRe = func(re *regexp.Regexp) StringPredicateFn {
+		return func(s string) bool {
+			return re.MatchString(s)
+		}
+	}
+)
+
+// TODO: merge with same thing from analytics package and move it to common place
+type StringPredicateFn func(string) bool
+
+func (eo *EnvOp) Do(ctx ActionContext) error {
+	var (
+		inclFn StringPredicateFn
+		exclFn StringPredicateFn
+		getter func() []string
+	)
+	getter = os.Environ
+	if eo.envGetter != nil {
+		getter = eo.envGetter
+	}
+	inclFn = MatchAny()
+	exclFn = MatchNone()
+	if eo.Include != nil {
+		inclFn = MatchRe(eo.Include)
+	}
+	if eo.Exclude != nil {
+		exclFn = MatchRe(eo.Exclude)
+	}
+	for _, env := range getter() {
+		parts := strings.SplitN(env, "=", 2)
+		if inclFn(parts[0]) && !exclFn(parts[0]) {
+			k := utils.ToPath(eo.Path, fmt.Sprintf("Env.%s", parts[0]))
+			ctx.Data().AddValueAt(k, dom.LeafNode(parts[1]))
+		}
+	}
+	return nil
+}
+
+func (eo *EnvOp) String() string {
+	return fmt.Sprintf("Env[path=%s,incl=%s,excl=%s]", eo.Path,
+		safeRegexpDeref(eo.Include), safeRegexpDeref(eo.Exclude))
+}
+
+func (eo *EnvOp) CloneWith(ctx ActionContext) Action {
+	return &EnvOp{
+		Include: eo.Include,
+		Exclude: eo.Exclude,
+		Path:    ctx.TemplateEngine().RenderLenient(eo.Path, ctx.Snapshot()),
+	}
+}

pipeline/env_op_test.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2024 Richard Kosegi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pipeline
+
+import (
+	"github.com/rkosegi/yaml-toolkit/dom"
+	"github.com/stretchr/testify/assert"
+	"regexp"
+	"testing"
+)
+
+func createRePtr(in string) *regexp.Regexp {
+	x := regexp.MustCompile(in)
+	return x
+}
+
+func TestEnvOpDo(t *testing.T) {
+	eo := EnvOp{
+		envGetter: func() []string {
+			return []string{
+				"MOCK1=val1",
+				"MOCK2=val2",
+				"XYZ=123",
+			}
+		},
+		Path:    "Sub",
+		Include: createRePtr(`MOCK\d+`),
+		Exclude: createRePtr("XYZ"),
+	}
+	d := b.Container()
+	err := eo.Do(mockActCtx(d))
+	assert.NoError(t, err)
+	assert.Equal(t, "val1", d.Lookup("Sub.Env.MOCK1").(dom.Leaf).Value())
+	assert.Equal(t, "val2", d.Lookup("Sub.Env.MOCK2").(dom.Leaf).Value())
+	assert.Contains(t, eo.String(), "Sub")
+}
+
+func TestEnvOpCloneWith(t *testing.T) {
+	eo := &EnvOp{
+		Path: "{{ .NewPath }}",
+	}
+	d := b.Container()
+	d.AddValue("NewPath", dom.LeafNode("root"))
+	eo = eo.CloneWith(mockActCtx(d)).(*EnvOp)
+	assert.Equal(t, "root", eo.Path)
+}
+
+func filterStrSlice(in []string, fn StringPredicateFn) []string {
+	result := make([]string, 0)
+	for _, e := range in {
+		if fn(e) {
+			result = append(result, e)
+		}
+	}
+	return result
+}
+
+func TestStringMatchFunc(t *testing.T) {
+	in := []string{"a", "b", "c"}
+	var res []string
+	res = filterStrSlice(in, MatchAny())
+	assert.Equal(t, in, res)
+	res = filterStrSlice(in, MatchNone())
+	assert.Equal(t, 0, len(res))
+	res = filterStrSlice(in, MatchRe(regexp.MustCompile(`a`)))
+	assert.Equal(t, 1, len(res))
+	res = filterStrSlice(in, MatchRe(regexp.MustCompile(`[ac]`)))
+	assert.Equal(t, 2, len(res))
+}

pipeline/export_op.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2024 Richard Kosegi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pipeline
+
+import (
+	"fmt"
+	"github.com/rkosegi/yaml-toolkit/dom"
+	"github.com/rkosegi/yaml-toolkit/props"
+	"os"
+)
+
+func (e *ExportOp) String() string {
+	return fmt.Sprintf("Export[file=%s,format=%s,path=%s]", e.File, e.Format, e.Path)
+}
+
+func (e *ExportOp) Do(ctx ActionContext) (err error) {
+	var (
+		d   dom.Node
+		enc dom.EncoderFunc
+	)
+	d = ctx.Data()
+	if len(e.Path) > 0 {
+		d = ctx.Data().Lookup(e.Path)
+	}
+	// use empty container, since path lookup didn't yield anything useful
+	if d == nil || !d.IsContainer() {
+		d = b.Container()
+	}
+	switch e.Format {
+	case OutputFormatYaml:
+		enc = dom.DefaultYamlEncoder
+	case OutputFormatJson:
+		enc = dom.DefaultJsonEncoder
+	case OutputFormatProperties:
+		enc = props.EncoderFn
+
+	default:
+		return fmt.Errorf("unknown output format: %s", e.Format)
+	}
+
+	f, err := os.OpenFile(e.File, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = f.Close()
+	}()
+	return enc(f, dom.DefaultNodeMappingFn(d.(dom.Container)))
+}
+
+func (e *ExportOp) CloneWith(ctx ActionContext) Action {
+	ss := ctx.Snapshot()
+	return &ExportOp{
+		File:   ctx.TemplateEngine().RenderLenient(e.File, ss),
+		Path:   ctx.TemplateEngine().RenderLenient(e.Path, ss),
+		Format: OutputFormat(ctx.TemplateEngine().RenderLenient(string(e.Format), ss)),
+	}
+}

pipeline/export_op_test.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2024 Richard Kosegi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pipeline
+
+import (
+	"github.com/rkosegi/yaml-toolkit/dom"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestExportOpDo(t *testing.T) {
+	var (
+		err error
+	)
+	f, err := os.CreateTemp("", "yt_export*.json")
+	assert.NoError(t, err)
+	if err != nil {
+		return
+	}
+	t.Cleanup(func() {
+		t.Logf("cleanup temporary file %s", f.Name())
+		_ = os.Remove(f.Name())
+	})
+	t.Logf("created temporary file: %s", f.Name())
+	eo := &ExportOp{
+		File:   f.Name(),
+		Path:   "root.sub1",
+		Format: OutputFormatJson,
+	}
+	assert.Contains(t, eo.String(), f.Name())
+	d := b.Container()
+	d.AddValueAt("root.sub1.sub2", dom.LeafNode(123))
+	err = eo.Do(mockActCtx(d))
+	assert.NoError(t, err)
+	fi, err := os.Stat(f.Name())
+	assert.NotNil(t, fi)
+	assert.NoError(t, err)
+}
+
+func TestExportOpDoInvalidDirectory(t *testing.T) {
+	eo := &ExportOp{
+		File:   "/invalid/dir/file.yaml",
+		Format: OutputFormatYaml,
+	}
+	assert.Error(t, eo.Do(mockEmptyActCtx()))
+}
+
+func TestExportOpDoInvalidOutFormat(t *testing.T) {
+	eo := &ExportOp{
+		Format: "invalid-format",
+	}
+	assert.Error(t, eo.Do(mockEmptyActCtx()))
+}
+
+func TestExportOpDoNonExistentPath(t *testing.T) {
+	f, err := os.CreateTemp("", "yt_export*.json")
+	assert.NoError(t, err)
+	if err != nil {
+		return
+	}
+	t.Cleanup(func() {
+		t.Logf("cleanup temporary file %s", f.Name())
+		_ = os.Remove(f.Name())
+	})
+	eo := &ExportOp{
+		File:   f.Name(),
+		Path:   "this.path.does.not.exist",
+		Format: OutputFormatProperties,
+	}
+	assert.NoError(t, eo.Do(mockEmptyActCtx()))
+}
+
+func TestExportOpCloneWith(t *testing.T) {
+	eo := &ExportOp{
+		File:   "/tmp/out.{{ .Format }}",
+		Path:   "root.sub10.{{ .Sub }}",
+		Format: "{{ .Format }}",
+	}
+	d := b.Container()
+	d.AddValueAt("Format", dom.LeafNode("yaml"))
+	d.AddValueAt("Sub", dom.LeafNode("sub20"))
+	eo = eo.CloneWith(mockActCtx(d)).(*ExportOp)
+	assert.Equal(t, "root.sub10.sub20", eo.Path)
+	assert.Equal(t, OutputFormatYaml, eo.Format)
+	assert.Equal(t, "/tmp/out.yaml", eo.File)
+}

pipeline/foreach_test.go

@@ -42,6 +42,11 @@ func TestForeachStringItem(t *testing.T) {
 					"X": "abc",
 				},
 			},
+			Env: &EnvOp{},
+			Export: &ExportOp{
+				File:   "/tmp/a-{{ .forEach }}.yaml",
+				Format: OutputFormatYaml,
+			},
 		},
 	}
 	d := b.Container()

pipeline/opspec.go

@@ -32,6 +32,12 @@ func (as OpSpec) toList() []Action {
 	if as.Template != nil {
 		actions = append(actions, as.Template)
 	}
+	if as.Export != nil {
+		actions = append(actions, as.Export)
+	}
+	if as.Env != nil {
+		actions = append(actions, as.Env)
+	}
 	if as.ForEach != nil {
 		actions = append(actions, as.ForEach)
 	}
@@ -65,10 +71,16 @@ func (as OpSpec) CloneWith(ctx ActionContext) Action {
 	if as.Template != nil {
 		r.Template = as.Template.CloneWith(ctx).(*TemplateOp)
 	}
+	if as.Export != nil {
+		r.Export = as.Export.CloneWith(ctx).(*ExportOp)
+	}
+	if as.Env != nil {
+		r.Env = as.Env.CloneWith(ctx).(*EnvOp)
+	}
 	return r
 }
 
 func (as OpSpec) String() string {
-	return fmt.Sprintf("OpSpec[ForEach=%v,Import=%v,Patch=%v,Set=%v,Template=%v]",
-		as.ForEach, as.Import, as.Patch, as.Set, as.Template)
+	return fmt.Sprintf("OpSpec[Env=%s,Export=%v,ForEach=%v,Import=%v,Patch=%v,Set=%v,Template=%v]",
+		as.Env, as.Export, as.ForEach, as.Import, as.Patch, as.Set, as.Template)
 }