diff --git a/pipeline/env_op.go b/pipeline/env_op.go new file mode 100644 index 0000000..b1313d8 --- /dev/null +++ b/pipeline/env_op.go @@ -0,0 +1,89 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "fmt" + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/rkosegi/yaml-toolkit/utils" + "os" + "regexp" + "strings" +) + +// TODO : move these predicates to common place, maybe utils package +var ( + MatchAny = func() StringPredicateFn { + return func(s string) bool { + return true + } + } + MatchNone = func() StringPredicateFn { + return func(s string) bool { + return false + } + } + MatchRe = func(re *regexp.Regexp) StringPredicateFn { + return func(s string) bool { + return re.MatchString(s) + } + } +) + +// TODO: merge with same thing from analytics package and move it to common place +type StringPredicateFn func(string) bool + +func (eo *EnvOp) Do(ctx ActionContext) error { + var ( + inclFn StringPredicateFn + exclFn StringPredicateFn + getter func() []string + ) + getter = os.Environ + if eo.envGetter != nil { + getter = eo.envGetter + } + inclFn = MatchAny() + exclFn = MatchNone() + if eo.Include != nil { + inclFn = MatchRe(eo.Include) + } + if eo.Exclude != nil { + exclFn = MatchRe(eo.Exclude) + } + for _, env := range getter() { + parts := strings.SplitN(env, "=", 2) + if inclFn(parts[0]) && !exclFn(parts[0]) { + k := utils.ToPath(eo.Path, fmt.Sprintf("Env.%s", parts[0])) + ctx.Data().AddValueAt(k, dom.LeafNode(parts[1])) + } + } + return nil +} + +func (eo *EnvOp) String() string { + return fmt.Sprintf("Env[path=%s,incl=%s,excl=%s]", eo.Path, + safeRegexpDeref(eo.Include), safeRegexpDeref(eo.Exclude)) +} + +func (eo *EnvOp) CloneWith(ctx ActionContext) Action { + return &EnvOp{ + Include: eo.Include, + Exclude: eo.Exclude, + Path: ctx.TemplateEngine().RenderLenient(eo.Path, ctx.Snapshot()), + } +} diff --git a/pipeline/env_op_test.go b/pipeline/env_op_test.go new file mode 100644 index 0000000..8762ca5 --- /dev/null +++ b/pipeline/env_op_test.go @@ -0,0 +1,83 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/stretchr/testify/assert" + "regexp" + "testing" +) + +func createRePtr(in string) *regexp.Regexp { + x := regexp.MustCompile(in) + return x +} + +func TestEnvOpDo(t *testing.T) { + eo := EnvOp{ + envGetter: func() []string { + return []string{ + "MOCK1=val1", + "MOCK2=val2", + "XYZ=123", + } + }, + Path: "Sub", + Include: createRePtr(`MOCK\d+`), + Exclude: createRePtr("XYZ"), + } + d := b.Container() + err := eo.Do(mockActCtx(d)) + assert.NoError(t, err) + assert.Equal(t, "val1", d.Lookup("Sub.Env.MOCK1").(dom.Leaf).Value()) + assert.Equal(t, "val2", d.Lookup("Sub.Env.MOCK2").(dom.Leaf).Value()) + assert.Contains(t, eo.String(), "Sub") +} + +func TestEnvOpCloneWith(t *testing.T) { + eo := &EnvOp{ + Path: "{{ .NewPath }}", + } + d := b.Container() + d.AddValue("NewPath", dom.LeafNode("root")) + eo = eo.CloneWith(mockActCtx(d)).(*EnvOp) + assert.Equal(t, "root", eo.Path) +} + +func filterStrSlice(in []string, fn StringPredicateFn) []string { + result := make([]string, 0) + for _, e := range in { + if fn(e) { + result = append(result, e) + } + } + return result +} + +func TestStringMatchFunc(t *testing.T) { + in := []string{"a", "b", "c"} + var res []string + res = filterStrSlice(in, MatchAny()) + assert.Equal(t, in, res) + res = filterStrSlice(in, MatchNone()) + assert.Equal(t, 0, len(res)) + res = filterStrSlice(in, MatchRe(regexp.MustCompile(`a`))) + assert.Equal(t, 1, len(res)) + res = filterStrSlice(in, MatchRe(regexp.MustCompile(`[ac]`))) + assert.Equal(t, 2, len(res)) +} diff --git a/pipeline/export_op.go b/pipeline/export_op.go new file mode 100644 index 0000000..b509e01 --- /dev/null +++ b/pipeline/export_op.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "fmt" + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/rkosegi/yaml-toolkit/props" + "os" +) + +func (e *ExportOp) String() string { + return fmt.Sprintf("Export[file=%s,format=%s,path=%s]", e.File, e.Format, e.Path) +} + +func (e *ExportOp) Do(ctx ActionContext) (err error) { + var ( + d dom.Node + enc dom.EncoderFunc + ) + d = ctx.Data() + if len(e.Path) > 0 { + d = ctx.Data().Lookup(e.Path) + } + // use empty container, since path lookup didn't yield anything useful + if d == nil || !d.IsContainer() { + d = b.Container() + } + switch e.Format { + case OutputFormatYaml: + enc = dom.DefaultYamlEncoder + case OutputFormatJson: + enc = dom.DefaultJsonEncoder + case OutputFormatProperties: + enc = props.EncoderFn + + default: + return fmt.Errorf("unknown output format: %s", e.Format) + } + + f, err := os.OpenFile(e.File, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer func() { + _ = f.Close() + }() + return enc(f, dom.DefaultNodeMappingFn(d.(dom.Container))) +} + +func (e *ExportOp) CloneWith(ctx ActionContext) Action { + ss := ctx.Snapshot() + return &ExportOp{ + File: ctx.TemplateEngine().RenderLenient(e.File, ss), + Path: ctx.TemplateEngine().RenderLenient(e.Path, ss), + Format: OutputFormat(ctx.TemplateEngine().RenderLenient(string(e.Format), ss)), + } +} diff --git a/pipeline/export_op_test.go b/pipeline/export_op_test.go new file mode 100644 index 0000000..62da20c --- /dev/null +++ b/pipeline/export_op_test.go @@ -0,0 +1,101 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func TestExportOpDo(t *testing.T) { + var ( + err error + ) + f, err := os.CreateTemp("", "yt_export*.json") + assert.NoError(t, err) + if err != nil { + return + } + t.Cleanup(func() { + t.Logf("cleanup temporary file %s", f.Name()) + _ = os.Remove(f.Name()) + }) + t.Logf("created temporary file: %s", f.Name()) + eo := &ExportOp{ + File: f.Name(), + Path: "root.sub1", + Format: OutputFormatJson, + } + assert.Contains(t, eo.String(), f.Name()) + d := b.Container() + d.AddValueAt("root.sub1.sub2", dom.LeafNode(123)) + err = eo.Do(mockActCtx(d)) + assert.NoError(t, err) + fi, err := os.Stat(f.Name()) + assert.NotNil(t, fi) + assert.NoError(t, err) +} + +func TestExportOpDoInvalidDirectory(t *testing.T) { + eo := &ExportOp{ + File: "/invalid/dir/file.yaml", + Format: OutputFormatYaml, + } + assert.Error(t, eo.Do(mockEmptyActCtx())) +} + +func TestExportOpDoInvalidOutFormat(t *testing.T) { + eo := &ExportOp{ + Format: "invalid-format", + } + assert.Error(t, eo.Do(mockEmptyActCtx())) +} + +func TestExportOpDoNonExistentPath(t *testing.T) { + f, err := os.CreateTemp("", "yt_export*.json") + assert.NoError(t, err) + if err != nil { + return + } + t.Cleanup(func() { + t.Logf("cleanup temporary file %s", f.Name()) + _ = os.Remove(f.Name()) + }) + eo := &ExportOp{ + File: f.Name(), + Path: "this.path.does.not.exist", + Format: OutputFormatProperties, + } + assert.NoError(t, eo.Do(mockEmptyActCtx())) +} + +func TestExportOpCloneWith(t *testing.T) { + eo := &ExportOp{ + File: "/tmp/out.{{ .Format }}", + Path: "root.sub10.{{ .Sub }}", + Format: "{{ .Format }}", + } + d := b.Container() + d.AddValueAt("Format", dom.LeafNode("yaml")) + d.AddValueAt("Sub", dom.LeafNode("sub20")) + eo = eo.CloneWith(mockActCtx(d)).(*ExportOp) + assert.Equal(t, "root.sub10.sub20", eo.Path) + assert.Equal(t, OutputFormatYaml, eo.Format) + assert.Equal(t, "/tmp/out.yaml", eo.File) +} diff --git a/pipeline/foreach_test.go b/pipeline/foreach_test.go index 0a3cddc..d19853f 100644 --- a/pipeline/foreach_test.go +++ b/pipeline/foreach_test.go @@ -42,6 +42,11 @@ func TestForeachStringItem(t *testing.T) { "X": "abc", }, }, + Env: &EnvOp{}, + Export: &ExportOp{ + File: "/tmp/a-{{ .forEach }}.yaml", + Format: OutputFormatYaml, + }, }, } d := b.Container() diff --git a/pipeline/opspec.go b/pipeline/opspec.go index 8908313..a0de13b 100644 --- a/pipeline/opspec.go +++ b/pipeline/opspec.go @@ -32,6 +32,12 @@ func (as OpSpec) toList() []Action { if as.Template != nil { actions = append(actions, as.Template) } + if as.Export != nil { + actions = append(actions, as.Export) + } + if as.Env != nil { + actions = append(actions, as.Env) + } if as.ForEach != nil { actions = append(actions, as.ForEach) } @@ -65,10 +71,16 @@ func (as OpSpec) CloneWith(ctx ActionContext) Action { if as.Template != nil { r.Template = as.Template.CloneWith(ctx).(*TemplateOp) } + if as.Export != nil { + r.Export = as.Export.CloneWith(ctx).(*ExportOp) + } + if as.Env != nil { + r.Env = as.Env.CloneWith(ctx).(*EnvOp) + } return r } func (as OpSpec) String() string { - return fmt.Sprintf("OpSpec[ForEach=%v,Import=%v,Patch=%v,Set=%v,Template=%v]", - as.ForEach, as.Import, as.Patch, as.Set, as.Template) + return fmt.Sprintf("OpSpec[Env=%s,Export=%v,ForEach=%v,Import=%v,Patch=%v,Set=%v,Template=%v]", + as.Env, as.Export, as.ForEach, as.Import, as.Patch, as.Set, as.Template) } diff --git a/pipeline/opspec_test.go b/pipeline/opspec_test.go index 2e8aea4..41850a4 100644 --- a/pipeline/opspec_test.go +++ b/pipeline/opspec_test.go @@ -43,6 +43,14 @@ func TestOpSpecCloneWith(t *testing.T) { Path: "{{ .Path }}", Mode: ParseFileModeYaml, }, + Env: &EnvOp{ + Path: "{{ .Path }}", + }, + Export: &ExportOp{ + File: "/tmp/file.yaml", + Path: "{{ .Path }}", + Format: OutputFormatYaml, + }, } a := o.CloneWith(mockActCtx(b.FromMap(map[string]interface{}{ @@ -54,4 +62,6 @@ func TestOpSpecCloneWith(t *testing.T) { assert.Equal(t, "root.sub2", a.Import.Path) assert.Equal(t, "/root/sub3", a.Patch.Path) assert.Equal(t, "root.sub2", a.Template.WriteTo) + assert.Equal(t, "root.sub2", a.Export.Path) + assert.Equal(t, "root.sub2", a.Env.Path) } diff --git a/pipeline/types.go b/pipeline/types.go index 701c3a5..095d653 100644 --- a/pipeline/types.go +++ b/pipeline/types.go @@ -21,11 +21,11 @@ import ( "fmt" "github.com/rkosegi/yaml-toolkit/dom" "github.com/rkosegi/yaml-toolkit/patch" + "regexp" ) var ( ErrNoDataToSet = errors.New("no data to set") - ErrPathMissing = errors.New("path can't be empty") ErrTemplateEmpty = errors.New("template cannot be empty") ErrWriteToEmpty = errors.New("writeTo cannot be empty") ErrNotContainer = errors.New("data element must be container when no path is provided") @@ -116,6 +116,11 @@ type OpSpec struct { // Template allows to render value at runtime Template *TemplateOp `yaml:"template,omitempty"` + // Env adds OS environment variables into data document + Env *EnvOp `yaml:"env,omitempty"` + + // Export exports data document into file + Export *ExportOp `yaml:"export,omitempty"` // ForEach execute same operation in a loop for every configured item ForEach *ForEachOp `yaml:"forEach,omitempty"` } @@ -144,6 +149,43 @@ type ActionMeta struct { When *string `yaml:"when,omitempty"` } +// EnvOp is used to import OS environment variables into data +type EnvOp struct { + // Optional regexp which defines what to include. Only item names matching this regexp are added into data document. + Include *regexp.Regexp `yaml:"include,omitempty"` + + // Optional regexp which defines what to exclude. Only item names NOT matching this regexp are added into data document. + // Exclusion is considered after inclusion regexp is processed. + Exclude *regexp.Regexp `yaml:"exclude,omitempty"` + + // Optional path within data tree under which "Env" container will be put. + // When omitted, then "Env" goes to root of data. + Path string `yaml:"path,omitempty"` + + // for mock purposes only. this could be used to override os.Environ() to arbitrary func + envGetter func() []string +} + +type OutputFormat string + +const ( + OutputFormatYaml = OutputFormat("yaml") + OutputFormatJson = OutputFormat("json") + OutputFormatProperties = OutputFormat("properties") +) + +// ExportOp allows to export data into file +type ExportOp struct { + // File to export data onto + File string + // Path within data tree pointing to dom.Container to export. Empty path denotes whole document. + // If path does not resolve or resolves to dom.Node that is not dom.Container, + // then empty document will be exported. + Path string + // Format of output file. + Format OutputFormat +} + type ForEachOp struct { Glob *string `yaml:"glob,omitempty"` Item *[]string `yaml:"item,omitempty"` diff --git a/pipeline/utils.go b/pipeline/utils.go index 4ceeec2..f78f8fd 100644 --- a/pipeline/utils.go +++ b/pipeline/utils.go @@ -19,6 +19,7 @@ package pipeline import ( "github.com/rkosegi/yaml-toolkit/dom" "os" + "regexp" "slices" "strings" ) @@ -45,6 +46,13 @@ func safeStrDeref(in *string) string { return *in } +func safeRegexpDeref(re *regexp.Regexp) string { + if re == nil { + return "" + } + return re.String() +} + func safeStrListSize(in *[]string) int { if in == nil { return 0 diff --git a/pipeline/utils_test.go b/pipeline/utils_test.go index e0858e3..dd07f7e 100644 --- a/pipeline/utils_test.go +++ b/pipeline/utils_test.go @@ -19,6 +19,7 @@ package pipeline import ( "github.com/rkosegi/yaml-toolkit/dom" "github.com/stretchr/testify/assert" + "regexp" "testing" ) @@ -49,3 +50,8 @@ func TestSafeStrListSize(t *testing.T) { assert.Equal(t, 0, safeStrListSize(nil)) assert.Equal(t, 1, safeStrListSize(&([]string{"a"}))) } + +func TestSafeRegexpDeref(t *testing.T) { + assert.Equal(t, "", safeRegexpDeref(nil)) + assert.Equal(t, "abc", safeRegexpDeref(regexp.MustCompile(`abc`))) +}