From 2084028ab1e4dca4c54a0d57f19d4873292f32c3 Mon Sep 17 00:00:00 2001 From: Richard Kosegi Date: Tue, 28 May 2024 07:10:46 +0200 Subject: [PATCH] Pipeline: Preliminary implementation Signed-off-by: Richard Kosegi --- go.mod | 1 + go.sum | 2 + pipeline/actions.go | 192 ++++++++++++++++++ pipeline/actions_test.go | 238 ++++++++++++++++++++++ pipeline/child_actions.go | 43 ++++ pipeline/child_actions_test.go | 61 ++++++ pipeline/executor.go | 109 ++++++++++ pipeline/executor_test.go | 332 +++++++++++++++++++++++++++++++ pipeline/foreach.go | 72 +++++++ pipeline/foreach_test.go | 114 +++++++++++ pipeline/opspec.go | 74 +++++++ pipeline/opspec_test.go | 57 ++++++ pipeline/template_engine.go | 78 ++++++++ pipeline/template_engine_test.go | 63 ++++++ pipeline/types.go | 193 ++++++++++++++++++ pipeline/utils.go | 72 +++++++ pipeline/utils_test.go | 51 +++++ testdata/doc1.json | 10 + testdata/pipeline1.yaml | 45 +++++ testdata/props1.properties | 2 + 20 files changed, 1809 insertions(+) create mode 100644 pipeline/actions.go create mode 100644 pipeline/actions_test.go create mode 100644 pipeline/child_actions.go create mode 100644 pipeline/child_actions_test.go create mode 100644 pipeline/executor.go create mode 100644 pipeline/executor_test.go create mode 100644 pipeline/foreach.go create mode 100644 pipeline/foreach_test.go create mode 100644 pipeline/opspec.go create mode 100644 pipeline/opspec_test.go create mode 100644 pipeline/template_engine.go create mode 100644 pipeline/template_engine_test.go create mode 100644 pipeline/types.go create mode 100644 pipeline/utils.go create mode 100644 pipeline/utils_test.go create mode 100644 testdata/doc1.json create mode 100644 testdata/pipeline1.yaml create mode 100644 testdata/props1.properties diff --git a/go.mod b/go.mod index 6697a3f..6ae5761 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ module github.com/rkosegi/yaml-toolkit go 1.21 require ( + github.com/go-task/slim-sprig/v3 v3.0.0 github.com/google/go-cmp v0.6.0 github.com/magiconair/properties v1.8.7 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index dda7e4d..84ce6bf 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= diff --git a/pipeline/actions.go b/pipeline/actions.go new file mode 100644 index 0000000..5317273 --- /dev/null +++ b/pipeline/actions.go @@ -0,0 +1,192 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "bytes" + "encoding/base64" + "fmt" + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/rkosegi/yaml-toolkit/patch" + "github.com/rkosegi/yaml-toolkit/props" +) + +func (am ActionMeta) String() string { + return fmt.Sprintf("[name=%s,order=%d,when=%s]", am.Name, am.Order, safeStrDeref(am.When)) +} + +func (s ActionSpec) CloneWith(ctx ActionContext) Action { + return ActionSpec{ + ActionMeta: s.ActionMeta, + Operations: s.Operations.CloneWith(ctx).(OpSpec), + Children: s.Children.CloneWith(ctx).(ChildActions), + } +} + +func (s ActionSpec) String() string { + return fmt.Sprintf("ActionSpec[meta=%v]", s.ActionMeta) +} + +func (s ActionSpec) Do(ctx ActionContext) error { + for _, a := range []Action{s.Operations, s.Children} { + if s.When != nil { + if ok, err := ctx.TemplateEngine().EvalBool(*s.When, ctx.Snapshot()); err != nil { + return err + } else if !ok { + return nil + } + } + err := ctx.Executor().Execute(a) + if err != nil { + return err + } + } + return nil +} + +func (pfm ParseFileMode) toValue(content []byte) (dom.Node, error) { + switch pfm { + case ParseFileModeBinary: + return dom.LeafNode(base64.StdEncoding.EncodeToString(content)), nil + case ParseFileModeText: + return dom.LeafNode(string(content)), nil + case ParseFileModeYaml: + return b.FromReader(bytes.NewReader(content), dom.DefaultYamlDecoder) + case ParseFileModeJson: + return b.FromReader(bytes.NewReader(content), dom.DefaultJsonDecoder) + case ParseFileModeProperties: + return b.FromReader(bytes.NewReader(content), props.DecoderFn) + default: + return nil, fmt.Errorf("invalid ParseFileMode: %v", pfm) + } +} + +func (ia *ImportOp) String() string { + return fmt.Sprintf("Import[file=%s,path=%s,mode=%s]", ia.File, ia.Path, ia.Mode) +} + +func (ia *ImportOp) Do(ctx ActionContext) error { + val, err := parseFile(ia.File, ia.Mode) + if err != nil { + return err + } + if len(ia.Path) > 0 { + ctx.Data().AddValueAt(ia.Path, val) + } else { + if !val.IsContainer() { + return ErrNotContainer + } else { + for k, v := range val.(dom.Container).Children() { + ctx.Data().AddValueAt(k, v) + } + } + } + return nil +} + +func (ia *ImportOp) CloneWith(ctx ActionContext) Action { + return &ImportOp{ + Mode: ia.Mode, + Path: ctx.TemplateEngine().RenderLenient(ia.Path, ctx.Snapshot()), + File: ctx.TemplateEngine().RenderLenient(ia.File, ctx.Snapshot()), + } +} + +func (ps *PatchOp) String() string { + return fmt.Sprintf("Patch[Op=%s,Path=%s]", ps.Op, ps.Path) +} + +func (ps *PatchOp) Do(ctx ActionContext) error { + oo := &patch.OpObj{ + Op: ps.Op, + } + path, err := patch.ParsePath(ps.Path) + if err != nil { + return err + } + oo.Path = path + oo.Value = b.FromMap(ps.Value) + if len(ps.From) > 0 { + from, err := patch.ParsePath(ps.From) + if err != nil { + return err + } + oo.From = &from + } + return patch.Do(oo, ctx.Data()) +} + +func (ps *PatchOp) CloneWith(ctx ActionContext) Action { + return &PatchOp{ + Op: ps.Op, + Value: ps.Value, + From: ctx.TemplateEngine().RenderLenient(ps.From, ctx.Snapshot()), + Path: ctx.TemplateEngine().RenderLenient(ps.Path, ctx.Snapshot()), + } +} + +func (sa *SetOp) String() string { + return fmt.Sprintf("Set[Path=%s]", sa.Path) +} + +func (sa *SetOp) Do(ctx ActionContext) error { + gd := ctx.Data() + if sa.Data == nil { + return ErrNoDataToSet + } + data := ctx.Factory().FromMap(sa.Data) + if len(sa.Path) > 0 { + gd.AddValueAt(sa.Path, data) + } else { + for k, v := range data.Children() { + gd.AddValueAt(k, v) + } + } + return nil +} + +func (sa *SetOp) CloneWith(ctx ActionContext) Action { + return &SetOp{ + Data: sa.Data, + Path: ctx.TemplateEngine().RenderLenient(sa.Path, ctx.Snapshot()), + } +} + +func (ts *TemplateOp) String() string { + return fmt.Sprintf("Template[WriteTo=%s]", ts.WriteTo) +} + +func (ts *TemplateOp) Do(ctx ActionContext) error { + if len(ts.Template) == 0 { + return ErrTemplateEmpty + } + if len(ts.WriteTo) == 0 { + return ErrWriteToEmpty + } + val, err := ctx.TemplateEngine().Render(ts.Template, map[string]interface{}{ + "Data": ctx.Snapshot(), + }) + ctx.Data().AddValueAt(ts.WriteTo, dom.LeafNode(val)) + return err +} + +func (ts *TemplateOp) CloneWith(ctx ActionContext) Action { + return &TemplateOp{ + Template: ts.Template, + WriteTo: ctx.TemplateEngine().RenderLenient(ts.WriteTo, ctx.Snapshot()), + } +} diff --git a/pipeline/actions_test.go b/pipeline/actions_test.go new file mode 100644 index 0000000..81d2434 --- /dev/null +++ b/pipeline/actions_test.go @@ -0,0 +1,238 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/rkosegi/yaml-toolkit/patch" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestExecuteImportOp(t *testing.T) { + var ( + is ImportOp + gd dom.ContainerBuilder + ) + gd = b.Container() + is = ImportOp{ + File: "../testdata/doc1.json", + Path: "step1.data", + Mode: ParseFileModeJson, + } + + assert.NoError(t, New(WithData(gd)).Execute(&is)) + assert.Equal(t, "c", gd.Lookup("step1.data.root.list1[2]").(dom.Leaf).Value()) + + // parsing YAML file as JSON should lead to error + is = ImportOp{ + File: "../testdata/doc1.yaml", + Mode: ParseFileModeJson, + } + assert.Error(t, New(WithData(gd)).Execute(&is)) + + gd = b.Container() + is = ImportOp{ + File: "../testdata/doc1.yaml", + Mode: ParseFileModeYaml, + Path: "step1.data", + } + assert.NoError(t, New(WithData(gd)).Execute(&is)) + assert.Equal(t, 456, gd.Lookup("step1.data.level1.level2a.level3b").(dom.Leaf).Value()) + + gd = b.Container() + is = ImportOp{ + File: "../testdata/doc1.yaml", + Mode: ParseFileModeText, + Path: "step3", + } + assert.NoError(t, New(WithData(gd)).Execute(&is)) + assert.NotEmpty(t, gd.Lookup("step3").(dom.Leaf).Value()) + assert.Contains(t, is.String(), "path=step3,mode=text") + + gd = b.Container() + is = ImportOp{ + File: "../testdata/doc1.yaml", + Mode: ParseFileModeBinary, + Path: "files.doc1", + } + assert.NoError(t, New(WithData(gd)).Execute(&is)) + assert.NotEmpty(t, gd.Lookup("files.doc1").(dom.Leaf).Value()) + + gd = b.Container() + is = ImportOp{ + File: "../testdata/doc1.json", + Path: "files.doc1_json", + } + assert.NoError(t, New(WithData(gd)).Execute(&is)) + assert.Contains(t, is.String(), "path=files.doc1_json,mode=") + + is = ImportOp{ + File: "non-existent-file.ext", + Path: "something", + } + assert.Error(t, New(WithData(gd)).Execute(&is)) + + is = ImportOp{ + File: "../testdata/props1.properties", + Mode: ParseFileModeProperties, + Path: "props", + } + assert.NoError(t, New(WithData(gd)).Execute(&is)) + + // no path provided and data is not a container - error + is = ImportOp{ + File: "../testdata/props1.properties", + Mode: ParseFileModeText, + } + assert.Error(t, New(WithData(gd)).Execute(&is)) + + // import directly to root (with no path) + is = ImportOp{ + File: "../testdata/doc1.json", + Mode: ParseFileModeJson, + } + assert.NoError(t, New(WithData(gd)).Execute(&is)) + + is = ImportOp{ + File: "../testdata/props1.properties", + Path: "something", + Mode: "invalid-mode", + } + assert.Error(t, New(WithData(gd)).Execute(&is)) +} + +func TestExecutePatchOp(t *testing.T) { + var ( + ps PatchOp + gd dom.ContainerBuilder + ) + + ps = PatchOp{ + Op: patch.OpAdd, + Path: "@#$%^&", + } + assert.Error(t, New(WithData(gd)).Execute(&ps)) + + gd = b.Container() + gd.AddValueAt("root.sub1.leaf2", dom.LeafNode("abcd")) + ps = PatchOp{ + Op: patch.OpReplace, + Path: "/root/sub1", + Value: map[string]interface{}{ + "leaf2": "xyz", + }, + } + assert.NoError(t, New(WithData(gd)).Execute(&ps)) + assert.Equal(t, "xyz", gd.Lookup("root.sub1.leaf2").(dom.Leaf).Value()) + assert.Contains(t, ps.String(), "Op=replace,Path=/root/sub1") + + gd = b.Container() + gd.AddValueAt("root.sub1.leaf3", dom.LeafNode("abcd")) + ps = PatchOp{ + Op: patch.OpMove, + From: "/root/sub1", + Path: "/root/sub2", + } + assert.NoError(t, New(WithData(gd)).Execute(&ps)) + assert.Equal(t, "abcd", gd.Lookup("root.sub2.leaf3").(dom.Leaf).Value()) + + gd = b.Container() + gd.AddValueAt("root.sub1.leaf3", dom.LeafNode("abcd")) + ps = PatchOp{ + Op: patch.OpMove, + From: "%#$&^^*&", + Path: "/root/sub2", + } + assert.Error(t, New(WithData(gd)).Execute(&ps)) +} + +func TestExecuteSetOp(t *testing.T) { + var ( + ss SetOp + gd dom.ContainerBuilder + err error + ) + ss = SetOp{ + Data: map[string]interface{}{ + "sub1": 123, + }, + } + gd = b.Container() + assert.NoError(t, New(WithData(gd)).Execute(&ss)) + assert.Equal(t, 123, gd.Lookup("sub1").(dom.Leaf).Value()) + + ss = SetOp{ + Data: map[string]interface{}{ + "sub1": 123, + }, + Path: "sub0", + } + gd = b.Container() + assert.NoError(t, New(WithData(gd)).Execute(&ss)) + assert.Equal(t, 123, gd.Lookup("sub0.sub1").(dom.Leaf).Value()) + assert.Contains(t, ss.String(), "sub0") + + ss = SetOp{} + err = New(WithData(gd)).Execute(&ss) + assert.Error(t, err) + assert.Equal(t, ErrNoDataToSet, err) +} + +func TestExecuteTemplateOp(t *testing.T) { + var ( + err error + ts TemplateOp + gd dom.ContainerBuilder + ) + + gd = b.Container() + gd.AddValueAt("root.leaf1", dom.LeafNode(123456)) + ts = TemplateOp{ + Template: `{{ (mul .Data.root.leaf1 2) | quote }}`, + WriteTo: "result.x1", + } + assert.NoError(t, New(WithData(gd)).Execute(&ts)) + assert.Equal(t, "\"246912\"", gd.Lookup("result.x1").(dom.Leaf).Value()) + assert.Contains(t, ts.String(), "result.x1") + + // empty template error + ts = TemplateOp{} + err = New(WithData(gd)).Execute(&ts) + assert.Error(t, err) + assert.Equal(t, ErrTemplateEmpty, err) + + // empty writeTo error + ts = TemplateOp{ + Template: `TEST`, + } + err = New(WithData(gd)).Execute(&ts) + assert.Error(t, err) + assert.Equal(t, ErrWriteToEmpty, err) + + ts = TemplateOp{ + Template: `{{}}{{`, + WriteTo: "result", + } + assert.Error(t, New(WithData(gd)).Execute(&ts)) + + ts = TemplateOp{ + Template: `{{ invalid_func }}`, + WriteTo: "result", + } + assert.Error(t, New(WithData(gd)).Execute(&ts)) +} diff --git a/pipeline/child_actions.go b/pipeline/child_actions.go new file mode 100644 index 0000000..6d04c8c --- /dev/null +++ b/pipeline/child_actions.go @@ -0,0 +1,43 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "fmt" +) + +func (na ChildActions) Do(ctx ActionContext) error { + for _, name := range sortActionNames(na) { + err := ctx.Executor().Execute(na[name]) + if err != nil { + return err + } + } + return nil +} + +func (na ChildActions) CloneWith(ctx ActionContext) Action { + r := make(ChildActions) + for k, v := range na { + r[k] = v.CloneWith(ctx).(ActionSpec) + } + return r +} + +func (na ChildActions) String() string { + return fmt.Sprintf("ChildActions[names=%s]", actionNames(na)) +} diff --git a/pipeline/child_actions_test.go b/pipeline/child_actions_test.go new file mode 100644 index 0000000..ed1ca11 --- /dev/null +++ b/pipeline/child_actions_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + sprig "github.com/go-task/slim-sprig/v3" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestSortActionNames(t *testing.T) { + ca := ChildActions{ + "a": ActionSpec{ActionMeta: ActionMeta{ + Order: 90, + }}, + "b": ActionSpec{ActionMeta: ActionMeta{ + Order: 50, + }}, + "c": ActionSpec{ActionMeta: ActionMeta{ + Order: 10, + }, + }, + } + assert.Equal(t, "c,b,a", actionNames(ca)) + assert.Contains(t, ca.String(), "c,b,a") +} + +func TestChildActionsCloneWith(t *testing.T) { + a := ChildActions{ + "step1": ActionSpec{ + Operations: OpSpec{ + Set: &SetOp{ + Data: map[string]interface{}{ + "abcd": 123, + }, + Path: "{{ .sub1.leaf1 }}", + }, + }, + }, + }.CloneWith(&actContext{d: b.FromMap(map[string]interface{}{ + "sub1": map[string]interface{}{ + "leaf1": "root.sub2", + }, + }), t: &templateEngine{fm: sprig.TxtFuncMap()}}) + assert.NotNil(t, a) + assert.Equal(t, "root.sub2", a.(ChildActions)["step1"].Operations.Set.Path) +} diff --git a/pipeline/executor.go b/pipeline/executor.go new file mode 100644 index 0000000..7632bfd --- /dev/null +++ b/pipeline/executor.go @@ -0,0 +1,109 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + sprig "github.com/go-task/slim-sprig/v3" + "github.com/rkosegi/yaml-toolkit/dom" +) + +var ( + b = dom.Builder() +) + +type exec struct { + gd dom.ContainerBuilder + l Listener + t TemplateEngine +} + +type actContext struct { + c Action + d dom.ContainerBuilder + e Executor + f dom.ContainerFactory + t TemplateEngine +} + +func (ac actContext) Data() dom.ContainerBuilder { return ac.d } +func (ac actContext) Factory() dom.ContainerFactory { return ac.f } +func (ac actContext) Executor() Executor { return ac.e } +func (ac actContext) TemplateEngine() TemplateEngine { return ac.t } +func (ac actContext) Snapshot() map[string]interface{} { + return dom.DefaultNodeMappingFn(ac.Data()).(map[string]interface{}) +} + +func (p *exec) newCtx() *actContext { + return &actContext{ + d: p.gd, + e: p, + f: b, + t: p.t, + } +} + +func (p *exec) Execute(act Action) (err error) { + ctx := p.newCtx() + p.l.OnBefore(ctx) + defer p.l.OnAfter(ctx, err) + err = act.Do(ctx) + return err +} + +type noopListener struct{} + +func (n *noopListener) OnBefore(ActionContext) {} +func (n *noopListener) OnAfter(ActionContext, error) {} + +type Opt func(*exec) + +func WithListener(l Listener) Opt { + return func(p *exec) { + p.l = l + } +} + +func WithData(gd dom.ContainerBuilder) Opt { + return func(p *exec) { + p.gd = gd + } +} + +func WithTemplateEngine(t TemplateEngine) Opt { + return func(p *exec) { + p.t = t + } +} + +var defOpts = []Opt{ + WithListener(&noopListener{}), + WithData(b.Container()), + WithTemplateEngine(&templateEngine{ + fm: sprig.TxtFuncMap(), + }), +} + +func New(opts ...Opt) Executor { + p := &exec{} + for _, opt := range defOpts { + opt(p) + } + for _, opt := range opts { + opt(p) + } + return p +} diff --git a/pipeline/executor_test.go b/pipeline/executor_test.go new file mode 100644 index 0000000..b2d854c --- /dev/null +++ b/pipeline/executor_test.go @@ -0,0 +1,332 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + sprig "github.com/go-task/slim-sprig/v3" + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/rkosegi/yaml-toolkit/patch" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + "strings" + "testing" +) + +var dummyExec = New().(*exec) + +func newTestExec(d dom.ContainerBuilder) *exec { + return New(WithData(d)).(*exec) +} + +func parse[T any](t *testing.T, source string) *T { + var x T + err := yaml.NewDecoder(strings.NewReader(source)).Decode(&x) + assert.NoError(t, err) + if err != nil { + return nil + } + return &x +} + +func TestBoolExpressionEval(t *testing.T) { + var ( + val bool + err error + ) + te := &templateEngine{ + fm: sprig.TxtFuncMap(), + } + expr := `{{ eq .Env "Development" }}` + val, err = te.EvalBool(expr, map[string]interface{}{ + "Env": "Development", + }) + assert.NoError(t, err) + assert.Equal(t, true, val) + + val, err = te.EvalBool(expr, map[string]interface{}{ + "Env": "Production", + }) + assert.NoError(t, err) + assert.Equal(t, false, val) + + _, err = te.EvalBool(`{{`, map[string]interface{}{}) + assert.Error(t, err) +} + +func TestParseStep(t *testing.T) { + pl := parse[ActionSpec](t, ` +--- +name: root step +set: + data: + root: + sub1: + leaf1: 123 + sub2: + - list_item1 + path: result +`) + assert.NotNil(t, pl) + assert.Contains(t, pl.String(), "root step") + assert.Equal(t, "root step", pl.Name) + assert.Equal(t, 123, pl.Operations.Set.Data["root"].(map[string]interface{})["sub1"].(map[string]interface{})["leaf1"]) + assert.Equal(t, "list_item1", pl.Operations.Set.Data["root"].(map[string]interface{})["sub2"].([]interface{})[0]) +} + +func TestExecute(t *testing.T) { + var ( + gd dom.ContainerBuilder + ex *exec + ) + gd = b.Container() + ex = newTestExec(gd) + assert.NoError(t, ex.Execute(&ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Operations: OpSpec{ + Set: &SetOp{ + Data: map[string]interface{}{ + "leaf": "abcd", + }, + }, + }, + Children: ChildActions{ + "sub1": { + Operations: OpSpec{ + Set: &SetOp{ + Path: "sub1.sub2", + Data: map[string]interface{}{ + "leaf": "abcd", + }, + }, + }, + }, + }, + })) + assert.Equal(t, "abcd", gd.Lookup("leaf").(dom.Leaf).Value()) + assert.Equal(t, "abcd", gd.Lookup("sub1.sub2.leaf").(dom.Leaf).Value()) + + gd = b.Container() + ex = newTestExec(gd) + assert.NoError(t, ex.Execute(&ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Children: ChildActions{ + "sub_step1": { + ActionMeta: ActionMeta{ + Name: "sub_step1", + }, + Operations: OpSpec{ + Template: &TemplateOp{ + Template: "{{ mul 1 2 3 4 5 6 }}", + WriteTo: "Results.Factorial", + }, + }, + }, + }, + })) + assert.Equal(t, "720", gd.Lookup("Results.Factorial").(dom.Leaf).Value()) + +} + +func TestExecuteImport(t *testing.T) { + var ( + gd dom.ContainerBuilder + ex *exec + ) + gd = b.Container() + ex = newTestExec(gd) + assert.NoError(t, ex.Execute(&ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Operations: OpSpec{ + Import: &ImportOp{ + File: "../testdata/props1.properties", + Path: "wrapped", + Mode: ParseFileModeProperties, + }, + }, + })) + assert.Equal(t, "abcdef", gd.Lookup("wrapped.root.sub1.leaf2").(dom.Leaf).Value()) +} + +func TestExecuteImportInvalid(t *testing.T) { + var ( + gd dom.ContainerBuilder + ex *exec + ) + gd = b.Container() + ex = newTestExec(gd) + assert.Error(t, ex.Execute(&ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Operations: OpSpec{ + Import: &ImportOp{}, + }, + })) +} + +func TestExecutePatch(t *testing.T) { + var ( + gd dom.ContainerBuilder + ex *exec + ) + gd = b.Container() + ex = newTestExec(gd) + assert.NoError(t, ex.Execute(&ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Operations: OpSpec{ + Patch: &PatchOp{ + Op: patch.OpAdd, + Path: "/root", + Value: map[string]interface{}{"leaf": "abcd"}, + }, + }, + })) + assert.Equal(t, "abcd", gd.Lookup("root.leaf").(dom.Leaf).Value()) +} + +func TestExecuteInnerSteps(t *testing.T) { + var ( + gd dom.ContainerBuilder + ex *exec + ) + gd = b.Container() + ex = newTestExec(gd) + assert.NoError(t, ex.Execute(&ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Children: ChildActions{ + "step20": { + ActionMeta: ActionMeta{ + Order: 20, + Name: "step 20", + }, + Operations: OpSpec{ + Set: &SetOp{ + Data: map[string]interface{}{ + "root.sub": 123, + }, + }, + }, + }, + "step30": { + ActionMeta: ActionMeta{ + Order: 30, + Name: "step 30", + }, + Operations: OpSpec{ + Set: &SetOp{ + Data: map[string]interface{}{ + "root.sub": 456, + }, + }, + }, + }, + }, + })) + assert.Equal(t, 456, gd.Lookup("root.sub").(dom.Leaf).Value()) +} + +func TestExecuteInnerStepsFail(t *testing.T) { + assert.Error(t, dummyExec.Execute(&ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Children: ChildActions{ + "step20": { + ActionMeta: ActionMeta{ + Order: 20, + Name: "step 20", + }, + Operations: OpSpec{ + Set: &SetOp{}, + }, + }, + }, + })) +} + +func TestExecuteInnerStepsSkipped(t *testing.T) { + assert.NoError(t, dummyExec.Execute(&ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Children: ChildActions{ + "step20": { + ActionMeta: ActionMeta{ + When: strPointer("{{ .Data.Skip | default false }}"), + Name: "step 20", + }, + }, + }, + })) +} + +func TestExecuteInnerStepsWhenInvalid(t *testing.T) { + assert.Error(t, dummyExec.Execute(&ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Children: ChildActions{ + "step20": { + ActionMeta: ActionMeta{ + When: strPointer("{{ .Data.Unknown.Field }}"), + Name: "step 20", + }, + }, + }, + })) +} + +func TestExecuteForEachFileGlob(t *testing.T) { + var ( + gd dom.ContainerBuilder + ex *exec + ) + gd = b.Container() + ex = newTestExec(gd) + fe := &ForEachOp{ + Glob: strPointer("../testdata/doc?.yaml"), + Action: OpSpec{ + Import: &ImportOp{ + File: "{{ .forEach }}", + Path: "import.files.{{ b64enc (osBase .forEach) }}", + Mode: ParseFileModeYaml, + }, + }, + } + + ss := &ActionSpec{ + ActionMeta: ActionMeta{ + Name: "root step", + }, + Operations: OpSpec{ + ForEach: fe, + }, + } + assert.NoError(t, ex.Execute(ss)) + assert.Equal(t, 2, len(gd.Lookup("import.files").(dom.Container).Children())) + assert.Contains(t, fe.String(), "doc?.yaml") +} diff --git a/pipeline/foreach.go b/pipeline/foreach.go new file mode 100644 index 0000000..c47a067 --- /dev/null +++ b/pipeline/foreach.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "fmt" + "github.com/rkosegi/yaml-toolkit/dom" + "path/filepath" +) + +func (fea *ForEachOp) Do(ctx ActionContext) error { + if nonEmpty(fea.Glob) { + if matches, err := filepath.Glob(*fea.Glob); err != nil { + return err + } else { + for _, match := range matches { + err = fea.performWithItem(ctx, match) + if err != nil { + return err + } + } + } + } else if fea.Item != nil { + for _, item := range *fea.Item { + err := fea.performWithItem(ctx, item) + if err != nil { + return err + } + } + } + return nil +} + +func (fea *ForEachOp) performWithItem(ctx ActionContext, item string) (err error) { + ctx.Data().AddValue("forEach", dom.LeafNode(item)) + defer ctx.Data().Remove("forEach") + + for _, act := range fea.Action.toList() { + act = act.CloneWith(ctx) + err = ctx.Executor().Execute(act) + if err != nil { + return err + } + } + return nil +} + +func (fea *ForEachOp) String() string { + return fmt.Sprintf("ForEach[Glob=%s,Items=%d]", safeStrDeref(fea.Glob), safeStrListSize(fea.Item)) +} + +func (fea *ForEachOp) CloneWith(ctx ActionContext) Action { + cp := new(ForEachOp) + cp.Glob = fea.Glob + cp.Item = fea.Item + cp.Action = OpSpec{}.CloneWith(ctx).(OpSpec) + return cp +} diff --git a/pipeline/foreach_test.go b/pipeline/foreach_test.go new file mode 100644 index 0000000..0a3cddc --- /dev/null +++ b/pipeline/foreach_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestForeachCloneWith(t *testing.T) { + op := ForEachOp{ + Item: &([]string{"a", "b", "c"}), + Action: OpSpec{}, + } + a := op.CloneWith(mockEmptyActCtx()).(*ForEachOp) + assert.NotNil(t, a) + assert.Equal(t, 3, len(*a.Item)) +} + +func TestForeachStringItem(t *testing.T) { + op := ForEachOp{ + Item: &([]string{"a", "b", "c"}), + Action: OpSpec{ + Set: &SetOp{ + Path: "{{ .forEach }}", + Data: map[string]interface{}{ + "X": "abc", + }, + }, + }, + } + d := b.Container() + err := op.Do(mockActCtx(d)) + assert.NoError(t, err) + assert.Equal(t, "abc", d.Lookup("a.X").(dom.Leaf).Value()) + assert.Equal(t, "abc", d.Lookup("b.X").(dom.Leaf).Value()) + assert.Equal(t, "abc", d.Lookup("c.X").(dom.Leaf).Value()) +} + +func TestForeachStringItemChildError(t *testing.T) { + op := ForEachOp{ + Item: &([]string{"a", "b", "c"}), + Action: OpSpec{ + Set: &SetOp{ + Path: "{{ .forEach }}", + }, + }, + } + d := b.Container() + err := op.Do(mockActCtx(d)) + assert.Error(t, err) +} + +func TestForeachGlob(t *testing.T) { + op := ForEachOp{ + Glob: strPointer("../testdata/doc?.yaml"), + Action: OpSpec{ + Import: &ImportOp{ + File: "{{ .forEach }}", + Path: "import.files.{{ b64enc (osBase .forEach) }}", + Mode: ParseFileModeYaml, + }, + }, + } + d := b.Container() + err := op.Do(mockActCtx(d)) + assert.NoError(t, err) + assert.Equal(t, 2, len(d.Lookup("import.files").(dom.Container).Children())) +} + +func TestForeachGlobChildError(t *testing.T) { + op := ForEachOp{ + Glob: strPointer("../testdata/doc?.yaml"), + Action: OpSpec{ + Set: &SetOp{ + Path: "{{ .forEach }}", + }, + }, + } + d := b.Container() + err := op.Do(mockActCtx(d)) + assert.Error(t, err) +} + +func TestForeachGlobInvalid(t *testing.T) { + op := ForEachOp{ + Glob: strPointer("[]]"), + Action: OpSpec{ + Import: &ImportOp{ + File: "{{ .forEach }}", + Path: "import.files.{{ b64enc (osBase .forEach) }}", + Mode: ParseFileModeYaml, + }, + }, + } + d := b.Container() + err := op.Do(mockActCtx(d)) + assert.Error(t, err) +} diff --git a/pipeline/opspec.go b/pipeline/opspec.go new file mode 100644 index 0000000..8908313 --- /dev/null +++ b/pipeline/opspec.go @@ -0,0 +1,74 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import "fmt" + +func (as OpSpec) toList() []Action { + actions := make([]Action, 0) + if as.Set != nil { + actions = append(actions, as.Set) + } + if as.Import != nil { + actions = append(actions, as.Import) + } + if as.Patch != nil { + actions = append(actions, as.Patch) + } + if as.Template != nil { + actions = append(actions, as.Template) + } + if as.ForEach != nil { + actions = append(actions, as.ForEach) + } + return actions +} + +func (as OpSpec) Do(ctx ActionContext) error { + for _, a := range as.toList() { + err := ctx.Executor().Execute(a) + if err != nil { + return err + } + } + return nil +} + +func (as OpSpec) CloneWith(ctx ActionContext) Action { + r := OpSpec{} + if as.ForEach != nil { + r.ForEach = as.ForEach.CloneWith(ctx).(*ForEachOp) + } + if as.Import != nil { + r.Import = as.Import.CloneWith(ctx).(*ImportOp) + } + if as.Patch != nil { + r.Patch = as.Patch.CloneWith(ctx).(*PatchOp) + } + if as.Set != nil { + r.Set = as.Set.CloneWith(ctx).(*SetOp) + } + if as.Template != nil { + r.Template = as.Template.CloneWith(ctx).(*TemplateOp) + } + return r +} + +func (as OpSpec) String() string { + return fmt.Sprintf("OpSpec[ForEach=%v,Import=%v,Patch=%v,Set=%v,Template=%v]", + as.ForEach, as.Import, as.Patch, as.Set, as.Template) +} diff --git a/pipeline/opspec_test.go b/pipeline/opspec_test.go new file mode 100644 index 0000000..2e8aea4 --- /dev/null +++ b/pipeline/opspec_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestOpSpecCloneWith(t *testing.T) { + o := OpSpec{ + Set: &SetOp{ + Data: map[string]interface{}{ + "a": 1, + }, + Path: "{{ .Path }}", + }, + Patch: &PatchOp{ + Path: "{{ .Path3 }}", + }, + ForEach: &ForEachOp{ + Item: &([]string{"left", "right"}), + Action: OpSpec{}, + }, + Template: &TemplateOp{ + WriteTo: "{{ .Path }}", + }, + Import: &ImportOp{ + Path: "{{ .Path }}", + Mode: ParseFileModeYaml, + }, + } + + a := o.CloneWith(mockActCtx(b.FromMap(map[string]interface{}{ + "Path": "root.sub2", + "Path3": "/root/sub3", + }))).(OpSpec) + t.Log(a.String()) + assert.Equal(t, "root.sub2", a.Set.Path) + assert.Equal(t, "root.sub2", a.Import.Path) + assert.Equal(t, "/root/sub3", a.Patch.Path) + assert.Equal(t, "root.sub2", a.Template.WriteTo) +} diff --git a/pipeline/template_engine.go b/pipeline/template_engine.go new file mode 100644 index 0000000..1b739b2 --- /dev/null +++ b/pipeline/template_engine.go @@ -0,0 +1,78 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "bytes" + "strconv" + "strings" + "text/template" +) + +type templateEngine struct { + fm template.FuncMap +} + +func renderTemplate(tmplStr string, data interface{}, fm template.FuncMap) (string, error) { + tmpl := template.New("tmpl").Funcs(fm) + _, err := tmpl.Parse(tmplStr) + if err != nil { + return "", err + } + var out bytes.Buffer + err = tmpl.Execute(&out, data) + if err != nil { + return "", err + } + return out.String(), nil +} + +func possiblyTemplate(in string) bool { + openIdx := strings.Index(in, "{{") + if openIdx == -1 { + return false + } + closeIdx := strings.Index(in[openIdx:], "}}") + return closeIdx > 0 +} + +func renderLenientTemplate(tmpl string, data map[string]interface{}, fm template.FuncMap) string { + if possiblyTemplate(tmpl) { + if val, err := renderTemplate(tmpl, data, fm); err != nil { + return tmpl + } else { + return val + } + } + return tmpl +} + +func (te templateEngine) RenderLenient(tmpl string, data map[string]interface{}) string { + return renderLenientTemplate(tmpl, data, te.fm) +} + +func (te templateEngine) Render(tmpl string, data map[string]interface{}) (string, error) { + return renderTemplate(tmpl, data, te.fm) +} + +func (te templateEngine) EvalBool(template string, data map[string]interface{}) (bool, error) { + val, err := te.Render(template, data) + if err != nil { + return false, err + } + return strconv.ParseBool(val) +} diff --git a/pipeline/template_engine_test.go b/pipeline/template_engine_test.go new file mode 100644 index 0000000..6ad70d4 --- /dev/null +++ b/pipeline/template_engine_test.go @@ -0,0 +1,63 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + sprig "github.com/go-task/slim-sprig/v3" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestPossiblyTemplate(t *testing.T) { + assert.True(t, possiblyTemplate("{{ . }}")) + assert.True(t, possiblyTemplate("{{data}}")) + assert.True(t, possiblyTemplate("{{}}")) + assert.False(t, possiblyTemplate("{{")) + assert.False(t, possiblyTemplate("345678")) +} + +func TestTemplateEngineRenderLenient(t *testing.T) { + te := &templateEngine{ + fm: sprig.TxtFuncMap(), + } + assert.Equal(t, "AAA", te.RenderLenient("AAA", nil)) + assert.Equal(t, "{{ data }}", te.RenderLenient("{{ data }}", nil)) + assert.Equal(t, "123", te.RenderLenient("{{ .data }}", map[string]interface{}{ + "data": 123, + })) +} + +func TestRenderTemplate(t *testing.T) { + var ( + out string + err error + ) + // invalid template syntax + _, err = renderTemplate("{{", map[string]interface{}{}, sprig.TxtFuncMap()) + assert.Error(t, err) + + // valid template, valid data + out, err = renderTemplate("{{ .X }}", map[string]interface{}{ + "X": "abcd", + }, sprig.TxtFuncMap()) + assert.NoError(t, err) + assert.Equal(t, "abcd", out) + + // invalid data + _, err = renderTemplate("{{ .a }}", "", sprig.TxtFuncMap()) + assert.Error(t, err) +} diff --git a/pipeline/types.go b/pipeline/types.go new file mode 100644 index 0000000..701c3a5 --- /dev/null +++ b/pipeline/types.go @@ -0,0 +1,193 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "errors" + "fmt" + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/rkosegi/yaml-toolkit/patch" +) + +var ( + ErrNoDataToSet = errors.New("no data to set") + ErrPathMissing = errors.New("path can't be empty") + ErrTemplateEmpty = errors.New("template cannot be empty") + ErrWriteToEmpty = errors.New("writeTo cannot be empty") + ErrNotContainer = errors.New("data element must be container when no path is provided") +) + +// ParseFileMode defines how the file is parsed before is put into data tree +type ParseFileMode string + +const ( + // ParseFileModeBinary File is read and encoded using base64 string into data tree + ParseFileModeBinary ParseFileMode = "binary" + + // ParseFileModeText File is read as-is and is assumed it represents utf-8 encoded byte stream + ParseFileModeText ParseFileMode = "text" + + // ParseFileModeYaml File is parsed as YAML document and put as child node into data tree + ParseFileModeYaml ParseFileMode = "yaml" + + // ParseFileModeJson File is parsed as JSON document and put as child node into data tree + ParseFileModeJson ParseFileMode = "json" + + // ParseFileModeProperties File is parsed as Java properties into map[string]interface{} and put as child node into data tree + ParseFileModeProperties ParseFileMode = "properties" +) + +// Executor interface is used by external callers to execute Action items +type Executor interface { + Execute(act Action) error +} + +type TemplateEngine interface { + Render(template string, data map[string]interface{}) (string, error) + // RenderLenient attempts to render given template using provided data, while swallowing any error. + // Value of template is first checked by simple means if it is actually template to avoid unnecessary errors. + // Use with caution. + RenderLenient(template string, data map[string]interface{}) string + EvalBool(template string, data map[string]interface{}) (bool, error) +} + +// Listener interface allows hook into execution of Action. +type Listener interface { + // OnBefore is called just before act is executed + OnBefore(ctx ActionContext) + // OnAfter is called sometime after act is executed, regardless of result. + // Any error returned by invoking Do() method is returned as last parameter. + OnAfter(ctx ActionContext, err error) +} + +// ActionContext is created by Executor implementation for sole purpose of invoking Action's Do function. +type ActionContext interface { + // Data exposes data document + Data() dom.ContainerBuilder + // Snapshot is read-only view of Data() in point in time + Snapshot() map[string]interface{} + // Factory give access to factory to create new documents + Factory() dom.ContainerFactory + // Executor returns reference to executor + Executor() Executor + // TemplateEngine return reference to TemplateEngine + TemplateEngine() TemplateEngine +} + +// Action is implemented by actions within ActionSpec +type Action interface { + fmt.Stringer + // Do will perform this action. + // This function is invoked by Executor implementation and as such it's not meant to be called by end user directly. + Do(ctx ActionContext) error + // CloneWith creates fresh clone of this Action with values of its fields templated. + // Data for template can be obtained by calling Snapshot() on provided context. + CloneWith(ctx ActionContext) Action +} + +// ChildActions is map of named actions that are executed as a part of parent action +type ChildActions map[string]ActionSpec + +// OpSpec is specification of operation. +type OpSpec struct { + // Set sets data in data document. + Set *SetOp `yaml:"set,omitempty"` + + // Patch performs RFC6902-style patch on data document. + Patch *PatchOp `yaml:"patch,omitempty"` + + // Import loads content of file into data document. + Import *ImportOp `yaml:"import,omitempty"` + + // Template allows to render value at runtime + Template *TemplateOp `yaml:"template,omitempty"` + + // ForEach execute same operation in a loop for every configured item + ForEach *ForEachOp `yaml:"forEach,omitempty"` +} + +type ActionSpec struct { + ActionMeta `yaml:",inline"` + // Operations to perform + Operations OpSpec `yaml:",inline"` + // Children element is an optional map of child actions that will be executed + // as a part of this action (after any of OpSpec in Operations are performed). + // Exact order of execution is given by Order field value (lower the value, sooner the execution will take place). + Children ChildActions `yaml:"steps,omitempty"` +} + +// ActionMeta holds action's metadata used by Executor +type ActionMeta struct { + // Name of this step, should be unique within current scope + Name string `yaml:"name,omitempty"` + + // Optional ordinal number that controls order of execution within parent step + Order int `yaml:"order,omitempty"` + + // Optional expression to make execution of this action conditional. + // Execution of this step is skipped when this expression is evaluated to false. + // If value of this field is omitted, then this action is executed. + When *string `yaml:"when,omitempty"` +} + +type ForEachOp struct { + Glob *string `yaml:"glob,omitempty"` + Item *[]string `yaml:"item,omitempty"` + // Action to perform for every item + Action OpSpec `yaml:"action"` +} + +// ImportOp reads content of file into data tree at given path +type ImportOp struct { + // File to read + File string `yaml:"file"` + + // Path at which to import data. + // If omitted, then data are merged into root of document + Path string `yaml:"path"` + + // How to parse file + Mode ParseFileMode `yaml:"mode,omitempty"` +} + +// PatchOp performs RFC6902-style patch on global data document. +// Check patch package for more details +type PatchOp struct { + Op patch.Op `yaml:"op"` + From string `yaml:"from,omitempty"` + Path string `yaml:"path"` + Value map[string]interface{} `yaml:"value,omitempty"` +} + +// SetOp sets data in global data document at given path. +type SetOp struct { + // Arbitrary data to put into data tree + Data map[string]interface{} `yaml:"data"` + + // Path at which to put data. + // If omitted, then data are merged into root of document + Path string `yaml:"path,omitempty"` +} + +// TemplateOp can be used to render value from data at runtime. +// Global data tree is available under .Data +type TemplateOp struct { + // template to render + Template string `yaml:"template"` + // path within global data tree where to set result at + WriteTo string `yaml:"writeTo"` +} diff --git a/pipeline/utils.go b/pipeline/utils.go new file mode 100644 index 0000000..4ceeec2 --- /dev/null +++ b/pipeline/utils.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "github.com/rkosegi/yaml-toolkit/dom" + "os" + "slices" + "strings" +) + +func parseFile(path string, mode ParseFileMode) (dom.Node, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if len(mode) == 0 { + mode = ParseFileModeText + } + val, err := mode.toValue(data) + if err != nil { + return nil, err + } + return val, nil +} + +func safeStrDeref(in *string) string { + if in == nil { + return "" + } + return *in +} + +func safeStrListSize(in *[]string) int { + if in == nil { + return 0 + } + return len(*in) +} + +func nonEmpty(in *string) bool { + return in != nil && len(*in) > 0 +} + +func sortActionNames(actions ChildActions) []string { + var keys []string + for n := range actions { + keys = append(keys, n) + } + slices.SortFunc(keys, func(a, b string) int { + return actions[a].Order - actions[b].Order + }) + return keys +} + +func actionNames(actions ChildActions) string { + return strings.Join(sortActionNames(actions), ",") +} diff --git a/pipeline/utils_test.go b/pipeline/utils_test.go new file mode 100644 index 0000000..e0858e3 --- /dev/null +++ b/pipeline/utils_test.go @@ -0,0 +1,51 @@ +/* +Copyright 2024 Richard Kosegi + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pipeline + +import ( + "github.com/rkosegi/yaml-toolkit/dom" + "github.com/stretchr/testify/assert" + "testing" +) + +func strPointer(str string) *string { + return &str +} + +func mockEmptyActCtx() ActionContext { + return mockActCtx(b.Container()) +} + +func mockActCtx(data dom.ContainerBuilder) ActionContext { + return New(WithData(data)).(*exec).newCtx() +} + +func TestNonEmpty(t *testing.T) { + assert.False(t, nonEmpty(strPointer(""))) + assert.False(t, nonEmpty(nil)) + assert.True(t, nonEmpty(strPointer("abcd"))) +} + +func TestSafeStrDeref(t *testing.T) { + assert.Equal(t, "", safeStrDeref(nil)) + assert.Equal(t, "aa", safeStrDeref(strPointer("aa"))) +} + +func TestSafeStrListSize(t *testing.T) { + assert.Equal(t, 0, safeStrListSize(nil)) + assert.Equal(t, 1, safeStrListSize(&([]string{"a"}))) +} diff --git a/testdata/doc1.json b/testdata/doc1.json new file mode 100644 index 0000000..7a44348 --- /dev/null +++ b/testdata/doc1.json @@ -0,0 +1,10 @@ +{ + "root" : { + "sub1": { + "leaf1": 123 + }, + "list1": [ + "a", "b", "c" + ] + } +} diff --git a/testdata/pipeline1.yaml b/testdata/pipeline1.yaml new file mode 100644 index 0000000..17b67c0 --- /dev/null +++ b/testdata/pipeline1.yaml @@ -0,0 +1,45 @@ +# Copyright 2024 Richard Kosegi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +name: root step +import: + file: envs.yaml + mode: yaml +set: + data: + url: https://default +steps: + env_dev: + name: env specific settings (dev) + when: '{{ eq .Env "dev" }}' + set: + data: + url: https://dev.acme.com + env_prod: + name: env specific settings (prod) + when: '{{ eq .Env "prod" }}' + set: + data: + url: https://prod.acme.com + spring_profiles: + name: Import each file with spring profile + order: 10000 + forEach: + glob: configs/profile-*.yaml + action: + import: + file: '{{ .forEach.key }}' + mode: yaml + path: '.Import.{{ .forEach.key }}' + diff --git a/testdata/props1.properties b/testdata/props1.properties new file mode 100644 index 0000000..d262aa6 --- /dev/null +++ b/testdata/props1.properties @@ -0,0 +1,2 @@ +root.sub1.leaf2=abcdef +root.sub2.leaf10=10.23