From e4e4d031d036648345f4f5668aa7060009053001 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Thu, 3 Oct 2024 02:57:49 -0400 Subject: [PATCH 01/31] feat: add chart --- pkg/chart/chart.go | 117 ++++++++++++++++++++++++++++++++++++++++ pkg/chart/chart_test.go | 38 +++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 pkg/chart/chart.go create mode 100644 pkg/chart/chart_test.go diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go new file mode 100644 index 00000000..0bf3702f --- /dev/null +++ b/pkg/chart/chart.go @@ -0,0 +1,117 @@ +package chart + +import ( + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/resource" + "github.com/siyul-park/uniflow/pkg/spec" +) + +// Chart defines how multiple nodes are combined into a cluster node. +type Chart struct { + // Unique identifier of the chart. + ID uuid.UUID `json:"id,omitempty" bson:"_id,omitempty" yaml:"id,omitempty" map:"id,omitempty"` + // Logical grouping or environment. + Namespace string `json:"namespace,omitempty" bson:"namespace,omitempty" yaml:"namespace,omitempty" map:"namespace,omitempty"` + // Name of the chart or cluster node (required). + Name string `json:"name" bson:"name" yaml:"name" map:"name"` + // Additional metadata. + Annotations map[string]string `json:"annotations,omitempty" bson:"annotations,omitempty" yaml:"annotations,omitempty" map:"annotations,omitempty"` + // Specifications that define the nodes and their configurations within the chart. + Specs []spec.Spec `json:"specs" bson:"specs" yaml:"specs" map:"specs"` + // Node connections within the chart. + Ports map[string][]Port `json:"ports,omitempty" bson:"ports,omitempty" yaml:"ports,omitempty" map:"ports,omitempty"` + // Sensitive configuration data or secrets. + Env map[string][]Secret `json:"env,omitempty" bson:"env,omitempty" yaml:"env,omitempty" map:"env,omitempty"` +} + +// Port represents a connection point for a node. +type Port struct { + // Unique identifier of the port. + ID uuid.UUID `json:"id,omitempty" bson:"_id,omitempty" yaml:"id,omitempty" map:"id,omitempty"` + // Name of the port. + Name string `json:"name,omitempty" bson:"name,omitempty" yaml:"name,omitempty" map:"name,omitempty"` + // Port number or identifier. + Port string `json:"port" bson:"port" yaml:"port" map:"port"` +} + +// Secret represents a sensitive value for a node. +type Secret struct { + // Unique identifier of the secret. + ID uuid.UUID `json:"id,omitempty" bson:"_id,omitempty" yaml:"id,omitempty" map:"id,omitempty"` + // Name of the secret. + Name string `json:"name,omitempty" bson:"name,omitempty" yaml:"name,omitempty" map:"name,omitempty"` + // Secret value. + Value any `json:"value" bson:"value" yaml:"value" map:"value"` +} + +var _ resource.Resource = (*Chart)(nil) + +// GetID returns the chart's ID. +func (c *Chart) GetID() uuid.UUID { + return c.ID +} + +// SetID sets the chart's ID. +func (c *Chart) SetID(val uuid.UUID) { + c.ID = val +} + +// GetNamespace returns the chart's namespace. +func (c *Chart) GetNamespace() string { + return c.Namespace +} + +// SetNamespace sets the chart's namespace. +func (c *Chart) SetNamespace(val string) { + c.Namespace = val +} + +// GetName returns the chart's name. +func (c *Chart) GetName() string { + return c.Name +} + +// SetName sets the chart's name. +func (c *Chart) SetName(val string) { + c.Name = val +} + +// GetAnnotations returns the chart's annotations. +func (c *Chart) GetAnnotations() map[string]string { + return c.Annotations +} + +// SetAnnotations sets the chart's annotations. +func (c *Chart) SetAnnotations(val map[string]string) { + c.Annotations = val +} + +// GetSpecs returns the chart's specs. +func (c *Chart) GetSpecs() []spec.Spec { + return c.Specs +} + +// SetSpecs sets the chart's specs. +func (c *Chart) SetSpecs(val []spec.Spec) { + c.Specs = val +} + +// GetPorts returns the chart's ports. +func (c *Chart) GetPorts() map[string][]Port { + return c.Ports +} + +// SetPorts sets the chart's ports. +func (c *Chart) SetPorts(val map[string][]Port) { + c.Ports = val +} + +// GetEnv returns the chart's environment data. +func (c *Chart) GetEnv() map[string][]Secret { + return c.Env +} + +// SetEnv sets the chart's environment data. +func (c *Chart) SetEnv(val map[string][]Secret) { + c.Env = val +} diff --git a/pkg/chart/chart_test.go b/pkg/chart/chart_test.go new file mode 100644 index 00000000..8c8169a6 --- /dev/null +++ b/pkg/chart/chart_test.go @@ -0,0 +1,38 @@ +package chart + +import ( + "testing" + + "github.com/go-faker/faker/v4" + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/resource" + "github.com/siyul-park/uniflow/pkg/spec" + "github.com/stretchr/testify/assert" +) + +func TestChart_GetSet(t *testing.T) { + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: "default", + Name: faker.Word(), + Annotations: map[string]string{"key": "value"}, + Specs: []spec.Spec{ + &spec.Meta{ + ID: uuid.Must(uuid.NewV7()), + Kind: faker.UUIDHyphenated(), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + }, + }, + Ports: map[string][]Port{"out": {{Name: faker.Word(), Port: "in"}}}, + Env: map[string][]Secret{"env1": {{Name: "secret1", Value: "value1"}}}, + } + + assert.Equal(t, chrt.ID, chrt.GetID()) + assert.Equal(t, chrt.Namespace, chrt.GetNamespace()) + assert.Equal(t, chrt.Name, chrt.GetName()) + assert.Equal(t, chrt.Annotations, chrt.GetAnnotations()) + assert.Equal(t, chrt.Specs, chrt.GetSpecs()) + assert.Equal(t, chrt.Ports, chrt.GetPorts()) + assert.Equal(t, chrt.Env, chrt.GetEnv()) +} From f5fa47531361c7341255abe6fe7e603d8fadcb66 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Thu, 3 Oct 2024 04:32:51 -0400 Subject: [PATCH 02/31] feat: add links and unlinks --- pkg/chart/table.go | 175 ++++++++++++++++++++++++++++++++++++++++ pkg/chart/table_test.go | 83 +++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 pkg/chart/table.go create mode 100644 pkg/chart/table_test.go diff --git a/pkg/chart/table.go b/pkg/chart/table.go new file mode 100644 index 00000000..b20630d6 --- /dev/null +++ b/pkg/chart/table.go @@ -0,0 +1,175 @@ +package chart + +import ( + "slices" + "sync" + + "github.com/gofrs/uuid" +) + +type Table struct { + charts map[uuid.UUID]*Chart + namespaces map[string]map[string]uuid.UUID + refences map[uuid.UUID][]uuid.UUID + mu sync.RWMutex +} + +func NewTable() *Table { + return &Table{ + charts: make(map[uuid.UUID]*Chart), + namespaces: make(map[string]map[string]uuid.UUID), + refences: make(map[uuid.UUID][]uuid.UUID), + } +} + +func (t *Table) Insert(chrt *Chart) error { + t.mu.Lock() + defer t.mu.Unlock() + + if _, err := t.free(chrt.GetID()); err != nil { + return err + } + return t.insert(chrt) +} + +func (t *Table) Free(id uuid.UUID) (bool, error) { + t.mu.Lock() + defer t.mu.Unlock() + + chrt, err := t.free(id) + if err != nil { + return false, err + } + return chrt != nil, nil +} + +func (t *Table) Lookup(id uuid.UUID) *Chart { + t.mu.RLock() + defer t.mu.RUnlock() + + return t.charts[id] +} + +func (t *Table) Close() error { + t.mu.Lock() + defer t.mu.Unlock() + + for id := range t.charts { + if _, err := t.free(id); err != nil { + return err + } + } + return nil +} + +func (t *Table) insert(chrt *Chart) error { + t.charts[chrt.GetID()] = chrt + + ns, ok := t.namespaces[chrt.GetNamespace()] + if !ok { + ns = make(map[string]uuid.UUID) + t.namespaces[chrt.GetNamespace()] = ns + } + ns[chrt.GetName()] = chrt.GetID() + + t.links(chrt) + return nil +} + +func (t *Table) free(id uuid.UUID) (*Chart, error) { + chrt, ok := t.charts[id] + if !ok { + return nil, nil + } + + t.unlinks(chrt) + + if ns, ok := t.namespaces[chrt.GetNamespace()]; ok { + delete(ns, chrt.GetName()) + if len(ns) == 0 { + delete(t.namespaces, chrt.GetNamespace()) + } + } + + delete(t.charts, id) + + return chrt, nil +} + +func (t *Table) links(chrt *Chart) { + for _, spec := range chrt.GetSpecs() { + id := t.lookup(chrt.GetNamespace(), spec.GetKind()) + if !slices.Contains(t.refences[id], chrt.GetID()) { + t.refences[id] = append(t.refences[id], chrt.GetID()) + } + } + + for _, ref := range t.charts { + for _, spec := range ref.GetSpecs() { + id := t.lookup(ref.GetNamespace(), spec.GetKind()) + if id == chrt.GetID() { + if !slices.Contains(t.refences[id], ref.GetID()) { + t.refences[id] = append(t.refences[id], ref.GetID()) + } + } + } + } +} + +func (t *Table) unlinks(chrt *Chart) { + for _, spec := range chrt.GetSpecs() { + id := t.lookup(chrt.GetNamespace(), spec.GetKind()) + + refences := t.refences[id] + for i := 0; i < len(refences); i++ { + if refences[i] == chrt.GetID() { + refences = append(refences[:i], refences[i+1:]...) + i-- + } + } + + if len(refences) > 0 { + t.refences[id] = refences + } else { + delete(t.refences, id) + } + } + + delete(t.refences, chrt.GetID()) +} + +func (t *Table) active(chrt *Chart) bool { + var linked []*Chart + + nexts := []*Chart{chrt} + for len(nexts) > 0 { + chrt := nexts[len(nexts)-1] + ok := true + for _, sp := range chrt.Specs { + id := t.lookup(chrt.GetNamespace(), sp.GetKind()) + next := t.charts[id] + + if next == nil || slices.Contains(nexts, next) { + return false + } + + ok = slices.Contains(linked, next) + if !ok { + nexts = append(nexts, next) + break + } + } + if ok { + nexts = nexts[0 : len(nexts)-1] + linked = append(linked, chrt) + } + } + return true +} + +func (t *Table) lookup(namespace, name string) uuid.UUID { + if ns, ok := t.namespaces[namespace]; ok { + return ns[name] + } + return uuid.Nil +} diff --git a/pkg/chart/table_test.go b/pkg/chart/table_test.go new file mode 100644 index 00000000..959e2b11 --- /dev/null +++ b/pkg/chart/table_test.go @@ -0,0 +1,83 @@ +package chart + +import ( + "testing" + + "github.com/go-faker/faker/v4" + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/resource" + "github.com/siyul-park/uniflow/pkg/spec" + "github.com/stretchr/testify/assert" +) + +func TestTable_Insert(t *testing.T) { + tb := NewTable() + defer tb.Close() + + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{ + &spec.Meta{ + ID: uuid.Must(uuid.NewV7()), + Kind: faker.UUIDHyphenated(), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + }, + }, + } + + err := tb.Insert(chrt) + assert.NoError(t, err) + assert.NotNil(t, tb.Lookup(chrt.GetID())) +} + +func TestTable_Free(t *testing.T) { + tb := NewTable() + defer tb.Close() + + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{ + &spec.Meta{ + ID: uuid.Must(uuid.NewV7()), + Kind: faker.UUIDHyphenated(), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + }, + }, + } + + err := tb.Insert(chrt) + assert.NoError(t, err) + + ok, err := tb.Free(chrt.GetID()) + assert.NoError(t, err) + assert.True(t, ok) +} + +func TestTable_Lookup(t *testing.T) { + tb := NewTable() + defer tb.Close() + + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{ + &spec.Meta{ + ID: uuid.Must(uuid.NewV7()), + Kind: faker.UUIDHyphenated(), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + }, + }, + } + + err := tb.Insert(chrt) + assert.NoError(t, err) + assert.Equal(t, chrt, tb.Lookup(chrt.GetID())) +} From b64e9a0a3dac45b802b7b077d8553f635feb95ac Mon Sep 17 00:00:00 2001 From: siyul-park Date: Mon, 7 Oct 2024 04:06:34 -0400 Subject: [PATCH 03/31] feat: add link hook --- pkg/chart/loadhook.go | 35 ++++++++++++++ pkg/chart/table.go | 103 +++++++++++++++++++++++++++++++++------- pkg/chart/table_test.go | 71 ++++++++++++++++++++++++--- pkg/chart/unloadhook.go | 36 ++++++++++++++ 4 files changed, 220 insertions(+), 25 deletions(-) create mode 100644 pkg/chart/loadhook.go create mode 100644 pkg/chart/unloadhook.go diff --git a/pkg/chart/loadhook.go b/pkg/chart/loadhook.go new file mode 100644 index 00000000..6228de2f --- /dev/null +++ b/pkg/chart/loadhook.go @@ -0,0 +1,35 @@ +package chart + +// LoadHook defines an interface for handling events when a symbol is loaded. +type LoadHook interface { + // Load is called to handle the loading of a symbol and may return an error. + Load(*Chart) error +} + +// LoadHooks is a slice of LoadHook interfaces, processed sequentially. +type LoadHooks []LoadHook + +type loadHook struct { + load func(*Chart) error +} + +var _ LoadHook = (LoadHooks)(nil) +var _ LoadHook = (*loadHook)(nil) + +// LoadFunc creates a new LoadHook from the provided function. +func LoadFunc(load func(*Chart) error) LoadHook { + return &loadHook{load: load} +} + +func (h LoadHooks) Load(chrt *Chart) error { + for _, hook := range h { + if err := hook.Load(chrt); err != nil { + return err + } + } + return nil +} + +func (h *loadHook) Load(chrt *Chart) error { + return h.load(chrt) +} diff --git a/pkg/chart/table.go b/pkg/chart/table.go index b20630d6..f996bf50 100644 --- a/pkg/chart/table.go +++ b/pkg/chart/table.go @@ -7,18 +7,35 @@ import ( "github.com/gofrs/uuid" ) +// TableOption holds configurations for a Table instance. +type TableOption struct { + LoadHooks []LoadHook // LoadHooks are functions executed when symbols are loaded. + UnloadHooks []UnloadHook // UnloadHooks are functions executed when symbols are unloaded. +} + type Table struct { - charts map[uuid.UUID]*Chart - namespaces map[string]map[string]uuid.UUID - refences map[uuid.UUID][]uuid.UUID - mu sync.RWMutex + charts map[uuid.UUID]*Chart + namespaces map[string]map[string]uuid.UUID + refences map[uuid.UUID][]uuid.UUID + loadHooks LoadHooks + unloadHooks UnloadHooks + mu sync.RWMutex } -func NewTable() *Table { +func NewTable(opts ...TableOption) *Table { + var loadHooks []LoadHook + var unloadHooks []UnloadHook + for _, opt := range opts { + loadHooks = append(loadHooks, opt.LoadHooks...) + unloadHooks = append(unloadHooks, opt.UnloadHooks...) + } + return &Table{ - charts: make(map[uuid.UUID]*Chart), - namespaces: make(map[string]map[string]uuid.UUID), - refences: make(map[uuid.UUID][]uuid.UUID), + charts: make(map[uuid.UUID]*Chart), + namespaces: make(map[string]map[string]uuid.UUID), + refences: make(map[uuid.UUID][]uuid.UUID), + loadHooks: loadHooks, + unloadHooks: unloadHooks, } } @@ -73,7 +90,7 @@ func (t *Table) insert(chrt *Chart) error { ns[chrt.GetName()] = chrt.GetID() t.links(chrt) - return nil + return t.load(chrt) } func (t *Table) free(id uuid.UUID) (*Chart, error) { @@ -82,6 +99,9 @@ func (t *Table) free(id uuid.UUID) (*Chart, error) { return nil, nil } + if err := t.unload(chrt); err != nil { + return nil, err + } t.unlinks(chrt) if ns, ok := t.namespaces[chrt.GetNamespace()]; ok { @@ -96,10 +116,35 @@ func (t *Table) free(id uuid.UUID) (*Chart, error) { return chrt, nil } +func (t *Table) load(chrt *Chart) error { + linked := t.linked(chrt) + for _, sb := range linked { + if t.active(sb) { + if err := t.loadHooks.Load(sb); err != nil { + return err + } + } + } + return nil +} + +func (t *Table) unload(chrt *Chart) error { + linked := t.linked(chrt) + for i := len(linked) - 1; i >= 0; i-- { + sb := linked[i] + if t.active(sb) { + if err := t.unloadHooks.Unload(sb); err != nil { + return err + } + } + } + return nil +} + func (t *Table) links(chrt *Chart) { for _, spec := range chrt.GetSpecs() { id := t.lookup(chrt.GetNamespace(), spec.GetKind()) - if !slices.Contains(t.refences[id], chrt.GetID()) { + if id != uuid.Nil && !slices.Contains(t.refences[id], chrt.GetID()) { t.refences[id] = append(t.refences[id], chrt.GetID()) } } @@ -107,7 +152,7 @@ func (t *Table) links(chrt *Chart) { for _, ref := range t.charts { for _, spec := range ref.GetSpecs() { id := t.lookup(ref.GetNamespace(), spec.GetKind()) - if id == chrt.GetID() { + if id != uuid.Nil && id == chrt.GetID() { if !slices.Contains(t.refences[id], ref.GetID()) { t.refences[id] = append(t.refences[id], ref.GetID()) } @@ -138,29 +183,51 @@ func (t *Table) unlinks(chrt *Chart) { delete(t.refences, chrt.GetID()) } -func (t *Table) active(chrt *Chart) bool { +func (t *Table) linked(chrt *Chart) []*Chart { var linked []*Chart + paths := []*Chart{chrt} + for len(paths) > 0 { + sb := paths[len(paths)-1] + ok := true + for _, id := range t.refences[sb.GetID()] { + next := t.charts[id] + ok = slices.Contains(paths, next) || slices.Contains(linked, next) + if !ok { + paths = append(paths, next) + break + } + } + if ok { + paths = paths[0 : len(paths)-1] + linked = append(linked, sb) + } + } + slices.Reverse(linked) + return linked +} - nexts := []*Chart{chrt} - for len(nexts) > 0 { - chrt := nexts[len(nexts)-1] +func (t *Table) active(chrt *Chart) bool { + var linked []*Chart + paths := []*Chart{chrt} + for len(paths) > 0 { + chrt := paths[len(paths)-1] ok := true for _, sp := range chrt.Specs { id := t.lookup(chrt.GetNamespace(), sp.GetKind()) next := t.charts[id] - if next == nil || slices.Contains(nexts, next) { + if next == nil || slices.Contains(paths, next) { return false } ok = slices.Contains(linked, next) if !ok { - nexts = append(nexts, next) + paths = append(paths, next) break } } if ok { - nexts = nexts[0 : len(nexts)-1] + paths = paths[0 : len(paths)-1] linked = append(linked, chrt) } } diff --git a/pkg/chart/table_test.go b/pkg/chart/table_test.go index 959e2b11..4c7b853e 100644 --- a/pkg/chart/table_test.go +++ b/pkg/chart/table_test.go @@ -14,23 +14,32 @@ func TestTable_Insert(t *testing.T) { tb := NewTable() defer tb.Close() - chrt := &Chart{ + chrt1 := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{}, + } + chrt2 := &Chart{ ID: uuid.Must(uuid.NewV7()), Namespace: resource.DefaultNamespace, Name: faker.UUIDHyphenated(), Specs: []spec.Spec{ &spec.Meta{ - ID: uuid.Must(uuid.NewV7()), - Kind: faker.UUIDHyphenated(), + Kind: chrt1.GetName(), Namespace: resource.DefaultNamespace, Name: faker.UUIDHyphenated(), }, }, } - err := tb.Insert(chrt) + err := tb.Insert(chrt1) + assert.NoError(t, err) + assert.NotNil(t, tb.Lookup(chrt1.GetID())) + + err = tb.Insert(chrt2) assert.NoError(t, err) - assert.NotNil(t, tb.Lookup(chrt.GetID())) + assert.NotNil(t, tb.Lookup(chrt2.GetID())) } func TestTable_Free(t *testing.T) { @@ -43,7 +52,6 @@ func TestTable_Free(t *testing.T) { Name: faker.UUIDHyphenated(), Specs: []spec.Spec{ &spec.Meta{ - ID: uuid.Must(uuid.NewV7()), Kind: faker.UUIDHyphenated(), Namespace: resource.DefaultNamespace, Name: faker.UUIDHyphenated(), @@ -69,7 +77,6 @@ func TestTable_Lookup(t *testing.T) { Name: faker.UUIDHyphenated(), Specs: []spec.Spec{ &spec.Meta{ - ID: uuid.Must(uuid.NewV7()), Kind: faker.UUIDHyphenated(), Namespace: resource.DefaultNamespace, Name: faker.UUIDHyphenated(), @@ -81,3 +88,53 @@ func TestTable_Lookup(t *testing.T) { assert.NoError(t, err) assert.Equal(t, chrt, tb.Lookup(chrt.GetID())) } + +func TestTable_Hook(t *testing.T) { + loaded := 0 + unloaded := 0 + + tb := NewTable(TableOption{ + LoadHooks: []LoadHook{ + LoadFunc(func(_ *Chart) error { + loaded += 1 + return nil + }), + }, + UnloadHooks: []UnloadHook{ + UnloadFunc(func(_ *Chart) error { + unloaded += 1 + return nil + }), + }, + }) + defer tb.Close() + + chrt1 := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{}, + } + chrt2 := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{ + &spec.Meta{ + Kind: chrt1.GetName(), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + }, + }, + } + + err := tb.Insert(chrt2) + assert.NoError(t, err) + assert.Equal(t, 0, loaded) + assert.Equal(t, 0, unloaded) + + err = tb.Insert(chrt1) + assert.NoError(t, err) + assert.Equal(t, 2, loaded) + assert.Equal(t, 0, unloaded) +} diff --git a/pkg/chart/unloadhook.go b/pkg/chart/unloadhook.go new file mode 100644 index 00000000..102bdf9c --- /dev/null +++ b/pkg/chart/unloadhook.go @@ -0,0 +1,36 @@ +package chart + +// UnloadHook defines an interface for handling events when a symbol is unloaded. +type UnloadHook interface { + // Unload is called when a symbol is unloaded and may return an error. + Unload(*Chart) error +} + +// UnloadHooks is a slice of UnloadHook interfaces, processed in reverse order. +type UnloadHooks []UnloadHook + +type unloadHook struct { + unload func(*Chart) error +} + +var _ UnloadHook = (UnloadHooks)(nil) +var _ UnloadHook = (*unloadHook)(nil) + +// UnloadFunc creates a new UnloadHook from the provided function. +func UnloadFunc(unload func(*Chart) error) UnloadHook { + return &unloadHook{unload: unload} +} + +func (h UnloadHooks) Unload(chrt *Chart) error { + for i := len(h) - 1; i >= 0; i-- { + hook := h[i] + if err := hook.Unload(chrt); err != nil { + return err + } + } + return nil +} + +func (h *unloadHook) Unload(chrt *Chart) error { + return h.unload(chrt) +} From 82fa520ff8ee77a53ea5d6a5cce35011aa3e2208 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Mon, 7 Oct 2024 04:13:46 -0400 Subject: [PATCH 04/31] feat: add scheme.Kinds --- pkg/scheme/scheme.go | 18 ++++++++++++++++++ pkg/scheme/scheme_test.go | 15 +++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/pkg/scheme/scheme.go b/pkg/scheme/scheme.go index d6bfe62f..75012b5f 100644 --- a/pkg/scheme/scheme.go +++ b/pkg/scheme/scheme.go @@ -2,6 +2,7 @@ package scheme import ( "reflect" + "slices" "sync" "github.com/pkg/errors" @@ -28,6 +29,23 @@ func New() *Scheme { } } +// Kinds returns all unique kinds from types and codecs. +func (s *Scheme) Kinds() []string { + s.mu.RLock() + defer s.mu.RUnlock() + + kinds := make([]string, 0, len(s.types)) + for kind := range s.types { + kinds = append(kinds, kind) + } + for kind := range s.codecs { + if !slices.Contains(kinds, kind) { + kinds = append(kinds, kind) + } + } + return kinds +} + // AddKnownType associates a spec type with a kind and returns true if successful. func (s *Scheme) AddKnownType(kind string, sp spec.Spec) bool { s.mu.Lock() diff --git a/pkg/scheme/scheme_test.go b/pkg/scheme/scheme_test.go index bfc7e912..61c97a42 100644 --- a/pkg/scheme/scheme_test.go +++ b/pkg/scheme/scheme_test.go @@ -10,6 +10,21 @@ import ( "github.com/stretchr/testify/assert" ) +func TestScheme_Kinds(t *testing.T) { + s := New() + kind := faker.UUIDHyphenated() + + c := CodecFunc(func(spec spec.Spec) (node.Node, error) { + return node.NewOneToOneNode(nil), nil + }) + + s.AddKnownType(kind, &spec.Meta{}) + s.AddCodec(kind, c) + + kinds := s.Kinds() + assert.Contains(t, kinds, kind) +} + func TestScheme_KnownType(t *testing.T) { s := New() kind := faker.UUIDHyphenated() From 7436d63dc883c16a511d79b4af8e27b9aa358dac Mon Sep 17 00:00:00 2001 From: siyul-park Date: Mon, 7 Oct 2024 04:19:06 -0400 Subject: [PATCH 05/31] docs: add comment --- pkg/chart/loadhook.go | 7 +++---- pkg/chart/table.go | 6 ++++++ pkg/chart/unloadhook.go | 9 +++++---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/chart/loadhook.go b/pkg/chart/loadhook.go index 6228de2f..e640b9eb 100644 --- a/pkg/chart/loadhook.go +++ b/pkg/chart/loadhook.go @@ -1,12 +1,11 @@ package chart -// LoadHook defines an interface for handling events when a symbol is loaded. +// LoadHook defines an interface for handling the loading of a chart. type LoadHook interface { - // Load is called to handle the loading of a symbol and may return an error. + // Load processes the loading of a chart and may return an error. Load(*Chart) error } -// LoadHooks is a slice of LoadHook interfaces, processed sequentially. type LoadHooks []LoadHook type loadHook struct { @@ -16,7 +15,7 @@ type loadHook struct { var _ LoadHook = (LoadHooks)(nil) var _ LoadHook = (*loadHook)(nil) -// LoadFunc creates a new LoadHook from the provided function. +// LoadFunc creates a LoadHook from the given function. func LoadFunc(load func(*Chart) error) LoadHook { return &loadHook{load: load} } diff --git a/pkg/chart/table.go b/pkg/chart/table.go index f996bf50..1db7e221 100644 --- a/pkg/chart/table.go +++ b/pkg/chart/table.go @@ -13,6 +13,7 @@ type TableOption struct { UnloadHooks []UnloadHook // UnloadHooks are functions executed when symbols are unloaded. } +// Table manages charts and their references, allowing insertion, lookup, and removal. type Table struct { charts map[uuid.UUID]*Chart namespaces map[string]map[string]uuid.UUID @@ -22,6 +23,7 @@ type Table struct { mu sync.RWMutex } +// NewTable creates and returns a new Table instance with the provided options. func NewTable(opts ...TableOption) *Table { var loadHooks []LoadHook var unloadHooks []UnloadHook @@ -39,6 +41,7 @@ func NewTable(opts ...TableOption) *Table { } } +// Insert adds a new chart to the table, freeing the previous chart if it exists. func (t *Table) Insert(chrt *Chart) error { t.mu.Lock() defer t.mu.Unlock() @@ -49,6 +52,7 @@ func (t *Table) Insert(chrt *Chart) error { return t.insert(chrt) } +// Free removes a chart from the table based on its UUID and unloads it. func (t *Table) Free(id uuid.UUID) (bool, error) { t.mu.Lock() defer t.mu.Unlock() @@ -60,6 +64,7 @@ func (t *Table) Free(id uuid.UUID) (bool, error) { return chrt != nil, nil } +// Lookup retrieves a chart from the table based on its UUID. func (t *Table) Lookup(id uuid.UUID) *Chart { t.mu.RLock() defer t.mu.RUnlock() @@ -67,6 +72,7 @@ func (t *Table) Lookup(id uuid.UUID) *Chart { return t.charts[id] } +// Close removes all charts from the table and unloads them. func (t *Table) Close() error { t.mu.Lock() defer t.mu.Unlock() diff --git a/pkg/chart/unloadhook.go b/pkg/chart/unloadhook.go index 102bdf9c..355d1b52 100644 --- a/pkg/chart/unloadhook.go +++ b/pkg/chart/unloadhook.go @@ -1,14 +1,15 @@ package chart -// UnloadHook defines an interface for handling events when a symbol is unloaded. +// UnloadHook defines an interface for handling the unloading of a chart. type UnloadHook interface { - // Unload is called when a symbol is unloaded and may return an error. + // Unload is called when a chart is unloaded and may return an error. Unload(*Chart) error } -// UnloadHooks is a slice of UnloadHook interfaces, processed in reverse order. +// UnloadHooks is a slice of UnloadHook, processed in reverse order. type UnloadHooks []UnloadHook +// unloadHook wraps an unload function to implement UnloadHook. type unloadHook struct { unload func(*Chart) error } @@ -16,7 +17,7 @@ type unloadHook struct { var _ UnloadHook = (UnloadHooks)(nil) var _ UnloadHook = (*unloadHook)(nil) -// UnloadFunc creates a new UnloadHook from the provided function. +// UnloadFunc creates an UnloadHook from the given function. func UnloadFunc(unload func(*Chart) error) UnloadHook { return &unloadHook{unload: unload} } From 01b7aeba6cd78995607ab1576f4ce394efaffe3d Mon Sep 17 00:00:00 2001 From: siyul-park Date: Mon, 7 Oct 2024 23:50:55 -0400 Subject: [PATCH 06/31] feat: add store --- pkg/chart/store.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 pkg/chart/store.go diff --git a/pkg/chart/store.go b/pkg/chart/store.go new file mode 100644 index 00000000..b71bfdc3 --- /dev/null +++ b/pkg/chart/store.go @@ -0,0 +1,13 @@ +package chart + +import "github.com/siyul-park/uniflow/pkg/resource" + +// Store is an alias for the resource.Store interface, specialized for Chart resources. +type Store resource.Store[*Chart] + +type Stream = resource.Stream + +// NewStore creates and returns a new instance of a Store for managing Chart resources. +func NewStore() Store { + return resource.NewStore[*Chart]() +} From 772ea13d417f549575579b10b8ea565cbf11a2f9 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 00:40:53 -0400 Subject: [PATCH 07/31] feat: support loader --- pkg/chart/chart.go | 78 +++++++++++++++++++++++++++--- pkg/chart/chart_test.go | 51 +++++++++++++++++++- pkg/chart/loader.go | 101 +++++++++++++++++++++++++++++++++++++++ pkg/chart/loader_test.go | 68 ++++++++++++++++++++++++++ pkg/chart/table.go | 12 +++++ pkg/chart/table_test.go | 23 +++++++++ 6 files changed, 325 insertions(+), 8 deletions(-) create mode 100644 pkg/chart/loader.go create mode 100644 pkg/chart/loader_test.go diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go index 0bf3702f..b4016641 100644 --- a/pkg/chart/chart.go +++ b/pkg/chart/chart.go @@ -2,17 +2,21 @@ package chart import ( "github.com/gofrs/uuid" + "github.com/pkg/errors" + "github.com/siyul-park/uniflow/pkg/encoding" "github.com/siyul-park/uniflow/pkg/resource" + "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" + "github.com/siyul-park/uniflow/pkg/template" ) -// Chart defines how multiple nodes are combined into a cluster node. +// Chart defines the structure that combines multiple nodes into a cluster node. type Chart struct { // Unique identifier of the chart. ID uuid.UUID `json:"id,omitempty" bson:"_id,omitempty" yaml:"id,omitempty" map:"id,omitempty"` // Logical grouping or environment. Namespace string `json:"namespace,omitempty" bson:"namespace,omitempty" yaml:"namespace,omitempty" map:"namespace,omitempty"` - // Name of the chart or cluster node (required). + // Name of the chart or cluster node. Name string `json:"name" bson:"name" yaml:"name" map:"name"` // Additional metadata. Annotations map[string]string `json:"annotations,omitempty" bson:"annotations,omitempty" yaml:"annotations,omitempty" map:"annotations,omitempty"` @@ -21,7 +25,7 @@ type Chart struct { // Node connections within the chart. Ports map[string][]Port `json:"ports,omitempty" bson:"ports,omitempty" yaml:"ports,omitempty" map:"ports,omitempty"` // Sensitive configuration data or secrets. - Env map[string][]Secret `json:"env,omitempty" bson:"env,omitempty" yaml:"env,omitempty" map:"env,omitempty"` + Env map[string][]Value `json:"env,omitempty" bson:"env,omitempty" yaml:"env,omitempty" map:"env,omitempty"` } // Port represents a connection point for a node. @@ -34,8 +38,8 @@ type Port struct { Port string `json:"port" bson:"port" yaml:"port" map:"port"` } -// Secret represents a sensitive value for a node. -type Secret struct { +// Value represents a sensitive value for a node. +type Value struct { // Unique identifier of the secret. ID uuid.UUID `json:"id,omitempty" bson:"_id,omitempty" yaml:"id,omitempty" map:"id,omitempty"` // Name of the secret. @@ -46,6 +50,66 @@ type Secret struct { var _ resource.Resource = (*Chart)(nil) +// IsBound checks whether any of the secrets are bound to the chart. +func IsBound(chrt *Chart, secrets ...*secret.Secret) bool { + for _, vals := range chrt.GetEnv() { + for _, val := range vals { + examples := make([]*secret.Secret, 0, 2) + if val.ID != uuid.Nil { + examples = append(examples, &secret.Secret{ID: val.ID}) + } + if val.Name != "" { + examples = append(examples, &secret.Secret{Namespace: chrt.GetNamespace(), Name: val.Name}) + } + + for _, sec := range secrets { + if len(resource.Match(sec, examples...)) > 0 { + return true + } + } + } + } + return false +} + +// Bind binds the chart's environment variables to the provided secrets. +func Bind(chrt *Chart, secrets ...*secret.Secret) (*Chart, error) { + for _, vals := range chrt.GetEnv() { + for i, val := range vals { + if val.ID != uuid.Nil || val.Name != "" { + example := &secret.Secret{ + ID: val.ID, + Namespace: chrt.GetNamespace(), + Name: val.Name, + } + + var sec *secret.Secret + for _, s := range secrets { + if len(resource.Match(s, example)) > 0 { + sec = s + break + } + } + if sec == nil { + return nil, errors.WithStack(encoding.ErrUnsupportedValue) + } + + v, err := template.Execute(val.Value, sec.Data) + if err != nil { + return nil, err + } + + val.ID = sec.GetID() + val.Name = sec.GetName() + val.Value = v + + vals[i] = val + } + } + } + return chrt, nil +} + // GetID returns the chart's ID. func (c *Chart) GetID() uuid.UUID { return c.ID @@ -107,11 +171,11 @@ func (c *Chart) SetPorts(val map[string][]Port) { } // GetEnv returns the chart's environment data. -func (c *Chart) GetEnv() map[string][]Secret { +func (c *Chart) GetEnv() map[string][]Value { return c.Env } // SetEnv sets the chart's environment data. -func (c *Chart) SetEnv(val map[string][]Secret) { +func (c *Chart) SetEnv(val map[string][]Value) { c.Env = val } diff --git a/pkg/chart/chart_test.go b/pkg/chart/chart_test.go index 8c8169a6..4dd2def3 100644 --- a/pkg/chart/chart_test.go +++ b/pkg/chart/chart_test.go @@ -6,10 +6,59 @@ import ( "github.com/go-faker/faker/v4" "github.com/gofrs/uuid" "github.com/siyul-park/uniflow/pkg/resource" + "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" "github.com/stretchr/testify/assert" ) +func TestIsBound(t *testing.T) { + sec1 := &secret.Secret{ + ID: uuid.Must(uuid.NewV7()), + } + sec2 := &secret.Secret{ + ID: uuid.Must(uuid.NewV7()), + } + + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Env: map[string][]Value{ + "FOO": { + { + ID: sec1.ID, + Value: "foo", + }, + }, + }, + } + + assert.True(t, IsBound(chrt, sec1)) + assert.False(t, IsBound(chrt, sec2)) +} + +func TestBind(t *testing.T) { + sec := &secret.Secret{ + ID: uuid.Must(uuid.NewV7()), + Data: "foo", + } + + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Env: map[string][]Value{ + "FOO": { + { + ID: sec.ID, + Value: "{{ . }}", + }, + }, + }, + } + + bind, err := Bind(chrt, sec) + assert.NoError(t, err) + assert.Equal(t, "foo", bind.GetEnv()["FOO"][0].Value) + assert.True(t, IsBound(bind, sec)) +} + func TestChart_GetSet(t *testing.T) { chrt := &Chart{ ID: uuid.Must(uuid.NewV7()), @@ -25,7 +74,7 @@ func TestChart_GetSet(t *testing.T) { }, }, Ports: map[string][]Port{"out": {{Name: faker.Word(), Port: "in"}}}, - Env: map[string][]Secret{"env1": {{Name: "secret1", Value: "value1"}}}, + Env: map[string][]Value{"env1": {{Name: "secret1", Value: "value1"}}}, } assert.Equal(t, chrt.ID, chrt.GetID()) diff --git a/pkg/chart/loader.go b/pkg/chart/loader.go new file mode 100644 index 00000000..6bd4db1b --- /dev/null +++ b/pkg/chart/loader.go @@ -0,0 +1,101 @@ +package chart + +import ( + "context" + "errors" + + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/resource" + "github.com/siyul-park/uniflow/pkg/secret" +) + +// LoaderConfig holds configuration for the Loader. +type LoaderConfig struct { + Table *Table // Symbol table for storing loaded symbols + ChartStore Store // ChartStore to retrieve charts from + SecretStore secret.Store // SecretStore to retrieve secrets from +} + +// Loader synchronizes with spec.Store to load spec.Spec into the Table. +type Loader struct { + table *Table + chartStore Store + secretStore secret.Store +} + +// NewLoader creates a new Loader instance with the provided configuration. +func NewLoader(config LoaderConfig) *Loader { + return &Loader{ + table: config.Table, + chartStore: config.ChartStore, + secretStore: config.SecretStore, + } +} + +func (l *Loader) Load(ctx context.Context, charts ...*Chart) error { + examples := charts + + charts, err := l.chartStore.Load(ctx, examples...) + if err != nil { + return err + } + + var secrets []*secret.Secret + for _, chrt := range charts { + for _, vals := range chrt.GetEnv() { + for _, val := range vals { + if val.ID == uuid.Nil && val.Name == "" { + continue + } + secrets = append(secrets, &secret.Secret{ + ID: val.ID, + Namespace: chrt.GetNamespace(), + Name: val.Name, + }) + } + } + } + + if len(secrets) > 0 { + secrets, err = l.secretStore.Load(ctx, secrets...) + if err != nil { + return err + } + } + + var errs []error + for _, chrt := range charts { + if chrt, err := Bind(chrt, secrets...); err != nil { + errs = append(errs, err) + } else if err := l.table.Insert(chrt); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + charts = nil + } + + for _, id := range l.table.Keys() { + chrt := l.table.Lookup(id) + if chrt != nil && len(resource.Match(chrt, examples...)) > 0 { + ok := false + for _, c := range charts { + if c.GetID() == id { + ok = true + break + } + } + if !ok { + if _, err := l.table.Free(id); err != nil { + return err + } + } + } + } + + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil +} diff --git a/pkg/chart/loader_test.go b/pkg/chart/loader_test.go new file mode 100644 index 00000000..2f850570 --- /dev/null +++ b/pkg/chart/loader_test.go @@ -0,0 +1,68 @@ +package chart + +import ( + "context" + "testing" + + "github.com/go-faker/faker/v4" + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/resource" + "github.com/siyul-park/uniflow/pkg/secret" + "github.com/siyul-park/uniflow/pkg/spec" + "github.com/stretchr/testify/assert" +) + +func TestLoader_Load(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + chartStore := NewStore() + secretStore := secret.NewStore() + + table := NewTable() + defer table.Close() + + loader := NewLoader(LoaderConfig{ + Table: table, + ChartStore: chartStore, + SecretStore: secretStore, + }) + + sec := &secret.Secret{ID: uuid.Must(uuid.NewV7())} + chrt1 := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{}, + Env: map[string][]Value{ + "key": { + { + ID: sec.GetID(), + Value: faker.Word(), + }, + }, + }, + } + chrt2 := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{ + &spec.Meta{ + Kind: chrt1.GetName(), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + }, + }, + } + + secretStore.Store(ctx, sec) + + chartStore.Store(ctx, chrt1) + chartStore.Store(ctx, chrt2) + + err := loader.Load(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.NotNil(t, table.Lookup(chrt1.GetID())) + assert.NotNil(t, table.Lookup(chrt2.GetID())) +} diff --git a/pkg/chart/table.go b/pkg/chart/table.go index 1db7e221..ba0f669d 100644 --- a/pkg/chart/table.go +++ b/pkg/chart/table.go @@ -72,6 +72,18 @@ func (t *Table) Lookup(id uuid.UUID) *Chart { return t.charts[id] } +// Keys returns all IDs of charts in the table. +func (t *Table) Keys() []uuid.UUID { + t.mu.RLock() + defer t.mu.RUnlock() + + var ids []uuid.UUID + for id := range t.charts { + ids = append(ids, id) + } + return ids +} + // Close removes all charts from the table and unloads them. func (t *Table) Close() error { t.mu.Lock() diff --git a/pkg/chart/table_test.go b/pkg/chart/table_test.go index 4c7b853e..b8fcdfc1 100644 --- a/pkg/chart/table_test.go +++ b/pkg/chart/table_test.go @@ -89,6 +89,29 @@ func TestTable_Lookup(t *testing.T) { assert.Equal(t, chrt, tb.Lookup(chrt.GetID())) } +func TestTable_Keys(t *testing.T) { + tb := NewTable() + defer tb.Close() + + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{ + &spec.Meta{ + Kind: faker.UUIDHyphenated(), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + }, + }, + } + + tb.Insert(chrt) + + ids := tb.Keys() + assert.Contains(t, ids, chrt.GetID()) +} + func TestTable_Hook(t *testing.T) { loaded := 0 unloaded := 0 From d5143bf9c48678842cd3ab7557e342e17e45f7cb Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 01:37:25 -0400 Subject: [PATCH 08/31] feat: add linker --- pkg/chart/linker.go | 137 +++++++++++++++++++++++++++++++++++++++ pkg/chart/linker_test.go | 89 +++++++++++++++++++++++++ pkg/symbol/loader.go | 3 + 3 files changed, 229 insertions(+) create mode 100644 pkg/chart/linker.go create mode 100644 pkg/chart/linker_test.go diff --git a/pkg/chart/linker.go b/pkg/chart/linker.go new file mode 100644 index 00000000..a746c9fb --- /dev/null +++ b/pkg/chart/linker.go @@ -0,0 +1,137 @@ +package chart + +import ( + "slices" + + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/hook" + "github.com/siyul-park/uniflow/pkg/node" + "github.com/siyul-park/uniflow/pkg/scheme" + "github.com/siyul-park/uniflow/pkg/spec" + "github.com/siyul-park/uniflow/pkg/symbol" + "github.com/siyul-park/uniflow/pkg/template" + "github.com/siyul-park/uniflow/pkg/types" +) + +// LinkerConfig holds the hook and scheme configuration. +type LinkerConfig struct { + Hook *hook.Hook // Manages symbol lifecycle events. + Scheme *scheme.Scheme // Defines symbol and node behavior. +} + +// Linker manages chart loading and unloading. +type Linker struct { + hook *hook.Hook + scheme *scheme.Scheme +} + +var _ LoadHook = (*Linker)(nil) +var _ UnloadHook = (*Linker)(nil) + +// NewLinker creates a new Linker. +func NewLinker(config LinkerConfig) *Linker { + return &Linker{ + hook: config.Hook, + scheme: config.Scheme, + } +} + +// Load loads the chart, creating nodes and symbols. +func (l *Linker) Load(chrt *Chart) error { + if slices.Contains(l.scheme.Kinds(), chrt.GetName()) { + return nil + } + + codec := scheme.CodecFunc(func(sp spec.Spec) (node.Node, error) { + doc, err := types.Marshal(sp) + if err != nil { + return nil, err + } + + env := map[string][]spec.Value{} + for key, vals := range chrt.GetEnv() { + for _, val := range vals { + if val.ID == uuid.Nil && val.Name == "" { + v, err := template.Execute(val.Value, types.InterfaceOf(doc)) + if err != nil { + return nil, err + } + val.Value = v + } + env[key] = append(env[key], spec.Value{Value: val.Value}) + } + } + + symbols := make([]*symbol.Symbol, 0, len(chrt.GetSpecs())) + for _, sp := range chrt.GetSpecs() { + doc, err := types.Marshal(sp) + if err != nil { + return nil, err + } + + unstructured := &spec.Unstructured{} + if err := types.Unmarshal(doc, unstructured); err != nil { + return nil, err + } + + unstructured.SetEnv(env) + + bind, err := spec.Bind(unstructured) + if err != nil { + return nil, err + } + + decode, err := l.scheme.Decode(bind) + if err != nil { + return nil, err + } + + n, err := l.scheme.Compile(decode) + if err != nil { + for _, sb := range symbols { + sb.Close() + } + return nil, err + } + + symbols = append(symbols, &symbol.Symbol{Spec: decode, Node: n}) + } + + var loadHooks []symbol.LoadHook + var unloadHook []symbol.UnloadHook + if l.hook != nil { + loadHooks = append(loadHooks, l.hook) + unloadHook = append(unloadHook, l.hook) + } + + table := symbol.NewTable(symbol.TableOption{ + LoadHooks: loadHooks, + UnloadHooks: unloadHook, + }) + + for _, sb := range symbols { + if err := table.Insert(sb); err != nil { + table.Close() + for _, sb := range symbols { + sb.Close() + } + return nil, err + } + } + + return nil, nil + }) + + l.scheme.AddKnownType(chrt.GetName(), &spec.Unstructured{}) + l.scheme.AddCodec(chrt.GetName(), codec) + + return nil +} + +// Unload removes the chart from the scheme. +func (l *Linker) Unload(chrt *Chart) error { + l.scheme.RemoveKnownType(chrt.GetName()) + l.scheme.RemoveCodec(chrt.GetName()) + + return nil +} diff --git a/pkg/chart/linker_test.go b/pkg/chart/linker_test.go new file mode 100644 index 00000000..f602adb0 --- /dev/null +++ b/pkg/chart/linker_test.go @@ -0,0 +1,89 @@ +package chart + +import ( + "testing" + + "github.com/go-faker/faker/v4" + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/node" + "github.com/siyul-park/uniflow/pkg/resource" + "github.com/siyul-park/uniflow/pkg/scheme" + "github.com/siyul-park/uniflow/pkg/secret" + "github.com/siyul-park/uniflow/pkg/spec" + "github.com/stretchr/testify/assert" +) + +func TestLinker_Load(t *testing.T) { + s := scheme.New() + kind := faker.UUIDHyphenated() + + s.AddKnownType(kind, &spec.Meta{}) + s.AddCodec(kind, scheme.CodecFunc(func(spec spec.Spec) (node.Node, error) { + return node.NewOneToOneNode(nil), nil + })) + + l := NewLinker(LinkerConfig{ + Scheme: s, + }) + + sec := &secret.Secret{ID: uuid.Must(uuid.NewV7())} + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{ + &spec.Meta{ + Kind: kind, + }, + }, + Env: map[string][]Value{ + "key1": { + { + ID: sec.GetID(), + Value: faker.Word(), + }, + }, + "key2": { + { + Value: "{{ .id }}", + }, + }, + }, + } + + meta := &spec.Meta{ + Kind: chrt.GetName(), + Namespace: resource.DefaultNamespace, + } + + err := l.Load(chrt) + assert.NoError(t, err) + assert.Contains(t, s.Kinds(), chrt.GetName()) + + _, err = s.Compile(meta) + assert.NoError(t, err) +} + +func TestLinker_Unload(t *testing.T) { + s := scheme.New() + + l := NewLinker(LinkerConfig{ + Scheme: s, + }) + + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{}, + } + + s.AddKnownType(chrt.GetName(), &spec.Meta{}) + s.AddCodec(chrt.GetName(), scheme.CodecFunc(func(spec spec.Spec) (node.Node, error) { + return node.NewOneToOneNode(nil), nil + })) + + err := l.Unload(chrt) + assert.NoError(t, err) + assert.NotContains(t, s.Kinds(), chrt.GetName()) +} diff --git a/pkg/symbol/loader.go b/pkg/symbol/loader.go index ef134c72..d65243b5 100644 --- a/pkg/symbol/loader.go +++ b/pkg/symbol/loader.go @@ -104,6 +104,9 @@ func (l *Loader) Load(ctx context.Context, specs ...spec.Spec) error { } if len(errs) > 0 { + for _, sb := range symbols { + sb.Close() + } symbols = nil } From 4fac10c30cfc03e3917993bf2ab69a469a1bd6c6 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 01:52:59 -0400 Subject: [PATCH 09/31] feat: add cluster node --- pkg/chart/cluster.go | 131 +++++++++++++++++++++++++++++++++++++++ pkg/chart/linker.go | 17 ++++- pkg/chart/linker_test.go | 9 +++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 pkg/chart/cluster.go diff --git a/pkg/chart/cluster.go b/pkg/chart/cluster.go new file mode 100644 index 00000000..51e63826 --- /dev/null +++ b/pkg/chart/cluster.go @@ -0,0 +1,131 @@ +package chart + +import ( + "sync" + + "github.com/siyul-park/uniflow/pkg/node" + "github.com/siyul-park/uniflow/pkg/port" + "github.com/siyul-park/uniflow/pkg/process" + "github.com/siyul-park/uniflow/pkg/symbol" +) + +// ClusterNode manages the ports and symbol table for the cluster. +type ClusterNode struct { + table *symbol.Table + inPorts map[string]*port.InPort + outPorts map[string]*port.OutPort + _inPorts map[string]*port.InPort + _outPorts map[string]*port.OutPort + mu sync.RWMutex +} + +var _ node.Node = (*ClusterNode)(nil) + +// NewClusterNode creates a new ClusterNode with the provided symbol table. +func NewClusterNode(table *symbol.Table) *ClusterNode { + return &ClusterNode{ + table: table, + inPorts: make(map[string]*port.InPort), + outPorts: make(map[string]*port.OutPort), + _inPorts: make(map[string]*port.InPort), + _outPorts: make(map[string]*port.OutPort), + } +} + +// Inbound sets up an input port and links it to the provided port. +func (n *ClusterNode) Inbound(name string, prt *port.InPort) { + n.mu.Lock() + defer n.mu.Unlock() + + inPort := port.NewIn() + outPort := port.NewOut() + + n.inPorts[node.PortErr] = inPort + n._outPorts[node.PortErr] = outPort + + outPort.Link(prt) + + inPort.AddListener(n.inbound(inPort, outPort)) + outPort.AddListener(n.outbound(inPort, outPort)) +} + +// Outbound sets up an output port and links it to the provided port. +func (n *ClusterNode) Outbound(name string, prt *port.OutPort) { + n.mu.Lock() + defer n.mu.Unlock() + + inPort := port.NewIn() + outPort := port.NewOut() + + n._inPorts[node.PortErr] = inPort + n.outPorts[node.PortErr] = outPort + + prt.Link(inPort) + + inPort.AddListener(n.inbound(inPort, outPort)) + outPort.AddListener(n.outbound(inPort, outPort)) +} + +// In returns the input port by name. +func (n *ClusterNode) In(name string) *port.InPort { + n.mu.RLock() + defer n.mu.RUnlock() + + return n.inPorts[name] +} + +// Out returns the output port by name. +func (n *ClusterNode) Out(name string) *port.OutPort { + n.mu.RLock() + defer n.mu.RUnlock() + + return n.outPorts[name] +} + +// Close shuts down all ports and the symbol table. +func (n *ClusterNode) Close() error { + n.mu.RLock() + defer n.mu.RUnlock() + + if err := n.table.Close(); err != nil { + return err + } + + for _, inPort := range n.inPorts { + inPort.Close() + } + for _, inPort := range n._inPorts { + inPort.Close() + } + for _, outPort := range n.outPorts { + outPort.Close() + } + for _, outPort := range n._outPorts { + outPort.Close() + } + return nil +} + +func (n *ClusterNode) inbound(inPort *port.InPort, outPort *port.OutPort) port.Listener { + return port.ListenFunc(func(proc *process.Process) { + reader := inPort.Open(proc) + writer := outPort.Open(proc) + + for inPck := range reader.Read() { + if writer.Write(inPck) == 0 { + reader.Receive(inPck) + } + } + }) +} + +func (n *ClusterNode) outbound(inPort *port.InPort, outPort *port.OutPort) port.Listener { + return port.ListenFunc(func(proc *process.Process) { + reader := inPort.Open(proc) + writer := outPort.Open(proc) + + for backPck := range writer.Receive() { + reader.Receive(backPck) + } + }) +} diff --git a/pkg/chart/linker.go b/pkg/chart/linker.go index a746c9fb..b80cc6f6 100644 --- a/pkg/chart/linker.go +++ b/pkg/chart/linker.go @@ -119,7 +119,22 @@ func (l *Linker) Load(chrt *Chart) error { } } - return nil, nil + n := NewClusterNode(table) + for name, ports := range chrt.GetPorts() { + for _, port := range ports { + for _, sb := range symbols { + if (sb.ID() == port.ID) || (sb.Name() != "" && sb.Name() == port.Name) { + if in := sb.In(port.Port); in != nil { + n.Inbound(name, in) + } + if out := sb.Out(port.Port); out != nil { + n.Outbound(name, out) + } + } + } + } + } + return n, nil }) l.scheme.AddKnownType(chrt.GetName(), &spec.Unstructured{}) diff --git a/pkg/chart/linker_test.go b/pkg/chart/linker_test.go index f602adb0..10c9f2a7 100644 --- a/pkg/chart/linker_test.go +++ b/pkg/chart/linker_test.go @@ -34,6 +34,7 @@ func TestLinker_Load(t *testing.T) { Specs: []spec.Spec{ &spec.Meta{ Kind: kind, + Name: "dummy", }, }, Env: map[string][]Value{ @@ -49,6 +50,14 @@ func TestLinker_Load(t *testing.T) { }, }, }, + Ports: map[string][]Port{ + node.PortIn: { + { + Name: "dummy", + Port: node.PortIn, + }, + }, + }, } meta := &spec.Meta{ From 4e8c890aaa9610a94eba7eb07dffd2d9eed27c56 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 02:28:47 -0400 Subject: [PATCH 10/31] refactor: divide build --- pkg/chart/chart.go | 66 +++++++++++++++++++++++++++++++++++------ pkg/chart/chart_test.go | 47 ++++++++++++++++++++++++----- pkg/chart/linker.go | 50 ++++--------------------------- pkg/chart/loader.go | 2 +- 4 files changed, 103 insertions(+), 62 deletions(-) diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go index b4016641..c50e0fa2 100644 --- a/pkg/chart/chart.go +++ b/pkg/chart/chart.go @@ -8,6 +8,7 @@ import ( "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" "github.com/siyul-park/uniflow/pkg/template" + "github.com/siyul-park/uniflow/pkg/types" ) // Chart defines the structure that combines multiple nodes into a cluster node. @@ -51,15 +52,15 @@ type Value struct { var _ resource.Resource = (*Chart)(nil) // IsBound checks whether any of the secrets are bound to the chart. -func IsBound(chrt *Chart, secrets ...*secret.Secret) bool { - for _, vals := range chrt.GetEnv() { +func (c *Chart) IsBound(secrets ...*secret.Secret) bool { + for _, vals := range c.GetEnv() { for _, val := range vals { examples := make([]*secret.Secret, 0, 2) if val.ID != uuid.Nil { examples = append(examples, &secret.Secret{ID: val.ID}) } if val.Name != "" { - examples = append(examples, &secret.Secret{Namespace: chrt.GetNamespace(), Name: val.Name}) + examples = append(examples, &secret.Secret{Namespace: c.GetNamespace(), Name: val.Name}) } for _, sec := range secrets { @@ -73,13 +74,13 @@ func IsBound(chrt *Chart, secrets ...*secret.Secret) bool { } // Bind binds the chart's environment variables to the provided secrets. -func Bind(chrt *Chart, secrets ...*secret.Secret) (*Chart, error) { - for _, vals := range chrt.GetEnv() { +func (c *Chart) Bind(secrets ...*secret.Secret) error { + for _, vals := range c.GetEnv() { for i, val := range vals { if val.ID != uuid.Nil || val.Name != "" { example := &secret.Secret{ ID: val.ID, - Namespace: chrt.GetNamespace(), + Namespace: c.GetNamespace(), Name: val.Name, } @@ -91,12 +92,12 @@ func Bind(chrt *Chart, secrets ...*secret.Secret) (*Chart, error) { } } if sec == nil { - return nil, errors.WithStack(encoding.ErrUnsupportedValue) + return errors.WithStack(encoding.ErrUnsupportedValue) } v, err := template.Execute(val.Value, sec.Data) if err != nil { - return nil, err + return err } val.ID = sec.GetID() @@ -107,7 +108,54 @@ func Bind(chrt *Chart, secrets ...*secret.Secret) (*Chart, error) { } } } - return chrt, nil + return nil +} + +// Build constructs a specs based on the given spec. +func (c *Chart) Build(sp spec.Spec) ([]spec.Spec, error) { + doc, err := types.Marshal(sp) + if err != nil { + return nil, err + } + + data := types.InterfaceOf(doc) + + env := map[string][]spec.Value{} + for key, vals := range c.GetEnv() { + for _, val := range vals { + if val.ID == uuid.Nil && val.Name == "" { + v, err := template.Execute(val.Value, data) + if err != nil { + return nil, err + } + val.Value = v + } + env[key] = append(env[key], spec.Value{Value: val.Value}) + } + } + + specs := make([]spec.Spec, 0, len(c.GetSpecs())) + for _, sp := range c.GetSpecs() { + doc, err := types.Marshal(sp) + if err != nil { + return nil, err + } + + unstructured := &spec.Unstructured{} + if err := types.Unmarshal(doc, unstructured); err != nil { + return nil, err + } + + unstructured.SetEnv(env) + + bind, err := spec.Bind(unstructured) + if err != nil { + return nil, err + } + + specs = append(specs, bind) + } + return specs, nil } // GetID returns the chart's ID. diff --git a/pkg/chart/chart_test.go b/pkg/chart/chart_test.go index 4dd2def3..8d83a5f7 100644 --- a/pkg/chart/chart_test.go +++ b/pkg/chart/chart_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestIsBound(t *testing.T) { +func TestChart_IsBound(t *testing.T) { sec1 := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), } @@ -31,11 +31,11 @@ func TestIsBound(t *testing.T) { }, } - assert.True(t, IsBound(chrt, sec1)) - assert.False(t, IsBound(chrt, sec2)) + assert.True(t, chrt.IsBound(sec1)) + assert.False(t, chrt.IsBound(sec2)) } -func TestBind(t *testing.T) { +func TestChart_Bind(t *testing.T) { sec := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), Data: "foo", @@ -53,10 +53,43 @@ func TestBind(t *testing.T) { }, } - bind, err := Bind(chrt, sec) + err := chrt.Bind(sec) assert.NoError(t, err) - assert.Equal(t, "foo", bind.GetEnv()["FOO"][0].Value) - assert.True(t, IsBound(bind, sec)) + assert.Equal(t, "foo", chrt.GetEnv()["FOO"][0].Value) +} + +func TestChart_Build(t *testing.T) { + chrt := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{ + &spec.Unstructured{ + Meta: spec.Meta{ + ID: uuid.Must(uuid.NewV7()), + Kind: faker.UUIDHyphenated(), + }, + Fields: map[string]any{ + "foo": "{{ .FOO }}", + }, + }, + }, + Env: map[string][]Value{ + "FOO": { + { + Value: "foo", + }, + }, + }, + } + + meta := &spec.Meta{ + Kind: chrt.GetName(), + Namespace: resource.DefaultNamespace, + } + + specs, err := chrt.Build(meta) + assert.NoError(t, err) + assert.Len(t, specs, 1) } func TestChart_GetSet(t *testing.T) { diff --git a/pkg/chart/linker.go b/pkg/chart/linker.go index b80cc6f6..e251d4ab 100644 --- a/pkg/chart/linker.go +++ b/pkg/chart/linker.go @@ -3,14 +3,11 @@ package chart import ( "slices" - "github.com/gofrs/uuid" "github.com/siyul-park/uniflow/pkg/hook" "github.com/siyul-park/uniflow/pkg/node" "github.com/siyul-park/uniflow/pkg/scheme" "github.com/siyul-park/uniflow/pkg/spec" "github.com/siyul-park/uniflow/pkg/symbol" - "github.com/siyul-park/uniflow/pkg/template" - "github.com/siyul-park/uniflow/pkg/types" ) // LinkerConfig holds the hook and scheme configuration. @@ -43,58 +40,21 @@ func (l *Linker) Load(chrt *Chart) error { } codec := scheme.CodecFunc(func(sp spec.Spec) (node.Node, error) { - doc, err := types.Marshal(sp) + specs, err := chrt.Build(sp) if err != nil { return nil, err } - env := map[string][]spec.Value{} - for key, vals := range chrt.GetEnv() { - for _, val := range vals { - if val.ID == uuid.Nil && val.Name == "" { - v, err := template.Execute(val.Value, types.InterfaceOf(doc)) - if err != nil { - return nil, err - } - val.Value = v - } - env[key] = append(env[key], spec.Value{Value: val.Value}) - } - } - - symbols := make([]*symbol.Symbol, 0, len(chrt.GetSpecs())) - for _, sp := range chrt.GetSpecs() { - doc, err := types.Marshal(sp) - if err != nil { - return nil, err - } - - unstructured := &spec.Unstructured{} - if err := types.Unmarshal(doc, unstructured); err != nil { - return nil, err - } - - unstructured.SetEnv(env) - - bind, err := spec.Bind(unstructured) - if err != nil { - return nil, err - } - - decode, err := l.scheme.Decode(bind) - if err != nil { - return nil, err - } - - n, err := l.scheme.Compile(decode) + symbols := make([]*symbol.Symbol, 0, len(specs)) + for _, sp := range specs { + n, err := l.scheme.Compile(sp) if err != nil { for _, sb := range symbols { sb.Close() } return nil, err } - - symbols = append(symbols, &symbol.Symbol{Spec: decode, Node: n}) + symbols = append(symbols, &symbol.Symbol{Spec: sp, Node: n}) } var loadHooks []symbol.LoadHook diff --git a/pkg/chart/loader.go b/pkg/chart/loader.go index 6bd4db1b..be26399d 100644 --- a/pkg/chart/loader.go +++ b/pkg/chart/loader.go @@ -65,7 +65,7 @@ func (l *Loader) Load(ctx context.Context, charts ...*Chart) error { var errs []error for _, chrt := range charts { - if chrt, err := Bind(chrt, secrets...); err != nil { + if err := chrt.Bind(secrets...); err != nil { errs = append(errs, err) } else if err := l.table.Insert(chrt); err != nil { errs = append(errs, err) From 739ebfe94af145d4c1b06d2dcfb6fdf8dad4d2a5 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 05:36:29 -0400 Subject: [PATCH 11/31] test: add test case --- pkg/chart/cluster.go | 8 +-- pkg/chart/cluster_test.go | 105 ++++++++++++++++++++++++++++++++++++++ pkg/chart/linker.go | 10 +++- pkg/chart/linker_test.go | 3 +- 4 files changed, 119 insertions(+), 7 deletions(-) create mode 100644 pkg/chart/cluster_test.go diff --git a/pkg/chart/cluster.go b/pkg/chart/cluster.go index 51e63826..c81315d3 100644 --- a/pkg/chart/cluster.go +++ b/pkg/chart/cluster.go @@ -40,8 +40,8 @@ func (n *ClusterNode) Inbound(name string, prt *port.InPort) { inPort := port.NewIn() outPort := port.NewOut() - n.inPorts[node.PortErr] = inPort - n._outPorts[node.PortErr] = outPort + n.inPorts[name] = inPort + n._outPorts[name] = outPort outPort.Link(prt) @@ -57,8 +57,8 @@ func (n *ClusterNode) Outbound(name string, prt *port.OutPort) { inPort := port.NewIn() outPort := port.NewOut() - n._inPorts[node.PortErr] = inPort - n.outPorts[node.PortErr] = outPort + n._inPorts[name] = inPort + n.outPorts[name] = outPort prt.Link(inPort) diff --git a/pkg/chart/cluster_test.go b/pkg/chart/cluster_test.go new file mode 100644 index 00000000..cad7c3fa --- /dev/null +++ b/pkg/chart/cluster_test.go @@ -0,0 +1,105 @@ +package chart + +import ( + "context" + "testing" + "time" + + "github.com/go-faker/faker/v4" + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/node" + "github.com/siyul-park/uniflow/pkg/packet" + "github.com/siyul-park/uniflow/pkg/port" + "github.com/siyul-park/uniflow/pkg/process" + "github.com/siyul-park/uniflow/pkg/spec" + "github.com/siyul-park/uniflow/pkg/symbol" + "github.com/siyul-park/uniflow/pkg/types" + "github.com/stretchr/testify/assert" +) + +func TestNewClusterNode(t *testing.T) { + n := NewClusterNode(symbol.NewTable()) + assert.NotNil(t, n) + assert.NoError(t, n.Close()) +} + +func TestClusterNode_Inbound(t *testing.T) { + tb := symbol.NewTable() + + sb := &symbol.Symbol{ + Spec: &spec.Meta{ + ID: uuid.Must(uuid.NewV7()), + Kind: faker.Word(), + }, + Node: node.NewOneToOneNode(nil), + } + tb.Insert(sb) + + n := NewClusterNode(tb) + defer n.Close() + + n.Inbound(node.PortIn, sb.In(node.PortIn)) + assert.NotNil(t, n.In(node.PortIn)) +} + +func TestClusterNode_Outbound(t *testing.T) { + tb := symbol.NewTable() + + sb := &symbol.Symbol{ + Spec: &spec.Meta{ + ID: uuid.Must(uuid.NewV7()), + Kind: faker.Word(), + }, + Node: node.NewOneToOneNode(nil), + } + tb.Insert(sb) + + n := NewClusterNode(tb) + defer n.Close() + + n.Outbound(node.PortOut, sb.Out(node.PortOut)) + assert.NotNil(t, n.Out(node.PortOut)) +} + +func NewClusterNode_SendAndReceive(t *testing.T) { + ctx, cancel := context.WithTimeout(context.TODO(), time.Second) + defer cancel() + + tb := symbol.NewTable() + + sb := &symbol.Symbol{ + Spec: &spec.Meta{ + ID: uuid.Must(uuid.NewV7()), + Kind: faker.Word(), + }, + Node: node.NewOneToOneNode(func(_ *process.Process, inPck *packet.Packet) (*packet.Packet, *packet.Packet) { + return inPck, nil + }), + } + tb.Insert(sb) + + n := NewClusterNode(tb) + defer n.Close() + + n.Inbound(node.PortIn, sb.In(node.PortIn)) + n.Outbound(node.PortOut, sb.Out(node.PortOut)) + + in := port.NewOut() + in.Link(n.In(node.PortIn)) + + proc := process.New() + defer proc.Exit(nil) + + inWriter := in.Open(proc) + + inPayload := types.NewString(faker.UUIDHyphenated()) + inPck := packet.New(inPayload) + + inWriter.Write(inPck) + + select { + case <-inWriter.Receive(): + case <-ctx.Done(): + assert.Fail(t, ctx.Err().Error()) + } +} diff --git a/pkg/chart/linker.go b/pkg/chart/linker.go index e251d4ab..73f9b098 100644 --- a/pkg/chart/linker.go +++ b/pkg/chart/linker.go @@ -54,7 +54,11 @@ func (l *Linker) Load(chrt *Chart) error { } return nil, err } - symbols = append(symbols, &symbol.Symbol{Spec: sp, Node: n}) + + symbols = append(symbols, &symbol.Symbol{ + Spec: sp, + Node: n, + }) } var loadHooks []symbol.LoadHook @@ -80,10 +84,11 @@ func (l *Linker) Load(chrt *Chart) error { } n := NewClusterNode(table) + for name, ports := range chrt.GetPorts() { for _, port := range ports { for _, sb := range symbols { - if (sb.ID() == port.ID) || (sb.Name() != "" && sb.Name() == port.Name) { + if sb.ID() == port.ID || sb.Name() == port.Name { if in := sb.In(port.Port); in != nil { n.Inbound(name, in) } @@ -94,6 +99,7 @@ func (l *Linker) Load(chrt *Chart) error { } } } + return n, nil }) diff --git a/pkg/chart/linker_test.go b/pkg/chart/linker_test.go index 10c9f2a7..54d56438 100644 --- a/pkg/chart/linker_test.go +++ b/pkg/chart/linker_test.go @@ -69,8 +69,9 @@ func TestLinker_Load(t *testing.T) { assert.NoError(t, err) assert.Contains(t, s.Kinds(), chrt.GetName()) - _, err = s.Compile(meta) + n, err := s.Compile(meta) assert.NoError(t, err) + assert.NotNil(t, n) } func TestLinker_Unload(t *testing.T) { From 517b6a98be85888a8c6a9654ed41a7bd6f8d222f Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 06:06:19 -0400 Subject: [PATCH 12/31] feat: link runtime --- pkg/runtime/runtime.go | 103 +++++++++++++++++++-- pkg/runtime/runtime_test.go | 177 +++++++++++++++++++++--------------- 2 files changed, 200 insertions(+), 80 deletions(-) diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index c60a0699..4c83fcca 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -5,12 +5,14 @@ import ( "sync" "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/hook" "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/scheme" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" "github.com/siyul-park/uniflow/pkg/symbol" + "golang.org/x/exp/slices" ) // Config defines configuration options for the Runtime. @@ -18,20 +20,25 @@ type Config struct { Namespace string // Namespace defines the isolated execution environment for workflows. Hook *hook.Hook // Hook is a collection of hook functions for managing symbols. Scheme *scheme.Scheme // Scheme defines the scheme and behaviors for symbols. - SpecStore spec.Store // SpecStore is responsible for persisting specifications. - SecretStore secret.Store // SecretStore is responsible for persisting secrets. + ChartStore chart.Store + SpecStore spec.Store // SpecStore is responsible for persisting specifications. + SecretStore secret.Store // SecretStore is responsible for persisting secrets. } // Runtime represents an environment for executing Workflows. type Runtime struct { namespace string scheme *scheme.Scheme + chartStore chart.Store specStore spec.Store secretStore secret.Store + chartStream chart.Stream specStream spec.Stream secretStream secret.Stream symbolTable *symbol.Table symbolLoader *symbol.Loader + chartTable *chart.Table + chartLoader *chart.Loader mu sync.RWMutex } @@ -46,6 +53,9 @@ func New(config Config) *Runtime { if config.Scheme == nil { config.Scheme = scheme.New() } + if config.ChartStore == nil { + config.ChartStore = chart.NewStore() + } if config.SpecStore == nil { config.SpecStore = spec.NewStore() } @@ -58,19 +68,44 @@ func New(config Config) *Runtime { UnloadHooks: []symbol.UnloadHook{config.Hook}, }) symbolLoader := symbol.NewLoader(symbol.LoaderConfig{ + Table: symbolTable, Scheme: config.Scheme, SpecStore: config.SpecStore, SecretStore: config.SecretStore, - Table: symbolTable, }) + chartLinker := chart.NewLinker(chart.LinkerConfig{ + Hook: config.Hook, + Scheme: config.Scheme, + }) + chartTable := chart.NewTable(chart.TableOption{ + LoadHooks: []chart.LoadHook{chartLinker}, + UnloadHooks: []chart.UnloadHook{chartLinker}, + }) + chartLoader := chart.NewLoader(chart.LoaderConfig{ + Table: chartTable, + ChartStore: config.ChartStore, + SecretStore: config.SecretStore, + }) + + for _, kind := range config.Scheme.Kinds() { + chartTable.Insert(&chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: config.Namespace, + Name: kind, + }) + } + return &Runtime{ namespace: config.Namespace, scheme: config.Scheme, + chartStore: config.ChartStore, specStore: config.SpecStore, secretStore: config.SecretStore, symbolTable: symbolTable, symbolLoader: symbolLoader, + chartTable: chartTable, + chartLoader: chartLoader, } } @@ -84,6 +119,17 @@ func (r *Runtime) Watch(ctx context.Context) error { r.mu.Lock() defer r.mu.Unlock() + if r.chartStream != nil { + if err := r.chartStream.Close(); err != nil { + return err + } + } + chartStream, err := r.chartStore.Watch(ctx, &chart.Chart{Namespace: r.namespace}) + if err != nil { + return err + } + r.chartStream = chartStream + if r.specStream != nil { if err := r.specStream.Close(); err != nil { return err @@ -113,12 +159,13 @@ func (r *Runtime) Watch(ctx context.Context) error { func (r *Runtime) Reconcile(ctx context.Context) error { r.mu.RLock() + chartStream := r.chartStream specStream := r.specStream secretStream := r.secretStream r.mu.RUnlock() - if specStream == nil || secretStream == nil { + if chartStream == nil || specStream == nil || secretStream == nil { return nil } @@ -128,6 +175,48 @@ func (r *Runtime) Reconcile(ctx context.Context) error { select { case <-ctx.Done(): return ctx.Err() + case event, ok := <-chartStream.Next(): + if !ok { + return nil + } + + charts, err := r.chartStore.Load(ctx, &chart.Chart{ID: event.ID}) + if err != nil { + return err + } + if len(charts) == 0 { + if chrt := r.chartTable.Lookup(event.ID); chrt != nil { + charts = append(charts, chrt) + } else { + charts = append(charts, &chart.Chart{ID: event.ID}) + } + } + + for _, chrt := range charts { + r.chartLoader.Load(ctx, chrt) + } + + kinds := make([]string, 0, len(charts)) + for _, chrt := range charts { + kinds = append(kinds, chrt.GetName()) + } + + for _, id := range r.symbolTable.Keys() { + sb := r.symbolTable.Lookup(id) + if sb != nil && slices.Contains(kinds, sb.Kind()) { + r.symbolTable.Free(sb.ID()) + unloaded[sb.ID()] = sb.Spec + } + } + for _, sp := range unloaded { + if slices.Contains(kinds, sp.GetKind()) { + if err := r.symbolLoader.Load(ctx, sp); err != nil { + unloaded[sp.GetID()] = sp + } else { + delete(unloaded, sp.GetID()) + } + } + } case event, ok := <-specStream.Next(): if !ok { return nil @@ -138,7 +227,9 @@ func (r *Runtime) Reconcile(ctx context.Context) error { return err } if len(specs) == 0 { - specs = append(specs, &spec.Meta{ID: event.ID}) + if sb := r.symbolTable.Lookup(event.ID); sb != nil { + specs = append(specs, sb.Spec) + } } for _, sp := range specs { @@ -165,7 +256,7 @@ func (r *Runtime) Reconcile(ctx context.Context) error { for _, id := range r.symbolTable.Keys() { sb := r.symbolTable.Lookup(id) if sb != nil && spec.IsBound(sb.Spec, secrets...) { - bounded[sb.Spec.GetID()] = sb.Spec + bounded[sb.ID()] = sb.Spec } } for _, sp := range unloaded { diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index e5da6823..3dd80f15 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -7,7 +7,7 @@ import ( "github.com/go-faker/faker/v4" "github.com/gofrs/uuid" - "github.com/siyul-park/uniflow/pkg/agent" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/hook" "github.com/siyul-park/uniflow/pkg/node" "github.com/siyul-park/uniflow/pkg/resource" @@ -30,11 +30,13 @@ func TestRuntime_Load(t *testing.T) { return node.NewOneToOneNode(nil), nil })) + chartStore := chart.NewStore() specStore := spec.NewStore() secretStore := secret.NewStore() r := New(Config{ Scheme: s, + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, }) @@ -53,7 +55,77 @@ func TestRuntime_Load(t *testing.T) { } func TestRuntime_Reconcile(t *testing.T) { - t.Run("ReconcileLoadedSpec", func(t *testing.T) { + t.Run("Chart", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + s := scheme.New() + kind := faker.UUIDHyphenated() + + chartStore := chart.NewStore() + specStore := spec.NewStore() + secretStore := secret.NewStore() + + h := hook.New() + symbols := make(chan *symbol.Symbol) + + h.AddLoadHook(symbol.LoadFunc(func(sb *symbol.Symbol) error { + symbols <- sb + return nil + })) + h.AddUnloadHook(symbol.UnloadFunc(func(sb *symbol.Symbol) error { + symbols <- sb + return nil + })) + + r := New(Config{ + Scheme: s, + Hook: h, + ChartStore: chartStore, + SpecStore: specStore, + SecretStore: secretStore, + }) + defer r.Close() + + err := r.Watch(ctx) + assert.NoError(t, err) + + go r.Reconcile(ctx) + + meta := &spec.Meta{ + ID: uuid.Must(uuid.NewV7()), + Kind: kind, + Namespace: resource.DefaultNamespace, + } + chrt := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: kind, + } + + specStore.Store(ctx, meta) + chartStore.Store(ctx, chrt) + + select { + case sb := <-symbols: + assert.Equal(t, meta.GetID(), sb.ID()) + case <-ctx.Done(): + assert.NoError(t, ctx.Err()) + return + } + + chartStore.Delete(ctx, chrt) + + select { + case sb := <-symbols: + assert.Equal(t, meta.GetID(), sb.ID()) + case <-ctx.Done(): + assert.NoError(t, ctx.Err()) + return + } + }) + + t.Run("Spec", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -65,11 +137,11 @@ func TestRuntime_Reconcile(t *testing.T) { return node.NewOneToOneNode(nil), nil })) + chartStore := chart.NewStore() specStore := spec.NewStore() secretStore := secret.NewStore() h := hook.New() - symbols := make(chan *symbol.Symbol) h.AddLoadHook(symbol.LoadFunc(func(sb *symbol.Symbol) error { @@ -84,6 +156,7 @@ func TestRuntime_Reconcile(t *testing.T) { r := New(Config{ Scheme: s, Hook: h, + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, }) @@ -94,31 +167,17 @@ func TestRuntime_Reconcile(t *testing.T) { go r.Reconcile(ctx) - sec := &secret.Secret{ - ID: uuid.Must(uuid.NewV7()), - Data: faker.Word(), - } meta := &spec.Meta{ ID: uuid.Must(uuid.NewV7()), Kind: kind, Namespace: resource.DefaultNamespace, - Env: map[string][]spec.Value{ - "key": { - { - ID: sec.GetID(), - Value: "{{ . }}", - }, - }, - }, } - secretStore.Store(ctx, sec) specStore.Store(ctx, meta) select { case sb := <-symbols: assert.Equal(t, meta.GetID(), sb.ID()) - assert.Equal(t, sec.Data, sb.Env()["key"][0].Value) case <-ctx.Done(): assert.NoError(t, ctx.Err()) return @@ -135,7 +194,7 @@ func TestRuntime_Reconcile(t *testing.T) { } }) - t.Run("ReconcileLoadedSecret", func(t *testing.T) { + t.Run("Secret", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -147,20 +206,26 @@ func TestRuntime_Reconcile(t *testing.T) { return node.NewOneToOneNode(nil), nil })) + chartStore := chart.NewStore() specStore := spec.NewStore() secretStore := secret.NewStore() h := hook.New() + symbols := make(chan *symbol.Symbol) - a := agent.New() - defer a.Close() - - h.AddLoadHook(a) - h.AddUnloadHook(a) + h.AddLoadHook(symbol.LoadFunc(func(sb *symbol.Symbol) error { + symbols <- sb + return nil + })) + h.AddUnloadHook(symbol.UnloadFunc(func(sb *symbol.Symbol) error { + symbols <- sb + return nil + })) r := New(Config{ Scheme: s, Hook: h, + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, }) @@ -191,59 +256,23 @@ func TestRuntime_Reconcile(t *testing.T) { specStore.Store(ctx, meta) secretStore.Store(ctx, sec) - func() { - for { - select { - case <-ctx.Done(): - assert.NoError(t, ctx.Err()) - return - default: - if sb := a.Symbol(meta.GetID()); sb != nil { - if sec.Data == sb.Env()["key"][0].Value { - return - } - } - - } - } - }() + select { + case sb := <-symbols: + assert.Equal(t, meta.GetID(), sb.ID()) + assert.Equal(t, sec.Data, sb.Env()["key"][0].Value) + case <-ctx.Done(): + assert.NoError(t, ctx.Err()) + return + } secretStore.Delete(ctx, sec) - func() { - for { - select { - case <-ctx.Done(): - assert.NoError(t, ctx.Err()) - return - default: - if sb := a.Symbol(meta.GetID()); sb == nil { - return - } - } - } - }() + select { + case sb := <-symbols: + assert.Equal(t, meta.GetID(), sb.ID()) + case <-ctx.Done(): + assert.NoError(t, ctx.Err()) + return + } }) } - -func BenchmarkNewRuntime(b *testing.B) { - kind := faker.UUIDHyphenated() - - s := scheme.New() - s.AddKnownType(kind, &spec.Meta{}) - s.AddCodec(kind, scheme.CodecFunc(func(spec spec.Spec) (node.Node, error) { - return node.NewOneToOneNode(nil), nil - })) - - specStore := spec.NewStore() - secretStore := secret.NewStore() - - for i := 0; i < b.N; i++ { - r := New(Config{ - Scheme: s, - SpecStore: specStore, - SecretStore: secretStore, - }) - r.Close() - } -} From 78f66164046a2183f06004c2fc295dcacb72d2d9 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 06:16:03 -0400 Subject: [PATCH 13/31] feat: support driver --- cmd/pkg/cli/apply.go | 16 +- cmd/pkg/cli/apply_test.go | 18 +- cmd/pkg/cli/delete.go | 6 +- cmd/pkg/cli/delete_test.go | 8 +- cmd/pkg/cli/get_test.go | 6 +- cmd/pkg/cli/start.go | 6 +- cmd/pkg/cli/start_test.go | 6 +- driver/mongo/pkg/chart/store.go | 298 ++++++++++++++++++++++++++ driver/mongo/pkg/chart/store_test.go | 189 ++++++++++++++++ driver/mongo/pkg/secret/store.go | 42 ++-- driver/mongo/pkg/secret/store_test.go | 47 ++-- ext/pkg/system/syscall_test.go | 20 +- pkg/chart/chart.go | 27 ++- pkg/chart/chart_test.go | 6 +- pkg/chart/linker_test.go | 4 +- pkg/chart/loader_test.go | 6 +- pkg/runtime/runtime_test.go | 10 +- pkg/secret/secret_test.go | 12 +- pkg/spec/spec.go | 16 +- pkg/spec/spec_test.go | 8 +- pkg/symbol/loader_test.go | 18 +- 21 files changed, 633 insertions(+), 136 deletions(-) create mode 100644 driver/mongo/pkg/chart/store.go create mode 100644 driver/mongo/pkg/chart/store_test.go diff --git a/cmd/pkg/cli/apply.go b/cmd/pkg/cli/apply.go index e04ba46e..08943ad8 100644 --- a/cmd/pkg/cli/apply.go +++ b/cmd/pkg/cli/apply.go @@ -97,9 +97,9 @@ func runApplyCommand(config ApplyConfig) func(cmd *cobra.Command, args []string) return err } - for _, sec := range secrets { - if sec.GetNamespace() == "" { - sec.SetNamespace(namespace) + for _, scrt := range secrets { + if scrt.GetNamespace() == "" { + scrt.SetNamespace(namespace) } } @@ -110,12 +110,12 @@ func runApplyCommand(config ApplyConfig) func(cmd *cobra.Command, args []string) var inserts []*secret.Secret var updates []*secret.Secret - for _, sec := range secrets { - if match := resourcebase.Match(sec, ok...); len(match) > 0 { - sec.SetID(match[0].GetID()) - updates = append(updates, sec) + for _, scrt := range secrets { + if match := resourcebase.Match(scrt, ok...); len(match) > 0 { + scrt.SetID(match[0].GetID()) + updates = append(updates, scrt) } else { - inserts = append(inserts, sec) + inserts = append(inserts, scrt) } } diff --git a/cmd/pkg/cli/apply_test.go b/cmd/pkg/cli/apply_test.go index bc9ff10e..24ba5ef7 100644 --- a/cmd/pkg/cli/apply_test.go +++ b/cmd/pkg/cli/apply_test.go @@ -72,13 +72,13 @@ func TestApplyCommand_Execute(t *testing.T) { filename := "secrets.json" - sec := &secret.Secret{ + scrt := &secret.Secret{ Namespace: resource.DefaultNamespace, Name: faker.UUIDHyphenated(), Data: faker.Word(), } - data, err := json.Marshal(sec) + data, err := json.Marshal(scrt) assert.NoError(t, err) file, err := fs.Create(filename) @@ -102,11 +102,11 @@ func TestApplyCommand_Execute(t *testing.T) { err = cmd.Execute() assert.NoError(t, err) - results, err := secretStore.Load(ctx, sec) + results, err := secretStore.Load(ctx, scrt) assert.NoError(t, err) assert.Len(t, results, 1) - assert.Contains(t, output.String(), sec.Name) + assert.Contains(t, output.String(), scrt.Name) }) t.Run("UpdateNodeSpec", func(t *testing.T) { @@ -163,16 +163,16 @@ func TestApplyCommand_Execute(t *testing.T) { filename := "secrets.json" - sec := &secret.Secret{ + scrt := &secret.Secret{ Namespace: resource.DefaultNamespace, Name: faker.UUIDHyphenated(), Data: faker.Word(), } - _, err := secretStore.Store(ctx, sec) + _, err := secretStore.Store(ctx, scrt) assert.NoError(t, err) - data, err := json.Marshal(sec) + data, err := json.Marshal(scrt) assert.NoError(t, err) file, err := fs.Create(filename) @@ -196,10 +196,10 @@ func TestApplyCommand_Execute(t *testing.T) { err = cmd.Execute() assert.NoError(t, err) - results, err := secretStore.Load(ctx, sec) + results, err := secretStore.Load(ctx, scrt) assert.NoError(t, err) assert.Len(t, results, 1) - assert.Contains(t, output.String(), sec.Name) + assert.Contains(t, output.String(), scrt.Name) }) } diff --git a/cmd/pkg/cli/delete.go b/cmd/pkg/cli/delete.go index 3d579a9e..519085f6 100644 --- a/cmd/pkg/cli/delete.go +++ b/cmd/pkg/cli/delete.go @@ -75,9 +75,9 @@ func runDeleteCommand(config DeleteConfig) func(cmd *cobra.Command, args []strin return err } - for _, sec := range secrets { - if sec.GetNamespace() == "" { - sec.SetNamespace(namespace) + for _, scrt := range secrets { + if scrt.GetNamespace() == "" { + scrt.SetNamespace(namespace) } } diff --git a/cmd/pkg/cli/delete_test.go b/cmd/pkg/cli/delete_test.go index 89721521..f80ea211 100644 --- a/cmd/pkg/cli/delete_test.go +++ b/cmd/pkg/cli/delete_test.go @@ -68,13 +68,13 @@ func TestDeleteCommand_Execute(t *testing.T) { filename := "secrets.json" - sec := &secret.Secret{ + scrt := &secret.Secret{ Namespace: resource.DefaultNamespace, Name: faker.UUIDHyphenated(), Data: faker.Word(), } - data, err := json.Marshal(sec) + data, err := json.Marshal(scrt) assert.NoError(t, err) file, err := fs.Create(filename) @@ -84,7 +84,7 @@ func TestDeleteCommand_Execute(t *testing.T) { _, err = file.Write(data) assert.NoError(t, err) - _, err = secretStore.Store(ctx, sec) + _, err = secretStore.Store(ctx, scrt) assert.NoError(t, err) cmd := NewDeleteCommand(DeleteConfig{ @@ -98,7 +98,7 @@ func TestDeleteCommand_Execute(t *testing.T) { err = cmd.Execute() assert.NoError(t, err) - rSecret, err := secretStore.Load(ctx, sec) + rSecret, err := secretStore.Load(ctx, scrt) assert.NoError(t, err) assert.Len(t, rSecret, 0) }) diff --git a/cmd/pkg/cli/get_test.go b/cmd/pkg/cli/get_test.go index f6dbb482..8bfb0fea 100644 --- a/cmd/pkg/cli/get_test.go +++ b/cmd/pkg/cli/get_test.go @@ -51,13 +51,13 @@ func TestGetCommand_Execute(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - sec := &secret.Secret{ + scrt := &secret.Secret{ Namespace: resource.DefaultNamespace, Name: faker.UUIDHyphenated(), Data: faker.Word(), } - _, err := secretStore.Store(ctx, sec) + _, err := secretStore.Store(ctx, scrt) assert.NoError(t, err) output := new(bytes.Buffer) @@ -73,6 +73,6 @@ func TestGetCommand_Execute(t *testing.T) { err = cmd.Execute() assert.NoError(t, err) - assert.Contains(t, output.String(), sec.Name) + assert.Contains(t, output.String(), scrt.Name) }) } diff --git a/cmd/pkg/cli/start.go b/cmd/pkg/cli/start.go index b3e5a523..52e0c448 100644 --- a/cmd/pkg/cli/start.go +++ b/cmd/pkg/cli/start.go @@ -116,9 +116,9 @@ func runStartCommand(config StartConfig) func(cmd *cobra.Command, args []string) return err } - for _, sec := range secrets { - if sec.GetNamespace() == "" { - sec.SetNamespace(namespace) + for _, scrt := range secrets { + if scrt.GetNamespace() == "" { + scrt.SetNamespace(namespace) } } diff --git a/cmd/pkg/cli/start_test.go b/cmd/pkg/cli/start_test.go index 7a650602..b13ba76f 100644 --- a/cmd/pkg/cli/start_test.go +++ b/cmd/pkg/cli/start_test.go @@ -93,13 +93,13 @@ func TestStartCommand_Execute(t *testing.T) { filename := "nodes.json" - sec := &secret.Secret{ + scrt := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), Namespace: resource.DefaultNamespace, Data: faker.Word(), } - data, _ := json.Marshal(sec) + data, _ := json.Marshal(scrt) f, _ := fs.Create(filename) f.Write(data) @@ -128,7 +128,7 @@ func TestStartCommand_Execute(t *testing.T) { assert.Fail(t, ctx.Err().Error()) return default: - if r, _ := secretStore.Load(ctx, sec); len(r) > 0 { + if r, _ := secretStore.Load(ctx, scrt); len(r) > 0 { return } } diff --git a/driver/mongo/pkg/chart/store.go b/driver/mongo/pkg/chart/store.go new file mode 100644 index 00000000..148ff833 --- /dev/null +++ b/driver/mongo/pkg/chart/store.go @@ -0,0 +1,298 @@ +package chart + +import ( + "context" + + "github.com/gofrs/uuid" + "github.com/pkg/errors" + _ "github.com/siyul-park/uniflow/driver/mongo/pkg/encoding" + "github.com/siyul-park/uniflow/pkg/chart" + "github.com/siyul-park/uniflow/pkg/resource" + "github.com/siyul-park/uniflow/pkg/secret" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// Store manages storage and retrieval of Spec objects in a MongoDB collection. +type Store struct { + collection *mongo.Collection +} + +// Stream represents a MongoDB change stream for tracking Spec changes. +type Stream struct { + stream *mongo.ChangeStream + ctx context.Context + cancel context.CancelFunc + out chan resource.Event +} + +type changeDocument struct { + OperationType string `bson:"operationType"` + DocumentKey struct { + ID uuid.UUID `bson:"_id"` + } `bson:"documentKey"` +} + +var _ chart.Store = (*Store)(nil) +var _ chart.Stream = (*Stream)(nil) + +// NewStore creates a new Store with the specified MongoDB collection. +func NewStore(collection *mongo.Collection) *Store { + return &Store{collection: collection} +} + +// Index ensures the collection has the required indexes and updates them if necessary. +func (s *Store) Index(ctx context.Context) error { + indexes := []mongo.IndexModel{ + { + Keys: bson.D{ + {Key: chart.KeyNamespace, Value: 1}, + {Key: chart.KeyName, Value: 1}, + }, + Options: options.Index().SetUnique(true).SetPartialFilterExpression(bson.M{ + chart.KeyName: bson.M{"$exists": true}, + }), + }, + } + + _, err := s.collection.Indexes().CreateMany(ctx, indexes) + return err +} + +// Watch returns a Stream that monitors changes matching the specified filter. +func (s *Store) Watch(ctx context.Context, charts ...*chart.Chart) (secret.Stream, error) { + filter := s.filter(charts...) + + opts := options.ChangeStream().SetFullDocument(options.UpdateLookup) + changeStream, err := s.collection.Watch(ctx, mongo.Pipeline{bson.D{{Key: "$match", Value: filter}}}, opts) + if err != nil { + return nil, err + } + + stream := newStream(changeStream) + + go func() { + select { + case <-ctx.Done(): + stream.Close() + case <-stream.Done(): + } + }() + + return stream, nil +} + +// Load retrieves Specs from the store that match the given criteria. +func (s *Store) Load(ctx context.Context, charts ...*chart.Chart) ([]*chart.Chart, error) { + filter := s.filter(charts...) + limit := int64(s.limit(charts...)) + + cursor, err := s.collection.Find(ctx, filter, &options.FindOptions{ + Limit: &limit, + }) + if err != nil { + return nil, err + } + defer cursor.Close(ctx) + + var result []*chart.Chart + for cursor.Next(ctx) { + chrt := &chart.Chart{} + if err := cursor.Decode(&chrt); err != nil { + return nil, err + } + result = append(result, chrt) + } + + if err := cursor.Err(); err != nil { + return nil, err + } + return result, nil +} + +// Store saves the given Specs into the database. +func (s *Store) Store(ctx context.Context, charts ...*chart.Chart) (int, error) { + var docs []any + for _, chrt := range charts { + if chrt.GetID() == uuid.Nil { + chrt.SetID(uuid.Must(uuid.NewV7())) + } + if chrt.GetNamespace() == "" { + chrt.SetNamespace(resource.DefaultNamespace) + } + + docs = append(docs, chrt) + } + + res, err := s.collection.InsertMany(ctx, docs) + if err != nil { + if mongo.IsDuplicateKeyError(err) { + return 0, errors.WithMessage(resource.ErrDuplicatedKey, err.Error()) + } + return 0, err + } + return len(res.InsertedIDs), nil +} + +// Swap updates existing Specs in the database with the provided data. +func (s *Store) Swap(ctx context.Context, charts ...*chart.Chart) (int, error) { + ids := make([]uuid.UUID, len(charts)) + for i, chrt := range charts { + if chrt.GetID() == uuid.Nil { + chrt.SetID(uuid.Must(uuid.NewV7())) + } + ids[i] = chrt.GetID() + } + + filter := bson.M{"_id": bson.M{"$in": ids}} + + cursor, err := s.collection.Find(ctx, filter) + if err != nil { + return 0, err + } + defer cursor.Close(ctx) + + ok := make(map[uuid.UUID]*chart.Chart) + for cursor.Next(ctx) { + chrt := &chart.Chart{} + if err := cursor.Decode(&chrt); err != nil { + return 0, err + } + ok[chrt.GetID()] = chrt + } + + count := 0 + for _, chrt := range charts { + exist, ok := ok[chrt.GetID()] + if !ok { + continue + } + + chrt.SetNamespace(exist.GetNamespace()) + + filter := bson.M{"_id": chrt.GetID()} + update := bson.M{"$set": chrt} + + res, err := s.collection.UpdateOne(ctx, filter, update) + if err != nil { + return 0, err + } + count += int(res.MatchedCount) + } + return count, nil +} + +// Delete removes Specs from the store based on the provided criteria. +func (s *Store) Delete(ctx context.Context, charts ...*chart.Chart) (int, error) { + filter := s.filter(charts...) + res, err := s.collection.DeleteMany(ctx, filter) + if err != nil { + return 0, err + } + return int(res.DeletedCount), nil +} + +func (s *Store) filter(charts ...*chart.Chart) bson.M { + var orFilters []bson.M + for _, v := range charts { + andFilters := bson.M{} + if v.GetID() != uuid.Nil { + andFilters["_id"] = v.GetID() + } + if v.GetNamespace() != "" { + andFilters[secret.KeyNamespace] = v.GetNamespace() + } + if v.GetName() != "" { + andFilters[secret.KeyName] = v.GetName() + } + if len(andFilters) > 0 { + orFilters = append(orFilters, andFilters) + } + } + + switch len(orFilters) { + case 0: + return bson.M{} + case 1: + return orFilters[0] + default: + return bson.M{"$or": orFilters} + } +} + +func (s *Store) limit(charts ...*chart.Chart) int { + limit := 0 + for _, v := range charts { + if v.GetID() != uuid.Nil || v.GetName() != "" { + limit += 1 + } else if v.GetNamespace() != "" { + return 0 + } + } + return limit +} + +// newStream creates and returns a new Stream. +func newStream(stream *mongo.ChangeStream) *Stream { + ctx, cancel := context.WithCancel(context.Background()) + + s := &Stream{ + stream: stream, + ctx: ctx, + cancel: cancel, + out: make(chan resource.Event), + } + + go func() { + defer close(s.out) + + for s.stream.Next(s.ctx) { + var doc changeDocument + if err := s.stream.Decode(&doc); err != nil { + continue + } + + var op resource.EventOP + switch doc.OperationType { + case "insert": + op = resource.EventStore + case "update": + op = resource.EventSwap + case "delete": + op = resource.EventDelete + default: + continue + } + + event := resource.Event{ + OP: op, + ID: doc.DocumentKey.ID, + } + + select { + case <-ctx.Done(): + return + case s.out <- event: + } + } + }() + + return s +} + +// Next returns a channel for receiving events from the stream. +func (s *Stream) Next() <-chan resource.Event { + return s.out +} + +// Done returns a channel that is closed when the stream is closed. +func (s *Stream) Done() <-chan struct{} { + return s.ctx.Done() +} + +// Close closes the stream and releases any resources. +func (s *Stream) Close() error { + s.cancel() + return nil +} diff --git a/driver/mongo/pkg/chart/store_test.go b/driver/mongo/pkg/chart/store_test.go new file mode 100644 index 00000000..f3f58a03 --- /dev/null +++ b/driver/mongo/pkg/chart/store_test.go @@ -0,0 +1,189 @@ +package chart + +import ( + "context" + "testing" + + "github.com/go-faker/faker/v4" + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/driver/mongo/pkg/server" + "github.com/siyul-park/uniflow/pkg/chart" + "github.com/siyul-park/uniflow/pkg/resource" + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func TestStore_Index(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + s := server.New() + defer server.Release(s) + + c, _ := mongo.Connect(ctx, options.Client().ApplyURI(s.URI())) + defer c.Disconnect(ctx) + + st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) + + err := st.Index(ctx) + assert.NoError(t, err) +} + +func TestStore_Watch(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + s := server.New() + defer server.Release(s) + + c, _ := mongo.Connect(ctx, options.Client().ApplyURI(s.URI())) + defer c.Disconnect(ctx) + + st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) + + stream, err := st.Watch(ctx) + assert.NoError(t, err) + assert.NotNil(t, stream) + + defer stream.Close() + + go func() { + for { + if event, ok := <-stream.Next(); ok { + assert.NotZero(t, event.ID) + } else { + return + } + } + }() + + chrt := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + } + + _, _ = st.Store(ctx, chrt) + _, _ = st.Store(ctx, chrt) + _, _ = st.Delete(ctx, chrt) +} + +func TestStore_Load(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + s := server.New() + defer server.Release(s) + + c, _ := mongo.Connect(ctx, options.Client().ApplyURI(s.URI())) + defer c.Disconnect(ctx) + + st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) + + chrt1 := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + } + chrt2 := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + } + + count, err := st.Store(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Equal(t, count, 2) + + loaded, err := st.Load(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Len(t, loaded, 2) +} + +func TestStore_Store(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + s := server.New() + defer server.Release(s) + + c, _ := mongo.Connect(ctx, options.Client().ApplyURI(s.URI())) + defer c.Disconnect(ctx) + + st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) + + chrt1 := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + } + chrt2 := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + } + + count, err := st.Store(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Equal(t, count, 2) + + loaded, err := st.Load(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Len(t, loaded, 2) +} + +func TestStore_Swap(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + s := server.New() + defer server.Release(s) + + c, _ := mongo.Connect(ctx, options.Client().ApplyURI(s.URI())) + defer c.Disconnect(ctx) + + st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) + + chrt1 := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + } + chrt2 := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + } + + count, err := st.Store(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Equal(t, count, 2) + + count, err = st.Swap(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Equal(t, count, 2) + + loaded, err := st.Load(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Len(t, loaded, 2) +} + +func TestMemStore_Delete(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + s := server.New() + defer server.Release(s) + + c, _ := mongo.Connect(ctx, options.Client().ApplyURI(s.URI())) + defer c.Disconnect(ctx) + + st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) + + chrt1 := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + } + chrt2 := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + } + + count, err := st.Store(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Equal(t, count, 2) + + count, err = st.Delete(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Equal(t, count, 2) + + loaded, err := st.Load(ctx, chrt1, chrt2) + assert.NoError(t, err) + assert.Len(t, loaded, 0) +} diff --git a/driver/mongo/pkg/secret/store.go b/driver/mongo/pkg/secret/store.go index 3a8c81cc..7c9f2f3a 100644 --- a/driver/mongo/pkg/secret/store.go +++ b/driver/mongo/pkg/secret/store.go @@ -97,11 +97,11 @@ func (s *Store) Load(ctx context.Context, secrets ...*secret.Secret) ([]*secret. var result []*secret.Secret for cursor.Next(ctx) { - sec := &secret.Secret{} - if err := cursor.Decode(&sec); err != nil { + scrt := &secret.Secret{} + if err := cursor.Decode(&scrt); err != nil { return nil, err } - result = append(result, sec) + result = append(result, scrt) } if err := cursor.Err(); err != nil { @@ -113,15 +113,15 @@ func (s *Store) Load(ctx context.Context, secrets ...*secret.Secret) ([]*secret. // Store saves the given Specs into the database. func (s *Store) Store(ctx context.Context, secrets ...*secret.Secret) (int, error) { var docs []any - for _, sec := range secrets { - if sec.GetID() == uuid.Nil { - sec.SetID(uuid.Must(uuid.NewV7())) + for _, scrt := range secrets { + if scrt.GetID() == uuid.Nil { + scrt.SetID(uuid.Must(uuid.NewV7())) } - if sec.GetNamespace() == "" { - sec.SetNamespace(resource.DefaultNamespace) + if scrt.GetNamespace() == "" { + scrt.SetNamespace(resource.DefaultNamespace) } - docs = append(docs, sec) + docs = append(docs, scrt) } res, err := s.collection.InsertMany(ctx, docs) @@ -137,11 +137,11 @@ func (s *Store) Store(ctx context.Context, secrets ...*secret.Secret) (int, erro // Swap updates existing Specs in the database with the provided data. func (s *Store) Swap(ctx context.Context, secrets ...*secret.Secret) (int, error) { ids := make([]uuid.UUID, len(secrets)) - for i, sec := range secrets { - if sec.GetID() == uuid.Nil { - sec.SetID(uuid.Must(uuid.NewV7())) + for i, scrt := range secrets { + if scrt.GetID() == uuid.Nil { + scrt.SetID(uuid.Must(uuid.NewV7())) } - ids[i] = sec.GetID() + ids[i] = scrt.GetID() } filter := bson.M{"_id": bson.M{"$in": ids}} @@ -154,24 +154,24 @@ func (s *Store) Swap(ctx context.Context, secrets ...*secret.Secret) (int, error ok := make(map[uuid.UUID]*secret.Secret) for cursor.Next(ctx) { - sec := &secret.Secret{} - if err := cursor.Decode(&sec); err != nil { + scrt := &secret.Secret{} + if err := cursor.Decode(&scrt); err != nil { return 0, err } - ok[sec.GetID()] = sec + ok[scrt.GetID()] = scrt } count := 0 - for _, sec := range secrets { - exist, ok := ok[sec.GetID()] + for _, scrt := range secrets { + exist, ok := ok[scrt.GetID()] if !ok { continue } - sec.SetNamespace(exist.GetNamespace()) + scrt.SetNamespace(exist.GetNamespace()) - filter := bson.M{"_id": sec.GetID()} - update := bson.M{"$set": sec} + filter := bson.M{"_id": scrt.GetID()} + update := bson.M{"$set": scrt} res, err := s.collection.UpdateOne(ctx, filter, update) if err != nil { diff --git a/driver/mongo/pkg/secret/store_test.go b/driver/mongo/pkg/secret/store_test.go index 4d65cbcf..b97867a0 100644 --- a/driver/mongo/pkg/secret/store_test.go +++ b/driver/mongo/pkg/secret/store_test.go @@ -58,15 +58,14 @@ func TestStore_Watch(t *testing.T) { } }() - secret := &secret.Secret{ - ID: uuid.Must(uuid.NewV7()), - + scrt := &secret.Secret{ + ID: uuid.Must(uuid.NewV7()), Namespace: resource.DefaultNamespace, } - _, _ = st.Store(ctx, secret) - _, _ = st.Store(ctx, secret) - _, _ = st.Delete(ctx, secret) + _, _ = st.Store(ctx, scrt) + _, _ = st.Store(ctx, scrt) + _, _ = st.Delete(ctx, scrt) } func TestStore_Load(t *testing.T) { @@ -81,18 +80,18 @@ func TestStore_Load(t *testing.T) { st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) - secret1 := &secret.Secret{ + scrt1 := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), } - secret2 := &secret.Secret{ + scrt2 := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), } - count, err := st.Store(ctx, secret1, secret2) + count, err := st.Store(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Equal(t, count, 2) - loaded, err := st.Load(ctx, secret1, secret2) + loaded, err := st.Load(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Len(t, loaded, 2) } @@ -109,18 +108,18 @@ func TestStore_Store(t *testing.T) { st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) - secret1 := &secret.Secret{ + scrt1 := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), } - secret2 := &secret.Secret{ + scrt2 := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), } - count, err := st.Store(ctx, secret1, secret2) + count, err := st.Store(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Equal(t, count, 2) - loaded, err := st.Load(ctx, secret1, secret2) + loaded, err := st.Load(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Len(t, loaded, 2) } @@ -137,22 +136,22 @@ func TestStore_Swap(t *testing.T) { st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) - secret1 := &secret.Secret{ + scrt1 := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), } - secret2 := &secret.Secret{ + scrt2 := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), } - count, err := st.Store(ctx, secret1, secret2) + count, err := st.Store(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Equal(t, count, 2) - count, err = st.Swap(ctx, secret1, secret2) + count, err = st.Swap(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Equal(t, count, 2) - loaded, err := st.Load(ctx, secret1, secret2) + loaded, err := st.Load(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Len(t, loaded, 2) } @@ -169,22 +168,22 @@ func TestMemStore_Delete(t *testing.T) { st := NewStore(c.Database(faker.UUIDHyphenated()).Collection(faker.UUIDHyphenated())) - secret1 := &secret.Secret{ + scrt1 := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), } - secret2 := &secret.Secret{ + scrt2 := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), } - count, err := st.Store(ctx, secret1, secret2) + count, err := st.Store(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Equal(t, count, 2) - count, err = st.Delete(ctx, secret1, secret2) + count, err = st.Delete(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Equal(t, count, 2) - loaded, err := st.Load(ctx, secret1, secret2) + loaded, err := st.Load(ctx, scrt1, scrt2) assert.NoError(t, err) assert.Len(t, loaded, 0) } diff --git a/ext/pkg/system/syscall_test.go b/ext/pkg/system/syscall_test.go index c13e3cca..b4eadfd0 100644 --- a/ext/pkg/system/syscall_test.go +++ b/ext/pkg/system/syscall_test.go @@ -182,7 +182,7 @@ func TestCreateSecrets(t *testing.T) { n, _ := NewNativeNode(CreateSecrets(st)) defer n.Close() - sec := &secret.Secret{ + scrt := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), Data: faker.Word(), } @@ -195,7 +195,7 @@ func TestCreateSecrets(t *testing.T) { inWriter := in.Open(proc) - inPayload, _ := types.Marshal(sec) + inPayload, _ := types.Marshal(scrt) inPck := packet.New(types.NewSlice(inPayload)) inWriter.Write(inPck) @@ -218,7 +218,7 @@ func TestReadSecrets(t *testing.T) { n, _ := NewNativeNode(ReadSecrets(st)) defer n.Close() - sec := &secret.Secret{ + scrt := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), Data: faker.Word(), } @@ -231,7 +231,7 @@ func TestReadSecrets(t *testing.T) { inWriter := in.Open(proc) - inPayload, _ := types.Marshal(sec) + inPayload, _ := types.Marshal(scrt) inPck := packet.New(inPayload) inWriter.Write(inPck) @@ -254,12 +254,12 @@ func TestUpdateSecrets(t *testing.T) { n, _ := NewNativeNode(UpdateSecrets(st)) defer n.Close() - sec := &secret.Secret{ + scrt := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), Data: faker.Word(), } - _, _ = st.Store(ctx, sec) + _, _ = st.Store(ctx, scrt) in := port.NewOut() in.Link(n.In(node.PortIn)) @@ -269,7 +269,7 @@ func TestUpdateSecrets(t *testing.T) { inWriter := in.Open(proc) - inPayload, _ := types.Marshal(sec) + inPayload, _ := types.Marshal(scrt) inPck := packet.New(types.NewSlice(inPayload)) inWriter.Write(inPck) @@ -292,12 +292,12 @@ func TestDeleteSecrets(t *testing.T) { n, _ := NewNativeNode(DeleteSecrets(st)) defer n.Close() - sec := &secret.Secret{ + scrt := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), Data: faker.Word(), } - _, _ = st.Store(ctx, sec) + _, _ = st.Store(ctx, scrt) in := port.NewOut() in.Link(n.In(node.PortIn)) @@ -307,7 +307,7 @@ func TestDeleteSecrets(t *testing.T) { inWriter := in.Open(proc) - inPayload, _ := types.Marshal(sec) + inPayload, _ := types.Marshal(scrt) inPck := packet.New(inPayload) inWriter.Write(inPck) diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go index c50e0fa2..5e821852 100644 --- a/pkg/chart/chart.go +++ b/pkg/chart/chart.go @@ -49,6 +49,17 @@ type Value struct { Value any `json:"value" bson:"value" yaml:"value" map:"value"` } +// Key constants for commonly used fields. +const ( + KeyID = "id" + KeyNamespace = "namespace" + KeyName = "name" + KeyAnnotations = "annotations" + KetSpecs = "specs" + KeyPorts = "ports" + KeyEnv = "env" +) + var _ resource.Resource = (*Chart)(nil) // IsBound checks whether any of the secrets are bound to the chart. @@ -63,8 +74,8 @@ func (c *Chart) IsBound(secrets ...*secret.Secret) bool { examples = append(examples, &secret.Secret{Namespace: c.GetNamespace(), Name: val.Name}) } - for _, sec := range secrets { - if len(resource.Match(sec, examples...)) > 0 { + for _, scrt := range secrets { + if len(resource.Match(scrt, examples...)) > 0 { return true } } @@ -84,24 +95,24 @@ func (c *Chart) Bind(secrets ...*secret.Secret) error { Name: val.Name, } - var sec *secret.Secret + var scrt *secret.Secret for _, s := range secrets { if len(resource.Match(s, example)) > 0 { - sec = s + scrt = s break } } - if sec == nil { + if scrt == nil { return errors.WithStack(encoding.ErrUnsupportedValue) } - v, err := template.Execute(val.Value, sec.Data) + v, err := template.Execute(val.Value, scrt.Data) if err != nil { return err } - val.ID = sec.GetID() - val.Name = sec.GetName() + val.ID = scrt.GetID() + val.Name = scrt.GetName() val.Value = v vals[i] = val diff --git a/pkg/chart/chart_test.go b/pkg/chart/chart_test.go index 8d83a5f7..9a2950f7 100644 --- a/pkg/chart/chart_test.go +++ b/pkg/chart/chart_test.go @@ -36,7 +36,7 @@ func TestChart_IsBound(t *testing.T) { } func TestChart_Bind(t *testing.T) { - sec := &secret.Secret{ + scrt := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), Data: "foo", } @@ -46,14 +46,14 @@ func TestChart_Bind(t *testing.T) { Env: map[string][]Value{ "FOO": { { - ID: sec.ID, + ID: scrt.ID, Value: "{{ . }}", }, }, }, } - err := chrt.Bind(sec) + err := chrt.Bind(scrt) assert.NoError(t, err) assert.Equal(t, "foo", chrt.GetEnv()["FOO"][0].Value) } diff --git a/pkg/chart/linker_test.go b/pkg/chart/linker_test.go index 54d56438..b52c7bad 100644 --- a/pkg/chart/linker_test.go +++ b/pkg/chart/linker_test.go @@ -26,7 +26,7 @@ func TestLinker_Load(t *testing.T) { Scheme: s, }) - sec := &secret.Secret{ID: uuid.Must(uuid.NewV7())} + scrt := &secret.Secret{ID: uuid.Must(uuid.NewV7())} chrt := &Chart{ ID: uuid.Must(uuid.NewV7()), Namespace: resource.DefaultNamespace, @@ -40,7 +40,7 @@ func TestLinker_Load(t *testing.T) { Env: map[string][]Value{ "key1": { { - ID: sec.GetID(), + ID: scrt.GetID(), Value: faker.Word(), }, }, diff --git a/pkg/chart/loader_test.go b/pkg/chart/loader_test.go index 2f850570..f49d6a1f 100644 --- a/pkg/chart/loader_test.go +++ b/pkg/chart/loader_test.go @@ -28,7 +28,7 @@ func TestLoader_Load(t *testing.T) { SecretStore: secretStore, }) - sec := &secret.Secret{ID: uuid.Must(uuid.NewV7())} + scrt := &secret.Secret{ID: uuid.Must(uuid.NewV7())} chrt1 := &Chart{ ID: uuid.Must(uuid.NewV7()), Namespace: resource.DefaultNamespace, @@ -37,7 +37,7 @@ func TestLoader_Load(t *testing.T) { Env: map[string][]Value{ "key": { { - ID: sec.GetID(), + ID: scrt.GetID(), Value: faker.Word(), }, }, @@ -56,7 +56,7 @@ func TestLoader_Load(t *testing.T) { }, } - secretStore.Store(ctx, sec) + secretStore.Store(ctx, scrt) chartStore.Store(ctx, chrt1) chartStore.Store(ctx, chrt2) diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index 3dd80f15..2d7f8396 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -235,7 +235,7 @@ func TestRuntime_Reconcile(t *testing.T) { go r.Reconcile(ctx) - sec := &secret.Secret{ + scrt := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), Data: faker.Word(), } @@ -246,7 +246,7 @@ func TestRuntime_Reconcile(t *testing.T) { Env: map[string][]spec.Value{ "key": { { - ID: sec.GetID(), + ID: scrt.GetID(), Value: "{{ . }}", }, }, @@ -254,18 +254,18 @@ func TestRuntime_Reconcile(t *testing.T) { } specStore.Store(ctx, meta) - secretStore.Store(ctx, sec) + secretStore.Store(ctx, scrt) select { case sb := <-symbols: assert.Equal(t, meta.GetID(), sb.ID()) - assert.Equal(t, sec.Data, sb.Env()["key"][0].Value) + assert.Equal(t, scrt.Data, sb.Env()["key"][0].Value) case <-ctx.Done(): assert.NoError(t, ctx.Err()) return } - secretStore.Delete(ctx, sec) + secretStore.Delete(ctx, scrt) select { case sb := <-symbols: diff --git a/pkg/secret/secret_test.go b/pkg/secret/secret_test.go index 41bb3031..d86e9590 100644 --- a/pkg/secret/secret_test.go +++ b/pkg/secret/secret_test.go @@ -9,7 +9,7 @@ import ( ) func TestSecret_GetSet(t *testing.T) { - sec := &Secret{ + scrt := &Secret{ ID: uuid.Must(uuid.NewV7()), Namespace: "default", Name: faker.Word(), @@ -17,9 +17,9 @@ func TestSecret_GetSet(t *testing.T) { Data: faker.Word(), } - assert.Equal(t, sec.ID, sec.GetID()) - assert.Equal(t, sec.Namespace, sec.GetNamespace()) - assert.Equal(t, sec.Name, sec.GetName()) - assert.Equal(t, sec.Annotations, sec.GetAnnotations()) - assert.Equal(t, sec.Data, sec.GetData()) + assert.Equal(t, scrt.ID, scrt.GetID()) + assert.Equal(t, scrt.Namespace, scrt.GetNamespace()) + assert.Equal(t, scrt.Name, scrt.GetName()) + assert.Equal(t, scrt.Annotations, scrt.GetAnnotations()) + assert.Equal(t, scrt.Data, scrt.GetData()) } diff --git a/pkg/spec/spec.go b/pkg/spec/spec.go index 134df801..17b482ac 100644 --- a/pkg/spec/spec.go +++ b/pkg/spec/spec.go @@ -95,8 +95,8 @@ func IsBound(sp Spec, secrets ...*secret.Secret) bool { examples = append(examples, &secret.Secret{Namespace: sp.GetNamespace(), Name: val.Name}) } - for _, sec := range secrets { - if len(resource.Match(sec, examples...)) > 0 { + for _, scrt := range secrets { + if len(resource.Match(scrt, examples...)) > 0 { return true } } @@ -127,24 +127,24 @@ func Bind(sp Spec, secrets ...*secret.Secret) (Spec, error) { Name: val.Name, } - var sec *secret.Secret + var scrt *secret.Secret for _, s := range secrets { if len(resource.Match(s, example)) > 0 { - sec = s + scrt = s break } } - if sec == nil { + if scrt == nil { continue } - v, err := template.Execute(val.Value, sec.Data) + v, err := template.Execute(val.Value, scrt.Data) if err != nil { return nil, err } - val.ID = sec.GetID() - val.Name = sec.GetName() + val.ID = scrt.GetID() + val.Name = scrt.GetName() val.Value = v vals[i] = val diff --git a/pkg/spec/spec_test.go b/pkg/spec/spec_test.go index 8f972687..9e8368fc 100644 --- a/pkg/spec/spec_test.go +++ b/pkg/spec/spec_test.go @@ -35,7 +35,7 @@ func TestIsBound(t *testing.T) { } func TestBind(t *testing.T) { - sec := &secret.Secret{ + scrt := &secret.Secret{ ID: uuid.Must(uuid.NewV7()), Data: "foo", } @@ -46,17 +46,17 @@ func TestBind(t *testing.T) { Env: map[string][]Value{ "FOO": { { - ID: sec.ID, + ID: scrt.ID, Value: "{{ . }}", }, }, }, } - bind, err := Bind(meta, sec) + bind, err := Bind(meta, scrt) assert.NoError(t, err) assert.Equal(t, "foo", bind.GetEnv()["FOO"][0].Value) - assert.True(t, IsBound(bind, sec)) + assert.True(t, IsBound(bind, scrt)) } func TestMeta_GetSet(t *testing.T) { diff --git a/pkg/symbol/loader_test.go b/pkg/symbol/loader_test.go index 6972c0ec..6ffa49e3 100644 --- a/pkg/symbol/loader_test.go +++ b/pkg/symbol/loader_test.go @@ -40,7 +40,7 @@ func TestLoader_Load(t *testing.T) { SecretStore: secretStore, }) - sec := &secret.Secret{ID: uuid.Must(uuid.NewV7())} + scrt := &secret.Secret{ID: uuid.Must(uuid.NewV7())} meta1 := &spec.Meta{ ID: uuid.Must(uuid.NewV7()), Kind: kind, @@ -49,7 +49,7 @@ func TestLoader_Load(t *testing.T) { Env: map[string][]spec.Value{ "key": { { - ID: sec.GetID(), + ID: scrt.GetID(), Value: faker.Word(), }, }, @@ -70,7 +70,7 @@ func TestLoader_Load(t *testing.T) { Ports: map[string][]spec.Port{node.PortIO: {{Name: meta2.GetName(), Port: node.PortIO}}}, } - secretStore.Store(ctx, sec) + secretStore.Store(ctx, scrt) specStore.Store(ctx, meta1) specStore.Store(ctx, meta2) specStore.Store(ctx, meta3) @@ -96,7 +96,7 @@ func TestLoader_Load(t *testing.T) { SecretStore: secretStore, }) - sec := &secret.Secret{ID: uuid.Must(uuid.NewV7())} + scrt := &secret.Secret{ID: uuid.Must(uuid.NewV7())} meta := &spec.Meta{ ID: uuid.Must(uuid.NewV7()), Kind: kind, @@ -104,14 +104,14 @@ func TestLoader_Load(t *testing.T) { Env: map[string][]spec.Value{ "key": { { - ID: sec.GetID(), + ID: scrt.GetID(), Value: faker.Word(), }, }, }, } - secretStore.Store(ctx, sec) + secretStore.Store(ctx, scrt) specStore.Store(ctx, meta) err := loader.Load(ctx, meta) @@ -135,7 +135,7 @@ func TestLoader_Load(t *testing.T) { SecretStore: secretStore, }) - sec := &secret.Secret{ID: uuid.Must(uuid.NewV7())} + scrt := &secret.Secret{ID: uuid.Must(uuid.NewV7())} meta := &spec.Meta{ ID: uuid.Must(uuid.NewV7()), Kind: kind, @@ -143,14 +143,14 @@ func TestLoader_Load(t *testing.T) { Env: map[string][]spec.Value{ "key": { { - ID: sec.GetID(), + ID: scrt.GetID(), Value: faker.Word(), }, }, }, } - secretStore.Store(ctx, sec) + secretStore.Store(ctx, scrt) specStore.Store(ctx, meta) err := loader.Load(ctx, meta) From 060a60b6e85f18f0dbe2864777ce18e21523c11b Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 06:36:41 -0400 Subject: [PATCH 14/31] feat: link start, apply cmd --- cmd/pkg/cli/apply.go | 40 ++++++++++- cmd/pkg/cli/apply_test.go | 134 ++++++++++++++++++++++++++++++++----- cmd/pkg/cli/arg.go | 1 + cmd/pkg/cli/flag.go | 1 + cmd/pkg/cli/start.go | 41 +++++++++++- cmd/pkg/cli/start_test.go | 58 +++++++++++++++- cmd/pkg/uniflow/main.go | 20 +++++- cmd/pkg/uniflowctl/main.go | 27 +++++--- pkg/chart/chart.go | 2 +- pkg/runtime/runtime.go | 9 +++ 10 files changed, 296 insertions(+), 37 deletions(-) diff --git a/cmd/pkg/cli/apply.go b/cmd/pkg/cli/apply.go index 08943ad8..304840b1 100644 --- a/cmd/pkg/cli/apply.go +++ b/cmd/pkg/cli/apply.go @@ -2,6 +2,7 @@ package cli import ( "github.com/siyul-park/uniflow/cmd/pkg/resource" + "github.com/siyul-park/uniflow/pkg/chart" resourcebase "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" @@ -11,6 +12,7 @@ import ( // ApplyConfig represents the configuration for the apply command. type ApplyConfig struct { + ChartStore chart.Store SpecStore spec.Store SecretStore secret.Store FS afero.Fs @@ -22,7 +24,7 @@ func NewApplyCommand(config ApplyConfig) *cobra.Command { Use: "apply", Short: "Apply resources to the specified namespace", Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), - ValidArgs: []string{argNodes, argSecrets}, + ValidArgs: []string{argCharts, argNodes, argSecrets}, RunE: runApplyCommand(config), } @@ -55,6 +57,42 @@ func runApplyCommand(config ApplyConfig) func(cmd *cobra.Command, args []string) writer := resource.NewWriter(cmd.OutOrStdout()) switch args[0] { + case argCharts: + var charts []*chart.Chart + if err := reader.Read(&charts); err != nil { + return err + } + + for _, scrt := range charts { + if scrt.GetNamespace() == "" { + scrt.SetNamespace(namespace) + } + } + + ok, err := config.ChartStore.Load(ctx, charts...) + if err != nil { + return err + } + + var inserts []*chart.Chart + var updates []*chart.Chart + for _, chrt := range charts { + if match := resourcebase.Match(chrt, ok...); len(match) > 0 { + chrt.SetID(match[0].GetID()) + updates = append(updates, chrt) + } else { + inserts = append(inserts, chrt) + } + } + + if _, err := config.ChartStore.Store(ctx, inserts...); err != nil { + return err + } + if _, err := config.ChartStore.Swap(ctx, updates...); err != nil { + return err + } + + return writer.Write(charts) case argNodes: var specs []spec.Spec if err := reader.Read(&specs); err != nil { diff --git a/cmd/pkg/cli/apply_test.go b/cmd/pkg/cli/apply_test.go index 24ba5ef7..2b328829 100644 --- a/cmd/pkg/cli/apply_test.go +++ b/cmd/pkg/cli/apply_test.go @@ -8,6 +8,8 @@ import ( "testing" "github.com/go-faker/faker/v4" + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" @@ -16,26 +18,72 @@ import ( ) func TestApplyCommand_Execute(t *testing.T) { + chartStore := chart.NewStore() specStore := spec.NewStore() secretStore := secret.NewStore() fs := afero.NewMemMapFs() - t.Run("InsertNodeSpec", func(t *testing.T) { + t.Run("InsertChart", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - filename := "nodes.json" + filename := "chart.json" - kind := faker.UUIDHyphenated() + chrt := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.Word(), + } - meta := &spec.Meta{ - Kind: kind, + data, err := json.Marshal(chrt) + assert.NoError(t, err) + + file, err := fs.Create(filename) + assert.NoError(t, err) + defer file.Close() + + _, err = file.Write(data) + assert.NoError(t, err) + + output := new(bytes.Buffer) + + cmd := NewApplyCommand(ApplyConfig{ + ChartStore: chartStore, + SpecStore: specStore, + SecretStore: secretStore, + FS: fs, + }) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs([]string{argCharts, fmt.Sprintf("--%s", flagFilename), filename}) + + err = cmd.Execute() + assert.NoError(t, err) + + results, err := chartStore.Load(ctx, chrt) + assert.NoError(t, err) + assert.Len(t, results, 1) + + assert.Contains(t, output.String(), chrt.Name) + }) + + t.Run("UpdateChart", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + filename := "chart.json" + + chrt := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), Namespace: resource.DefaultNamespace, - Name: faker.UUIDHyphenated(), + Name: faker.Word(), } - data, err := json.Marshal(meta) + _, err := chartStore.Store(ctx, chrt) + assert.NoError(t, err) + + data, err := json.Marshal(chrt) assert.NoError(t, err) file, err := fs.Create(filename) @@ -48,37 +96,40 @@ func TestApplyCommand_Execute(t *testing.T) { output := new(bytes.Buffer) cmd := NewApplyCommand(ApplyConfig{ + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, FS: fs, }) cmd.SetOut(output) cmd.SetErr(output) - cmd.SetArgs([]string{argNodes, fmt.Sprintf("--%s", flagFilename), filename}) + cmd.SetArgs([]string{argCharts, fmt.Sprintf("--%s", flagFilename), filename}) err = cmd.Execute() assert.NoError(t, err) - results, err := specStore.Load(ctx, meta) + results, err := chartStore.Load(ctx, chrt) assert.NoError(t, err) assert.Len(t, results, 1) - assert.Contains(t, output.String(), meta.Name) + assert.Contains(t, output.String(), chrt.Name) }) - t.Run("InsertSecret", func(t *testing.T) { + t.Run("InsertNodeSpec", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - filename := "secrets.json" + filename := "nodes.json" - scrt := &secret.Secret{ + kind := faker.UUIDHyphenated() + + meta := &spec.Meta{ + Kind: kind, Namespace: resource.DefaultNamespace, Name: faker.UUIDHyphenated(), - Data: faker.Word(), } - data, err := json.Marshal(scrt) + data, err := json.Marshal(meta) assert.NoError(t, err) file, err := fs.Create(filename) @@ -91,22 +142,23 @@ func TestApplyCommand_Execute(t *testing.T) { output := new(bytes.Buffer) cmd := NewApplyCommand(ApplyConfig{ + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, FS: fs, }) cmd.SetOut(output) cmd.SetErr(output) - cmd.SetArgs([]string{argSecrets, fmt.Sprintf("--%s", flagFilename), filename}) + cmd.SetArgs([]string{argNodes, fmt.Sprintf("--%s", flagFilename), filename}) err = cmd.Execute() assert.NoError(t, err) - results, err := secretStore.Load(ctx, scrt) + results, err := specStore.Load(ctx, meta) assert.NoError(t, err) assert.Len(t, results, 1) - assert.Contains(t, output.String(), scrt.Name) + assert.Contains(t, output.String(), meta.Name) }) t.Run("UpdateNodeSpec", func(t *testing.T) { @@ -139,6 +191,7 @@ func TestApplyCommand_Execute(t *testing.T) { output := new(bytes.Buffer) cmd := NewApplyCommand(ApplyConfig{ + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, FS: fs, @@ -157,6 +210,50 @@ func TestApplyCommand_Execute(t *testing.T) { assert.Contains(t, output.String(), meta.Name) }) + t.Run("InsertSecret", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + filename := "secrets.json" + + scrt := &secret.Secret{ + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Data: faker.Word(), + } + + data, err := json.Marshal(scrt) + assert.NoError(t, err) + + file, err := fs.Create(filename) + assert.NoError(t, err) + defer file.Close() + + _, err = file.Write(data) + assert.NoError(t, err) + + output := new(bytes.Buffer) + + cmd := NewApplyCommand(ApplyConfig{ + ChartStore: chartStore, + SpecStore: specStore, + SecretStore: secretStore, + FS: fs, + }) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs([]string{argSecrets, fmt.Sprintf("--%s", flagFilename), filename}) + + err = cmd.Execute() + assert.NoError(t, err) + + results, err := secretStore.Load(ctx, scrt) + assert.NoError(t, err) + assert.Len(t, results, 1) + + assert.Contains(t, output.String(), scrt.Name) + }) + t.Run("UpdateSecret", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -185,6 +282,7 @@ func TestApplyCommand_Execute(t *testing.T) { output := new(bytes.Buffer) cmd := NewApplyCommand(ApplyConfig{ + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, FS: fs, diff --git a/cmd/pkg/cli/arg.go b/cmd/pkg/cli/arg.go index ac6e3fb9..4f30c9ed 100644 --- a/cmd/pkg/cli/arg.go +++ b/cmd/pkg/cli/arg.go @@ -1,6 +1,7 @@ package cli const ( + argCharts = "charts" argNodes = "nodes" argSecrets = "secrets" ) diff --git a/cmd/pkg/cli/flag.go b/cmd/pkg/cli/flag.go index 366dae80..45bbfed0 100644 --- a/cmd/pkg/cli/flag.go +++ b/cmd/pkg/cli/flag.go @@ -4,6 +4,7 @@ const ( flagNamespace = "namespace" flagFilename = "filename" + flagFromCharts = "from-charts" flagFromNodes = "from-nodes" flagFromSecrets = "from-secrets" diff --git a/cmd/pkg/cli/start.go b/cmd/pkg/cli/start.go index 52e0c448..64aece36 100644 --- a/cmd/pkg/cli/start.go +++ b/cmd/pkg/cli/start.go @@ -8,6 +8,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/siyul-park/uniflow/cmd/pkg/resource" "github.com/siyul-park/uniflow/pkg/agent" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/hook" resourcebase "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/runtime" @@ -22,6 +23,7 @@ import ( type StartConfig struct { Scheme *scheme.Scheme Hook *hook.Hook + ChartStore chart.Store SpecStore spec.Store SecretStore secret.Store FS afero.Fs @@ -36,7 +38,8 @@ func NewStartCommand(config StartConfig) *cobra.Command { } cmd.PersistentFlags().StringP(flagNamespace, toShorthand(flagNamespace), resourcebase.DefaultNamespace, "Set the namespace for running") - cmd.PersistentFlags().String(flagFromNodes, "", "Specify the file path containing node specs") + cmd.PersistentFlags().String(flagFromCharts, "", "Specify the file path containing charts") + cmd.PersistentFlags().String(flagFromNodes, "", "Specify the file path containing specs") cmd.PersistentFlags().String(flagFromSecrets, "", "Specify the file path containing secrets") cmd.PersistentFlags().Bool(flagDebug, false, "Enable debug mode") @@ -53,6 +56,11 @@ func runStartCommand(config StartConfig) func(cmd *cobra.Command, args []string) return err } + fromCharts, err := cmd.Flags().GetString(flagFromCharts) + if err != nil { + return err + } + fromNodes, err := cmd.Flags().GetString(flagFromNodes) if err != nil { return err @@ -68,6 +76,36 @@ func runStartCommand(config StartConfig) func(cmd *cobra.Command, args []string) return err } + if fromCharts != "" { + charts, err := config.ChartStore.Load(ctx, &chart.Chart{Namespace: namespace}) + if err != nil { + return err + } + + if len(charts) == 0 { + file, err := config.FS.Open(fromCharts) + if err != nil { + return err + } + defer file.Close() + + reader := resource.NewReader(file) + if err := reader.Read(&charts); err != nil { + return err + } + + for _, chrt := range charts { + if chrt.GetNamespace() == "" { + chrt.SetNamespace(namespace) + } + } + + if _, err := config.ChartStore.Store(ctx, charts...); err != nil { + return err + } + } + } + if fromNodes != "" { specs, err := config.SpecStore.Load(ctx, &spec.Meta{Namespace: namespace}) if err != nil { @@ -144,6 +182,7 @@ func runStartCommand(config StartConfig) func(cmd *cobra.Command, args []string) Namespace: namespace, Scheme: config.Scheme, Hook: h, + ChartStore: config.ChartStore, SpecStore: config.SpecStore, SecretStore: config.SecretStore, }) diff --git a/cmd/pkg/cli/start_test.go b/cmd/pkg/cli/start_test.go index b13ba76f..30fd66d1 100644 --- a/cmd/pkg/cli/start_test.go +++ b/cmd/pkg/cli/start_test.go @@ -10,6 +10,7 @@ import ( "github.com/go-faker/faker/v4" "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/hook" "github.com/siyul-park/uniflow/pkg/node" "github.com/siyul-park/uniflow/pkg/resource" @@ -24,6 +25,7 @@ func TestStartCommand_Execute(t *testing.T) { s := scheme.New() h := hook.New() + chartStore := chart.NewStore() specStore := spec.NewStore() secretStore := secret.NewStore() @@ -38,7 +40,57 @@ func TestStartCommand_Execute(t *testing.T) { s.AddKnownType(kind, &spec.Meta{}) s.AddCodec(kind, codec) - t.Run("ExecuteFromNodes", func(t *testing.T) { + t.Run(flagFromCharts, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + filename := "charts.json" + + chrt := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.Word(), + } + + data, _ := json.Marshal(chrt) + + f, _ := fs.Create(filename) + f.Write(data) + + output := new(bytes.Buffer) + + cmd := NewStartCommand(StartConfig{ + Scheme: s, + Hook: h, + FS: fs, + ChartStore: chartStore, + SpecStore: specStore, + SecretStore: secretStore, + }) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetContext(ctx) + + cmd.SetArgs([]string{fmt.Sprintf("--%s", flagFromCharts), filename}) + + go func() { + _ = cmd.Execute() + }() + + for { + select { + case <-ctx.Done(): + assert.Fail(t, ctx.Err().Error()) + return + default: + if r, _ := chartStore.Load(ctx, chrt); len(r) > 0 { + return + } + } + } + }) + + t.Run(flagFromNodes, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -61,6 +113,7 @@ func TestStartCommand_Execute(t *testing.T) { Scheme: s, Hook: h, FS: fs, + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, }) @@ -87,7 +140,7 @@ func TestStartCommand_Execute(t *testing.T) { } }) - t.Run("ExecuteFromSecrets", func(t *testing.T) { + t.Run(flagFromSecrets, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -110,6 +163,7 @@ func TestStartCommand_Execute(t *testing.T) { Scheme: s, Hook: h, FS: fs, + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, }) diff --git a/cmd/pkg/uniflow/main.go b/cmd/pkg/uniflow/main.go index 2b7268d4..6a1d4dac 100644 --- a/cmd/pkg/uniflow/main.go +++ b/cmd/pkg/uniflow/main.go @@ -7,6 +7,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/siyul-park/uniflow/cmd/pkg/cli" + mongochart "github.com/siyul-park/uniflow/driver/mongo/pkg/chart" mongosecret "github.com/siyul-park/uniflow/driver/mongo/pkg/secret" mongoserver "github.com/siyul-park/uniflow/driver/mongo/pkg/server" mongospec "github.com/siyul-park/uniflow/driver/mongo/pkg/spec" @@ -21,6 +22,7 @@ import ( "github.com/siyul-park/uniflow/ext/pkg/language/yaml" "github.com/siyul-park/uniflow/ext/pkg/network" "github.com/siyul-park/uniflow/ext/pkg/system" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/hook" "github.com/siyul-park/uniflow/pkg/scheme" "github.com/siyul-park/uniflow/pkg/secret" @@ -36,6 +38,7 @@ const configFile = ".uniflow.toml" const ( flagDatabaseURL = "database.url" flagDatabaseName = "database.name" + flagCollectionCharts = "collection.charts" flagCollectionNodes = "collection.nodes" flagCollectionSecrets = "collection.secrets" ) @@ -43,7 +46,6 @@ const ( func init() { viper.SetConfigFile(configFile) viper.AutomaticEnv() - viper.ReadInConfig() } @@ -51,10 +53,14 @@ func main() { ctx := context.Background() databaseURL := viper.GetString(flagDatabaseURL) - databaseName := viper.GetString(flagDatabaseName) + databaseName := viper.GetString(flagCollectionCharts) + collectionCharts := viper.GetString(flagCollectionNodes) collectionNodes := viper.GetString(flagCollectionNodes) collectionSecrets := viper.GetString(flagCollectionSecrets) + if collectionCharts == "" { + collectionCharts = "charts" + } if collectionNodes == "" { collectionNodes = "nodes" } @@ -69,6 +75,7 @@ func main() { databaseURL = server.URI() } + var chartStore chart.Store var specStore spec.Store var secretStore secret.Store if strings.HasPrefix(databaseURL, "mongodb://") { @@ -81,7 +88,13 @@ func main() { } defer client.Disconnect(ctx) - collection := client.Database(databaseName).Collection(collectionNodes) + collection := client.Database(databaseName).Collection(collectionCharts) + chartStore = mongochart.NewStore(collection) + if err := chartStore.(*mongochart.Store).Index(ctx); err != nil { + log.Fatal(err) + } + + collection = client.Database(databaseName).Collection(collectionNodes) specStore = mongospec.NewStore(collection) if err := specStore.(*mongospec.Store).Index(ctx); err != nil { log.Fatal(err) @@ -145,6 +158,7 @@ func main() { cmd.AddCommand(cli.NewStartCommand(cli.StartConfig{ Scheme: scheme, Hook: hook, + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, FS: fs, diff --git a/cmd/pkg/uniflowctl/main.go b/cmd/pkg/uniflowctl/main.go index 448fb7f6..f3003d5a 100644 --- a/cmd/pkg/uniflowctl/main.go +++ b/cmd/pkg/uniflowctl/main.go @@ -7,9 +7,10 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/siyul-park/uniflow/cmd/pkg/cli" + mongochart "github.com/siyul-park/uniflow/driver/mongo/pkg/chart" mongosecret "github.com/siyul-park/uniflow/driver/mongo/pkg/secret" - mongoserver "github.com/siyul-park/uniflow/driver/mongo/pkg/server" mongospec "github.com/siyul-park/uniflow/driver/mongo/pkg/spec" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" "github.com/spf13/afero" @@ -23,6 +24,7 @@ const configFile = ".uniflow.toml" const ( flagDatabaseURL = "database.url" flagDatabaseName = "database.name" + flagCollectionCharts = "collection.charts" flagCollectionNodes = "collection.nodes" flagCollectionSecrets = "collection.secrets" ) @@ -30,7 +32,6 @@ const ( func init() { viper.SetConfigFile(configFile) viper.AutomaticEnv() - viper.ReadInConfig() } @@ -38,10 +39,14 @@ func main() { ctx := context.Background() databaseURL := viper.GetString(flagDatabaseURL) - databaseName := viper.GetString(flagDatabaseName) + databaseName := viper.GetString(flagCollectionCharts) + collectionCharts := viper.GetString(flagCollectionNodes) collectionNodes := viper.GetString(flagCollectionNodes) collectionSecrets := viper.GetString(flagCollectionSecrets) + if collectionCharts == "" { + collectionCharts = "charts" + } if collectionNodes == "" { collectionNodes = "nodes" } @@ -49,13 +54,7 @@ func main() { collectionSecrets = "secrets" } - if strings.HasPrefix(databaseURL, "memongodb://") { - server := mongoserver.New() - defer server.Stop() - - databaseURL = server.URI() - } - + var chartStore chart.Store var specStore spec.Store var secretStore secret.Store if strings.HasPrefix(databaseURL, "mongodb://") { @@ -68,7 +67,13 @@ func main() { } defer client.Disconnect(ctx) - collection := client.Database(databaseName).Collection(collectionNodes) + collection := client.Database(databaseName).Collection(collectionCharts) + chartStore = mongochart.NewStore(collection) + if err := chartStore.(*mongochart.Store).Index(ctx); err != nil { + log.Fatal(err) + } + + collection = client.Database(databaseName).Collection(collectionNodes) specStore = mongospec.NewStore(collection) if err := specStore.(*mongospec.Store).Index(ctx); err != nil { log.Fatal(err) diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go index 5e821852..d78adff1 100644 --- a/pkg/chart/chart.go +++ b/pkg/chart/chart.go @@ -22,7 +22,7 @@ type Chart struct { // Additional metadata. Annotations map[string]string `json:"annotations,omitempty" bson:"annotations,omitempty" yaml:"annotations,omitempty" map:"annotations,omitempty"` // Specifications that define the nodes and their configurations within the chart. - Specs []spec.Spec `json:"specs" bson:"specs" yaml:"specs" map:"specs"` + Specs []spec.Spec `json:"specs,omitempty" bson:"specs,omitempty" yaml:"specs,omitempty" map:"specs,omitempty"` // Node connections within the chart. Ports map[string][]Port `json:"ports,omitempty" bson:"ports,omitempty" yaml:"ports,omitempty" map:"ports,omitempty"` // Sensitive configuration data or secrets. diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 4c83fcca..5ff1cd87 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -281,6 +281,12 @@ func (r *Runtime) Close() error { r.mu.Lock() defer r.mu.Unlock() + if r.chartStream != nil { + if err := r.chartStream.Close(); err != nil { + return err + } + r.chartStream = nil + } if r.specStream != nil { if err := r.specStream.Close(); err != nil { return err @@ -294,5 +300,8 @@ func (r *Runtime) Close() error { r.secretStream = nil } + if err := r.chartTable.Close(); err != nil { + return err + } return r.symbolTable.Close() } From f192e7f9408f8c96f48d87a8ef11d063a393f1fc Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 06:42:13 -0400 Subject: [PATCH 15/31] feat: link delete and get --- cmd/pkg/cli/delete.go | 19 +++++++++++++-- cmd/pkg/cli/delete_test.go | 48 ++++++++++++++++++++++++++++++++++++++ cmd/pkg/cli/get.go | 2 ++ cmd/pkg/cli/get_test.go | 35 +++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 2 deletions(-) diff --git a/cmd/pkg/cli/delete.go b/cmd/pkg/cli/delete.go index 519085f6..6f598aee 100644 --- a/cmd/pkg/cli/delete.go +++ b/cmd/pkg/cli/delete.go @@ -2,6 +2,7 @@ package cli import ( "github.com/siyul-park/uniflow/cmd/pkg/resource" + "github.com/siyul-park/uniflow/pkg/chart" resourcebase "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" @@ -11,6 +12,7 @@ import ( // DeleteConfig represents the configuration for the delete command. type DeleteConfig struct { + ChartStore chart.Store SpecStore spec.Store SecretStore secret.Store FS afero.Fs @@ -22,7 +24,7 @@ func NewDeleteCommand(config DeleteConfig) *cobra.Command { Use: "delete", Short: "Delete resources from the specified namespace", Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), - ValidArgs: []string{argNodes, argSecrets}, + ValidArgs: []string{argCharts, argNodes, argSecrets}, RunE: runDeleteCommand(config), } @@ -54,6 +56,20 @@ func runDeleteCommand(config DeleteConfig) func(cmd *cobra.Command, args []strin reader := resource.NewReader(file) switch args[0] { + case argCharts: + var charts []*chart.Chart + if err := reader.Read(&charts); err != nil { + return err + } + + for _, chrt := range charts { + if chrt.GetNamespace() == "" { + chrt.SetNamespace(namespace) + } + } + + _, err := config.ChartStore.Delete(ctx, charts...) + return err case argNodes: var specs []spec.Spec if err := reader.Read(&specs); err != nil { @@ -68,7 +84,6 @@ func runDeleteCommand(config DeleteConfig) func(cmd *cobra.Command, args []strin _, err := config.SpecStore.Delete(ctx, specs...) return err - case argSecrets: var secrets []*secret.Secret if err := reader.Read(&secrets); err != nil { diff --git a/cmd/pkg/cli/delete_test.go b/cmd/pkg/cli/delete_test.go index f80ea211..2e3f89ee 100644 --- a/cmd/pkg/cli/delete_test.go +++ b/cmd/pkg/cli/delete_test.go @@ -7,6 +7,8 @@ import ( "testing" "github.com/go-faker/faker/v4" + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" @@ -15,10 +17,54 @@ import ( ) func TestDeleteCommand_Execute(t *testing.T) { + chartStore := chart.NewStore() specStore := spec.NewStore() secretStore := secret.NewStore() + fs := afero.NewMemMapFs() + t.Run("DeleteChart", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + filename := "chart.json" + + chrt := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.Word(), + } + + data, err := json.Marshal(chrt) + assert.NoError(t, err) + + file, err := fs.Create(filename) + assert.NoError(t, err) + defer file.Close() + + _, err = file.Write(data) + assert.NoError(t, err) + + _, err = chartStore.Store(ctx, chrt) + assert.NoError(t, err) + + cmd := NewDeleteCommand(DeleteConfig{ + ChartStore: chartStore, + SpecStore: specStore, + SecretStore: secretStore, + FS: fs, + }) + + cmd.SetArgs([]string{argCharts, fmt.Sprintf("--%s", flagFilename), filename}) + + err = cmd.Execute() + assert.NoError(t, err) + + r, err := chartStore.Load(ctx, chrt) + assert.NoError(t, err) + assert.Len(t, r, 0) + }) + t.Run("DeleteNodeSpec", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -47,6 +93,7 @@ func TestDeleteCommand_Execute(t *testing.T) { assert.NoError(t, err) cmd := NewDeleteCommand(DeleteConfig{ + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, FS: fs, @@ -88,6 +135,7 @@ func TestDeleteCommand_Execute(t *testing.T) { assert.NoError(t, err) cmd := NewDeleteCommand(DeleteConfig{ + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, FS: fs, diff --git a/cmd/pkg/cli/get.go b/cmd/pkg/cli/get.go index 9a3526b0..77ef7719 100644 --- a/cmd/pkg/cli/get.go +++ b/cmd/pkg/cli/get.go @@ -2,6 +2,7 @@ package cli import ( "github.com/siyul-park/uniflow/cmd/pkg/resource" + "github.com/siyul-park/uniflow/pkg/chart" resourcebase "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" @@ -10,6 +11,7 @@ import ( // GetConfig represents the configuration for the get command. type GetConfig struct { + ChartStore chart.Store SpecStore spec.Store SecretStore secret.Store } diff --git a/cmd/pkg/cli/get_test.go b/cmd/pkg/cli/get_test.go index 8bfb0fea..d7d3f36b 100644 --- a/cmd/pkg/cli/get_test.go +++ b/cmd/pkg/cli/get_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/go-faker/faker/v4" + "github.com/gofrs/uuid" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" @@ -13,9 +15,40 @@ import ( ) func TestGetCommand_Execute(t *testing.T) { + chartStore := chart.NewStore() specStore := spec.NewStore() secretStore := secret.NewStore() + t.Run("GetChart", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + chrt := &chart.Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.Word(), + } + + _, err := chartStore.Store(ctx, chrt) + assert.NoError(t, err) + + output := new(bytes.Buffer) + + cmd := NewGetCommand(GetConfig{ + ChartStore: chartStore, + SpecStore: specStore, + SecretStore: secretStore, + }) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs([]string{argCharts}) + + err = cmd.Execute() + assert.NoError(t, err) + + assert.Contains(t, output.String(), chrt.Name) + }) + t.Run("GetNodeSpec", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -34,6 +67,7 @@ func TestGetCommand_Execute(t *testing.T) { output := new(bytes.Buffer) cmd := NewGetCommand(GetConfig{ + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, }) @@ -63,6 +97,7 @@ func TestGetCommand_Execute(t *testing.T) { output := new(bytes.Buffer) cmd := NewGetCommand(GetConfig{ + ChartStore: chartStore, SpecStore: specStore, SecretStore: secretStore, }) From af2af58453da529b227aab3a117e562f5698af95 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 06:44:11 -0400 Subject: [PATCH 16/31] feat: link get --- cmd/pkg/cli/apply.go | 1 - cmd/pkg/cli/delete.go | 1 - cmd/pkg/cli/get.go | 18 ++++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/pkg/cli/apply.go b/cmd/pkg/cli/apply.go index 304840b1..9f88d54d 100644 --- a/cmd/pkg/cli/apply.go +++ b/cmd/pkg/cli/apply.go @@ -166,7 +166,6 @@ func runApplyCommand(config ApplyConfig) func(cmd *cobra.Command, args []string) return writer.Write(secrets) } - return nil } } diff --git a/cmd/pkg/cli/delete.go b/cmd/pkg/cli/delete.go index 6f598aee..068257c2 100644 --- a/cmd/pkg/cli/delete.go +++ b/cmd/pkg/cli/delete.go @@ -99,7 +99,6 @@ func runDeleteCommand(config DeleteConfig) func(cmd *cobra.Command, args []strin _, err := config.SecretStore.Delete(ctx, secrets...) return err } - return nil } } diff --git a/cmd/pkg/cli/get.go b/cmd/pkg/cli/get.go index 77ef7719..13f43a49 100644 --- a/cmd/pkg/cli/get.go +++ b/cmd/pkg/cli/get.go @@ -22,7 +22,7 @@ func NewGetCommand(config GetConfig) *cobra.Command { Use: "get", Short: "Get resources from the specified namespace", Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), - ValidArgs: []string{argNodes, argSecrets}, + ValidArgs: []string{argCharts, argNodes, argSecrets}, RunE: runGetCommand(config), } @@ -43,26 +43,28 @@ func runGetCommand(config GetConfig) func(cmd *cobra.Command, args []string) err writer := resource.NewWriter(cmd.OutOrStdout()) switch args[0] { + case argCharts: + charts, err := config.ChartStore.Load(ctx, &chart.Chart{Namespace: namespace}) + if err != nil { + return err + } + + return writer.Write(charts) case argNodes: - specs, err := config.SpecStore.Load(ctx, &spec.Meta{ - Namespace: namespace, - }) + specs, err := config.SpecStore.Load(ctx, &spec.Meta{Namespace: namespace}) if err != nil { return err } return writer.Write(specs) case argSecrets: - secrets, err := config.SecretStore.Load(ctx, &secret.Secret{ - Namespace: namespace, - }) + secrets, err := config.SecretStore.Load(ctx, &secret.Secret{Namespace: namespace}) if err != nil { return err } return writer.Write(secrets) } - return nil } } From dde76b345b6e8d52c8bd233e5b80e244237d3dad Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 07:19:30 -0400 Subject: [PATCH 17/31] docs: add document --- docs/getting_started.md | 96 +++++++++++++++++------------- docs/getting_started_kr.md | 76 +++++++++++++---------- docs/key_concepts.md | 119 ++++++++++++++++++++++++------------- docs/key_concepts_kr.md | 35 +++++++++++ pkg/runtime/runtime.go | 3 + 5 files changed, 217 insertions(+), 112 deletions(-) diff --git a/docs/getting_started.md b/docs/getting_started.md index 5a640ade..b78cb1f4 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -1,134 +1,150 @@ # 🚀 Getting Started -This guide provides detailed instructions on installing, configuring, and managing workflows using the [Command Line Interface (CLI)](../cmd/README.md). It covers the entire process from installation to workflow control and configuration. +This guide will walk you through installing, configuring, and managing workflows using the [Command Line Interface (CLI)](../cmd/README.md). It covers the full process, from installation to controlling and configuring workflows. ## Installing from Source -To begin, set up the [CLI](../cmd/README.md) along with the [built-in extensions](../ext/README.md). Before starting the installation, ensure that [Go 1.23](https://go.dev/doc/install) or higher is installed on your system. +First, set up the [CLI](../cmd/README.md) along with the [core extensions](../ext/README.md). Make sure your system has [Go 1.23](https://go.dev/doc/install) or a later version installed. -### Cloning the Repository +### Clone the Repository -To clone the source code, run: +To download the source code, run the following command in your terminal: ```sh git clone https://github.com/siyul-park/uniflow ``` -Navigate to the cloned directory: +Navigate to the downloaded folder: ```sh cd uniflow ``` -### Installing Dependencies and Building +### Install Dependencies and Build -To install dependencies and build the project, execute: +To install the required dependencies and build the project, run the following commands: ```sh make init make build ``` -Once the build is complete, the executable will be located in the `dist` folder. +After the build completes, the executable files will be available in the `dist` folder. ### Configuration -You can modify settings flexibly via the `.uniflow.toml` file or system environment variables. Key configuration options include: +Settings can be modified using the `.uniflow.toml` file or system environment variables. The key configuration options are: -| TOML Key | Environment Variable Key | Example | -|-----------------------|----------------------------|----------------------------| -| `database.url` | `DATABASE.URL` | `mem://` or `mongodb://` | -| `database.name` | `DATABASE.NAME` | - | -| `collection.nodes` | `COLLECTION.NODES` | `nodes` | -| `collection.secrets` | `COLLECTION.SECRETS` | `secrets` | +| TOML Key | Environment Variable Key | Example | +|----------------------|--------------------------|-----------------------------| +| `database.url` | `DATABASE.URL` | `mem://` or `mongodb://` | +| `database.name` | `DATABASE.NAME` | - | +| `collection.nodes` | `COLLECTION.NODES` | `nodes` | +| `collection.secrets` | `COLLECTION.SECRETS` | `secrets` | -If using [MongoDB](https://www.mongodb.com/), enable [Change Streams](https://www.mongodb.com/docs/manual/changeStreams/) to allow the engine to track node specifications and secret changes. This requires setting up a [Replica Set](https://www.mongodb.com/docs/manual/replication/). +If you are using [MongoDB](https://www.mongodb.com/), enable [Change Streams](https://www.mongodb.com/docs/manual/changeStreams/) to track resource changes in real time. This requires setting up a [replica set](https://www.mongodb.com/docs/manual/replication/). -## Uniflow +## Using Uniflow -`uniflow` is primarily used to start and manage runtime environments. +`uniflow` is primarily used to start and manage the runtime environment. -### Start +### Start Command -The `start` command initiates the runtime with node specifications for a specific namespace. Basic usage is as follows: +The `start` command executes all node specifications in the specified namespace. If no namespace is provided, the default namespace (`default`) is used. ```sh ./dist/uniflow start --namespace default ``` -If the namespace is empty, you can provide initial node specifications using the `--from-nodes` flag: +If the namespace is empty, you can provide an initial node specification using the `--from-nodes` flag: ```sh ./dist/uniflow start --namespace default --from-nodes examples/nodes.yaml ``` -To provide initial secrets, use the `--from-secrets` flag: +You can specify an initial secrets file with the `--from-secrets` flag: ```sh ./dist/uniflow start --namespace default --from-secrets examples/secrets.yaml ``` -This command executes all node specifications for the specified namespace. If no namespace is specified, the `default` namespace is used. +Charts can be initialized using the `--from-charts` flag: -## Uniflowctl +```sh +./dist/uniflow start --namespace default --from-charts examples/charts.yaml +``` + +## Using Uniflowctl -`uniflowctl` is used to manage node specifications and secrets within a namespace. +`uniflowctl` is a command used to manage resources within a namespace. -### Apply +### Apply Command -The `apply` command adds or updates node specifications or secrets in a namespace. Usage examples are: +The `apply` command applies the contents of a specified file to the namespace. If no namespace is specified, the `default` namespace is used. ```sh ./dist/uniflowctl apply nodes --namespace default --filename examples/nodes.yaml ``` -or +To apply secrets: ```sh ./dist/uniflowctl apply secrets --namespace default --filename examples/secrets.yaml ``` -This command applies the contents of the specified file to the namespace. If no namespace is specified, the `default` namespace is used by default. +To apply charts: + +```sh +./dist/uniflowctl apply charts --namespace default --filename examples/charts.yaml +``` -### Delete +### Delete Command -The `delete` command removes node specifications or secrets from a namespace. Usage examples are: +The `delete` command removes all resources defined in the specified file. If no namespace is specified, the `default` namespace is used. ```sh ./dist/uniflowctl delete nodes --namespace default --filename examples/nodes.yaml ``` -or +To delete secrets: ```sh ./dist/uniflowctl delete secrets --namespace default --filename examples/secrets.yaml ``` -This command removes all node specifications or secrets defined in the specified file. If no namespace is specified, the `default` namespace is used. +To delete charts: -### Get +```sh +./dist/uniflowctl delete charts --namespace default --filename examples/charts.yaml +``` + +### Get Command -The `get` command retrieves node specifications or secrets from a namespace. Usage examples are: +The `get` command retrieves all resources in the specified namespace. If no namespace is specified, the `default` namespace is used. ```sh ./dist/uniflowctl get nodes --namespace default ``` -or +To retrieve secrets: ```sh ./dist/uniflowctl get secrets --namespace default ``` -This command displays all node specifications or secrets for the specified namespace. If no namespace is specified, the `default` namespace is used. +To retrieve charts: + +```sh +./dist/uniflowctl get charts --namespace default +``` -## HTTP API Integration +## Integrating HTTP API -To modify node specifications through the HTTP API, set up a workflow that exposes this functionality. You can use the `native` node included in the [basic extensions](../ext/README.md): +To modify node specifications via the HTTP API, set up workflows accordingly. You can use the `native` node provided in the [core extensions](../ext/README.md): ```yaml kind: native opcode: nodes.create # or nodes.read, nodes.update, nodes.delete ``` -To get started, refer to the [workflow example](../examples/system.yaml). You may need to add authentication and authorization processes to this workflow as needed. Typically, such runtime control workflows are defined in the `system` namespace. +Refer to the [workflow examples](../examples/system.yaml) to get started. If needed, you can add authentication and authorization processes. These runtime control workflows are typically defined in the `system` namespace. diff --git a/docs/getting_started_kr.md b/docs/getting_started_kr.md index 103cbf1a..ef5f3c0a 100644 --- a/docs/getting_started_kr.md +++ b/docs/getting_started_kr.md @@ -1,20 +1,20 @@ # 🚀 시작하기 -이 안내서는 [명령줄 인터페이스(CLI)](../cmd/README_kr.md)를 설치하고 구성하며 워크플로우를 관리하는 방법을 자세히 설명합니다. 설치부터 워크플로우 제어 및 설정까지의 전반적인 과정을 다룹니다. +이 가이드는 [명령줄 인터페이스(CLI)](../cmd/README_kr.md)의 설치, 설정, 그리고 워크플로우 관리 방법을 쉽게 따라 할 수 있도록 설명합니다. 설치 과정부터 워크플로우의 제어 및 설정 방법까지, 필요한 모든 단계를 다룹니다. ## 소스에서 설치하기 -먼저 [기본 확장 기능](../ext/README_kr.md)과 함께 제공되는 [CLI](../cmd/README_kr.md)를 설정해야 합니다. 설치를 시작하기 전에 시스템에 [Go 1.23](https://go.dev/doc/install) 이상이 설치되어 있는지 확인하세요. +먼저 [기본 확장 기능](../ext/README_kr.md)과 함께 제공되는 [CLI](../cmd/README_kr.md)를 설정해야 합니다. 시작하기 전에, 시스템에 [Go 1.23](https://go.dev/doc/install) 이상의 버전이 설치되어 있는지 확인하세요. ### 리포지토리 클론 -소스 코드를 클론하려면 다음 명령어를 실행합니다: +소스 코드를 다운로드하려면, 터미널에서 아래 명령어를 입력하세요: ```sh git clone https://github.com/siyul-park/uniflow ``` -클론한 디렉토리로 이동합니다: +다운로드한 폴더로 이동합니다: ```sh cd uniflow @@ -22,7 +22,7 @@ cd uniflow ### 의존성 설치 및 빌드 -의존성을 설치하고 프로젝트를 빌드하려면 다음 명령어를 실행합니다: +필요한 의존성을 설치하고 프로젝트를 빌드하려면, 아래 명령어를 실행하세요: ```sh make init @@ -33,7 +33,7 @@ make build ### 설정 -`.uniflow.toml` 파일이나 시스템 환경 변수를 통해 설정을 유연하게 변경할 수 있습니다. 주요 구성 옵션은 다음과 같습니다: +설정은 `.uniflow.toml` 파일이나 시스템 환경 변수를 사용해 유연하게 변경할 수 있습니다. 주요 설정 항목은 다음과 같습니다: | TOML 키 | 환경 변수 키 | 예시 | |----------------------|-------------------------|----------------------------| @@ -42,93 +42,109 @@ make build | `collection.nodes` | `COLLECTION.NODES` | `nodes` | | `collection.secrets` | `COLLECTION.SECRETS` | `secrets` | -[MongoDB](https://www.mongodb.com/)를 사용하는 경우, 엔진이 노드 명세 및 시크릿 변경을 추적할 수 있도록 [변경 스트림](https://www.mongodb.com/docs/manual/changeStreams/)을 활성화해야 합니다. 이를 위해 [복제 세트](https://www.mongodb.com/docs/manual/replication/) 설정이 필요합니다. +만약 [MongoDB](https://www.mongodb.com/)를 사용한다면, 리소스의 변경 사항을 실시간으로 추적하기 위해 [변경 스트림](https://www.mongodb.com/docs/manual/changeStreams/)을 활성화해야 합니다. 이를 위해서는 [복제 세트](https://www.mongodb.com/docs/manual/replication/) 설정이 필요합니다. -## Uniflow +## Uniflow 사용하기 -`uniflow`는 주로 런타임 환경을 시작하고 관리하는 데 사용됩니다. +`uniflow`는 주로 런타임 환경을 시작하고 관리하는 명령어입니다. -### Start +### Start 명령어 -`start` 명령어는 특정 네임스페이스의 노드 명세로 런타임을 시작합니다. 기본 사용법은 다음과 같습니다: +`start` 명령어는 지정된 네임스페이스 내의 모든 노드 명세를 실행합니다. 네임스페이스가 지정되지 않으면 기본적으로 `default` 네임스페이스가 사용됩니다. ```sh ./dist/uniflow start --namespace default ``` -네임스페이스가 비어 있는 경우, `--from-nodes` 플래그를 사용하여 초기 노드 명세를 제공할 수 있습니다: +네임스페이스가 비어 있을 경우, 초기 노드 명세를 `--from-nodes` 플래그로 제공할 수 있습니다: ```sh ./dist/uniflow start --namespace default --from-nodes examples/nodes.yaml ``` -초기 시크릿을 제공하려면 `--from-secrets` 플래그를 사용할 수 있습니다: +초기 시크릿 파일은 `--from-secrets` 플래그로 설정할 수 있습니다: ```sh ./dist/uniflow start --namespace default --from-secrets examples/secrets.yaml ``` -이 명령어는 지정된 네임스페이스의 모든 노드 명세를 실행합니다. 네임스페이스를 지정하지 않으면 `default` 네임스페이스가 사용됩니다. +초기 차트 파일은 `--from-charts` 플래그로 제공할 수 있습니다: -## Uniflowctl +```sh +./dist/uniflow start --namespace default --from-charts examples/charts.yaml +``` + +## Uniflowctl 사용하기 -`uniflowctl`는 네임스페이스 내에서 노드 명세와 시크릿을 관리하는 데 사용됩니다. +`uniflowctl`는 네임스페이스 내에서 리소스를 관리하는 명령어입니다. -### Apply +### Apply 명령어 -`apply` 명령어는 네임스페이스에 노드 명세 또는 시크릿을 추가하거나 업데이트합니다. 사용 예시는 다음과 같습니다: +`apply` 명령어는 지정된 파일 내용을 네임스페이스에 적용합니다. 네임스페이스를 지정하지 않으면 기본적으로 `default` 네임스페이스가 사용됩니다. ```sh ./dist/uniflowctl apply nodes --namespace default --filename examples/nodes.yaml ``` -또는 +시크릿을 적용하려면: ```sh ./dist/uniflowctl apply secrets --namespace default --filename examples/secrets.yaml ``` -이 명령어는 지정된 파일의 내용을 네임스페이스에 적용합니다. 네임스페이스를 지정하지 않으면 기본적으로 `default` 네임스페이스가 사용됩니다. +차트를 적용하려면: + +```sh +./dist/uniflowctl apply charts --namespace default --filename examples/charts.yaml +``` -### Delete +### Delete 명령어 -`delete` 명령어는 네임스페이스에서 노드 명세 또는 시크릿을 제거합니다. 사용 예시는 다음과 같습니다: +`delete` 명령어는 지정된 파일에 정의된 모든 리소스를 삭제합니다. 네임스페이스를 지정하지 않으면 기본적으로 `default` 네임스페이스가 사용됩니다. ```sh ./dist/uniflowctl delete nodes --namespace default --filename examples/nodes.yaml ``` -또는 +시크릿을 삭제하려면: ```sh ./dist/uniflowctl delete secrets --namespace default --filename examples/secrets.yaml ``` -이 명령어는 지정된 파일에 정의된 모든 노드 명세 또는 시크릿을 제거합니다. 네임스페이스를 지정하지 않으면 `default` 네임스페이스가 사용됩니다. +차트를 삭제하려면: -### Get +```sh +./dist/uniflowctl delete charts --namespace default --filename examples/charts.yaml +``` + +### Get 명령어 -`get` 명령어는 네임스페이스에서 노드 명세 또는 시크릿을 조회합니다. 사용 예시는 다음과 같습니다: +`get` 명령어는 지정된 네임스페이스 내 모든 리소스를 조회합니다. 네임스페이스가 지정되지 않으면 기본적으로 `default` 네임스페이스가 사용됩니다. ```sh ./dist/uniflowctl get nodes --namespace default ``` -또는 +시크릿을 조회하려면: ```sh ./dist/uniflowctl get secrets --namespace default ``` -이 명령어는 지정된 네임스페이스의 모든 노드 명세 또는 시크릿을 표시합니다. 네임스페이스를 지정하지 않으면 `default` 네임스페이스가 사용됩니다. +차트를 조회하려면: + +```sh +./dist/uniflowctl get charts --namespace default +``` ## HTTP API 통합 -HTTP API를 통해 노드 명세를 수정하려면 해당 기능을 노출하는 워크플로우를 설정해야 합니다. 이를 위해 [기본 확장](../ext/README_kr.md)에 포함된 `native` 노드를 활용할 수 있습니다: +HTTP API를 통해 노드 명세를 수정하려면, 관련 워크플로우를 설정해야 합니다. 이를 위해 [기본 확장](../ext/README_kr.md)에 포함된 `native` 노드를 사용할 수 있습니다: ```yaml kind: native opcode: nodes.create # 또는 nodes.read, nodes.update, nodes.delete ``` -시작하려면 [워크플로우 예제](../examples/system.yaml)를 참고하세요. 필요에 따라 이 워크플로우에 인증 및 권한 부여 프로세스를 추가할 수 있습니다. 일반적으로 이러한 런타임 제어 워크플로우는 `system` 네임스페이스에 정의됩니다. +시작하려면 [워크플로우 예제](../examples/system.yaml)를 참고하세요. 필요한 경우, 인증 및 권한 관리 프로세스를 추가할 수 있습니다. 이러한 런타임 제어 워크플로우는 보통 `system` 네임스페이스에 정의됩니다. diff --git a/docs/key_concepts.md b/docs/key_concepts.md index c661d905..94b6d5bb 100644 --- a/docs/key_concepts.md +++ b/docs/key_concepts.md @@ -1,10 +1,10 @@ # 📚 Key Concepts -This guide provides detailed explanations of the key terms and concepts used in the system. +This guide provides a detailed explanation of the key terms and concepts used within the system. ## Node Specification -A node specification declaratively defines the behavior and connections of each node. The engine compiles this specification into executable nodes. +A node specification declaratively define how each node operates and connects. The engine compiles these specifications into executable nodes. ```yaml id: 01908c74-8b22-7cbf-a475-6b6bc871b01a @@ -27,18 +27,18 @@ env: ``` - `id`: A unique identifier in UUID format. UUID V7 is recommended. -- `kind`: Specifies the type of node. This example is a `listener`. Additional fields may vary based on the node type. -- `namespace`: The namespace to which the node belongs; the default is `default`. -- `name`: The name of the node, which must be unique within the namespace. -- `annotations`: Additional metadata about the node, including user-defined key-value pairs like description and version. -- `protocol`: Specifies the protocol used by the listener. This field is required for `listener` nodes. -- `port`: Specifies the port used by the listener. This field is required for `listener` nodes. -- `ports`: Defines the connection scheme of ports. `out` defines an output port named `proxy`, which connects to the `in` port of another node. -- `env`: Specifies environment variables needed by the node. Here, `PORT` is dynamically set from a secret. +- `kind`: Specifies the type of the node. Additional fields may vary based on the node type. +- `namespace`: Specifies the namespace to which the node belongs, defaulting to `default`. +- `name`: Specifies the name of the node, which must be unique within the same namespace. +- `annotations`: Additional metadata for the node, including user-defined key-value pairs such as description and version. +- `protocol`: Specifies the protocol used by the listener. This is an additional required field for nodes of the `listener` type. +- `port`: Specifies the port used by the listener. This is an additional required field for nodes of the `listener` type. +- `ports`: Defines how the ports are connected. `out` defines an output port named `proxy`, which connects to the `in` port of another node. +- `env`: Specifies the environment variables required by the node. In this case, `PORT` is dynamically set from a secret. ## Secret -A secret securely stores sensitive information needed by nodes, such as passwords and API keys. +A secret securely store sensitive information needed by nodes, such as passwords and API keys. ```yaml id: 01908c74-8b22-7cbf-a475-6b6bc871b01b @@ -51,65 +51,100 @@ data: ``` - `id`: A unique identifier in UUID format. UUID V7 is recommended. -- `namespace`: The namespace to which the secret belongs; the default is `default`. -- `name`: The name of the secret, which must be unique within the namespace. -- `annotations`: Additional metadata about the secret, including user-defined key-value pairs like description and version. -- `data`: Contains the secret data in key-value pairs. +- `namespace`: Specifies the namespace to which the secret belongs, defaulting to `default`. +- `name`: Specifies the name of the secret, which must be unique within the same namespace. +- `annotations`: Additional metadata for the secret, including user-defined key-value pairs such as description and version. +- `data`: Contains the secret data structured as key-value pairs. + +## Chart + +A chart defines a node that combines multiple node specifications to perform more complex operations. Charts are used to set up interactions between nodes. + +```yaml +id: 01908c74-8b22-7cbf-a475-6b6bc871b01b +namespace: default +name: sqlite +annotations: + version: "v1.0.0" +specs: + - kind: sql + name: sql + driver: sqlite3 + source: file::{{ .FILENAME }}:?cache=shared +ports: + in: + - name: sql + port: in + out: + - name: sql + port: out +env: + FILENAME: + value: "{{ .filename }}" +``` + +- `id`: A unique identifier in UUID format. UUID V7 is recommended. +- `namespace`: Specifies the namespace to which the chart belongs, defaulting to `default`. +- `name`: Specifies the name of the chart, which must be unique within the same namespace. This name becomes the type of the node specification. +- `annotations`: Additional metadata for the chart, including user-defined key-value pairs such as description and version. +- `specs`: Defines the node specifications that make up the chart. +- `ports`: Defines how the chart's ports connect. It specifies how external ports should connect to internal nodes. +- `env`: Specifies the environment variables required by the chart. If `id` and `name` are empty, this is used as an argument for node specifications that utilize this chart. ## Node -A node is an entity that processes data, exchanging packets through connected ports to execute workflows. Each node has an independent processing loop and communicates asynchronously with other nodes. +A node is an object that processes data, executing workflows by sending and receiving packets through connected ports. Each node has its own independent processing loop and communicates asynchronously with other nodes. -Nodes are classified based on their packet processing methods: -- `ZeroToOne`: Generates initial packets to start a workflow. -- `OneToOne`: Receives a packet from an input port, processes it, and sends it to an output port. -- `OneToMany`: Receives a packet from an input port and sends it to multiple output ports. -- `ManyToOne`: Receives packets from multiple input ports and sends a single packet to an output port. -- `Other`: Includes nodes that manage state and interactions beyond simple packet forwarding. +Nodes are classified based on how they process packets: +- `ZeroToOne`: A node that generates an initial packet to start the workflow. +- `OneToOne`: A node that receives packets from an input port, processes them, and sends them to an output port. +- `OneToMany`: A node that receives packets from an input port and sends them to multiple output ports. +- `ManyToOne`: A node that receives packets from multiple input ports and sends them to a single output port. +- `Other`: A node that includes state management and interaction beyond simple packet forwarding. ## Port -Ports are connection points for exchanging packets between nodes. There are two types of ports: `InPort` and `OutPort`. Packets sent to a port are delivered to all connected ports. +Ports are connection points for sending and receiving packets between nodes. There are two types of ports: `InPort` and `OutPort`, and packets are transmitted by connecting them. A packet sent to one port is forwarded to all connected ports. -Common port names include: -- `init`: A special port used to initialize a node. When the node becomes available, workflows connected to the `init` port are executed. -- `io`: Processes and immediately returns packets. -- `in`: Receives packets for processing and sends results to `out` or `error`. If `out` or `error` ports are not connected, the result is returned directly. -- `out`: Sends processed packets. The results can be sent back to another `in` port. -- `error`: Sends packets containing error information. Error handling results can be sent back to an `in` port. +Commonly used port names include: +- `init`: A special port used to initialize nodes. When the node becomes available, the workflow connected to the `init` port executes. +- `io`: Processes packets and returns them immediately. +- `in`: Receives packets for processing and sends the results to `out` or `error`. If there are no connected `out` or `error` ports, the result is returned directly. +- `out`: Sends processed packets. The transmitted result can be sent to other `in` ports. +- `error`: Sends errors encountered during packet processing. Error handling results can be sent back to an `in` port. -When multiple ports with the same role are needed, they can be expressed as `in[0]`, `in[1]`, `out[0]`, `out[1]`, etc. +When multiple ports with the same role are needed, they are expressed as `in[0]`, `in[1]`, `out[0]`, `out[1]`, etc. ## Packet -A packet is a unit of data exchanged between ports. Each packet contains a payload, which nodes process and transmit. +A packet is a unit of data exchanged between ports. Each packet includes a payload, which the node processes before transmission. -Nodes must return response packets in the order of received request packets. When connected to multiple ports, all response packets are collected and returned as a new response packet. +Nodes must return response packets in the order of the request packets. If connected to multiple ports, all response packets are gathered and returned as a single new response packet. -A special `None` packet indicates no response, simply acknowledging the packet's acceptance. +A special `None` packet indicates that there is no response, merely indicating that the packet was accepted. ## Process -A process is the fundamental unit of execution, managed independently. Processes can have parent processes, and when a parent process terminates, its child processes also terminate. +A process is the basic unit of execution, managed independently. A process may have a parent process, and when the parent terminates, the child process also terminates. -Processes have their own storage to retain values that are difficult to transmit via packets. This storage operates using a Copy-On-Write (COW) mechanism to efficiently share data from parent processes. +Processes maintain their own storage to store values that are difficult to transmit as packets. This storage operates on a Copy-On-Write (COW) basis, efficiently sharing data from the parent process. -A new workflow begins by creating a process. When a process ends, all resources used by it are released. +A new workflow is initiated by creating a process. When the process terminates, all resources used are released. -Processes can have multiple root packets, but root packets must originate from the same node. If they come from different nodes, a new child process must be created to handle them. +Processes can have two or more root packets, but the root packets must be generated by the same node. If generated by different nodes, a new child process must be created to handle them. ## Workflow -A workflow is defined as a directed graph with multiple interconnected nodes. Each node processes data, and packets flow between nodes. +A workflow is defined as a directed graph, a structure where multiple nodes are interconnected. In this graph, each node is responsible for data processing, and packets are transmitted between nodes. -Workflows consist of multiple stages, with data processed and transmitted according to defined rules at each stage. Data can be processed sequentially or in parallel during this process. +Workflows consist of multiple stages, where data is processed and passed according to defined rules. In this process, data can be processed sequentially or in parallel. -For example, given initial data, it is processed by the first node and then forwarded to the next node. Each node receives input, processes it, and sends the processed result to the next stage. +For example, given initial data, it is processed by the first node before being passed to the next node. Each node receives input, processes it, and sends the result to the next stage. ## Namespace -A namespace manages workflows in isolation, providing an independent execution environment. Each namespace can contain multiple workflows, and nodes within a namespace cannot reference nodes from another namespace. Each namespace independently manages its data and resources. +A namespace serves to isolate and manage workflows, offering independent execution environments. It can house multiple workflows, ensuring that nodes within a namespace cannot reference nodes from other namespaces. Each namespace independently manages its own data and resources, allowing for streamlined operations and clear boundaries between workflows. ## Runtime Environment -A runtime environment is an independent space where each namespace is executed. The engine loads all nodes within the namespace to build the environment and execute workflows. This prevents conflicts during workflow execution and ensures a stable execution environment. +The runtime environment provides an independent execution context for each namespace, ensuring that workflows within namespaces operate without interference. This environment includes the necessary resources and configurations for the execution of nodes and processes, facilitating the smooth operation of workflows within the system. diff --git a/docs/key_concepts_kr.md b/docs/key_concepts_kr.md index 5380e595..1080849c 100644 --- a/docs/key_concepts_kr.md +++ b/docs/key_concepts_kr.md @@ -56,6 +56,41 @@ data: - `annotations`: 시크릿에 대한 추가 메타데이터입니다. 설명, 버전 등 사용자 정의 키-값 쌍을 포함할 수 있습니다. - `data`: 키-값 쌍으로 구성된 시크릿 데이터를 포함합니다. +## 차트 + +차트는 여러 개의 노드 명세를 결합하여 더 복잡한 동작을 수행하는 노드를 정의합니다. 차트는 노드들 간의 상호 작용을 설정하는 데 사용됩니다. + +```yaml +id: 01908c74-8b22-7cbf-a475-6b6bc871b01b +namespace: default +name: sqlite +annotations: + version: "v1.0.0" +specs: + - kind: sql + name: sql + driver: sqlite3 + source: file::{{ .FILENAME }}:?cache=shared +ports: + in: + - name: sql + port: in + out: + - name: sql + port: out +env: + FILENAME: + value: "{{ .filename }}" +``` + +- `id`: UUID 형식의 고유 식별자입니다. UUID V7을 권장합니다. +- `namespace`: 차트가 속한 네임스페이스를 지정하며, 기본값은 `default`입니다. +- `name`: 차트의 이름을 지정하며, 동일한 네임스페이스 내에서 고유해야 합니다. 이 이름이 노드 명세의 유형이 됩니다. +- `annotations`: 차트에 대한 추가 메타데이터입니다. 설명, 버전 등 사용자 정의 키-값 쌍을 포함할 수 있습니다. +- `specs`: 차트를 구성하는 노드 명세를 정의합니다. +- `ports`: 차트의 연결 방식을 정의합니다. 외부로 노출될 포트가 내부의 어떤 노드와 연결되어야 하는지 정의합니다. +- `env`: 차트에 필요한 환경 변수를 지정합니다. `id`와 `name`이 비어져 있다면 이 차트를 활용하는 노드 명세가 값을 평가하기 위한 인자로 사용됩니다. + ## 노드 노드는 데이터를 처리하는 객체로, 서로 연결된 포트를 통해 패킷을 주고받으며 워크플로우를 실행합니다. 각 노드는 독립적인 처리 루프를 가지며, 비동기적으로 다른 노드와 통신합니다. diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 5ff1cd87..2ba125e9 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -111,6 +111,9 @@ func New(config Config) *Runtime { // Load loads symbols from the spec store into the symbol table. func (r *Runtime) Load(ctx context.Context) error { + if err := r.chartLoader.Load(ctx, &chart.Chart{Namespace: r.namespace}); err != nil { + return err + } return r.symbolLoader.Load(ctx, &spec.Meta{Namespace: r.namespace}) } From 87b76c97aff544d5e387b595879fd77090f4504a Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 07:23:09 -0400 Subject: [PATCH 18/31] docs: more readable --- cmd/README.md | 3 ++- cmd/README_kr.md | 3 ++- docs/getting_started.md | 1 + docs/getting_started_kr.md | 1 + docs/key_concepts.md | 2 +- docs/key_concepts_kr.md | 2 +- 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/README.md b/cmd/README.md index 9fb6e2db..44c6ee49 100644 --- a/cmd/README.md +++ b/cmd/README.md @@ -10,7 +10,8 @@ Before running commands, configure your system using environment variables. You |-----------------------|----------------------------|----------------------------| | `database.url` | `DATABASE.URL` | `mem://` or `mongodb://` | | `database.name` | `DATABASE.NAME` | - | +| `collection.charts` | `COLLECTION.CHARTS` | `charts` | | `collection.nodes` | `COLLECTION.NODES` | `nodes` | | `collection.secrets` | `COLLECTION.SECRETS` | `secrets` | -If using [MongoDB](https://www.mongodb.com/), ensure that [Change Streams](https://www.mongodb.com/docs/manual/changeStreams/) are enabled so that the engine can track changes to node specifications. To utilize Change Streams, set up a [Replica Set](https://www.mongodb.com/docs/manual/replication/#std-label-replication). +If using [MongoDB](https://www.mongodb.com/), ensure that [Change Streams](https://www.mongodb.com/docs/manual/changeStreams/) are enabled so that the engine can track changes to resources. To utilize Change Streams, set up a [Replica Set](https://www.mongodb.com/docs/manual/replication/#std-label-replication). diff --git a/cmd/README_kr.md b/cmd/README_kr.md index 3540fe3d..1f0f3ddd 100644 --- a/cmd/README_kr.md +++ b/cmd/README_kr.md @@ -10,7 +10,8 @@ |----------------------|--------------------------|----------------------------| | `database.url` | `DATABASE.URL` | `mem://` 또는 `mongodb://` | | `database.name` | `DATABASE.NAME` | - | +| `collection.charts` | `COLLECTION.CHARTS` | `charts` | | `collection.nodes` | `COLLECTION.NODES` | `nodes` | | `collection.secrets` | `COLLECTION.SECRETS` | `secrets` | -[MongoDB](https://www.mongodb.com/)를 사용할 경우, 엔진이 노드 명세의 변경을 추적할 수 있도록 [변경 스트림](https://www.mongodb.com/docs/manual/changeStreams/)을 활성화해야 합니다. 변경 스트림을 이용하려면 [복제본 세트](https://www.mongodb.com/ko-kr/docs/manual/replication/#std-label-replication)를 설정하세요. +[MongoDB](https://www.mongodb.com/)를 사용할 경우, 엔진이 리소스의 변경을 추적할 수 있도록 [변경 스트림](https://www.mongodb.com/docs/manual/changeStreams/)을 활성화해야 합니다. 변경 스트림을 이용하려면 [복제본 세트](https://www.mongodb.com/ko-kr/docs/manual/replication/#std-label-replication)를 설정하세요. diff --git a/docs/getting_started.md b/docs/getting_started.md index b78cb1f4..aaf59765 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -39,6 +39,7 @@ Settings can be modified using the `.uniflow.toml` file or system environment va |----------------------|--------------------------|-----------------------------| | `database.url` | `DATABASE.URL` | `mem://` or `mongodb://` | | `database.name` | `DATABASE.NAME` | - | +| `collection.charts` | `COLLECTION.CHARTS` | `charts` | | `collection.nodes` | `COLLECTION.NODES` | `nodes` | | `collection.secrets` | `COLLECTION.SECRETS` | `secrets` | diff --git a/docs/getting_started_kr.md b/docs/getting_started_kr.md index ef5f3c0a..c70530a1 100644 --- a/docs/getting_started_kr.md +++ b/docs/getting_started_kr.md @@ -39,6 +39,7 @@ make build |----------------------|-------------------------|----------------------------| | `database.url` | `DATABASE.URL` | `mem://` 또는 `mongodb://` | | `database.name` | `DATABASE.NAME` | - | +| `collection.charts` | `COLLECTION.CHARTS` | `charts` | | `collection.nodes` | `COLLECTION.NODES` | `nodes` | | `collection.secrets` | `COLLECTION.SECRETS` | `secrets` | diff --git a/docs/key_concepts.md b/docs/key_concepts.md index 94b6d5bb..ac8384cf 100644 --- a/docs/key_concepts.md +++ b/docs/key_concepts.md @@ -58,7 +58,7 @@ data: ## Chart -A chart defines a node that combines multiple node specifications to perform more complex operations. Charts are used to set up interactions between nodes. +A chart defines a node that combines multiple nodes to perform more complex operations. Charts are used to set up interactions between nodes. ```yaml id: 01908c74-8b22-7cbf-a475-6b6bc871b01b diff --git a/docs/key_concepts_kr.md b/docs/key_concepts_kr.md index 1080849c..6f1abfd2 100644 --- a/docs/key_concepts_kr.md +++ b/docs/key_concepts_kr.md @@ -58,7 +58,7 @@ data: ## 차트 -차트는 여러 개의 노드 명세를 결합하여 더 복잡한 동작을 수행하는 노드를 정의합니다. 차트는 노드들 간의 상호 작용을 설정하는 데 사용됩니다. +차트는 여러 개의 노드를 결합하여 더 복잡한 동작을 수행하는 노드를 정의합니다. 차트는 노드들 간의 상호 작용을 설정하는 데 사용됩니다. ```yaml id: 01908c74-8b22-7cbf-a475-6b6bc871b01b From 7ab3731fd333742f8976ceb367934e8301f8fbb5 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 07:27:07 -0400 Subject: [PATCH 19/31] fix: support memongo --- cmd/pkg/uniflowctl/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/pkg/uniflowctl/main.go b/cmd/pkg/uniflowctl/main.go index f3003d5a..88080a6d 100644 --- a/cmd/pkg/uniflowctl/main.go +++ b/cmd/pkg/uniflowctl/main.go @@ -8,6 +8,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/siyul-park/uniflow/cmd/pkg/cli" mongochart "github.com/siyul-park/uniflow/driver/mongo/pkg/chart" + mongoserver "github.com/siyul-park/uniflow/driver/mongo/pkg/server" mongosecret "github.com/siyul-park/uniflow/driver/mongo/pkg/secret" mongospec "github.com/siyul-park/uniflow/driver/mongo/pkg/spec" "github.com/siyul-park/uniflow/pkg/chart" @@ -54,6 +55,13 @@ func main() { collectionSecrets = "secrets" } + if strings.HasPrefix(databaseURL, "memongodb://") { + server := mongoserver.New() + defer server.Stop() + + databaseURL = server.URI() + } + var chartStore chart.Store var specStore spec.Store var secretStore secret.Store From 2d36910b90e138700c0d22fafb117628e82d68da Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 07:28:39 -0400 Subject: [PATCH 20/31] style: make lint --- cmd/pkg/uniflowctl/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pkg/uniflowctl/main.go b/cmd/pkg/uniflowctl/main.go index 88080a6d..5a408e4d 100644 --- a/cmd/pkg/uniflowctl/main.go +++ b/cmd/pkg/uniflowctl/main.go @@ -8,8 +8,8 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/siyul-park/uniflow/cmd/pkg/cli" mongochart "github.com/siyul-park/uniflow/driver/mongo/pkg/chart" - mongoserver "github.com/siyul-park/uniflow/driver/mongo/pkg/server" mongosecret "github.com/siyul-park/uniflow/driver/mongo/pkg/secret" + mongoserver "github.com/siyul-park/uniflow/driver/mongo/pkg/server" mongospec "github.com/siyul-park/uniflow/driver/mongo/pkg/spec" "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/secret" From f5e3672900cabd410ce08189cb0b99c71f3a5a37 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Tue, 8 Oct 2024 22:42:38 -0400 Subject: [PATCH 21/31] fix: remain native codec --- pkg/chart/linker.go | 33 +++++++++++++++++++++++++-------- pkg/chart/linker_test.go | 5 +---- pkg/chart/table.go | 12 ++++++++++++ pkg/chart/table_test.go | 33 +++++++++++++++++++++++++++++++++ pkg/runtime/runtime.go | 34 ++++++++++++++++++---------------- pkg/runtime/runtime_test.go | 1 - pkg/scheme/codec.go | 23 +++++++++++++++-------- 7 files changed, 104 insertions(+), 37 deletions(-) diff --git a/pkg/chart/linker.go b/pkg/chart/linker.go index 73f9b098..026554e6 100644 --- a/pkg/chart/linker.go +++ b/pkg/chart/linker.go @@ -1,7 +1,7 @@ package chart import ( - "slices" + "sync" "github.com/siyul-park/uniflow/pkg/hook" "github.com/siyul-park/uniflow/pkg/node" @@ -20,6 +20,8 @@ type LinkerConfig struct { type Linker struct { hook *hook.Hook scheme *scheme.Scheme + codecs map[string]scheme.Codec + mu sync.RWMutex } var _ LoadHook = (*Linker)(nil) @@ -30,16 +32,22 @@ func NewLinker(config LinkerConfig) *Linker { return &Linker{ hook: config.Hook, scheme: config.Scheme, + codecs: make(map[string]scheme.Codec), } } // Load loads the chart, creating nodes and symbols. func (l *Linker) Load(chrt *Chart) error { - if slices.Contains(l.scheme.Kinds(), chrt.GetName()) { + l.mu.Lock() + defer l.mu.Unlock() + + kind := chrt.GetName() + codec := l.codecs[kind] + if l.scheme.Codec(kind) != codec { return nil } - codec := scheme.CodecFunc(func(sp spec.Spec) (node.Node, error) { + codec = scheme.CodecFunc(func(sp spec.Spec) (node.Node, error) { specs, err := chrt.Build(sp) if err != nil { return nil, err @@ -103,16 +111,25 @@ func (l *Linker) Load(chrt *Chart) error { return n, nil }) - l.scheme.AddKnownType(chrt.GetName(), &spec.Unstructured{}) - l.scheme.AddCodec(chrt.GetName(), codec) - + l.scheme.AddKnownType(kind, &spec.Unstructured{}) + l.scheme.AddCodec(kind, codec) + l.codecs[kind] = codec return nil } // Unload removes the chart from the scheme. func (l *Linker) Unload(chrt *Chart) error { - l.scheme.RemoveKnownType(chrt.GetName()) - l.scheme.RemoveCodec(chrt.GetName()) + l.mu.Lock() + defer l.mu.Unlock() + + kind := chrt.GetName() + codec := l.codecs[kind] + if l.scheme.Codec(kind) != codec { + return nil + } + l.scheme.RemoveKnownType(kind) + l.scheme.RemoveCodec(kind) + delete(l.codecs, kind) return nil } diff --git a/pkg/chart/linker_test.go b/pkg/chart/linker_test.go index b52c7bad..35272d4a 100644 --- a/pkg/chart/linker_test.go +++ b/pkg/chart/linker_test.go @@ -88,10 +88,7 @@ func TestLinker_Unload(t *testing.T) { Specs: []spec.Spec{}, } - s.AddKnownType(chrt.GetName(), &spec.Meta{}) - s.AddCodec(chrt.GetName(), scheme.CodecFunc(func(spec spec.Spec) (node.Node, error) { - return node.NewOneToOneNode(nil), nil - })) + l.Load(chrt) err := l.Unload(chrt) assert.NoError(t, err) diff --git a/pkg/chart/table.go b/pkg/chart/table.go index ba0f669d..47d4e779 100644 --- a/pkg/chart/table.go +++ b/pkg/chart/table.go @@ -72,6 +72,18 @@ func (t *Table) Lookup(id uuid.UUID) *Chart { return t.charts[id] } +// Links returns the charts linked to the chart specified by its UUID. +func (t *Table) Links(id uuid.UUID) []*Chart { + t.mu.RLock() + defer t.mu.RUnlock() + + chrt, ok := t.charts[id] + if !ok { + return nil + } + return t.linked(chrt) +} + // Keys returns all IDs of charts in the table. func (t *Table) Keys() []uuid.UUID { t.mu.RLock() diff --git a/pkg/chart/table_test.go b/pkg/chart/table_test.go index b8fcdfc1..b23e3493 100644 --- a/pkg/chart/table_test.go +++ b/pkg/chart/table_test.go @@ -89,6 +89,39 @@ func TestTable_Lookup(t *testing.T) { assert.Equal(t, chrt, tb.Lookup(chrt.GetID())) } +func TestTable_Links(t *testing.T) { + tb := NewTable() + defer tb.Close() + + chrt1 := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{}, + } + chrt2 := &Chart{ + ID: uuid.Must(uuid.NewV7()), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + Specs: []spec.Spec{ + &spec.Meta{ + Kind: chrt1.GetName(), + Namespace: resource.DefaultNamespace, + Name: faker.UUIDHyphenated(), + }, + }, + } + + tb.Insert(chrt1) + tb.Insert(chrt2) + + links := tb.Links(chrt1.GetID()) + assert.Equal(t, []*Chart{chrt1, chrt2}, links) + + links = tb.Links(chrt2.GetID()) + assert.Equal(t, []*Chart{chrt2}, links) +} + func TestTable_Keys(t *testing.T) { tb := NewTable() defer tb.Close() diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 2ba125e9..97ef197d 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -20,9 +20,9 @@ type Config struct { Namespace string // Namespace defines the isolated execution environment for workflows. Hook *hook.Hook // Hook is a collection of hook functions for managing symbols. Scheme *scheme.Scheme // Scheme defines the scheme and behaviors for symbols. - ChartStore chart.Store - SpecStore spec.Store // SpecStore is responsible for persisting specifications. - SecretStore secret.Store // SecretStore is responsible for persisting secrets. + ChartStore chart.Store // ChartStore is responsible for persisting charts. + SpecStore spec.Store // SpecStore is responsible for persisting specifications. + SecretStore secret.Store // SecretStore is responsible for persisting secrets. } // Runtime represents an environment for executing Workflows. @@ -183,35 +183,37 @@ func (r *Runtime) Reconcile(ctx context.Context) error { return nil } - charts, err := r.chartStore.Load(ctx, &chart.Chart{ID: event.ID}) - if err != nil { - return err - } + charts := r.chartTable.Links(event.ID) if len(charts) == 0 { - if chrt := r.chartTable.Lookup(event.ID); chrt != nil { - charts = append(charts, chrt) - } else { - charts = append(charts, &chart.Chart{ID: event.ID}) + var err error + charts, err = r.chartStore.Load(ctx, &chart.Chart{ID: event.ID}) + if err != nil { + return err } } - for _, chrt := range charts { - r.chartLoader.Load(ctx, chrt) - } - kinds := make([]string, 0, len(charts)) for _, chrt := range charts { kinds = append(kinds, chrt.GetName()) } + bounded := make(map[uuid.UUID]spec.Spec) for _, id := range r.symbolTable.Keys() { sb := r.symbolTable.Lookup(id) if sb != nil && slices.Contains(kinds, sb.Kind()) { + bounded[sb.ID()] = sb.Spec r.symbolTable.Free(sb.ID()) - unloaded[sb.ID()] = sb.Spec } } for _, sp := range unloaded { + if slices.Contains(kinds, sp.GetKind()) { + bounded[sp.GetID()] = sp + } + } + + r.chartLoader.Load(ctx, &chart.Chart{ID: event.ID}) + + for _, sp := range bounded { if slices.Contains(kinds, sp.GetKind()) { if err := r.symbolLoader.Load(ctx, sp); err != nil { unloaded[sp.GetID()] = sp diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index 2d7f8396..47fee7b5 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -40,7 +40,6 @@ func TestRuntime_Load(t *testing.T) { SpecStore: specStore, SecretStore: secretStore, }) - defer r.Close() meta := &spec.Meta{ diff --git a/pkg/scheme/codec.go b/pkg/scheme/codec.go index e0f8d23b..1721b7d2 100644 --- a/pkg/scheme/codec.go +++ b/pkg/scheme/codec.go @@ -7,16 +7,24 @@ import ( "github.com/siyul-park/uniflow/pkg/spec" ) -// Codec defines the interface for decoding spec.Spec into a node.Node. +// Codec defines the interface for converting a spec.Spec into a node.Node. type Codec interface { - // Compile compiles the given spec.Spec into a node.Node. + // Compile converts the given spec.Spec into a node.Node. Compile(sp spec.Spec) (node.Node, error) } -// CodecFunc represents a function type that implements the Codec interface. -type CodecFunc func(sp spec.Spec) (node.Node, error) +type codec struct { + compile func(sp spec.Spec) (node.Node, error) +} + +var _ Codec = (*codec)(nil) + +// CodecFunc takes a compile function and returns a struct that implements the Codec interface. +func CodecFunc(compile func(sp spec.Spec) (node.Node, error)) Codec { + return &codec{compile: compile} +} -// CodecWithType creates a new CodecFunc for the specified type T. +// CodecWithType creates a Codec that works with a specific type T. func CodecWithType[T spec.Spec](compile func(spec T) (node.Node, error)) Codec { return CodecFunc(func(spec spec.Spec) (node.Node, error) { if converted, ok := spec.(T); ok { @@ -26,7 +34,6 @@ func CodecWithType[T spec.Spec](compile func(spec T) (node.Node, error)) Codec { }) } -// Compile implements the Compile method for CodecFunc. -func (f CodecFunc) Compile(sp spec.Spec) (node.Node, error) { - return f(sp) +func (c *codec) Compile(sp spec.Spec) (node.Node, error) { + return c.compile(sp) } From 8c1ced2399d1b2ed7efb97152047f9e8caf85dad Mon Sep 17 00:00:00 2001 From: siyul-park Date: Wed, 9 Oct 2024 03:47:04 -0400 Subject: [PATCH 22/31] refactor: more readable --- pkg/runtime/runtime.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 97ef197d..b964bb25 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -202,7 +202,6 @@ func (r *Runtime) Reconcile(ctx context.Context) error { sb := r.symbolTable.Lookup(id) if sb != nil && slices.Contains(kinds, sb.Kind()) { bounded[sb.ID()] = sb.Spec - r.symbolTable.Free(sb.ID()) } } for _, sp := range unloaded { @@ -211,15 +210,17 @@ func (r *Runtime) Reconcile(ctx context.Context) error { } } + for _, sp := range bounded { + r.symbolTable.Free(sp.GetID()) + } + r.chartLoader.Load(ctx, &chart.Chart{ID: event.ID}) for _, sp := range bounded { - if slices.Contains(kinds, sp.GetKind()) { - if err := r.symbolLoader.Load(ctx, sp); err != nil { - unloaded[sp.GetID()] = sp - } else { - delete(unloaded, sp.GetID()) - } + if err := r.symbolLoader.Load(ctx, sp); err != nil { + unloaded[sp.GetID()] = sp + } else { + delete(unloaded, sp.GetID()) } } case event, ok := <-specStream.Next(): From e66b1e587c82e5a4693515fc00d12ae9261418d9 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Wed, 9 Oct 2024 04:06:05 -0400 Subject: [PATCH 23/31] fix: reuse port in cluster --- pkg/chart/chart.go | 10 ++++----- pkg/chart/cluster.go | 48 +++++++++++++++++++++++++++++++------------- pkg/chart/linker.go | 4 +++- pkg/chart/loader.go | 9 ++++----- 4 files changed, 46 insertions(+), 25 deletions(-) diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go index d78adff1..d7ff53a5 100644 --- a/pkg/chart/chart.go +++ b/pkg/chart/chart.go @@ -64,7 +64,7 @@ var _ resource.Resource = (*Chart)(nil) // IsBound checks whether any of the secrets are bound to the chart. func (c *Chart) IsBound(secrets ...*secret.Secret) bool { - for _, vals := range c.GetEnv() { + for _, vals := range c.Env { for _, val := range vals { examples := make([]*secret.Secret, 0, 2) if val.ID != uuid.Nil { @@ -86,7 +86,7 @@ func (c *Chart) IsBound(secrets ...*secret.Secret) bool { // Bind binds the chart's environment variables to the provided secrets. func (c *Chart) Bind(secrets ...*secret.Secret) error { - for _, vals := range c.GetEnv() { + for _, vals := range c.Env { for i, val := range vals { if val.ID != uuid.Nil || val.Name != "" { example := &secret.Secret{ @@ -132,7 +132,7 @@ func (c *Chart) Build(sp spec.Spec) ([]spec.Spec, error) { data := types.InterfaceOf(doc) env := map[string][]spec.Value{} - for key, vals := range c.GetEnv() { + for key, vals := range c.Env { for _, val := range vals { if val.ID == uuid.Nil && val.Name == "" { v, err := template.Execute(val.Value, data) @@ -145,8 +145,8 @@ func (c *Chart) Build(sp spec.Spec) ([]spec.Spec, error) { } } - specs := make([]spec.Spec, 0, len(c.GetSpecs())) - for _, sp := range c.GetSpecs() { + specs := make([]spec.Spec, 0, len(c.Specs)) + for _, sp := range c.Specs { doc, err := types.Marshal(sp) if err != nil { return nil, err diff --git a/pkg/chart/cluster.go b/pkg/chart/cluster.go index c81315d3..e5cf7c4b 100644 --- a/pkg/chart/cluster.go +++ b/pkg/chart/cluster.go @@ -37,16 +37,26 @@ func (n *ClusterNode) Inbound(name string, prt *port.InPort) { n.mu.Lock() defer n.mu.Unlock() - inPort := port.NewIn() - outPort := port.NewOut() + inPort, ok1 := n.inPorts[name] + if !ok1 { + inPort = port.NewIn() + n.inPorts[name] = inPort + } - n.inPorts[name] = inPort - n._outPorts[name] = outPort + outPort, ok2 := n._outPorts[name] + if !ok2 { + outPort = port.NewOut() + n._outPorts[name] = outPort + } - outPort.Link(prt) + if !ok1 { + inPort.AddListener(n.inbound(inPort, outPort)) + } + if !ok2 { + outPort.AddListener(n.outbound(inPort, outPort)) + } - inPort.AddListener(n.inbound(inPort, outPort)) - outPort.AddListener(n.outbound(inPort, outPort)) + outPort.Link(prt) } // Outbound sets up an output port and links it to the provided port. @@ -54,16 +64,26 @@ func (n *ClusterNode) Outbound(name string, prt *port.OutPort) { n.mu.Lock() defer n.mu.Unlock() - inPort := port.NewIn() - outPort := port.NewOut() + inPort, ok1 := n._inPorts[name] + if !ok1 { + inPort = port.NewIn() + n._inPorts[name] = inPort + } - n._inPorts[name] = inPort - n.outPorts[name] = outPort + outPort, ok2 := n.outPorts[name] + if !ok2 { + outPort = port.NewOut() + n.outPorts[name] = outPort + } - prt.Link(inPort) + if !ok1 { + inPort.AddListener(n.inbound(inPort, outPort)) + } + if !ok2 { + outPort.AddListener(n.outbound(inPort, outPort)) + } - inPort.AddListener(n.inbound(inPort, outPort)) - outPort.AddListener(n.outbound(inPort, outPort)) + prt.Link(inPort) } // In returns the input port by name. diff --git a/pkg/chart/linker.go b/pkg/chart/linker.go index 026554e6..abd79a6f 100644 --- a/pkg/chart/linker.go +++ b/pkg/chart/linker.go @@ -43,6 +43,7 @@ func (l *Linker) Load(chrt *Chart) error { kind := chrt.GetName() codec := l.codecs[kind] + if l.scheme.Codec(kind) != codec { return nil } @@ -96,7 +97,7 @@ func (l *Linker) Load(chrt *Chart) error { for name, ports := range chrt.GetPorts() { for _, port := range ports { for _, sb := range symbols { - if sb.ID() == port.ID || sb.Name() == port.Name { + if port.Name == "" || sb.ID() == port.ID || sb.Name() == port.Name { if in := sb.In(port.Port); in != nil { n.Inbound(name, in) } @@ -124,6 +125,7 @@ func (l *Linker) Unload(chrt *Chart) error { kind := chrt.GetName() codec := l.codecs[kind] + if l.scheme.Codec(kind) != codec { return nil } diff --git a/pkg/chart/loader.go b/pkg/chart/loader.go index be26399d..42408dd7 100644 --- a/pkg/chart/loader.go +++ b/pkg/chart/loader.go @@ -64,23 +64,22 @@ func (l *Loader) Load(ctx context.Context, charts ...*Chart) error { } var errs []error + loaded := make([]*Chart, 0, len(charts)) for _, chrt := range charts { if err := chrt.Bind(secrets...); err != nil { errs = append(errs, err) } else if err := l.table.Insert(chrt); err != nil { errs = append(errs, err) + } else { + loaded = append(loaded, chrt) } } - if len(errs) > 0 { - charts = nil - } - for _, id := range l.table.Keys() { chrt := l.table.Lookup(id) if chrt != nil && len(resource.Match(chrt, examples...)) > 0 { ok := false - for _, c := range charts { + for _, c := range loaded { if c.GetID() == id { ok = true break From 3eba618ef93a4bc6d42496f565890fca490e03f1 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Wed, 9 Oct 2024 06:26:35 -0400 Subject: [PATCH 24/31] test: add more test case --- cmd/pkg/cli/apply_test.go | 31 ++++++++++++------------------- cmd/pkg/cli/delete_test.go | 16 ++++++---------- cmd/pkg/cli/get_test.go | 16 ++++++---------- cmd/pkg/cli/start_test.go | 38 ++++++++++++++++++++++++++++++++++++++ pkg/chart/loader.go | 2 +- pkg/symbol/loader.go | 2 +- 6 files changed, 64 insertions(+), 41 deletions(-) diff --git a/cmd/pkg/cli/apply_test.go b/cmd/pkg/cli/apply_test.go index 2b328829..bb1fc35b 100644 --- a/cmd/pkg/cli/apply_test.go +++ b/cmd/pkg/cli/apply_test.go @@ -10,7 +10,6 @@ import ( "github.com/go-faker/faker/v4" "github.com/gofrs/uuid" "github.com/siyul-park/uniflow/pkg/chart" - "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" "github.com/spf13/afero" @@ -31,9 +30,8 @@ func TestApplyCommand_Execute(t *testing.T) { filename := "chart.json" chrt := &chart.Chart{ - ID: uuid.Must(uuid.NewV7()), - Namespace: resource.DefaultNamespace, - Name: faker.Word(), + ID: uuid.Must(uuid.NewV7()), + Name: faker.Word(), } data, err := json.Marshal(chrt) @@ -75,9 +73,8 @@ func TestApplyCommand_Execute(t *testing.T) { filename := "chart.json" chrt := &chart.Chart{ - ID: uuid.Must(uuid.NewV7()), - Namespace: resource.DefaultNamespace, - Name: faker.Word(), + ID: uuid.Must(uuid.NewV7()), + Name: faker.Word(), } _, err := chartStore.Store(ctx, chrt) @@ -124,9 +121,8 @@ func TestApplyCommand_Execute(t *testing.T) { kind := faker.UUIDHyphenated() meta := &spec.Meta{ - Kind: kind, - Namespace: resource.DefaultNamespace, - Name: faker.UUIDHyphenated(), + Kind: kind, + Name: faker.UUIDHyphenated(), } data, err := json.Marshal(meta) @@ -170,9 +166,8 @@ func TestApplyCommand_Execute(t *testing.T) { kind := faker.UUIDHyphenated() meta := &spec.Meta{ - Kind: kind, - Namespace: resource.DefaultNamespace, - Name: faker.UUIDHyphenated(), + Kind: kind, + Name: faker.UUIDHyphenated(), } _, err := specStore.Store(ctx, meta) @@ -217,9 +212,8 @@ func TestApplyCommand_Execute(t *testing.T) { filename := "secrets.json" scrt := &secret.Secret{ - Namespace: resource.DefaultNamespace, - Name: faker.UUIDHyphenated(), - Data: faker.Word(), + Name: faker.UUIDHyphenated(), + Data: faker.Word(), } data, err := json.Marshal(scrt) @@ -261,9 +255,8 @@ func TestApplyCommand_Execute(t *testing.T) { filename := "secrets.json" scrt := &secret.Secret{ - Namespace: resource.DefaultNamespace, - Name: faker.UUIDHyphenated(), - Data: faker.Word(), + Name: faker.UUIDHyphenated(), + Data: faker.Word(), } _, err := secretStore.Store(ctx, scrt) diff --git a/cmd/pkg/cli/delete_test.go b/cmd/pkg/cli/delete_test.go index 2e3f89ee..52227658 100644 --- a/cmd/pkg/cli/delete_test.go +++ b/cmd/pkg/cli/delete_test.go @@ -9,7 +9,6 @@ import ( "github.com/go-faker/faker/v4" "github.com/gofrs/uuid" "github.com/siyul-park/uniflow/pkg/chart" - "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" "github.com/spf13/afero" @@ -30,9 +29,8 @@ func TestDeleteCommand_Execute(t *testing.T) { filename := "chart.json" chrt := &chart.Chart{ - ID: uuid.Must(uuid.NewV7()), - Namespace: resource.DefaultNamespace, - Name: faker.Word(), + ID: uuid.Must(uuid.NewV7()), + Name: faker.Word(), } data, err := json.Marshal(chrt) @@ -74,9 +72,8 @@ func TestDeleteCommand_Execute(t *testing.T) { kind := faker.UUIDHyphenated() meta := &spec.Meta{ - Kind: kind, - Namespace: resource.DefaultNamespace, - Name: faker.UUIDHyphenated(), + Kind: kind, + Name: faker.UUIDHyphenated(), } data, err := json.Marshal(meta) @@ -116,9 +113,8 @@ func TestDeleteCommand_Execute(t *testing.T) { filename := "secrets.json" scrt := &secret.Secret{ - Namespace: resource.DefaultNamespace, - Name: faker.UUIDHyphenated(), - Data: faker.Word(), + Name: faker.UUIDHyphenated(), + Data: faker.Word(), } data, err := json.Marshal(scrt) diff --git a/cmd/pkg/cli/get_test.go b/cmd/pkg/cli/get_test.go index d7d3f36b..bdef2454 100644 --- a/cmd/pkg/cli/get_test.go +++ b/cmd/pkg/cli/get_test.go @@ -8,7 +8,6 @@ import ( "github.com/go-faker/faker/v4" "github.com/gofrs/uuid" "github.com/siyul-park/uniflow/pkg/chart" - "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/secret" "github.com/siyul-park/uniflow/pkg/spec" "github.com/stretchr/testify/assert" @@ -24,9 +23,8 @@ func TestGetCommand_Execute(t *testing.T) { defer cancel() chrt := &chart.Chart{ - ID: uuid.Must(uuid.NewV7()), - Namespace: resource.DefaultNamespace, - Name: faker.Word(), + ID: uuid.Must(uuid.NewV7()), + Name: faker.Word(), } _, err := chartStore.Store(ctx, chrt) @@ -56,9 +54,8 @@ func TestGetCommand_Execute(t *testing.T) { kind := faker.UUIDHyphenated() meta := &spec.Meta{ - Kind: kind, - Namespace: resource.DefaultNamespace, - Name: faker.UUIDHyphenated(), + Kind: kind, + Name: faker.UUIDHyphenated(), } _, err := specStore.Store(ctx, meta) @@ -86,9 +83,8 @@ func TestGetCommand_Execute(t *testing.T) { defer cancel() scrt := &secret.Secret{ - Namespace: resource.DefaultNamespace, - Name: faker.UUIDHyphenated(), - Data: faker.Word(), + Name: faker.UUIDHyphenated(), + Data: faker.Word(), } _, err := secretStore.Store(ctx, scrt) diff --git a/cmd/pkg/cli/start_test.go b/cmd/pkg/cli/start_test.go index 30fd66d1..b8800309 100644 --- a/cmd/pkg/cli/start_test.go +++ b/cmd/pkg/cli/start_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "testing" "time" @@ -40,6 +41,43 @@ func TestStartCommand_Execute(t *testing.T) { s.AddKnownType(kind, &spec.Meta{}) s.AddCodec(kind, codec) + t.Run(flagDebug, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + output := new(bytes.Buffer) + + cmd := NewStartCommand(StartConfig{ + Scheme: s, + Hook: h, + FS: fs, + ChartStore: chartStore, + SpecStore: specStore, + SecretStore: secretStore, + }) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetContext(ctx) + + cmd.SetArgs([]string{fmt.Sprintf("--%s", flagDebug)}) + + go func() { + _ = cmd.Execute() + }() + + for { + select { + case <-ctx.Done(): + assert.Fail(t, ctx.Err().Error()) + return + default: + if strings.Contains(output.String(), "debug") { + return + } + } + } + }) + t.Run(flagFromCharts, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() diff --git a/pkg/chart/loader.go b/pkg/chart/loader.go index 42408dd7..848385d4 100644 --- a/pkg/chart/loader.go +++ b/pkg/chart/loader.go @@ -87,7 +87,7 @@ func (l *Loader) Load(ctx context.Context, charts ...*Chart) error { } if !ok { if _, err := l.table.Free(id); err != nil { - return err + errs = append(errs, err) } } } diff --git a/pkg/symbol/loader.go b/pkg/symbol/loader.go index d65243b5..1ce604cb 100644 --- a/pkg/symbol/loader.go +++ b/pkg/symbol/loader.go @@ -122,7 +122,7 @@ func (l *Loader) Load(ctx context.Context, specs ...spec.Spec) error { } if !ok { if _, err := l.table.Free(id); err != nil { - return err + errs = append(errs, err) } } } From 1e0914ed2974fb6da724d3fdf026f03c680bedc4 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Wed, 9 Oct 2024 06:29:48 -0400 Subject: [PATCH 25/31] fix: rollback loaded chart --- pkg/chart/loader.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/chart/loader.go b/pkg/chart/loader.go index 848385d4..df84f304 100644 --- a/pkg/chart/loader.go +++ b/pkg/chart/loader.go @@ -75,6 +75,10 @@ func (l *Loader) Load(ctx context.Context, charts ...*Chart) error { } } + if len(errs) > 0 { + loaded = nil + } + for _, id := range l.table.Keys() { chrt := l.table.Lookup(id) if chrt != nil && len(resource.Match(chrt, examples...)) > 0 { From 850802ecbe15fc83225598c87ef7fea7c6ba18d1 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Wed, 9 Oct 2024 06:34:09 -0400 Subject: [PATCH 26/31] docs: add comment --- pkg/chart/loader.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/chart/loader.go b/pkg/chart/loader.go index df84f304..70e47c7e 100644 --- a/pkg/chart/loader.go +++ b/pkg/chart/loader.go @@ -32,6 +32,7 @@ func NewLoader(config LoaderConfig) *Loader { } } +// Load loads charts and binds them with secrets, then inserts them into the table. func (l *Loader) Load(ctx context.Context, charts ...*Chart) error { examples := charts From 725d259a720a12452f9647f9ad17b33ace2e13d6 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Wed, 9 Oct 2024 06:44:44 -0400 Subject: [PATCH 27/31] fix: race condition --- cmd/pkg/cli/start_test.go | 81 +++++++++------------------------------ 1 file changed, 18 insertions(+), 63 deletions(-) diff --git a/cmd/pkg/cli/start_test.go b/cmd/pkg/cli/start_test.go index b8800309..6dc31f6e 100644 --- a/cmd/pkg/cli/start_test.go +++ b/cmd/pkg/cli/start_test.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "strings" "testing" "time" @@ -41,43 +40,6 @@ func TestStartCommand_Execute(t *testing.T) { s.AddKnownType(kind, &spec.Meta{}) s.AddCodec(kind, codec) - t.Run(flagDebug, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - output := new(bytes.Buffer) - - cmd := NewStartCommand(StartConfig{ - Scheme: s, - Hook: h, - FS: fs, - ChartStore: chartStore, - SpecStore: specStore, - SecretStore: secretStore, - }) - cmd.SetOut(output) - cmd.SetErr(output) - cmd.SetContext(ctx) - - cmd.SetArgs([]string{fmt.Sprintf("--%s", flagDebug)}) - - go func() { - _ = cmd.Execute() - }() - - for { - select { - case <-ctx.Done(): - assert.Fail(t, ctx.Err().Error()) - return - default: - if strings.Contains(output.String(), "debug") { - return - } - } - } - }) - t.Run(flagFromCharts, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -111,20 +73,17 @@ func TestStartCommand_Execute(t *testing.T) { cmd.SetArgs([]string{fmt.Sprintf("--%s", flagFromCharts), filename}) + chartStream, _ := chartStore.Watch(ctx) + defer chartStream.Close() + go func() { _ = cmd.Execute() }() - for { - select { - case <-ctx.Done(): - assert.Fail(t, ctx.Err().Error()) - return - default: - if r, _ := chartStore.Load(ctx, chrt); len(r) > 0 { - return - } - } + select { + case <-chartStream.Next(): + case <-ctx.Done(): + assert.Fail(t, ctx.Err().Error()) } }) @@ -161,20 +120,17 @@ func TestStartCommand_Execute(t *testing.T) { cmd.SetArgs([]string{fmt.Sprintf("--%s", flagFromNodes), filename}) + specStream, _ := specStore.Watch(ctx) + defer specStream.Close() + go func() { _ = cmd.Execute() }() - for { - select { - case <-ctx.Done(): - assert.Fail(t, ctx.Err().Error()) - return - default: - if r, _ := specStore.Load(ctx, meta); len(r) > 0 { - return - } - } + select { + case <-specStream.Next(): + case <-ctx.Done(): + assert.Fail(t, ctx.Err().Error()) } }) @@ -211,18 +167,17 @@ func TestStartCommand_Execute(t *testing.T) { cmd.SetArgs([]string{fmt.Sprintf("--%s", flagFromSecrets), filename}) + secretStream, _ := secretStore.Watch(ctx) + defer secretStream.Close() + go func() { _ = cmd.Execute() }() select { + case <-secretStream.Next(): case <-ctx.Done(): assert.Fail(t, ctx.Err().Error()) - return - default: - if r, _ := secretStore.Load(ctx, scrt); len(r) > 0 { - return - } } }) } From 9ed9d05cd0b0e1c4168ff9c5c7cbdc5620d0610e Mon Sep 17 00:00:00 2001 From: siyul-park Date: Wed, 9 Oct 2024 06:49:50 -0400 Subject: [PATCH 28/31] refactor: remove unnessart --- pkg/runtime/runtime_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index 47fee7b5..b98cb5f0 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -110,7 +110,6 @@ func TestRuntime_Reconcile(t *testing.T) { assert.Equal(t, meta.GetID(), sb.ID()) case <-ctx.Done(): assert.NoError(t, ctx.Err()) - return } chartStore.Delete(ctx, chrt) @@ -120,7 +119,6 @@ func TestRuntime_Reconcile(t *testing.T) { assert.Equal(t, meta.GetID(), sb.ID()) case <-ctx.Done(): assert.NoError(t, ctx.Err()) - return } }) @@ -179,7 +177,6 @@ func TestRuntime_Reconcile(t *testing.T) { assert.Equal(t, meta.GetID(), sb.ID()) case <-ctx.Done(): assert.NoError(t, ctx.Err()) - return } specStore.Delete(ctx, meta) @@ -189,7 +186,6 @@ func TestRuntime_Reconcile(t *testing.T) { assert.Equal(t, meta.GetID(), sb.ID()) case <-ctx.Done(): assert.NoError(t, ctx.Err()) - return } }) @@ -261,7 +257,6 @@ func TestRuntime_Reconcile(t *testing.T) { assert.Equal(t, scrt.Data, sb.Env()["key"][0].Value) case <-ctx.Done(): assert.NoError(t, ctx.Err()) - return } secretStore.Delete(ctx, scrt) @@ -271,7 +266,6 @@ func TestRuntime_Reconcile(t *testing.T) { assert.Equal(t, meta.GetID(), sb.ID()) case <-ctx.Done(): assert.NoError(t, ctx.Err()) - return } }) } From 5e32f89fe8c7367af93507c982a0b4ae684993f7 Mon Sep 17 00:00:00 2001 From: siyul-park Date: Wed, 9 Oct 2024 06:52:04 -0400 Subject: [PATCH 29/31] refactor: change method name --- pkg/chart/chart_test.go | 2 +- pkg/secret/secret_test.go | 2 +- pkg/spec/spec_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/chart/chart_test.go b/pkg/chart/chart_test.go index 9a2950f7..44a5ab0d 100644 --- a/pkg/chart/chart_test.go +++ b/pkg/chart/chart_test.go @@ -92,7 +92,7 @@ func TestChart_Build(t *testing.T) { assert.Len(t, specs, 1) } -func TestChart_GetSet(t *testing.T) { +func TestChart_Get(t *testing.T) { chrt := &Chart{ ID: uuid.Must(uuid.NewV7()), Namespace: "default", diff --git a/pkg/secret/secret_test.go b/pkg/secret/secret_test.go index d86e9590..8260af77 100644 --- a/pkg/secret/secret_test.go +++ b/pkg/secret/secret_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestSecret_GetSet(t *testing.T) { +func TestSecret_Get(t *testing.T) { scrt := &Secret{ ID: uuid.Must(uuid.NewV7()), Namespace: "default", diff --git a/pkg/spec/spec_test.go b/pkg/spec/spec_test.go index 9e8368fc..3fa5b9f5 100644 --- a/pkg/spec/spec_test.go +++ b/pkg/spec/spec_test.go @@ -59,7 +59,7 @@ func TestBind(t *testing.T) { assert.True(t, IsBound(bind, scrt)) } -func TestMeta_GetSet(t *testing.T) { +func TestMeta_Get(t *testing.T) { meta := &Meta{ ID: uuid.Must(uuid.NewV7()), Kind: faker.Word(), From 94de6aaaaf65b990faf014826cb69f219f5823ae Mon Sep 17 00:00:00 2001 From: siyul-park Date: Wed, 9 Oct 2024 07:08:10 -0400 Subject: [PATCH 30/31] feat: support link unlink hook --- pkg/chart/linker.go | 47 +++++++++++++++----------------- pkg/chart/linker_test.go | 6 ++--- pkg/chart/linkhook.go | 34 +++++++++++++++++++++++ pkg/chart/loadhook.go | 34 ----------------------- pkg/chart/table.go | 24 ++++++++--------- pkg/chart/table_test.go | 8 +++--- pkg/chart/unlinkhook.go | 37 +++++++++++++++++++++++++ pkg/chart/unloadhook.go | 37 ------------------------- pkg/hook/hook.go | 58 +++++++++++++++++++++++++++++----------- pkg/hook/hook_test.go | 33 +++++++++++++++++++++++ pkg/runtime/runtime.go | 9 ++++--- 11 files changed, 191 insertions(+), 136 deletions(-) create mode 100644 pkg/chart/linkhook.go delete mode 100644 pkg/chart/loadhook.go create mode 100644 pkg/chart/unlinkhook.go delete mode 100644 pkg/chart/unloadhook.go diff --git a/pkg/chart/linker.go b/pkg/chart/linker.go index abd79a6f..7a96d43d 100644 --- a/pkg/chart/linker.go +++ b/pkg/chart/linker.go @@ -3,41 +3,43 @@ package chart import ( "sync" - "github.com/siyul-park/uniflow/pkg/hook" "github.com/siyul-park/uniflow/pkg/node" "github.com/siyul-park/uniflow/pkg/scheme" "github.com/siyul-park/uniflow/pkg/spec" "github.com/siyul-park/uniflow/pkg/symbol" ) -// LinkerConfig holds the hook and scheme configuration. +// LinkerConfig holds the configuration for the linker, including the scheme and hooks for loading/unloading symbols. type LinkerConfig struct { - Hook *hook.Hook // Manages symbol lifecycle events. - Scheme *scheme.Scheme // Defines symbol and node behavior. + Scheme *scheme.Scheme // Specifies the scheme, which defines symbol and node behavior. + LoadHooks []symbol.LoadHook // A list of hooks to be executed when symbols are loaded. + UnloadHooks []symbol.UnloadHook // A list of hooks to be executed when symbols are unloaded. } // Linker manages chart loading and unloading. type Linker struct { - hook *hook.Hook - scheme *scheme.Scheme - codecs map[string]scheme.Codec - mu sync.RWMutex + scheme *scheme.Scheme + codecs map[string]scheme.Codec + loadHooks []symbol.LoadHook + unloadHooks []symbol.UnloadHook + mu sync.RWMutex } -var _ LoadHook = (*Linker)(nil) -var _ UnloadHook = (*Linker)(nil) +var _ LinkHook = (*Linker)(nil) +var _ UnlinkHook = (*Linker)(nil) // NewLinker creates a new Linker. func NewLinker(config LinkerConfig) *Linker { return &Linker{ - hook: config.Hook, - scheme: config.Scheme, - codecs: make(map[string]scheme.Codec), + scheme: config.Scheme, + codecs: make(map[string]scheme.Codec), + loadHooks: config.LoadHooks, + unloadHooks: config.UnloadHooks, } } -// Load loads the chart, creating nodes and symbols. -func (l *Linker) Load(chrt *Chart) error { +// Link loads the chart, creating nodes and symbols. +func (l *Linker) Link(chrt *Chart) error { l.mu.Lock() defer l.mu.Unlock() @@ -70,16 +72,9 @@ func (l *Linker) Load(chrt *Chart) error { }) } - var loadHooks []symbol.LoadHook - var unloadHook []symbol.UnloadHook - if l.hook != nil { - loadHooks = append(loadHooks, l.hook) - unloadHook = append(unloadHook, l.hook) - } - table := symbol.NewTable(symbol.TableOption{ - LoadHooks: loadHooks, - UnloadHooks: unloadHook, + LoadHooks: l.loadHooks, + UnloadHooks: l.unloadHooks, }) for _, sb := range symbols { @@ -118,8 +113,8 @@ func (l *Linker) Load(chrt *Chart) error { return nil } -// Unload removes the chart from the scheme. -func (l *Linker) Unload(chrt *Chart) error { +// Unlink removes the chart from the scheme. +func (l *Linker) Unlink(chrt *Chart) error { l.mu.Lock() defer l.mu.Unlock() diff --git a/pkg/chart/linker_test.go b/pkg/chart/linker_test.go index 35272d4a..5d2ac1a2 100644 --- a/pkg/chart/linker_test.go +++ b/pkg/chart/linker_test.go @@ -65,7 +65,7 @@ func TestLinker_Load(t *testing.T) { Namespace: resource.DefaultNamespace, } - err := l.Load(chrt) + err := l.Link(chrt) assert.NoError(t, err) assert.Contains(t, s.Kinds(), chrt.GetName()) @@ -88,9 +88,9 @@ func TestLinker_Unload(t *testing.T) { Specs: []spec.Spec{}, } - l.Load(chrt) + l.Link(chrt) - err := l.Unload(chrt) + err := l.Unlink(chrt) assert.NoError(t, err) assert.NotContains(t, s.Kinds(), chrt.GetName()) } diff --git a/pkg/chart/linkhook.go b/pkg/chart/linkhook.go new file mode 100644 index 00000000..b62c282d --- /dev/null +++ b/pkg/chart/linkhook.go @@ -0,0 +1,34 @@ +package chart + +// LinkHook defines an interface for handling the loading of a chart. +type LinkHook interface { + // Link processes the loading of a chart and may return an error. + Link(*Chart) error +} + +type LinkHooks []LinkHook + +type linkHook struct { + link func(*Chart) error +} + +var _ LinkHook = (LinkHooks)(nil) +var _ LinkHook = (*linkHook)(nil) + +// LinkFunc creates a LoadHook from the given function. +func LinkFunc(link func(*Chart) error) LinkHook { + return &linkHook{link: link} +} + +func (h LinkHooks) Link(chrt *Chart) error { + for _, hook := range h { + if err := hook.Link(chrt); err != nil { + return err + } + } + return nil +} + +func (h *linkHook) Link(chrt *Chart) error { + return h.link(chrt) +} diff --git a/pkg/chart/loadhook.go b/pkg/chart/loadhook.go deleted file mode 100644 index e640b9eb..00000000 --- a/pkg/chart/loadhook.go +++ /dev/null @@ -1,34 +0,0 @@ -package chart - -// LoadHook defines an interface for handling the loading of a chart. -type LoadHook interface { - // Load processes the loading of a chart and may return an error. - Load(*Chart) error -} - -type LoadHooks []LoadHook - -type loadHook struct { - load func(*Chart) error -} - -var _ LoadHook = (LoadHooks)(nil) -var _ LoadHook = (*loadHook)(nil) - -// LoadFunc creates a LoadHook from the given function. -func LoadFunc(load func(*Chart) error) LoadHook { - return &loadHook{load: load} -} - -func (h LoadHooks) Load(chrt *Chart) error { - for _, hook := range h { - if err := hook.Load(chrt); err != nil { - return err - } - } - return nil -} - -func (h *loadHook) Load(chrt *Chart) error { - return h.load(chrt) -} diff --git a/pkg/chart/table.go b/pkg/chart/table.go index 47d4e779..d245f04c 100644 --- a/pkg/chart/table.go +++ b/pkg/chart/table.go @@ -9,8 +9,8 @@ import ( // TableOption holds configurations for a Table instance. type TableOption struct { - LoadHooks []LoadHook // LoadHooks are functions executed when symbols are loaded. - UnloadHooks []UnloadHook // UnloadHooks are functions executed when symbols are unloaded. + LinkHooks []LinkHook // LoadHooks are functions executed when symbols are loaded. + UnlinkHooks []UnlinkHook // UnloadHooks are functions executed when symbols are unloaded. } // Table manages charts and their references, allowing insertion, lookup, and removal. @@ -18,26 +18,26 @@ type Table struct { charts map[uuid.UUID]*Chart namespaces map[string]map[string]uuid.UUID refences map[uuid.UUID][]uuid.UUID - loadHooks LoadHooks - unloadHooks UnloadHooks + linkHooks LinkHooks + unlinkHooks UnlinkHooks mu sync.RWMutex } // NewTable creates and returns a new Table instance with the provided options. func NewTable(opts ...TableOption) *Table { - var loadHooks []LoadHook - var unloadHooks []UnloadHook + var linkHooks []LinkHook + var unlinkHooks []UnlinkHook for _, opt := range opts { - loadHooks = append(loadHooks, opt.LoadHooks...) - unloadHooks = append(unloadHooks, opt.UnloadHooks...) + linkHooks = append(linkHooks, opt.LinkHooks...) + unlinkHooks = append(unlinkHooks, opt.UnlinkHooks...) } return &Table{ charts: make(map[uuid.UUID]*Chart), namespaces: make(map[string]map[string]uuid.UUID), refences: make(map[uuid.UUID][]uuid.UUID), - loadHooks: loadHooks, - unloadHooks: unloadHooks, + linkHooks: linkHooks, + unlinkHooks: unlinkHooks, } } @@ -150,7 +150,7 @@ func (t *Table) load(chrt *Chart) error { linked := t.linked(chrt) for _, sb := range linked { if t.active(sb) { - if err := t.loadHooks.Load(sb); err != nil { + if err := t.linkHooks.Link(sb); err != nil { return err } } @@ -163,7 +163,7 @@ func (t *Table) unload(chrt *Chart) error { for i := len(linked) - 1; i >= 0; i-- { sb := linked[i] if t.active(sb) { - if err := t.unloadHooks.Unload(sb); err != nil { + if err := t.unlinkHooks.Unlink(sb); err != nil { return err } } diff --git a/pkg/chart/table_test.go b/pkg/chart/table_test.go index b23e3493..be35218b 100644 --- a/pkg/chart/table_test.go +++ b/pkg/chart/table_test.go @@ -150,14 +150,14 @@ func TestTable_Hook(t *testing.T) { unloaded := 0 tb := NewTable(TableOption{ - LoadHooks: []LoadHook{ - LoadFunc(func(_ *Chart) error { + LinkHooks: []LinkHook{ + LinkFunc(func(_ *Chart) error { loaded += 1 return nil }), }, - UnloadHooks: []UnloadHook{ - UnloadFunc(func(_ *Chart) error { + UnlinkHooks: []UnlinkHook{ + UnlinkFunc(func(_ *Chart) error { unloaded += 1 return nil }), diff --git a/pkg/chart/unlinkhook.go b/pkg/chart/unlinkhook.go new file mode 100644 index 00000000..aa294873 --- /dev/null +++ b/pkg/chart/unlinkhook.go @@ -0,0 +1,37 @@ +package chart + +// UnlinkHook defines an interface for handling the unloading of a chart. +type UnlinkHook interface { + // Unlink is called when a chart is unloaded and may return an error. + Unlink(*Chart) error +} + +// UnlinkHooks is a slice of UnloadHook, processed in reverse order. +type UnlinkHooks []UnlinkHook + +// unlinkHook wraps an unload function to implement UnloadHook. +type unlinkHook struct { + unlink func(*Chart) error +} + +var _ UnlinkHook = (UnlinkHooks)(nil) +var _ UnlinkHook = (*unlinkHook)(nil) + +// UnlinkFunc creates an UnloadHook from the given function. +func UnlinkFunc(unlink func(*Chart) error) UnlinkHook { + return &unlinkHook{unlink: unlink} +} + +func (h UnlinkHooks) Unlink(chrt *Chart) error { + for i := len(h) - 1; i >= 0; i-- { + hook := h[i] + if err := hook.Unlink(chrt); err != nil { + return err + } + } + return nil +} + +func (h *unlinkHook) Unlink(chrt *Chart) error { + return h.unlink(chrt) +} diff --git a/pkg/chart/unloadhook.go b/pkg/chart/unloadhook.go deleted file mode 100644 index 355d1b52..00000000 --- a/pkg/chart/unloadhook.go +++ /dev/null @@ -1,37 +0,0 @@ -package chart - -// UnloadHook defines an interface for handling the unloading of a chart. -type UnloadHook interface { - // Unload is called when a chart is unloaded and may return an error. - Unload(*Chart) error -} - -// UnloadHooks is a slice of UnloadHook, processed in reverse order. -type UnloadHooks []UnloadHook - -// unloadHook wraps an unload function to implement UnloadHook. -type unloadHook struct { - unload func(*Chart) error -} - -var _ UnloadHook = (UnloadHooks)(nil) -var _ UnloadHook = (*unloadHook)(nil) - -// UnloadFunc creates an UnloadHook from the given function. -func UnloadFunc(unload func(*Chart) error) UnloadHook { - return &unloadHook{unload: unload} -} - -func (h UnloadHooks) Unload(chrt *Chart) error { - for i := len(h) - 1; i >= 0; i-- { - hook := h[i] - if err := hook.Unload(chrt); err != nil { - return err - } - } - return nil -} - -func (h *unloadHook) Unload(chrt *Chart) error { - return h.unload(chrt) -} diff --git a/pkg/hook/hook.go b/pkg/hook/hook.go index 61799637..1afaeda2 100644 --- a/pkg/hook/hook.go +++ b/pkg/hook/hook.go @@ -3,16 +3,21 @@ package hook import ( "sync" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/symbol" ) -// Hook represents a collection of hook functions that can be executed on symbols. +// Hook represents a collection of hook functions that can be executed on charts and symbols. type Hook struct { - loadHooks []symbol.LoadHook - unloadHooks []symbol.UnloadHook + linkHooks chart.LinkHooks + unlinkHooks chart.UnlinkHooks + loadHooks symbol.LoadHooks + unloadHooks symbol.UnloadHooks mu sync.RWMutex } +var _ chart.LinkHook = (*Hook)(nil) +var _ chart.UnlinkHook = (*Hook)(nil) var _ symbol.LoadHook = (*Hook)(nil) var _ symbol.UnloadHook = (*Hook)(nil) @@ -21,6 +26,22 @@ func New() *Hook { return &Hook{} } +// AddLinkHook adds a LinkHook function to the Hook. +func (h *Hook) AddLinkHook(hook chart.LinkHook) { + h.mu.Lock() + defer h.mu.Unlock() + + h.linkHooks = append(h.linkHooks, hook) +} + +// AddUnlinkHook adds an UnlinkHook function to the Hook. +func (h *Hook) AddUnlinkHook(hook chart.UnlinkHook) { + h.mu.Lock() + defer h.mu.Unlock() + + h.unlinkHooks = append(h.unlinkHooks, hook) +} + // AddLoadHook adds a LoadHook function to the Hook. func (h *Hook) AddLoadHook(hook symbol.LoadHook) { h.mu.Lock() @@ -37,17 +58,28 @@ func (h *Hook) AddUnloadHook(hook symbol.UnloadHook) { h.unloadHooks = append(h.unloadHooks, hook) } +// Link executes all LinkHooks registered in the Hook on the provided chart. +func (h *Hook) Link(chrt *chart.Chart) error { + h.mu.RLock() + defer h.mu.RUnlock() + + return h.linkHooks.Link(chrt) +} + +// Unlink executes all UnlinkHooks registered in the Hook on the provided chart. +func (h *Hook) Unlink(chrt *chart.Chart) error { + h.mu.RLock() + defer h.mu.RUnlock() + + return h.unlinkHooks.Unlink(chrt) +} + // Load executes all LoadHooks registered in the Hook on the provided symbol. func (h *Hook) Load(sb *symbol.Symbol) error { h.mu.RLock() defer h.mu.RUnlock() - for _, hook := range h.loadHooks { - if err := hook.Load(sb); err != nil { - return err - } - } - return nil + return h.loadHooks.Load(sb) } // Unload executes all UnloadHooks registered in the Hook on the provided symbol. @@ -55,11 +87,5 @@ func (h *Hook) Unload(sb *symbol.Symbol) error { h.mu.RLock() defer h.mu.RUnlock() - for i := len(h.unloadHooks) - 1; i >= 0; i-- { - hook := h.unloadHooks[i] - if err := hook.Unload(sb); err != nil { - return err - } - } - return nil + return h.unloadHooks.Unload(sb) } diff --git a/pkg/hook/hook_test.go b/pkg/hook/hook_test.go index 3b2e35b0..5ebb64a5 100644 --- a/pkg/hook/hook_test.go +++ b/pkg/hook/hook_test.go @@ -3,12 +3,45 @@ package hook import ( "testing" + "github.com/siyul-park/uniflow/pkg/chart" "github.com/siyul-park/uniflow/pkg/node" "github.com/siyul-park/uniflow/pkg/spec" "github.com/siyul-park/uniflow/pkg/symbol" "github.com/stretchr/testify/assert" ) +func TestHook_LinkHook(t *testing.T) { + hooks := New() + + count := 0 + h := chart.LinkFunc(func(_ *chart.Chart) error { + count += 1 + return nil + }) + + hooks.AddLinkHook(h) + + err := hooks.Link(&chart.Chart{}) + assert.NoError(t, err) + assert.Equal(t, 1, count) +} + +func TestHook_UnlinkHook(t *testing.T) { + hooks := New() + + count := 0 + h := chart.UnlinkFunc(func(_ *chart.Chart) error { + count += 1 + return nil + }) + + hooks.AddUnlinkHook(h) + + err := hooks.Unlink(&chart.Chart{}) + assert.NoError(t, err) + assert.Equal(t, 1, count) +} + func TestHook_LoadHook(t *testing.T) { hooks := New() diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index b964bb25..0513aca2 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -75,12 +75,13 @@ func New(config Config) *Runtime { }) chartLinker := chart.NewLinker(chart.LinkerConfig{ - Hook: config.Hook, - Scheme: config.Scheme, + LoadHooks: []symbol.LoadHook{config.Hook}, + UnloadHooks: []symbol.UnloadHook{config.Hook}, + Scheme: config.Scheme, }) chartTable := chart.NewTable(chart.TableOption{ - LoadHooks: []chart.LoadHook{chartLinker}, - UnloadHooks: []chart.UnloadHook{chartLinker}, + LinkHooks: []chart.LinkHook{chartLinker, config.Hook}, + UnlinkHooks: []chart.UnlinkHook{chartLinker, config.Hook}, }) chartLoader := chart.NewLoader(chart.LoaderConfig{ Table: chartTable, From 8306bd4eb07faf76532a7ebe514e9c48a998aaad Mon Sep 17 00:00:00 2001 From: siyul-park Date: Thu, 10 Oct 2024 04:37:48 -0400 Subject: [PATCH 31/31] feat: support chart syscall --- cmd/pkg/uniflow/main.go | 37 ++++-- docs/architecture.md | 95 --------------- docs/architecture_kr.md | 95 --------------- examples/system.yaml | 95 +++++++++++++++ ext/pkg/system/syscall.go | 87 ++++---------- ext/pkg/system/syscall_test.go | 207 ++++----------------------------- pkg/chart/linker.go | 2 - 7 files changed, 169 insertions(+), 449 deletions(-) diff --git a/cmd/pkg/uniflow/main.go b/cmd/pkg/uniflow/main.go index 6a1d4dac..4208a763 100644 --- a/cmd/pkg/uniflow/main.go +++ b/cmd/pkg/uniflow/main.go @@ -43,6 +43,23 @@ const ( flagCollectionSecrets = "collection.secrets" ) +const ( + opCreateCharts = "charts.create" + opReadCharts = "charts.read" + opUpdateCharts = "charts.update" + opDeleteCharts = "charts.delete" + + opCreateNodes = "nodes.create" + opReadNodes = "nodes.read" + opUpdateNodes = "nodes.update" + opDeleteNodes = "nodes.delete" + + opCreateSecrets = "secrets.create" + opReadSecrets = "secrets.read" + opUpdateSecrets = "secrets.update" + opDeleteSecrets = "secrets.delete" +) + func init() { viper.SetConfigFile(configFile) viper.AutomaticEnv() @@ -122,14 +139,18 @@ func main() { langs.Store(typescript.Language, typescript.NewCompiler()) nativeTable := system.NewNativeTable() - nativeTable.Store(system.CodeCreateNodes, system.CreateNodes(specStore)) - nativeTable.Store(system.CodeReadNodes, system.ReadNodes(specStore)) - nativeTable.Store(system.CodeUpdateNodes, system.UpdateNodes(specStore)) - nativeTable.Store(system.CodeDeleteNodes, system.DeleteNodes(specStore)) - nativeTable.Store(system.CodeCreateSecrets, system.CreateSecrets(secretStore)) - nativeTable.Store(system.CodeReadSecrets, system.ReadSecrets(secretStore)) - nativeTable.Store(system.CodeUpdateSecrets, system.UpdateSecrets(secretStore)) - nativeTable.Store(system.CodeDeleteSecrets, system.DeleteSecrets(secretStore)) + nativeTable.Store(opCreateCharts, system.CreateResource(chartStore)) + nativeTable.Store(opReadCharts, system.ReadResource(chartStore)) + nativeTable.Store(opUpdateCharts, system.UpdateResource(chartStore)) + nativeTable.Store(opDeleteCharts, system.DeleteResource(chartStore)) + nativeTable.Store(opCreateNodes, system.CreateResource(specStore)) + nativeTable.Store(opReadNodes, system.ReadResource(specStore)) + nativeTable.Store(opUpdateNodes, system.UpdateResource(specStore)) + nativeTable.Store(opDeleteNodes, system.DeleteResource(specStore)) + nativeTable.Store(opCreateSecrets, system.CreateResource(secretStore)) + nativeTable.Store(opReadSecrets, system.ReadResource(secretStore)) + nativeTable.Store(opUpdateSecrets, system.UpdateResource(secretStore)) + nativeTable.Store(opDeleteSecrets, system.DeleteResource(secretStore)) schemeBuilder.Register(control.AddToScheme(langs, cel.Language)) schemeBuilder.Register(io.AddToScheme(io.NewOSFileSystem())) diff --git a/docs/architecture.md b/docs/architecture.md index 55c204f2..1b129c11 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -62,18 +62,6 @@ Users can update node specifications by using a Command-Line Interface (CLI) or - method: DELETE path: /v1/nodes port: out[3] - - method: POST - path: /v1/secrets - port: out[4] - - method: GET - path: /v1/secrets - port: out[5] - - method: PATCH - path: /v1/secrets - port: out[6] - - method: DELETE - path: /v1/secrets - port: out[7] ports: out[0]: - name: nodes_create @@ -87,18 +75,6 @@ Users can update node specifications by using a Command-Line Interface (CLI) or out[3]: - name: nodes_delete port: in - out[4]: - - name: secrets_create - port: in - out[5]: - - name: secrets_read - port: in - out[6]: - - name: secrets_update - port: in - out[7]: - - name: secrets_delete - port: in - kind: block name: nodes_create @@ -171,77 +147,6 @@ Users can update node specifications by using a Command-Line Interface (CLI) or }; } -- kind: block - name: secrets_create - specs: - - kind: snippet - language: cel - code: 'has(self.body) ? self.body : null' - - kind: native - opcode: secrets.create - - kind: snippet - language: javascript - code: | - export default function (args) { - return { - body: args, - status: 201 - }; - } - -- kind: block - name: secrets_read - specs: - - kind: snippet - language: json - code: 'null' - - kind: native - opcode: secrets.read - - kind: snippet - language: javascript - code: | - export default function (args) { - return { - body: args, - status: 200 - }; - } - -- kind: block - name: secrets_update - specs: - - kind: snippet - language: cel - code: 'has(self.body) ? self.body : null' - - kind: native - opcode: secrets.update - - kind: snippet - language: javascript - code: | - export default function (args) { - return { - body: args, - status: 200 - }; - } - -- kind: block - name: secrets_delete - specs: - - kind: snippet - language: json - code: 'null' - - kind: native - opcode: secrets.delete - - kind: snippet - language: javascript - code: | - export default function (args) { - return { - status: 204 - }; - } - - kind: switch name: catch matches: diff --git a/docs/architecture_kr.md b/docs/architecture_kr.md index 5595093a..94aee235 100644 --- a/docs/architecture_kr.md +++ b/docs/architecture_kr.md @@ -60,18 +60,6 @@ - method: DELETE path: /v1/nodes port: out[3] - - method: POST - path: /v1/secrets - port: out[4] - - method: GET - path: /v1/secrets - port: out[5] - - method: PATCH - path: /v1/secrets - port: out[6] - - method: DELETE - path: /v1/secrets - port: out[7] ports: out[0]: - name: nodes_create @@ -85,18 +73,6 @@ out[3]: - name: nodes_delete port: in - out[4]: - - name: secrets_create - port: in - out[5]: - - name: secrets_read - port: in - out[6]: - - name: secrets_update - port: in - out[7]: - - name: secrets_delete - port: in - kind: block name: nodes_create @@ -169,77 +145,6 @@ }; } -- kind: block - name: secrets_create - specs: - - kind: snippet - language: cel - code: 'has(self.body) ? self.body : null' - - kind: native - opcode: secrets.create - - kind: snippet - language: javascript - code: | - export default function (args) { - return { - body: args, - status: 201 - }; - } - -- kind: block - name: secrets_read - specs: - - kind: snippet - language: json - code: 'null' - - kind: native - opcode: secrets.read - - kind: snippet - language: javascript - code: | - export default function (args) { - return { - body: args, - status: 200 - }; - } - -- kind: block - name: secrets_update - specs: - - kind: snippet - language: cel - code: 'has(self.body) ? self.body : null' - - kind: native - opcode: secrets.update - - kind: snippet - language: javascript - code: | - export default function (args) { - return { - body: args, - status: 200 - }; - } - -- kind: block - name: secrets_delete - specs: - - kind: snippet - language: json - code: 'null' - - kind: native - opcode: secrets.delete - - kind: snippet - language: javascript - code: | - export default function (args) { - return { - status: 204 - }; - } - - kind: switch name: catch matches: diff --git a/examples/system.yaml b/examples/system.yaml index bf6d7204..dc189fe6 100644 --- a/examples/system.yaml +++ b/examples/system.yaml @@ -37,6 +37,18 @@ - method: DELETE path: /v1/secrets port: out[7] + - method: POST + path: /v1/charts + port: out[8] + - method: GET + path: /v1/charts + port: out[9] + - method: PATCH + path: /v1/charts + port: out[10] + - method: DELETE + path: /v1/charts + port: out[11] ports: out[0]: - name: nodes_create @@ -62,6 +74,18 @@ out[7]: - name: secrets_delete port: in + out[8]: + - name: charts_create + port: in + out[9]: + - name: charts_read + port: in + out[10]: + - name: charts_update + port: in + out[11]: + - name: charts_delete + port: in - kind: block name: nodes_create @@ -205,6 +229,77 @@ }; } +- kind: block + name: charts_create + specs: + - kind: snippet + language: cel + code: 'has(self.body) ? self.body : null' + - kind: native + opcode: charts.create + - kind: snippet + language: javascript + code: | + export default function (args) { + return { + body: args, + status: 201 + }; + } + +- kind: block + name: charts_read + specs: + - kind: snippet + language: json + code: 'null' + - kind: native + opcode: charts.read + - kind: snippet + language: javascript + code: | + export default function (args) { + return { + body: args, + status: 200 + }; + } + +- kind: block + name: charts_update + specs: + - kind: snippet + language: cel + code: 'has(self.body) ? self.body : null' + - kind: native + opcode: charts.update + - kind: snippet + language: javascript + code: | + export default function (args) { + return { + body: args, + status: 200 + }; + } + +- kind: block + name: charts_delete + specs: + - kind: snippet + language: json + code: 'null' + - kind: native + opcode: charts.delete + - kind: snippet + language: javascript + code: | + export default function (args) { + return { + status: 204 + }; + } + - kind: switch name: catch matches: diff --git a/ext/pkg/system/syscall.go b/ext/pkg/system/syscall.go index c9284236..bb37b50f 100644 --- a/ext/pkg/system/syscall.go +++ b/ext/pkg/system/syscall.go @@ -3,91 +3,44 @@ package system import ( "context" - "github.com/siyul-park/uniflow/pkg/secret" - "github.com/siyul-park/uniflow/pkg/spec" + "github.com/siyul-park/uniflow/pkg/resource" ) -const ( - CodeCreateNodes = "nodes.create" - CodeReadNodes = "nodes.read" - CodeUpdateNodes = "nodes.update" - CodeDeleteNodes = "nodes.delete" - - CodeCreateSecrets = "secrets.create" - CodeReadSecrets = "secrets.read" - CodeUpdateSecrets = "secrets.update" - CodeDeleteSecrets = "secrets.delete" -) - -func CreateNodes(s spec.Store) func(context.Context, []spec.Spec) ([]spec.Spec, error) { - return func(ctx context.Context, specs []spec.Spec) ([]spec.Spec, error) { - if _, err := s.Store(ctx, specs...); err != nil { - return nil, err - } - return s.Load(ctx, specs...) - - } -} - -func ReadNodes(s spec.Store) func(context.Context, []spec.Spec) ([]spec.Spec, error) { - return func(ctx context.Context, specs []spec.Spec) ([]spec.Spec, error) { - return s.Load(ctx, specs...) - } -} - -func UpdateNodes(s spec.Store) func(context.Context, []spec.Spec) ([]spec.Spec, error) { - return func(ctx context.Context, specs []spec.Spec) ([]spec.Spec, error) { - if _, err := s.Swap(ctx, specs...); err != nil { - return nil, err - } - return s.Load(ctx, specs...) - } -} - -func DeleteNodes(s spec.Store) func(context.Context, []spec.Spec) ([]spec.Spec, error) { - return func(ctx context.Context, specs []spec.Spec) ([]spec.Spec, error) { - ok, err := s.Load(ctx, specs...) - if err != nil { - return nil, err - } - if _, err := s.Delete(ctx, ok...); err != nil { - return nil, err - } - return ok, nil - } -} - -func CreateSecrets(s secret.Store) func(context.Context, []*secret.Secret) ([]*secret.Secret, error) { - return func(ctx context.Context, secrets []*secret.Secret) ([]*secret.Secret, error) { - if _, err := s.Store(ctx, secrets...); err != nil { +// CreateResource is a generic function to store and load resources. +func CreateResource[T resource.Resource](store resource.Store[T]) func(context.Context, []T) ([]T, error) { + return func(ctx context.Context, resources []T) ([]T, error) { + if _, err := store.Store(ctx, resources...); err != nil { return nil, err } - return s.Load(ctx, secrets...) + return store.Load(ctx, resources...) } } -func ReadSecrets(s secret.Store) func(context.Context, []*secret.Secret) ([]*secret.Secret, error) { - return func(ctx context.Context, secrets []*secret.Secret) ([]*secret.Secret, error) { - return s.Load(ctx, secrets...) +// ReadResource is a generic function to load resources. +func ReadResource[T resource.Resource](store resource.Store[T]) func(context.Context, []T) ([]T, error) { + return func(ctx context.Context, resources []T) ([]T, error) { + return store.Load(ctx, resources...) } } -func UpdateSecrets(s secret.Store) func(context.Context, []*secret.Secret) ([]*secret.Secret, error) { - return func(ctx context.Context, secrets []*secret.Secret) ([]*secret.Secret, error) { - if _, err := s.Swap(ctx, secrets...); err != nil { +// UpdateResource is a generic function to swap and load resources. +func UpdateResource[T resource.Resource](store resource.Store[T]) func(context.Context, []T) ([]T, error) { + return func(ctx context.Context, resources []T) ([]T, error) { + if _, err := store.Swap(ctx, resources...); err != nil { return nil, err } - return s.Load(ctx, secrets...) + return store.Load(ctx, resources...) } } -func DeleteSecrets(s secret.Store) func(context.Context, []*secret.Secret) ([]*secret.Secret, error) { - return func(ctx context.Context, secrets []*secret.Secret) ([]*secret.Secret, error) { - ok, err := s.Load(ctx, secrets...) +// DeleteResource is a generic function to load and delete resources. +func DeleteResource[T resource.Resource](store resource.Store[T]) func(context.Context, []T) ([]T, error) { + return func(ctx context.Context, resources []T) ([]T, error) { + ok, err := store.Load(ctx, resources...) if err != nil { return nil, err } - if _, err := s.Delete(ctx, ok...); err != nil { + if _, err := store.Delete(ctx, ok...); err != nil { return nil, err } return ok, nil diff --git a/ext/pkg/system/syscall_test.go b/ext/pkg/system/syscall_test.go index b4eadfd0..3d87b949 100644 --- a/ext/pkg/system/syscall_test.go +++ b/ext/pkg/system/syscall_test.go @@ -11,26 +11,23 @@ import ( "github.com/siyul-park/uniflow/pkg/packet" "github.com/siyul-park/uniflow/pkg/port" "github.com/siyul-park/uniflow/pkg/process" - "github.com/siyul-park/uniflow/pkg/secret" - "github.com/siyul-park/uniflow/pkg/spec" + "github.com/siyul-park/uniflow/pkg/resource" "github.com/siyul-park/uniflow/pkg/types" "github.com/stretchr/testify/assert" ) -func TestCreateNodes(t *testing.T) { +func TestCreateResource(t *testing.T) { ctx, cancel := context.WithTimeout(context.TODO(), time.Second) defer cancel() - kind := faker.UUIDHyphenated() + st := resource.NewStore[*resource.Meta]() - st := spec.NewStore() - - n, _ := NewNativeNode(CreateNodes(st)) + n, _ := NewNativeNode(CreateResource(st)) defer n.Close() - meta := &spec.Meta{ + meta := &resource.Meta{ ID: uuid.Must(uuid.NewV7()), - Kind: kind, + Name: faker.Word(), } in := port.NewOut() @@ -48,27 +45,25 @@ func TestCreateNodes(t *testing.T) { select { case outPck := <-inWriter.Receive(): - var outPayload []*spec.Meta + var outPayload []*resource.Meta assert.NoError(t, types.Unmarshal(outPck.Payload(), &outPayload)) case <-ctx.Done(): assert.Fail(t, ctx.Err().Error()) } } -func TestReadNodes(t *testing.T) { +func TestReadResource(t *testing.T) { ctx, cancel := context.WithTimeout(context.TODO(), time.Second) defer cancel() - kind := faker.UUIDHyphenated() - - st := spec.NewStore() + st := resource.NewStore[*resource.Meta]() - n, _ := NewNativeNode(ReadNodes(st)) + n, _ := NewNativeNode(ReadResource(st)) defer n.Close() - meta := &spec.Meta{ + meta := &resource.Meta{ ID: uuid.Must(uuid.NewV7()), - Kind: kind, + Name: faker.Word(), } in := port.NewOut() @@ -86,27 +81,25 @@ func TestReadNodes(t *testing.T) { select { case outPck := <-inWriter.Receive(): - var outPayload []*spec.Meta + var outPayload []*resource.Meta assert.NoError(t, types.Unmarshal(outPck.Payload(), &outPayload)) case <-ctx.Done(): assert.Fail(t, ctx.Err().Error()) } } -func TestUpdateNodes(t *testing.T) { +func TestUpdateResource(t *testing.T) { ctx, cancel := context.WithTimeout(context.TODO(), time.Second) defer cancel() - kind := faker.UUIDHyphenated() + st := resource.NewStore[*resource.Meta]() - st := spec.NewStore() - - n, _ := NewNativeNode(UpdateNodes(st)) + n, _ := NewNativeNode(UpdateResource(st)) defer n.Close() - meta := &spec.Meta{ + meta := &resource.Meta{ ID: uuid.Must(uuid.NewV7()), - Kind: kind, + Name: faker.Word(), } _, _ = st.Store(ctx, meta) @@ -126,27 +119,25 @@ func TestUpdateNodes(t *testing.T) { select { case outPck := <-inWriter.Receive(): - var outPayload []*spec.Meta + var outPayload []*resource.Meta assert.NoError(t, types.Unmarshal(outPck.Payload(), &outPayload)) case <-ctx.Done(): assert.Fail(t, ctx.Err().Error()) } } -func TestDeleteNodes(t *testing.T) { +func TestDeleteResource(t *testing.T) { ctx, cancel := context.WithTimeout(context.TODO(), time.Second) defer cancel() - kind := faker.UUIDHyphenated() - - st := spec.NewStore() + st := resource.NewStore[*resource.Meta]() - n, _ := NewNativeNode(DeleteNodes(st)) + n, _ := NewNativeNode(DeleteResource(st)) defer n.Close() - meta := &spec.Meta{ + meta := &resource.Meta{ ID: uuid.Must(uuid.NewV7()), - Kind: kind, + Name: faker.Word(), } _, _ = st.Store(ctx, meta) @@ -166,155 +157,7 @@ func TestDeleteNodes(t *testing.T) { select { case outPck := <-inWriter.Receive(): - var outPayload []*spec.Meta - assert.NoError(t, types.Unmarshal(outPck.Payload(), &outPayload)) - case <-ctx.Done(): - assert.Fail(t, ctx.Err().Error()) - } -} - -func TestCreateSecrets(t *testing.T) { - ctx, cancel := context.WithTimeout(context.TODO(), time.Second) - defer cancel() - - st := secret.NewStore() - - n, _ := NewNativeNode(CreateSecrets(st)) - defer n.Close() - - scrt := &secret.Secret{ - ID: uuid.Must(uuid.NewV7()), - Data: faker.Word(), - } - - in := port.NewOut() - in.Link(n.In(node.PortIn)) - - proc := process.New() - defer proc.Exit(nil) - - inWriter := in.Open(proc) - - inPayload, _ := types.Marshal(scrt) - inPck := packet.New(types.NewSlice(inPayload)) - - inWriter.Write(inPck) - - select { - case outPck := <-inWriter.Receive(): - var outPayload []*secret.Secret - assert.NoError(t, types.Unmarshal(outPck.Payload(), &outPayload)) - case <-ctx.Done(): - assert.Fail(t, ctx.Err().Error()) - } -} - -func TestReadSecrets(t *testing.T) { - ctx, cancel := context.WithTimeout(context.TODO(), time.Second) - defer cancel() - - st := secret.NewStore() - - n, _ := NewNativeNode(ReadSecrets(st)) - defer n.Close() - - scrt := &secret.Secret{ - ID: uuid.Must(uuid.NewV7()), - Data: faker.Word(), - } - - in := port.NewOut() - in.Link(n.In(node.PortIn)) - - proc := process.New() - defer proc.Exit(nil) - - inWriter := in.Open(proc) - - inPayload, _ := types.Marshal(scrt) - inPck := packet.New(inPayload) - - inWriter.Write(inPck) - - select { - case outPck := <-inWriter.Receive(): - var outPayload []*secret.Secret - assert.NoError(t, types.Unmarshal(outPck.Payload(), &outPayload)) - case <-ctx.Done(): - assert.Fail(t, ctx.Err().Error()) - } -} - -func TestUpdateSecrets(t *testing.T) { - ctx, cancel := context.WithTimeout(context.TODO(), time.Second) - defer cancel() - - st := secret.NewStore() - - n, _ := NewNativeNode(UpdateSecrets(st)) - defer n.Close() - - scrt := &secret.Secret{ - ID: uuid.Must(uuid.NewV7()), - Data: faker.Word(), - } - - _, _ = st.Store(ctx, scrt) - - in := port.NewOut() - in.Link(n.In(node.PortIn)) - - proc := process.New() - defer proc.Exit(nil) - - inWriter := in.Open(proc) - - inPayload, _ := types.Marshal(scrt) - inPck := packet.New(types.NewSlice(inPayload)) - - inWriter.Write(inPck) - - select { - case outPck := <-inWriter.Receive(): - var outPayload []*secret.Secret - assert.NoError(t, types.Unmarshal(outPck.Payload(), &outPayload)) - case <-ctx.Done(): - assert.Fail(t, ctx.Err().Error()) - } -} - -func TestDeleteSecrets(t *testing.T) { - ctx, cancel := context.WithTimeout(context.TODO(), time.Second) - defer cancel() - - st := secret.NewStore() - - n, _ := NewNativeNode(DeleteSecrets(st)) - defer n.Close() - - scrt := &secret.Secret{ - ID: uuid.Must(uuid.NewV7()), - Data: faker.Word(), - } - - _, _ = st.Store(ctx, scrt) - - in := port.NewOut() - in.Link(n.In(node.PortIn)) - - proc := process.New() - defer proc.Exit(nil) - - inWriter := in.Open(proc) - - inPayload, _ := types.Marshal(scrt) - inPck := packet.New(inPayload) - - inWriter.Write(inPck) - - select { - case outPck := <-inWriter.Receive(): - var outPayload []*secret.Secret + var outPayload []*resource.Meta assert.NoError(t, types.Unmarshal(outPck.Payload(), &outPayload)) case <-ctx.Done(): assert.Fail(t, ctx.Err().Error()) diff --git a/pkg/chart/linker.go b/pkg/chart/linker.go index 7a96d43d..7d033cd6 100644 --- a/pkg/chart/linker.go +++ b/pkg/chart/linker.go @@ -107,7 +107,6 @@ func (l *Linker) Link(chrt *Chart) error { return n, nil }) - l.scheme.AddKnownType(kind, &spec.Unstructured{}) l.scheme.AddCodec(kind, codec) l.codecs[kind] = codec return nil @@ -125,7 +124,6 @@ func (l *Linker) Unlink(chrt *Chart) error { return nil } - l.scheme.RemoveKnownType(kind) l.scheme.RemoveCodec(kind) delete(l.codecs, kind) return nil