feat: support customized state backend (#124)

docs/backend.md

@@ -0,0 +1,61 @@
+# backend
+kusion state backends定义state存储位置，默认情况下，kusion使用local类型存储state在本地磁盘上，对于团队协作项目，state可存储在远程服务上，允许多人使用
+
+## backend 配置
+### 配置文件
+
+kusion 通过 project.yaml 中 backend 配置储存，例如
+```
+backend:
+  storageType: local
+  config:
+    path: kusion_state.json
+```
+* storageType - 声明储存类型
+* config - 声明对应存储类型所需参数
+### 命令行配置
+```
+kusion apply --backend-type local --backend-config path=kusion-state.json
+```
+### 合并配置
+当配置文件中 config 和命令行中 --backend-config 同时配置时，整个配置合并配置文件和命令行配置，例如
+```
+backend:
+  storageType: local
+  config:
+    path: kusion_state.json
+```
+```
+kusion apply --backend-config path-state=kusion-state.json
+```
+合并后 backend config 为
+```
+backend:
+  storageType: local
+  config:
+    path: kusion_state.json
+    path-state: kusion-state.json
+```
+## 可用Backend
+- local
+
+### 默认Backend
+
+当配置文件及命令行都没有声明 Backend 配置时，默认使用 [local](#local)
+
+### local
+local类型存储state在本地文件系统上，在本地操作，不适用于多人协同
+
+配置示例:
+```
+backend:
+  storageType: local
+  config:
+    path: kusion_state.json
+```
+* storageType - local, 表示使用本地文件系统
+* path - (可选) 配置 state 本地存储文件
+
+
+
+

pkg/engine/backend/backend.go

@@ -0,0 +1,125 @@
+package backend
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/gocty"
+	backendInit "kusionstack.io/kusion/pkg/engine/backend/init"
+	"kusionstack.io/kusion/pkg/engine/states"
+	"kusionstack.io/kusion/pkg/engine/states/local"
+	"kusionstack.io/kusion/pkg/util/i18n"
+)
+
+// backend config state storage type
+type Storage struct {
+	Type   string                 `json:"storageType,omitempty" yaml:"storageType,omitempty"`
+	Config map[string]interface{} `json:"config,omitempty" yaml:"config,omitempty"`
+}
+
+// BackendOps kusion cli backend override config
+type BackendOps struct {
+	// Config is a series of backend configurations,
+	// such as ["path=kusion_state.json"]
+	Config []string
+
+	// Type is the type of backend, currently supported:
+	//    local - state is stored to a local file
+	// TODO: support other storage type
+	Type string
+}
+
+func (o *BackendOps) AddBackendFlags(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&o.Type, "backend-type", "",
+		i18n.T("backend-type specify state storage backend"))
+	cmd.Flags().StringSliceVarP(&o.Config, "backend-config", "C", []string{},
+		i18n.T("backend-config config state storage backend"))
+}
+
+// MergeConfig merge project backend config and cli backend config
+func MergeConfig(config, override map[string]interface{}) map[string]interface{} {
+	content := make(map[string]interface{})
+	for k, v := range config {
+		content[k] = v
+	}
+	for k, v := range override {
+		content[k] = v
+	}
+	return content
+}
+
+// NewDefalutBackend return defalut backend.
+// default backend is local filesystem
+func NewDefaultBackend(dir string, fileName string) *Storage {
+	return &Storage{
+		Type: "local",
+		Config: map[string]interface{}{
+			"path": filepath.Join(dir, fileName),
+		},
+	}
+}
+
+// BackendFromConfig return stateStorage, this func handler
+// backend config merge and configure backend.
+// return a StateStorage to manage State
+func BackendFromConfig(config *Storage, override BackendOps, dir string) (states.StateStorage, error) {
+	var backendConfig Storage
+	if config == nil {
+		config = NewDefaultBackend(dir, local.KusionState)
+	}
+	if config.Type != "" {
+		backendConfig.Type = config.Type
+	}
+
+	if override.Type != "" {
+		backendConfig.Type = override.Type
+	}
+	configOverride := make(map[string]interface{})
+	for _, v := range override.Config {
+		bk := strings.Split(v, "=")
+		if len(bk) != 2 {
+			return nil, fmt.Errorf("kusion cli backend config should be path=kusion_state.json")
+		}
+		configOverride[bk[0]] = bk[1]
+	}
+	if config.Config != nil || override.Config != nil {
+		backendConfig.Config = MergeConfig(config.Config, configOverride)
+	}
+
+	backendFunc := backendInit.GetBackend(backendConfig.Type)
+	if backendFunc == nil {
+		return nil, fmt.Errorf("kusion backend storage: %s not support, please check storageType config", backendConfig.Type)
+	}
+
+	bf := backendFunc()
+
+	backendSchema := bf.ConfigSchema()
+	err := validBackendConfig(backendConfig.Config, backendSchema)
+	if err != nil {
+		return nil, err
+	}
+	ctyBackend, err := gocty.ToCtyValue(backendConfig.Config, backendSchema)
+	if err != nil {
+		return nil, err
+	}
+
+	err = bf.Configure(ctyBackend)
+	if err != nil {
+		return nil, err
+	}
+
+	return bf.StateStorage(), nil
+}
+
+// validBackendConfig check backend config.
+func validBackendConfig(config map[string]interface{}, schema cty.Type) error {
+	for k := range config {
+		if !schema.HasAttribute(k) {
+			return fmt.Errorf("not support %s in backend config", k)
+		}
+	}
+	return nil
+}

pkg/engine/backend/backend_test.go

@@ -0,0 +1,128 @@
+package backend
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/zclconf/go-cty/cty"
+	_ "kusionstack.io/kusion/pkg/engine/backend/init"
+	"kusionstack.io/kusion/pkg/engine/states"
+	"kusionstack.io/kusion/pkg/engine/states/local"
+)
+
+func TestMergeConfig(t *testing.T) {
+	type args struct {
+		config   map[string]interface{}
+		override map[string]interface{}
+	}
+	type want struct {
+		content map[string]interface{}
+	}
+
+	tests := map[string]struct {
+		args
+		want
+	}{
+		"MergeConfig": {
+			args: args{
+				config: map[string]interface{}{
+					"path": "kusion_state.json",
+				},
+				override: map[string]interface{}{
+					"config": "kusion_config.json",
+				},
+			},
+			want: want{
+				content: map[string]interface{}{
+					"path":   "kusion_state.json",
+					"config": "kusion_config.json",
+				},
+			},
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			mergeConfig := MergeConfig(tt.config, tt.override)
+			if diff := cmp.Diff(tt.want.content, mergeConfig); diff != "" {
+				t.Errorf("\nWrapMergeConfigFailed(...): -want message, +got message:\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestBackendFromConfig(t *testing.T) {
+	type args struct {
+		config   *Storage
+		override BackendOps
+	}
+	type want struct {
+		storage states.StateStorage
+		err     error
+	}
+	tests := map[string]struct {
+		args
+		want
+	}{
+		"BackendFromConfig": {
+			args: args{
+				config: &Storage{
+					Type: "local",
+					Config: map[string]interface{}{
+						"path": "kusion_state.json",
+					},
+				},
+				override: BackendOps{
+					Config: []string{
+						"path=kusion_local.json",
+					},
+				},
+			},
+			want: want{
+				storage: &local.FileSystemState{Path: "kusion_local.json"},
+				err:     nil,
+			},
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			storage, _ := BackendFromConfig(tt.config, tt.override, "./")
+			if diff := cmp.Diff(tt.want.storage, storage); diff != "" {
+				t.Errorf("\nWrapBackendFromConfigFailed(...): -want message, +got message:\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestValidBackendConfig(t *testing.T) {
+	type args struct {
+		config map[string]interface{}
+		schema cty.Type
+	}
+	type want struct {
+		errMsg string
+	}
+	tests := map[string]struct {
+		args
+		want
+	}{
+		"InValidBackendConfig": {
+			args: args{
+				config: map[string]interface{}{
+					"kusionPath": "kusion_state.json",
+				},
+				schema: cty.Object(map[string]cty.Type{"path": cty.String}),
+			},
+			want: want{
+				errMsg: "not support kusionPath in backend config",
+			},
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			err := validBackendConfig(tt.config, tt.schema)
+			if diff := cmp.Diff(tt.want.errMsg, err.Error()); diff != "" {
+				t.Errorf("\nWrapvalidBackendConfigFailed(...): -want message, +got message:\n%s", diff)
+			}
+		})
+	}
+}

pkg/engine/backend/init/init.go

@@ -0,0 +1,21 @@
+package init
+
+import (
+	"kusionstack.io/kusion/pkg/engine/states"
+	"kusionstack.io/kusion/pkg/engine/states/local"
+)
+
+// backends store all available backend
+var backends map[string]func() states.Backend
+
+// init backends map with all support backend
+func init() {
+	backends = map[string]func() states.Backend{
+		"local": local.NewLocalBackend,
+	}
+}
+
+// GetBackend return backend, or nil if not exists
+func GetBackend(name string) func() states.Backend {
+	return backends[name]
+}

pkg/engine/states/backend.go

@@ -13,9 +13,3 @@ type Backend interface {
 	// StateStorage return a StateStorage to manage State
 	StateStorage() StateStorage
 }
-
-var Backends = make(map[string]func() StateStorage)
-
-func AddToBackends(name string, storage func() StateStorage) {
-	Backends[name] = storage
-}

pkg/engine/states/local/backend.go

@@ -0,0 +1,35 @@
+package local
+
+import (
+	"github.com/zclconf/go-cty/cty"
+	"kusionstack.io/kusion/pkg/engine/states"
+)
+
+type LocalBackend struct {
+	FileSystemState
+}
+
+func NewLocalBackend() states.Backend {
+	return &LocalBackend{}
+}
+
+func (f *LocalBackend) StateStorage() states.StateStorage {
+	return &FileSystemState{f.Path}
+}
+
+func (f *LocalBackend) ConfigSchema() cty.Type {
+	config := map[string]cty.Type{
+		"path": cty.String,
+	}
+	return cty.Object(config)
+}
+
+func (f *LocalBackend) Configure(obj cty.Value) error {
+	var path cty.Value
+	if path = obj.GetAttr("path"); !path.IsNull() && path.AsString() != "" {
+		f.Path = path.AsString()
+	} else {
+		f.Path = KusionState
+	}
+	return nil
+}