-
Notifications
You must be signed in to change notification settings - Fork 78
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: support customized state backend (#124)
- Loading branch information
Showing
16 changed files
with
494 additions
and
137 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
# backend | ||
kusion state backends定义state存储位置,默认情况下,kusion使用local类型存储state在本地磁盘上,对于团队协作项目,state可存储在远程服务上,允许多人使用 | ||
|
||
## backend 配置 | ||
### 配置文件 | ||
|
||
kusion 通过 project.yaml 中 backend 配置储存,例如 | ||
``` | ||
backend: | ||
storageType: local | ||
config: | ||
path: kusion_state.json | ||
``` | ||
* storageType - 声明储存类型 | ||
* config - 声明对应存储类型所需参数 | ||
### 命令行配置 | ||
``` | ||
kusion apply --backend-type local --backend-config path=kusion-state.json | ||
``` | ||
### 合并配置 | ||
当配置文件中 config 和命令行中 --backend-config 同时配置时,整个配置合并配置文件和命令行配置,例如 | ||
``` | ||
backend: | ||
storageType: local | ||
config: | ||
path: kusion_state.json | ||
``` | ||
``` | ||
kusion apply --backend-config path-state=kusion-state.json | ||
``` | ||
合并后 backend config 为 | ||
``` | ||
backend: | ||
storageType: local | ||
config: | ||
path: kusion_state.json | ||
path-state: kusion-state.json | ||
``` | ||
## 可用Backend | ||
- local | ||
|
||
### 默认Backend | ||
|
||
当配置文件及命令行都没有声明 Backend 配置时,默认使用 [local](#local) | ||
|
||
### local | ||
local类型存储state在本地文件系统上,在本地操作,不适用于多人协同 | ||
|
||
配置示例: | ||
``` | ||
backend: | ||
storageType: local | ||
config: | ||
path: kusion_state.json | ||
``` | ||
* storageType - local, 表示使用本地文件系统 | ||
* path - (可选) 配置 state 本地存储文件 | ||
|
||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,125 @@ | ||
package backend | ||
|
||
import ( | ||
"fmt" | ||
"path/filepath" | ||
"strings" | ||
|
||
"github.com/spf13/cobra" | ||
"github.com/zclconf/go-cty/cty" | ||
"github.com/zclconf/go-cty/cty/gocty" | ||
backendInit "kusionstack.io/kusion/pkg/engine/backend/init" | ||
"kusionstack.io/kusion/pkg/engine/states" | ||
"kusionstack.io/kusion/pkg/engine/states/local" | ||
"kusionstack.io/kusion/pkg/util/i18n" | ||
) | ||
|
||
// backend config state storage type | ||
type Storage struct { | ||
Type string `json:"storageType,omitempty" yaml:"storageType,omitempty"` | ||
Config map[string]interface{} `json:"config,omitempty" yaml:"config,omitempty"` | ||
} | ||
|
||
// BackendOps kusion cli backend override config | ||
type BackendOps struct { | ||
// Config is a series of backend configurations, | ||
// such as ["path=kusion_state.json"] | ||
Config []string | ||
|
||
// Type is the type of backend, currently supported: | ||
// local - state is stored to a local file | ||
// TODO: support other storage type | ||
Type string | ||
} | ||
|
||
func (o *BackendOps) AddBackendFlags(cmd *cobra.Command) { | ||
cmd.Flags().StringVar(&o.Type, "backend-type", "", | ||
i18n.T("backend-type specify state storage backend")) | ||
cmd.Flags().StringSliceVarP(&o.Config, "backend-config", "C", []string{}, | ||
i18n.T("backend-config config state storage backend")) | ||
} | ||
|
||
// MergeConfig merge project backend config and cli backend config | ||
func MergeConfig(config, override map[string]interface{}) map[string]interface{} { | ||
content := make(map[string]interface{}) | ||
for k, v := range config { | ||
content[k] = v | ||
} | ||
for k, v := range override { | ||
content[k] = v | ||
} | ||
return content | ||
} | ||
|
||
// NewDefalutBackend return defalut backend. | ||
// default backend is local filesystem | ||
func NewDefaultBackend(dir string, fileName string) *Storage { | ||
return &Storage{ | ||
Type: "local", | ||
Config: map[string]interface{}{ | ||
"path": filepath.Join(dir, fileName), | ||
}, | ||
} | ||
} | ||
|
||
// BackendFromConfig return stateStorage, this func handler | ||
// backend config merge and configure backend. | ||
// return a StateStorage to manage State | ||
func BackendFromConfig(config *Storage, override BackendOps, dir string) (states.StateStorage, error) { | ||
var backendConfig Storage | ||
if config == nil { | ||
config = NewDefaultBackend(dir, local.KusionState) | ||
} | ||
if config.Type != "" { | ||
backendConfig.Type = config.Type | ||
} | ||
|
||
if override.Type != "" { | ||
backendConfig.Type = override.Type | ||
} | ||
configOverride := make(map[string]interface{}) | ||
for _, v := range override.Config { | ||
bk := strings.Split(v, "=") | ||
if len(bk) != 2 { | ||
return nil, fmt.Errorf("kusion cli backend config should be path=kusion_state.json") | ||
} | ||
configOverride[bk[0]] = bk[1] | ||
} | ||
if config.Config != nil || override.Config != nil { | ||
backendConfig.Config = MergeConfig(config.Config, configOverride) | ||
} | ||
|
||
backendFunc := backendInit.GetBackend(backendConfig.Type) | ||
if backendFunc == nil { | ||
return nil, fmt.Errorf("kusion backend storage: %s not support, please check storageType config", backendConfig.Type) | ||
} | ||
|
||
bf := backendFunc() | ||
|
||
backendSchema := bf.ConfigSchema() | ||
err := validBackendConfig(backendConfig.Config, backendSchema) | ||
if err != nil { | ||
return nil, err | ||
} | ||
ctyBackend, err := gocty.ToCtyValue(backendConfig.Config, backendSchema) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
err = bf.Configure(ctyBackend) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
return bf.StateStorage(), nil | ||
} | ||
|
||
// validBackendConfig check backend config. | ||
func validBackendConfig(config map[string]interface{}, schema cty.Type) error { | ||
for k := range config { | ||
if !schema.HasAttribute(k) { | ||
return fmt.Errorf("not support %s in backend config", k) | ||
} | ||
} | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,128 @@ | ||
package backend | ||
|
||
import ( | ||
"testing" | ||
|
||
"github.com/google/go-cmp/cmp" | ||
"github.com/zclconf/go-cty/cty" | ||
_ "kusionstack.io/kusion/pkg/engine/backend/init" | ||
"kusionstack.io/kusion/pkg/engine/states" | ||
"kusionstack.io/kusion/pkg/engine/states/local" | ||
) | ||
|
||
func TestMergeConfig(t *testing.T) { | ||
type args struct { | ||
config map[string]interface{} | ||
override map[string]interface{} | ||
} | ||
type want struct { | ||
content map[string]interface{} | ||
} | ||
|
||
tests := map[string]struct { | ||
args | ||
want | ||
}{ | ||
"MergeConfig": { | ||
args: args{ | ||
config: map[string]interface{}{ | ||
"path": "kusion_state.json", | ||
}, | ||
override: map[string]interface{}{ | ||
"config": "kusion_config.json", | ||
}, | ||
}, | ||
want: want{ | ||
content: map[string]interface{}{ | ||
"path": "kusion_state.json", | ||
"config": "kusion_config.json", | ||
}, | ||
}, | ||
}, | ||
} | ||
for name, tt := range tests { | ||
t.Run(name, func(t *testing.T) { | ||
mergeConfig := MergeConfig(tt.config, tt.override) | ||
if diff := cmp.Diff(tt.want.content, mergeConfig); diff != "" { | ||
t.Errorf("\nWrapMergeConfigFailed(...): -want message, +got message:\n%s", diff) | ||
} | ||
}) | ||
} | ||
} | ||
|
||
func TestBackendFromConfig(t *testing.T) { | ||
type args struct { | ||
config *Storage | ||
override BackendOps | ||
} | ||
type want struct { | ||
storage states.StateStorage | ||
err error | ||
} | ||
tests := map[string]struct { | ||
args | ||
want | ||
}{ | ||
"BackendFromConfig": { | ||
args: args{ | ||
config: &Storage{ | ||
Type: "local", | ||
Config: map[string]interface{}{ | ||
"path": "kusion_state.json", | ||
}, | ||
}, | ||
override: BackendOps{ | ||
Config: []string{ | ||
"path=kusion_local.json", | ||
}, | ||
}, | ||
}, | ||
want: want{ | ||
storage: &local.FileSystemState{Path: "kusion_local.json"}, | ||
err: nil, | ||
}, | ||
}, | ||
} | ||
for name, tt := range tests { | ||
t.Run(name, func(t *testing.T) { | ||
storage, _ := BackendFromConfig(tt.config, tt.override, "./") | ||
if diff := cmp.Diff(tt.want.storage, storage); diff != "" { | ||
t.Errorf("\nWrapBackendFromConfigFailed(...): -want message, +got message:\n%s", diff) | ||
} | ||
}) | ||
} | ||
} | ||
|
||
func TestValidBackendConfig(t *testing.T) { | ||
type args struct { | ||
config map[string]interface{} | ||
schema cty.Type | ||
} | ||
type want struct { | ||
errMsg string | ||
} | ||
tests := map[string]struct { | ||
args | ||
want | ||
}{ | ||
"InValidBackendConfig": { | ||
args: args{ | ||
config: map[string]interface{}{ | ||
"kusionPath": "kusion_state.json", | ||
}, | ||
schema: cty.Object(map[string]cty.Type{"path": cty.String}), | ||
}, | ||
want: want{ | ||
errMsg: "not support kusionPath in backend config", | ||
}, | ||
}, | ||
} | ||
for name, tt := range tests { | ||
t.Run(name, func(t *testing.T) { | ||
err := validBackendConfig(tt.config, tt.schema) | ||
if diff := cmp.Diff(tt.want.errMsg, err.Error()); diff != "" { | ||
t.Errorf("\nWrapvalidBackendConfigFailed(...): -want message, +got message:\n%s", diff) | ||
} | ||
}) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
package init | ||
|
||
import ( | ||
"kusionstack.io/kusion/pkg/engine/states" | ||
"kusionstack.io/kusion/pkg/engine/states/local" | ||
) | ||
|
||
// backends store all available backend | ||
var backends map[string]func() states.Backend | ||
|
||
// init backends map with all support backend | ||
func init() { | ||
backends = map[string]func() states.Backend{ | ||
"local": local.NewLocalBackend, | ||
} | ||
} | ||
|
||
// GetBackend return backend, or nil if not exists | ||
func GetBackend(name string) func() states.Backend { | ||
return backends[name] | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
package local | ||
|
||
import ( | ||
"github.com/zclconf/go-cty/cty" | ||
"kusionstack.io/kusion/pkg/engine/states" | ||
) | ||
|
||
type LocalBackend struct { | ||
FileSystemState | ||
} | ||
|
||
func NewLocalBackend() states.Backend { | ||
return &LocalBackend{} | ||
} | ||
|
||
func (f *LocalBackend) StateStorage() states.StateStorage { | ||
return &FileSystemState{f.Path} | ||
} | ||
|
||
func (f *LocalBackend) ConfigSchema() cty.Type { | ||
config := map[string]cty.Type{ | ||
"path": cty.String, | ||
} | ||
return cty.Object(config) | ||
} | ||
|
||
func (f *LocalBackend) Configure(obj cty.Value) error { | ||
var path cty.Value | ||
if path = obj.GetAttr("path"); !path.IsNull() && path.AsString() != "" { | ||
f.Path = path.AsString() | ||
} else { | ||
f.Path = KusionState | ||
} | ||
return nil | ||
} |
Oops, something went wrong.