feat: add function StateStorage in backend, and implement it (#909)

pkg/backend/backend.go

@@ -7,14 +7,16 @@ import (
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
 	"kusionstack.io/kusion/pkg/backend/storages"
 	"kusionstack.io/kusion/pkg/config"
+	"kusionstack.io/kusion/pkg/engine/state"
 )
 
 // Backend is used to provide the storage service for Workspace, Spec and State.
 type Backend interface {
 	// todo: add functions to parse storage for workspace, spec and state, the format is like the following:
 	// WorkspaceStorage() workspace.Storage
-	// StateStorage(projectName, stackName string) state.Storage
 	// SpecStorage(projectName, stackName string) spec.Storage
+
+	StateStorage(project, stack, workspace string) state.Storage
 }
 
 // NewBackend creates the Backend with the configuration set in the Kusion configuration file. If the
@@ -34,14 +36,13 @@ func NewBackend(name string) (Backend, error) {
 	}
 
 	var bkCfg *v1.BackendConfig
-	if name == "" {
-		if emptyCfg || cfg.Backends.Current == "" {
-			// if empty backends config or empty current backend, use the default storage
-			return storages.NewDefaultStorage(), nil
-		}
-		name = cfg.Backends.Current
-		bkCfg = cfg.Backends.Backends[name]
+	if name == "" && (emptyCfg || cfg.Backends.Current == "") {
+		// if empty backends config or empty current backend, use default local storage
+		bkCfg = &v1.BackendConfig{Type: v1.BackendTypeLocal}
 	} else {
+		if name == "" {
+			name = cfg.Backends.Current
+		}
 		bkCfg = cfg.Backends.Backends[name]
 		if bkCfg == nil {
 			return nil, fmt.Errorf("config of backend %s does not exist", name)
@@ -52,12 +53,15 @@ func NewBackend(name string) (Backend, error) {
 	switch bkCfg.Type {
 	case v1.BackendTypeLocal:
 		bkConfig := bkCfg.ToLocalBackend()
+		if err = storages.CompleteLocalConfig(bkConfig); err != nil {
+			return nil, fmt.Errorf("complete local config failed, %w", err)
+		}
 		return storages.NewLocalStorage(bkConfig), nil
 	case v1.BackendTypeMysql:
 		bkConfig := bkCfg.ToMysqlBackend()
 		storages.CompleteMysqlConfig(bkConfig)
 		if err = storages.ValidateMysqlConfig(bkConfig); err != nil {
-			return nil, fmt.Errorf("invalid config of backend %s: %w", name, err)
+			return nil, fmt.Errorf("invalid config of backend %s, %w", name, err)
 		}
 		storage, err = storages.NewMysqlStorage(bkConfig)
 		if err != nil {
@@ -67,7 +71,7 @@ func NewBackend(name string) (Backend, error) {
 		bkConfig := bkCfg.ToOssBackend()
 		storages.CompleteOssConfig(bkConfig)
 		if err = storages.ValidateOssConfig(bkConfig); err != nil {
-			return nil, fmt.Errorf("invalid config of backend %s: %w", name, err)
+			return nil, fmt.Errorf("invalid config of backend %s, %w", name, err)
 		}
 		storage, err = storages.NewOssStorage(bkConfig)
 		if err != nil {

pkg/backend/backend_test.go

@@ -51,6 +51,10 @@ func mockConfig() *v1.Config {
 	}
 }
 
+func mockCompleteLocalStorage() {
+	mockey.Mock(storages.CompleteLocalConfig).Return(nil).Build()
+}
+
 func mockNewStorage() {
 	mockey.Mock(storages.NewLocalStorage).Return(&storages.LocalStorage{}).Build()
 	mockey.Mock(storages.NewMysqlStorage).Return(&storages.MysqlStorage{}, nil).Build()
@@ -142,6 +146,7 @@ func TestNewBackend(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			mockey.PatchConvey("mock config", t, func() {
 				mockey.Mock(config.GetConfig).Return(tc.cfg, nil).Build()
+				mockCompleteLocalStorage()
 				mockNewStorage()
 				for k, v := range tc.envs {
 					_ = os.Setenv(k, v)

pkg/backend/storages/completion.go

@@ -4,9 +4,23 @@ import (
 	"os"
 
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/util/kfile"
 )
 
-// CompleteMysqlConfig sets default value of mysql config if not set.
+// CompleteLocalConfig sets default value of path if not set, which uses the path of kusion data folder.
+func CompleteLocalConfig(config *v1.BackendLocalConfig) error {
+	if config.Path == "" {
+		path, err := kfile.KusionDataFolder()
+		if err != nil {
+			return err
+		}
+		config.Path = path
+	}
+	return nil
+}
+
+// CompleteMysqlConfig sets default value of port if not set, which is 3306, and fulfills password from environment
+// variables if set.
 func CompleteMysqlConfig(config *v1.BackendMysqlConfig) {
 	if config.Port == 0 {
 		config.Port = v1.DefaultMysqlPort
@@ -17,7 +31,7 @@ func CompleteMysqlConfig(config *v1.BackendMysqlConfig) {
 	}
 }
 
-// CompleteOssConfig constructs the whole oss config by environment variables if set.
+// CompleteOssConfig fulfills the whole oss config from environment variables if set.
 func CompleteOssConfig(config *v1.BackendOssConfig) {
 	accessKeyID := os.Getenv(v1.EnvOssAccessKeyID)
 	accessKeySecret := os.Getenv(v1.EnvOssAccessKeySecret)
@@ -30,7 +44,7 @@ func CompleteOssConfig(config *v1.BackendOssConfig) {
 	}
 }
 
-// CompleteS3Config constructs the whole s3 config by environment variables if set.
+// CompleteS3Config fulfills the whole s3 config from environment variables if set.
 func CompleteS3Config(config *v1.BackendS3Config) {
 	accessKeyID := os.Getenv(v1.EnvAwsAccessKeyID)
 	accessKeySecret := os.Getenv(v1.EnvAwsSecretAccessKey)

pkg/backend/storages/completion_test.go

@@ -4,11 +4,44 @@ import (
 	"os"
 	"testing"
 
+	"github.com/bytedance/mockey"
 	"github.com/stretchr/testify/assert"
 
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/util/kfile"
 )
 
+func TestCompleteLocalConfig(t *testing.T) {
+	testcases := []struct {
+		name                 string
+		success              bool
+		config               *v1.BackendLocalConfig
+		mockKusionDataFolder string
+		completeConfig       *v1.BackendLocalConfig
+	}{
+		{
+			name:                 "complete local config",
+			success:              true,
+			config:               &v1.BackendLocalConfig{},
+			mockKusionDataFolder: "/etc",
+			completeConfig: &v1.BackendLocalConfig{
+				Path: "/etc",
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			mockey.PatchConvey("mock kusion data folder", t, func() {
+				mockey.Mock(kfile.KusionDataFolder).Return(tc.mockKusionDataFolder, nil).Build()
+				err := CompleteLocalConfig(tc.config)
+				assert.Equal(t, tc.success, err == nil)
+				assert.Equal(t, tc.completeConfig, tc.config)
+			})
+		})
+	}
+}
+
 func TestCompleteMysqlConfig(t *testing.T) {
 	testcases := []struct {
 		name           string

pkg/backend/storages/local.go

@@ -2,6 +2,8 @@ package storages
 
 import (
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/engine/state"
+	statestorages "kusionstack.io/kusion/pkg/engine/state/storages"
 )
 
 // LocalStorage is an implementation of backend.Backend which uses local filesystem as storage.
@@ -15,6 +17,6 @@ func NewLocalStorage(config *v1.BackendLocalConfig) *LocalStorage {
 	return &LocalStorage{path: config.Path}
 }
 
-func NewDefaultStorage() *LocalStorage {
-	return NewLocalStorage(&v1.BackendLocalConfig{})
+func (s *LocalStorage) StateStorage(project, stack, workspace string) state.Storage {
+	return statestorages.NewLocalStorage(statestorages.GenStateFilePath(s.path, project, stack, workspace))
 }

pkg/backend/storages/local_test.go

@@ -1,11 +1,14 @@
 package storages
 
 import (
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/engine/state"
+	statestorages "kusionstack.io/kusion/pkg/engine/state/storages"
 )
 
 func TestNewLocalStorage(t *testing.T) {
@@ -28,3 +31,32 @@ func TestNewLocalStorage(t *testing.T) {
 		})
 	}
 }
+
+func TestLocalStorage_StateStorage(t *testing.T) {
+	testcases := []struct {
+		name                      string
+		localStorage              *LocalStorage
+		project, stack, workspace string
+		stateStorage              state.Storage
+	}{
+		{
+			name: "state storage from s3 backend",
+			localStorage: &LocalStorage{
+				path: "kusion",
+			},
+			project:   "wordpress",
+			stack:     "dev",
+			workspace: "dev",
+			stateStorage: statestorages.NewLocalStorage(
+				filepath.Join("kusion", "states", "wordpress", "dev", "dev", "state.yaml"),
+			),
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			stateStorage := tc.localStorage.StateStorage(tc.project, tc.stack, tc.workspace)
+			assert.Equal(t, stateStorage, tc.stateStorage)
+		})
+	}
+}

pkg/backend/storages/mysql.go

@@ -8,6 +8,8 @@ import (
 	"gorm.io/gorm"
 
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/engine/state"
+	statestorages "kusionstack.io/kusion/pkg/engine/state/storages"
 )
 
 // MysqlStorage is an implementation of backend.Backend which uses mysql as storage.
@@ -35,3 +37,7 @@ func NewMysqlStorage(config *v1.BackendMysqlConfig) (*MysqlStorage, error) {
 
 	return &MysqlStorage{db: db}, nil
 }
+
+func (s *MysqlStorage) StateStorage(project, stack, workspace string) state.Storage {
+	return statestorages.NewMysqlStorage(s.db, project, stack, workspace)
+}

pkg/backend/storages/mysql_test.go

@@ -8,6 +8,8 @@ import (
 	"gorm.io/gorm"
 
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/engine/state"
+	statestorages "kusionstack.io/kusion/pkg/engine/state/storages"
 )
 
 func TestNewMysqlStorage(t *testing.T) {
@@ -38,3 +40,35 @@ func TestNewMysqlStorage(t *testing.T) {
 		})
 	}
 }
+
+func TestMysqlStorage_StateStorage(t *testing.T) {
+	testcases := []struct {
+		name                      string
+		mysqlStorage              *MysqlStorage
+		project, stack, workspace string
+		stateStorage              state.Storage
+	}{
+		{
+			name: "state storage from mysql",
+			mysqlStorage: &MysqlStorage{
+				db: &gorm.DB{},
+			},
+			project:   "wordpress",
+			stack:     "dev",
+			workspace: "dev",
+			stateStorage: statestorages.NewMysqlStorage(
+				&gorm.DB{},
+				"wordpress",
+				"dev",
+				"dev",
+			),
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			stateStorage := tc.mysqlStorage.StateStorage(tc.project, tc.stack, tc.workspace)
+			assert.Equal(t, stateStorage, tc.stateStorage)
+		})
+	}
+}

pkg/backend/storages/oss.go

@@ -4,6 +4,8 @@ import (
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/engine/state"
+	statestorages "kusionstack.io/kusion/pkg/engine/state/storages"
 )
 
 // OssStorage is an implementation of backend.Backend which uses oss as storage.
@@ -26,3 +28,7 @@ func NewOssStorage(config *v1.BackendOssConfig) (*OssStorage, error) {
 
 	return &OssStorage{bucket: bucket, prefix: config.Prefix}, nil
 }
+
+func (s *OssStorage) StateStorage(project, stack, workspace string) state.Storage {
+	return statestorages.NewOssStorage(s.bucket, statestorages.GenGenericOssStateFileKey(s.prefix, project, stack, workspace))
+}

pkg/backend/storages/oss_test.go

@@ -8,6 +8,8 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/engine/state"
+	statestorages "kusionstack.io/kusion/pkg/engine/state/storages"
 )
 
 func TestNewOssStorage(t *testing.T) {
@@ -40,3 +42,34 @@ func TestNewOssStorage(t *testing.T) {
 		})
 	}
 }
+
+func TestOssStorage_StateStorage(t *testing.T) {
+	testcases := []struct {
+		name                      string
+		ossStorage                *OssStorage
+		project, stack, workspace string
+		stateStorage              state.Storage
+	}{
+		{
+			name: "state storage from oss backend",
+			ossStorage: &OssStorage{
+				bucket: &oss.Bucket{},
+				prefix: "kusion",
+			},
+			project:   "wordpress",
+			stack:     "dev",
+			workspace: "dev",
+			stateStorage: statestorages.NewOssStorage(
+				&oss.Bucket{},
+				"kusion/states/wordpress/dev/dev/state.yaml",
+			),
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			stateStorage := tc.ossStorage.StateStorage(tc.project, tc.stack, tc.workspace)
+			assert.Equal(t, stateStorage, tc.stateStorage)
+		})
+	}
+}

pkg/backend/storages/s3.go

@@ -4,13 +4,16 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
 
 	v1 "kusionstack.io/kusion/pkg/apis/core/v1"
+	"kusionstack.io/kusion/pkg/engine/state"
+	statestorages "kusionstack.io/kusion/pkg/engine/state/storages"
 )
 
 // S3Storage is an implementation of backend.Backend which uses s3 as storage.
 type S3Storage struct {
-	sess   *session.Session
+	s3     *s3.S3
 	bucket string
 
 	// prefix will be added to the object storage key, so that all the files are stored under the prefix.
@@ -33,8 +36,12 @@ func NewS3Storage(config *v1.BackendS3Config) (*S3Storage, error) {
 	}
 
 	return &S3Storage{
-		sess:   sess,
+		s3:     s3.New(sess),
 		bucket: config.Bucket,
 		prefix: config.Prefix,
 	}, nil
 }
+
+func (s *S3Storage) StateStorage(project, stack, workspace string) state.Storage {
+	return statestorages.NewS3Storage(s.s3, s.bucket, statestorages.GenGenericOssStateFileKey(s.prefix, project, stack, workspace))
+}