From eeb26187e84c81cfa3d5d8f4c6f620eee42c4dc4 Mon Sep 17 00:00:00 2001 From: healthjyk Date: Fri, 15 Mar 2024 17:03:28 +0800 Subject: [PATCH] feat: add workspace storages. including local, oss, s3, mysql --- pkg/workspace/storage.go | 32 ++ pkg/workspace/storages/local.go | 191 ++++++++ pkg/workspace/storages/local_test.go | 409 ++++++++++++++++++ pkg/workspace/storages/mysql.go | 231 ++++++++++ pkg/workspace/storages/mysql_test.go | 127 ++++++ pkg/workspace/storages/oss.go | 212 +++++++++ pkg/workspace/storages/oss_test.go | 264 +++++++++++ pkg/workspace/storages/s3.go | 239 ++++++++++ pkg/workspace/storages/s3_test.go | 266 ++++++++++++ .../for_create_workspaces/.metadata.yml | 3 + .../for_create_workspaces/default.yaml | 1 + .../for_delete_workspaces/.metadata.yml | 4 + .../for_delete_workspaces/default.yaml | 1 + .../testdata/for_delete_workspaces/dev.yaml | 22 + .../for_set_current_workspaces/.metadata.yml | 4 + .../for_set_current_workspaces/default.yaml | 1 + .../for_set_current_workspaces/dev.yaml | 22 + .../invalid_metadata_workspaces/.metadata.yml | 1 + .../testdata/workspaces/.metadata.yml | 5 + .../storages/testdata/workspaces/default.yaml | 1 + .../storages/testdata/workspaces/dev.yaml | 22 + .../storages/testdata/workspaces/prod.yaml | 22 + pkg/workspace/storages/util.go | 58 +++ pkg/workspace/storages/util_test.go | 146 +++++++ 24 files changed, 2284 insertions(+) create mode 100644 pkg/workspace/storage.go create mode 100644 pkg/workspace/storages/local.go create mode 100644 pkg/workspace/storages/local_test.go create mode 100644 pkg/workspace/storages/mysql.go create mode 100644 pkg/workspace/storages/mysql_test.go create mode 100644 pkg/workspace/storages/oss.go create mode 100644 pkg/workspace/storages/oss_test.go create mode 100644 pkg/workspace/storages/s3.go create mode 100644 pkg/workspace/storages/s3_test.go create mode 100644 pkg/workspace/storages/testdata/for_create_workspaces/.metadata.yml create mode 100644 pkg/workspace/storages/testdata/for_create_workspaces/default.yaml create mode 100644 pkg/workspace/storages/testdata/for_delete_workspaces/.metadata.yml create mode 100644 pkg/workspace/storages/testdata/for_delete_workspaces/default.yaml create mode 100755 pkg/workspace/storages/testdata/for_delete_workspaces/dev.yaml create mode 100644 pkg/workspace/storages/testdata/for_set_current_workspaces/.metadata.yml create mode 100644 pkg/workspace/storages/testdata/for_set_current_workspaces/default.yaml create mode 100644 pkg/workspace/storages/testdata/for_set_current_workspaces/dev.yaml create mode 100644 pkg/workspace/storages/testdata/invalid_metadata_workspaces/.metadata.yml create mode 100644 pkg/workspace/storages/testdata/workspaces/.metadata.yml create mode 100644 pkg/workspace/storages/testdata/workspaces/default.yaml create mode 100644 pkg/workspace/storages/testdata/workspaces/dev.yaml create mode 100644 pkg/workspace/storages/testdata/workspaces/prod.yaml create mode 100644 pkg/workspace/storages/util.go create mode 100644 pkg/workspace/storages/util_test.go diff --git a/pkg/workspace/storage.go b/pkg/workspace/storage.go new file mode 100644 index 00000000..b4bc5f8b --- /dev/null +++ b/pkg/workspace/storage.go @@ -0,0 +1,32 @@ +package workspace + +import ( + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +// Storage is used to provide the storage service for multiple workspaces. +type Storage interface { + // Get returns the specified workspace configurations. + Get(name string) (*v1.Workspace, error) + + // Create the workspace. + Create(ws *v1.Workspace) error + + // Update the workspace. + Update(ws *v1.Workspace) error + + // Delete deletes the specified workspace. + Delete(name string) error + + // Exist returns the specified workspace exists or not. + Exist(name string) (bool, error) + + // GetNames returns the names of all the existing workspaces. + GetNames() ([]string, error) + + // GetCurrent gets the name of the current workspace. + GetCurrent() (string, error) + + // SetCurrent sets the specified workspace as the current workspace. + SetCurrent(name string) error +} diff --git a/pkg/workspace/storages/local.go b/pkg/workspace/storages/local.go new file mode 100644 index 00000000..f69f6c9d --- /dev/null +++ b/pkg/workspace/storages/local.go @@ -0,0 +1,191 @@ +package storages + +import ( + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +// LocalStorage is an implementation of workspace.Storage which uses local filesystem as storage. +type LocalStorage struct { + // The directory path to store the workspace files. + path string +} + +// NewLocalStorage news local workspace storage and init default workspace. +func NewLocalStorage(path string) (*LocalStorage, error) { + s := &LocalStorage{path: path} + + // create the workspace directory + if err := os.MkdirAll(s.path, os.ModePerm); err != nil { + return nil, fmt.Errorf("create workspace directory failed, %w", err) + } + + return s, s.initDefaultWorkspaceIf() +} + +func (s *LocalStorage) Get(name string) (*v1.Workspace, error) { + exist, err := s.Exist(name) + if err != nil { + return nil, err + } + if !exist { + return nil, ErrWorkspaceNotExist + } + + content, err := os.ReadFile(filepath.Join(s.path, name+yamlSuffix)) + if err != nil { + return nil, fmt.Errorf("read workspace file failed: %w", err) + } + + ws := &v1.Workspace{} + if err = yaml.Unmarshal(content, ws); err != nil { + return nil, fmt.Errorf("yaml unmarshal workspace failed: %w", err) + } + ws.Name = name + return ws, nil +} + +func (s *LocalStorage) Create(ws *v1.Workspace) error { + meta, err := s.readMeta() + if err != nil { + return err + } + if checkWorkspaceExistence(meta, ws.Name) { + return ErrWorkspaceAlreadyExist + } + + if err = s.writeWorkspace(ws); err != nil { + return err + } + + addAvailableWorkspaces(meta, ws.Name) + return s.writeMeta(meta) +} + +func (s *LocalStorage) Update(ws *v1.Workspace) error { + exist, err := s.Exist(ws.Name) + if err != nil { + return err + } + if !exist { + return ErrWorkspaceNotExist + } + + return s.writeWorkspace(ws) +} + +func (s *LocalStorage) Delete(name string) error { + meta, err := s.readMeta() + if err != nil { + return err + } + if !checkWorkspaceExistence(meta, name) { + return nil + } + + if err = os.Remove(filepath.Join(s.path, name+yamlSuffix)); err != nil { + return fmt.Errorf("remove workspace file failed: %w", err) + } + + removeAvailableWorkspaces(meta, name) + return s.writeMeta(meta) +} + +func (s *LocalStorage) Exist(name string) (bool, error) { + meta, err := s.readMeta() + if err != nil { + return false, err + } + return checkWorkspaceExistence(meta, name), nil +} + +func (s *LocalStorage) GetNames() ([]string, error) { + meta, err := s.readMeta() + if err != nil { + return nil, err + } + return meta.AvailableWorkspaces, nil +} + +func (s *LocalStorage) GetCurrent() (string, error) { + meta, err := s.readMeta() + if err != nil { + return "", err + } + return meta.Current, nil +} + +func (s *LocalStorage) SetCurrent(name string) error { + meta, err := s.readMeta() + if err != nil { + return err + } + if !checkWorkspaceExistence(meta, name) { + return ErrWorkspaceNotExist + } + meta.Current = name + return s.writeMeta(meta) +} + +func (s *LocalStorage) initDefaultWorkspaceIf() error { + meta, err := s.readMeta() + if err != nil { + return err + } + if !checkWorkspaceExistence(meta, defaultWorkspace) { + // if there is no default workspace, create one with empty workspace. + if err = s.writeWorkspace(&v1.Workspace{Name: defaultWorkspace}); err != nil { + return err + } + addAvailableWorkspaces(meta, defaultWorkspace) + } + + if meta.Current == "" { + meta.Current = defaultWorkspace + } + return s.writeMeta(meta) +} + +func (s *LocalStorage) readMeta() (*workspacesMetaData, error) { + content, err := os.ReadFile(filepath.Join(s.path, metadataFile)) + if os.IsNotExist(err) { + return &workspacesMetaData{}, nil + } else if err != nil { + return nil, fmt.Errorf("read workspace meta data file failed: %w", err) + } + + meta := &workspacesMetaData{} + if err = yaml.Unmarshal(content, meta); err != nil { + return nil, fmt.Errorf("yaml unmarshal workspaces meta data failed: %w", err) + } + return meta, nil +} + +func (s *LocalStorage) writeMeta(meta *workspacesMetaData) error { + content, err := yaml.Marshal(meta) + if err != nil { + return fmt.Errorf("yaml marshal workspaces meta data failed: %w", err) + } + + if err = os.WriteFile(filepath.Join(s.path, metadataFile), content, os.ModePerm); err != nil { + return fmt.Errorf("write workspaces meta data file failed: %w", err) + } + return nil +} + +func (s *LocalStorage) writeWorkspace(ws *v1.Workspace) error { + content, err := yaml.Marshal(ws) + if err != nil { + return fmt.Errorf("yaml marshal workspace failed: %w", err) + } + + if err = os.WriteFile(filepath.Join(s.path, ws.Name+yamlSuffix), content, os.ModePerm); err != nil { + return fmt.Errorf("write workspace file failed: %w", err) + } + return nil +} diff --git a/pkg/workspace/storages/local_test.go b/pkg/workspace/storages/local_test.go new file mode 100644 index 00000000..00dcc5a6 --- /dev/null +++ b/pkg/workspace/storages/local_test.go @@ -0,0 +1,409 @@ +package storages + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +func testDataFolder(path string) string { + pwd, _ := os.Getwd() + return filepath.Join(pwd, "testdata", path) +} + +func mockWorkspace(name string) *v1.Workspace { + return &v1.Workspace{ + Name: name, + Modules: map[string]*v1.ModuleConfig{ + "database": { + Default: v1.GenericConfig{ + "type": "aws", + "version": "5.7", + "instanceType": "db.t3.micro", + }, + ModulePatcherConfigs: v1.ModulePatcherConfigs{ + "smallClass": { + GenericConfig: v1.GenericConfig{ + "instanceType": "db.t3.small", + }, + ProjectSelector: []string{"foo", "bar"}, + }, + }, + }, + "port": { + Default: v1.GenericConfig{ + "type": "aws", + }, + }, + }, + Runtimes: &v1.RuntimeConfigs{ + Kubernetes: &v1.KubernetesConfig{ + KubeConfig: "/etc/kubeconfig.yaml", + }, + Terraform: v1.TerraformConfig{ + "aws": { + Source: "hashicorp/aws", + Version: "1.0.4", + GenericConfig: v1.GenericConfig{ + "region": "us-east-1", + }, + }, + }, + }, + } +} + +func mockWorkspaceContent() string { + return ` +modules: + database: + default: + instanceType: db.t3.micro + type: aws + version: "5.7" + smallClass: + projectSelector: + - foo + - bar + instanceType: db.t3.small + port: + default: + type: aws +runtimes: + kubernetes: + kubeConfig: /etc/kubeconfig.yaml + terraform: + aws: + source: hashicorp/aws + version: 1.0.4 + region: us-east-1 +` +} + +func TestLocalStorageOperation(t *testing.T) { + testcases := []struct { + name string + success bool + path string + expectedMeta *workspacesMetaData + deletePath bool + }{ + { + name: "new local storage with empty directory", + success: true, + path: testDataFolder("empty_workspaces"), + expectedMeta: &workspacesMetaData{ + Current: "default", + AvailableWorkspaces: []string{"default"}, + }, + deletePath: true, + }, + { + name: "new local storage with exist directory", + success: true, + path: testDataFolder("workspaces"), + expectedMeta: mockWorkspacesMetaData(), + deletePath: false, + }, + { + name: "new local storage failed", + success: false, + path: testDataFolder("invalid_metadata_workspaces"), + expectedMeta: nil, + deletePath: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + s, err := NewLocalStorage(tc.path) + assert.Equal(t, tc.success, err == nil) + if tc.success { + var meta *workspacesMetaData + meta, err = s.readMeta() + assert.NoError(t, err) + assert.Equal(t, tc.expectedMeta, meta) + } + if tc.deletePath { + _ = os.RemoveAll(tc.path) + } + }) + } +} + +func TestLocalStorage_Get(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + expectedWorkspace *v1.Workspace + }{ + { + name: "get workspace successfully", + success: true, + wsName: "dev", + expectedWorkspace: mockWorkspace("dev"), + }, + { + name: "get workspace failed not exist", + success: false, + wsName: "pre", + expectedWorkspace: nil, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + s, err := NewLocalStorage(testDataFolder("workspaces")) + assert.NoError(t, err) + workspace, err := s.Get(tc.wsName) + assert.Equal(t, tc.success, err == nil) + if tc.success { + assert.Equal(t, tc.expectedWorkspace, workspace) + } + }) + } +} + +func TestLocalStorage_Create(t *testing.T) { + testcases := []struct { + name string + success bool + path string + workspace *v1.Workspace + expectedMeta *workspacesMetaData + }{ + { + name: "create workspace successfully", + success: true, + path: testDataFolder("for_create_workspaces"), + workspace: mockWorkspace("dev"), + expectedMeta: &workspacesMetaData{ + Current: "default", + AvailableWorkspaces: []string{"default", "dev"}, + }, + }, + { + name: "create workspace failed already exist", + success: false, + path: testDataFolder("workspaces"), + workspace: mockWorkspace("prod"), + expectedMeta: nil, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + s, err := NewLocalStorage(tc.path) + assert.NoError(t, err) + err = s.Create(tc.workspace) + assert.Equal(t, tc.success, err == nil) + if tc.success { + var meta *workspacesMetaData + meta, err = s.readMeta() + assert.NoError(t, err) + assert.Equal(t, tc.expectedMeta, meta) + _ = s.Delete(tc.workspace.Name) + } + }) + } +} + +func TestLocalStorage_Update(t *testing.T) { + testcases := []struct { + name string + success bool + workspace *v1.Workspace + expectedWorkspace *v1.Workspace + }{ + { + name: "update workspace successfully", + success: true, + workspace: &v1.Workspace{Name: "default"}, + expectedWorkspace: &v1.Workspace{Name: "default"}, + }, + { + name: "update workspace failed not exist", + success: false, + workspace: mockWorkspace("pre"), + expectedWorkspace: nil, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + s, err := NewLocalStorage(testDataFolder("workspaces")) + assert.NoError(t, err) + err = s.Update(tc.workspace) + assert.Equal(t, tc.success, err == nil) + if tc.success { + var workspace *v1.Workspace + workspace, err = s.Get(tc.workspace.Name) + assert.NoError(t, err) + assert.Equal(t, tc.expectedWorkspace, workspace) + } + }) + } +} + +func TestLocalStorage_Delete(t *testing.T) { + testcases := []struct { + name string + success bool + path string + wsName string + expectedMeta *workspacesMetaData + }{ + { + name: "delete workspace successfully", + success: true, + path: testDataFolder("for_delete_workspaces"), + wsName: "dev", + expectedMeta: &workspacesMetaData{ + Current: "default", + AvailableWorkspaces: []string{"default"}, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + s, err := NewLocalStorage(tc.path) + assert.NoError(t, err) + err = s.Delete(tc.wsName) + assert.Equal(t, tc.success, err == nil) + if tc.success { + var meta *workspacesMetaData + meta, err = s.readMeta() + assert.NoError(t, err) + assert.Equal(t, tc.expectedMeta, meta) + _ = s.Create(mockWorkspace(tc.wsName)) + } + }) + } +} + +func TestLocalStorage_Exist(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + expectedExist bool + }{ + { + name: "exist workspace", + success: true, + wsName: "dev", + expectedExist: true, + }, + { + name: "not exist workspace", + success: true, + wsName: "pre", + expectedExist: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + s, err := NewLocalStorage(testDataFolder("workspaces")) + assert.NoError(t, err) + exist, err := s.Exist(tc.wsName) + assert.Equal(t, tc.success, err == nil) + if tc.success { + assert.Equal(t, tc.expectedExist, exist) + } + }) + } +} + +func TestLocalStorage_GetNames(t *testing.T) { + testcases := []struct { + name string + success bool + expectedNames []string + }{ + { + name: "exist workspace", + success: true, + expectedNames: []string{"default", "dev", "prod"}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + s, err := NewLocalStorage(testDataFolder("workspaces")) + assert.NoError(t, err) + names, err := s.GetNames() + assert.Equal(t, tc.success, err == nil) + if tc.success { + assert.Equal(t, tc.expectedNames, names) + } + }) + } +} + +func TestLocalStorage_GetCurrent(t *testing.T) { + testcases := []struct { + name string + success bool + expectedCurrent string + }{ + { + name: "current default workspace", + success: true, + expectedCurrent: "dev", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + s, err := NewLocalStorage(testDataFolder("workspaces")) + assert.NoError(t, err) + current, err := s.GetCurrent() + assert.Equal(t, tc.success, err == nil) + if tc.success { + assert.Equal(t, tc.expectedCurrent, current) + } + }) + } +} + +func TestLocalStorage_SetCurrent(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + }{ + { + name: "set current workspace successfully", + success: true, + wsName: "dev", + }, + { + name: "failed to set current workspace not exist", + success: false, + wsName: "pre", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + s, err := NewLocalStorage(testDataFolder("for_set_current_workspaces")) + assert.NoError(t, err) + err = s.SetCurrent(tc.wsName) + assert.Equal(t, tc.success, err == nil) + if tc.success { + var current string + current, err = s.GetCurrent() + assert.NoError(t, err) + assert.Equal(t, tc.wsName, current) + _ = s.SetCurrent("default") + } + }) + } +} diff --git a/pkg/workspace/storages/mysql.go b/pkg/workspace/storages/mysql.go new file mode 100644 index 00000000..c95e9bb9 --- /dev/null +++ b/pkg/workspace/storages/mysql.go @@ -0,0 +1,231 @@ +package storages + +import ( + "errors" + "fmt" + + "gopkg.in/yaml.v3" + "gorm.io/gorm" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +// MysqlStorage is an implementation of workspace.Storage which uses mysql as storage. +type MysqlStorage struct { + db *gorm.DB +} + +// NewMysqlStorage news mysql workspace storage and init default workspace. +func NewMysqlStorage(db *gorm.DB) (*MysqlStorage, error) { + s := &MysqlStorage{db: db} + + return s, s.initDefaultWorkspaceIf() +} + +func (s *MysqlStorage) Get(name string) (*v1.Workspace, error) { + exist, err := checkWorkspaceExistenceInMysql(s.db, name) + if err != nil { + return nil, err + } + if !exist { + return nil, ErrWorkspaceNotExist + } + + w, err := getWorkspaceFromMysql(s.db, name) + if err != nil { + return nil, fmt.Errorf("get workspace from mysql database failed: %w", err) + } + ws, err := convertFromMysqlDO(w) + if err != nil { + return nil, err + } + ws.Name = name + return ws, nil +} + +func (s *MysqlStorage) Create(ws *v1.Workspace) error { + exist, err := checkWorkspaceExistenceInMysql(s.db, ws.Name) + if err != nil { + return err + } + if exist { + return ErrWorkspaceAlreadyExist + } + + w, err := convertToMysqlDO(ws) + if err != nil { + return err + } + return createWorkspaceInMysql(s.db, w) +} + +func (s *MysqlStorage) Update(ws *v1.Workspace) error { + exist, err := checkWorkspaceExistenceInMysql(s.db, ws.Name) + if err != nil { + return err + } + if !exist { + return ErrWorkspaceNotExist + } + + w, err := convertToMysqlDO(ws) + if err != nil { + return err + } + return updateWorkspaceInMysql(s.db, w) +} + +func (s *MysqlStorage) Delete(name string) error { + return deleteWorkspaceInMysql(s.db, name) +} + +func (s *MysqlStorage) Exist(name string) (bool, error) { + return checkWorkspaceExistenceInMysql(s.db, name) +} + +func (s *MysqlStorage) GetNames() ([]string, error) { + return getWorkspaceNamesFromMysql(s.db) +} + +func (s *MysqlStorage) GetCurrent() (string, error) { + return getCurrentWorkspaceNameFromMysql(s.db) +} + +func (s *MysqlStorage) SetCurrent(name string) error { + exist, err := checkWorkspaceExistenceInMysql(s.db, name) + if err != nil { + return err + } + if !exist { + return ErrWorkspaceNotExist + } + + return alterCurrentWorkspaceInMysql(s.db, name) +} + +func (s *MysqlStorage) initDefaultWorkspaceIf() error { + exist, err := checkWorkspaceExistenceInMysql(s.db, defaultWorkspace) + if err != nil { + return err + } + if exist { + return nil + } + + w := &WorkspaceMysqlDO{Name: defaultWorkspace} + currentName, err := getCurrentWorkspaceNameFromMysql(s.db) + if err != nil { + return err + } + if currentName == "" { + isCurrent := true + w.IsCurrent = &isCurrent + } + + return createWorkspaceInMysql(s.db, w) +} + +// WorkspaceMysqlDO is the data object stored in the mysql db. +type WorkspaceMysqlDO struct { + Name string + Content string + IsCurrent *bool +} + +func (s WorkspaceMysqlDO) TableName() string { + return workspaceTable +} + +func getWorkspaceFromMysql(db *gorm.DB, name string) (*WorkspaceMysqlDO, error) { + q := &WorkspaceMysqlDO{Name: name} + w := &WorkspaceMysqlDO{} + result := db.Where(q).First(w) + return w, result.Error +} + +func getCurrentWorkspaceNameFromMysql(db *gorm.DB) (string, error) { + isCurrent := true + q := &WorkspaceMysqlDO{IsCurrent: &isCurrent} + w := &WorkspaceMysqlDO{} + result := db.Select("name").Where(q).First(w) + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return "", nil + } + return w.Name, result.Error +} + +func getWorkspaceNamesFromMysql(db *gorm.DB) ([]string, error) { + var wList []*WorkspaceMysqlDO + result := db.Select("name").Find(wList) + if result.Error != nil { + return nil, result.Error + } + names := make([]string, len(wList)) + for i, w := range wList { + names[i] = w.Name + } + return names, nil +} + +func checkWorkspaceExistenceInMysql(db *gorm.DB, name string) (bool, error) { + q := &WorkspaceMysqlDO{Name: name} + w := &WorkspaceMysqlDO{} + result := db.Select("name").Where(q).First(w) + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return false, nil + } + return result.Error == nil, result.Error +} + +func createWorkspaceInMysql(db *gorm.DB, w *WorkspaceMysqlDO) error { + return db.Create(w).Error +} + +func updateWorkspaceInMysql(db *gorm.DB, w *WorkspaceMysqlDO) error { + q := &WorkspaceMysqlDO{Name: w.Name} + return db.Where(q).Updates(w).Error +} + +func deleteWorkspaceInMysql(db *gorm.DB, name string) error { + q := &WorkspaceMysqlDO{Name: name} + return db.Where(q).Delete(&WorkspaceMysqlDO{}).Error +} + +func alterCurrentWorkspaceInMysql(db *gorm.DB, name string) error { + return db.Transaction(func(tx *gorm.DB) error { + // set the current workspace now to not current + isCurrent := true + q := &WorkspaceMysqlDO{IsCurrent: &isCurrent} + notCurrent := false + w := &WorkspaceMysqlDO{IsCurrent: ¬Current} + result := tx.Where(q).Updates(w) + if result.Error != nil { + return result.Error + } + + // set current of the specified workspace + q = &WorkspaceMysqlDO{Name: name} + w = &WorkspaceMysqlDO{IsCurrent: &isCurrent} + result = tx.Where(q).Updates(w) + return result.Error + }) +} + +func convertToMysqlDO(ws *v1.Workspace) (*WorkspaceMysqlDO, error) { + content, err := yaml.Marshal(ws) + if err != nil { + return nil, fmt.Errorf("yaml marshal workspace failed: %w", err) + } + return &WorkspaceMysqlDO{ + Name: ws.Name, + Content: string(content), + }, nil +} + +func convertFromMysqlDO(w *WorkspaceMysqlDO) (*v1.Workspace, error) { + ws := &v1.Workspace{} + if err := yaml.Unmarshal([]byte(w.Content), ws); err != nil { + return nil, fmt.Errorf("yaml unmarshal workspace failed: %w", err) + } + return ws, nil +} diff --git a/pkg/workspace/storages/mysql_test.go b/pkg/workspace/storages/mysql_test.go new file mode 100644 index 00000000..2cfff1fc --- /dev/null +++ b/pkg/workspace/storages/mysql_test.go @@ -0,0 +1,127 @@ +package storages + +import ( + "testing" + + "github.com/bytedance/mockey" + "github.com/stretchr/testify/assert" + "gorm.io/gorm" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +func mockWorkspaceMysqlDO(name string) *WorkspaceMysqlDO { + return &WorkspaceMysqlDO{ + Name: name, + Content: mockWorkspaceContent(), + } +} + +func mockMysqlStorage() *MysqlStorage { + return &MysqlStorage{db: &gorm.DB{}} +} + +func TestMysqlStorage_Get(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + content string + expectedWorkspace *v1.Workspace + }{ + { + name: "get workspace successfully", + success: true, + wsName: "dev", + content: mockWorkspaceContent(), + expectedWorkspace: mockWorkspace("dev"), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock gorm operation", t, func() { + mockey.Mock(checkWorkspaceExistenceInMysql).Return(true, nil).Build() + mockey.Mock(getWorkspaceFromMysql).Return(&WorkspaceMysqlDO{Content: tc.content}, nil).Build() + workspace, err := mockMysqlStorage().Get(tc.wsName) + assert.Equal(t, tc.success, err == nil) + assert.Equal(t, tc.expectedWorkspace, workspace) + }) + }) + } +} + +func TestMysqlStorage_Create(t *testing.T) { + testcases := []struct { + name string + success bool + workspace *v1.Workspace + }{ + { + name: "create workspace successfully", + success: true, + workspace: mockWorkspace("pre"), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock gorm operation", t, func() { + mockey.Mock(checkWorkspaceExistenceInMysql).Return(false, nil).Build() + mockey.Mock(createWorkspaceInMysql).Return(nil).Build() + err := mockMysqlStorage().Create(tc.workspace) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} + +func TestMysqlStorage_Update(t *testing.T) { + testcases := []struct { + name string + success bool + workspace *v1.Workspace + }{ + { + name: "update workspace successfully", + success: true, + workspace: mockWorkspace("dev"), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock gorm operation", t, func() { + mockey.Mock(checkWorkspaceExistenceInMysql).Return(true, nil).Build() + mockey.Mock(updateWorkspaceInMysql).Return(nil).Build() + err := mockMysqlStorage().Update(tc.workspace) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} + +func TestMysqlStorage_SetCurrent(t *testing.T) { + testcases := []struct { + name string + success bool + current string + }{ + { + name: "set current workspace successfully", + success: true, + current: "prod", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock gorm operation", t, func() { + mockey.Mock(checkWorkspaceExistenceInMysql).Return(true, nil).Build() + mockey.Mock(alterCurrentWorkspaceInMysql).Return(nil).Build() + err := mockMysqlStorage().SetCurrent(tc.current) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} diff --git a/pkg/workspace/storages/oss.go b/pkg/workspace/storages/oss.go new file mode 100644 index 00000000..4b6e35e1 --- /dev/null +++ b/pkg/workspace/storages/oss.go @@ -0,0 +1,212 @@ +package storages + +import ( + "bytes" + "fmt" + "io" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "gopkg.in/yaml.v3" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +// OssStorage is an implementation of workspace.Storage which uses oss as storage. +type OssStorage struct { + bucket *oss.Bucket + + // The prefix to store the workspaces files. + prefix string +} + +// NewOssStorage news oss workspace storage and init default workspace. +func NewOssStorage(bucket *oss.Bucket, prefix string) (*OssStorage, error) { + s := &OssStorage{ + bucket: bucket, + prefix: prefix, + } + return s, s.initDefaultWorkspaceIf() +} + +func (s *OssStorage) Get(name string) (*v1.Workspace, error) { + exist, err := s.Exist(name) + if err != nil { + return nil, err + } + if !exist { + return nil, ErrWorkspaceNotExist + } + + body, err := s.bucket.GetObject(s.prefix + "/" + name + yamlSuffix) + if err != nil { + return nil, fmt.Errorf("get workspace from oss failed: %w", err) + } + defer func() { + _ = body.Close() + }() + content, err := io.ReadAll(body) + if err != nil { + return nil, fmt.Errorf("read workspace failed: %w", err) + } + + ws := &v1.Workspace{} + if err = yaml.Unmarshal(content, ws); err != nil { + return nil, fmt.Errorf("yaml unmarshal workspace failed: %w", err) + } + ws.Name = name + return ws, nil +} + +func (s *OssStorage) Create(ws *v1.Workspace) error { + meta, err := s.readMeta() + if err != nil { + return err + } + if checkWorkspaceExistence(meta, ws.Name) { + return ErrWorkspaceAlreadyExist + } + + if err = s.writeWorkspace(ws); err != nil { + return err + } + + addAvailableWorkspaces(meta, ws.Name) + return s.writeMeta(meta) +} + +func (s *OssStorage) Update(ws *v1.Workspace) error { + exist, err := s.Exist(ws.Name) + if err != nil { + return err + } + if !exist { + return ErrWorkspaceNotExist + } + + return s.writeWorkspace(ws) +} + +func (s *OssStorage) Delete(name string) error { + meta, err := s.readMeta() + if err != nil { + return err + } + if !checkWorkspaceExistence(meta, name) { + return nil + } + + if err = s.bucket.DeleteObject(s.prefix + "/" + name + yamlSuffix); err != nil { + return fmt.Errorf("remove workspace in oss failed: %w", err) + } + + removeAvailableWorkspaces(meta, name) + return s.writeMeta(meta) +} + +func (s *OssStorage) Exist(name string) (bool, error) { + meta, err := s.readMeta() + if err != nil { + return false, err + } + return checkWorkspaceExistence(meta, name), nil +} + +func (s *OssStorage) GetNames() ([]string, error) { + meta, err := s.readMeta() + if err != nil { + return nil, err + } + return meta.AvailableWorkspaces, nil +} + +func (s *OssStorage) GetCurrent() (string, error) { + meta, err := s.readMeta() + if err != nil { + return "", err + } + return meta.Current, nil +} + +func (s *OssStorage) SetCurrent(name string) error { + meta, err := s.readMeta() + if err != nil { + return err + } + if !checkWorkspaceExistence(meta, name) { + return ErrWorkspaceNotExist + } + meta.Current = name + return s.writeMeta(meta) +} + +func (s *OssStorage) initDefaultWorkspaceIf() error { + meta, err := s.readMeta() + if err != nil { + return err + } + if !checkWorkspaceExistence(meta, defaultWorkspace) { + // if there is no default workspace, create one with empty workspace. + if err = s.writeWorkspace(&v1.Workspace{Name: defaultWorkspace}); err != nil { + return err + } + addAvailableWorkspaces(meta, defaultWorkspace) + } + + if meta.Current == "" { + meta.Current = defaultWorkspace + } + return s.writeMeta(meta) +} + +func (s *OssStorage) readMeta() (*workspacesMetaData, error) { + body, err := s.bucket.GetObject(s.prefix + "/" + metadataFile) + if err != nil { + ossErr, ok := err.(oss.ServiceError) + // error code ref: github.com/aliyun/aliyun-oss-go-sdk@v2.1.8+incompatible/oss/bucket.go:553 + if ok && ossErr.StatusCode == 404 { + return &workspacesMetaData{}, nil + } + return nil, fmt.Errorf("get workspaces meta data from oss failed: %w", err) + } + defer func() { + _ = body.Close() + }() + + content, err := io.ReadAll(body) + if err != nil { + return nil, fmt.Errorf("read workspaces meta data failed: %w", err) + } + if len(content) == 0 { + return &workspacesMetaData{}, nil + } + + meta := &workspacesMetaData{} + if err = yaml.Unmarshal(content, meta); err != nil { + return nil, fmt.Errorf("yaml unmarshal workspaces meta data failed: %w", err) + } + return meta, nil +} + +func (s *OssStorage) writeMeta(meta *workspacesMetaData) error { + content, err := yaml.Marshal(meta) + if err != nil { + return fmt.Errorf("yaml marshal workspaces meta data failed: %w", err) + } + + if err = s.bucket.PutObject(s.prefix+"/"+metadataFile, bytes.NewReader(content)); err != nil { + return fmt.Errorf("put workspaces meta data to oss failed: %w", err) + } + return nil +} + +func (s *OssStorage) writeWorkspace(ws *v1.Workspace) error { + content, err := yaml.Marshal(ws) + if err != nil { + return fmt.Errorf("yaml marshal workspace failed: %w", err) + } + + if err = s.bucket.PutObject(s.prefix+"/"+ws.Name+yamlSuffix, bytes.NewReader(content)); err != nil { + return fmt.Errorf("put workspace to oss failed: %w", err) + } + return nil +} diff --git a/pkg/workspace/storages/oss_test.go b/pkg/workspace/storages/oss_test.go new file mode 100644 index 00000000..9f1e49c2 --- /dev/null +++ b/pkg/workspace/storages/oss_test.go @@ -0,0 +1,264 @@ +package storages + +import ( + "bytes" + "io" + "testing" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/bytedance/mockey" + "github.com/stretchr/testify/assert" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +func mockOssStorage() *OssStorage { + return &OssStorage{bucket: &oss.Bucket{}} +} + +func mockOssStorageReadMeta(meta *workspacesMetaData) { + mockey.Mock((*OssStorage).readMeta).Return(meta, nil).Build() +} + +func mockOssStorageWriteMeta() { + mockey.Mock((*OssStorage).writeMeta).Return(nil).Build() +} + +func mockOssStorageWriteWorkspace() { + mockey.Mock((*OssStorage).writeWorkspace).Return(nil).Build() +} + +func TestOssStorage_Get(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + content []byte + expectedWorkspace *v1.Workspace + }{ + { + name: "get workspace successfully", + success: true, + wsName: "dev", + content: []byte(mockWorkspaceContent()), + expectedWorkspace: mockWorkspace("dev"), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock oss operation", t, func() { + mockey.Mock(oss.Bucket.GetObject).Return(io.NopCloser(bytes.NewReader([]byte(""))), nil).Build() + mockey.Mock(io.ReadAll).Return(tc.content, nil).Build() + mockOssStorageReadMeta(mockWorkspacesMetaData()) + workspace, err := mockOssStorage().Get(tc.wsName) + assert.Equal(t, tc.success, err == nil) + assert.Equal(t, tc.expectedWorkspace, workspace) + }) + }) + } +} + +func TestOssStorage_Create(t *testing.T) { + testcases := []struct { + name string + success bool + workspace *v1.Workspace + }{ + { + name: "create workspace successfully", + success: true, + workspace: mockWorkspace("pre"), + }, + { + name: "failed to create workspace already exist", + success: false, + workspace: mockWorkspace("dev"), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock oss operation", t, func() { + mockOssStorageReadMeta(mockWorkspacesMetaData()) + mockOssStorageWriteMeta() + mockOssStorageWriteWorkspace() + err := mockOssStorage().Create(tc.workspace) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} + +func TestOssStorage_Update(t *testing.T) { + testcases := []struct { + name string + success bool + workspace *v1.Workspace + }{ + { + name: "update workspace successfully", + success: true, + workspace: mockWorkspace("dev"), + }, + { + name: "failed to update workspace not exist", + success: false, + workspace: mockWorkspace("pre"), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock oss operation", t, func() { + mockOssStorageReadMeta(mockWorkspacesMetaData()) + mockOssStorageWriteWorkspace() + err := mockOssStorage().Update(tc.workspace) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} + +func TestOssStorage_Delete(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + }{ + { + name: "delete workspace successfully", + success: true, + wsName: "dev", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock oss operation", t, func() { + mockey.Mock(oss.Bucket.DeleteObject).Return(nil).Build() + mockOssStorageReadMeta(mockWorkspacesMetaData()) + mockOssStorageWriteMeta() + err := mockOssStorage().Delete(tc.wsName) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} + +func TestOssStorage_Exist(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + expectedExist bool + }{ + { + name: "exist workspace", + success: true, + wsName: "dev", + expectedExist: true, + }, + { + name: "not exist workspace", + success: true, + wsName: "pre", + expectedExist: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock oss operation", t, func() { + mockOssStorageReadMeta(mockWorkspacesMetaData()) + exist, err := mockOssStorage().Exist(tc.wsName) + assert.Equal(t, tc.success, err == nil) + assert.Equal(t, tc.expectedExist, exist) + }) + }) + } +} + +func TestOssStorage_GetNames(t *testing.T) { + testcases := []struct { + name string + success bool + expectedNames []string + }{ + { + name: "get all workspace names successfully", + success: true, + expectedNames: []string{"default", "dev", "prod"}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock oss operation", t, func() { + mockOssStorageReadMeta(mockWorkspacesMetaData()) + wsNames, err := mockOssStorage().GetNames() + assert.Equal(t, tc.success, err == nil) + if tc.success { + assert.Equal(t, tc.expectedNames, wsNames) + } + }) + }) + } +} + +func TestOssStorage_GetCurrent(t *testing.T) { + testcases := []struct { + name string + success bool + expectedCurrent string + }{ + { + name: "get current workspace successfully", + success: true, + expectedCurrent: "dev", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock oss operation", t, func() { + mockOssStorageReadMeta(mockWorkspacesMetaData()) + current, err := mockOssStorage().GetCurrent() + assert.Equal(t, tc.success, err == nil) + if tc.success { + assert.Equal(t, tc.expectedCurrent, current) + } + }) + }) + } +} + +func TestOssStorage_SetCurrent(t *testing.T) { + testcases := []struct { + name string + success bool + current string + }{ + { + name: "set current workspace successfully", + success: true, + current: "prod", + }, + { + name: "failed to set current workspace not exist", + success: false, + current: "pre", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock oss operation", t, func() { + mockOssStorageReadMeta(mockWorkspacesMetaData()) + mockOssStorageWriteMeta() + err := mockOssStorage().SetCurrent(tc.current) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} diff --git a/pkg/workspace/storages/s3.go b/pkg/workspace/storages/s3.go new file mode 100644 index 00000000..6ed518e9 --- /dev/null +++ b/pkg/workspace/storages/s3.go @@ -0,0 +1,239 @@ +package storages + +import ( + "bytes" + "fmt" + "io" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "gopkg.in/yaml.v3" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +// S3Storage is an implementation of workspace.Storage which uses s3 as storage. +type S3Storage struct { + s3 *s3.S3 + bucket string + + // The prefix to store the workspaces files. + prefix string +} + +// NewS3Storage news s3 workspace storage and init default workspace. +func NewS3Storage(s3 *s3.S3, bucket, prefix string) (*S3Storage, error) { + s := &S3Storage{ + s3: s3, + bucket: bucket, + prefix: prefix, + } + return s, s.initDefaultWorkspaceIf() +} + +func (s *S3Storage) Get(name string) (*v1.Workspace, error) { + exist, err := s.Exist(name) + if err != nil { + return nil, err + } + if !exist { + return nil, ErrWorkspaceNotExist + } + + key := s.prefix + "/" + name + yamlSuffix + input := &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: &key, + } + output, err := s.s3.GetObject(input) + if err != nil { + return nil, fmt.Errorf("get workspace from s3 failed: %w", err) + } + defer func() { + _ = output.Body.Close() + }() + content, err := io.ReadAll(output.Body) + if err != nil { + return nil, fmt.Errorf("read workspace failed: %w", err) + } + + ws := &v1.Workspace{} + if err = yaml.Unmarshal(content, ws); err != nil { + return nil, fmt.Errorf("yaml unmarshal workspace failed: %w", err) + } + ws.Name = name + return ws, nil +} + +func (s *S3Storage) Create(ws *v1.Workspace) error { + meta, err := s.readMeta() + if err != nil { + return err + } + if checkWorkspaceExistence(meta, ws.Name) { + return ErrWorkspaceAlreadyExist + } + + if err = s.writeWorkspace(ws); err != nil { + return err + } + + addAvailableWorkspaces(meta, ws.Name) + return s.writeMeta(meta) +} + +func (s *S3Storage) Update(ws *v1.Workspace) error { + exist, err := s.Exist(ws.Name) + if err != nil { + return err + } + if !exist { + return ErrWorkspaceNotExist + } + + return s.writeWorkspace(ws) +} + +func (s *S3Storage) Delete(name string) error { + meta, err := s.readMeta() + if err != nil { + return err + } + if !checkWorkspaceExistence(meta, name) { + return nil + } + + input := &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(s.prefix + "/" + name + yamlSuffix), + } + if _, err = s.s3.DeleteObject(input); err != nil { + return fmt.Errorf("remove workspace in s3 failed: %w", err) + } + + removeAvailableWorkspaces(meta, name) + return s.writeMeta(meta) +} + +func (s *S3Storage) Exist(name string) (bool, error) { + meta, err := s.readMeta() + if err != nil { + return false, err + } + return checkWorkspaceExistence(meta, name), nil +} + +func (s *S3Storage) GetNames() ([]string, error) { + meta, err := s.readMeta() + if err != nil { + return nil, err + } + return meta.AvailableWorkspaces, nil +} + +func (s *S3Storage) GetCurrent() (string, error) { + meta, err := s.readMeta() + if err != nil { + return "", err + } + return meta.Current, nil +} + +func (s *S3Storage) SetCurrent(name string) error { + meta, err := s.readMeta() + if err != nil { + return err + } + if !checkWorkspaceExistence(meta, name) { + return ErrWorkspaceNotExist + } + meta.Current = name + return s.writeMeta(meta) +} + +func (s *S3Storage) initDefaultWorkspaceIf() error { + meta, err := s.readMeta() + if err != nil { + return err + } + if !checkWorkspaceExistence(meta, defaultWorkspace) { + // if there is no default workspace, create one with empty workspace. + if err = s.writeWorkspace(&v1.Workspace{Name: defaultWorkspace}); err != nil { + return err + } + addAvailableWorkspaces(meta, defaultWorkspace) + } + + if meta.Current == "" { + meta.Current = defaultWorkspace + } + return s.writeMeta(meta) +} + +func (s *S3Storage) readMeta() (*workspacesMetaData, error) { + key := s.prefix + "/" + metadataFile + input := &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: &key, + } + output, err := s.s3.GetObject(input) + if err != nil { + awsErr, ok := err.(awserr.Error) + if ok && awsErr.Code() == s3.ErrCodeNoSuchKey { + return &workspacesMetaData{}, nil + } + return nil, fmt.Errorf("get workspaces meta data from s3 failed: %w", err) + } + defer func() { + _ = output.Body.Close() + }() + + content, err := io.ReadAll(output.Body) + if err != nil { + return nil, fmt.Errorf("read workspaces meta data failed: %w", err) + } + if len(content) == 0 { + return &workspacesMetaData{}, nil + } + + meta := &workspacesMetaData{} + if err = yaml.Unmarshal(content, meta); err != nil { + return nil, fmt.Errorf("yaml unmarshal workspaces meta data failed: %w", err) + } + return meta, nil +} + +func (s *S3Storage) writeMeta(meta *workspacesMetaData) error { + content, err := yaml.Marshal(meta) + if err != nil { + return fmt.Errorf("yaml marshal workspaces meta data failed: %w", err) + } + + input := &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(s.prefix + "/" + metadataFile), + Body: bytes.NewReader(content), + } + if _, err = s.s3.PutObject(input); err != nil { + return fmt.Errorf("put workspaces meta data to s3 failed: %w", err) + } + return nil +} + +func (s *S3Storage) writeWorkspace(ws *v1.Workspace) error { + content, err := yaml.Marshal(ws) + if err != nil { + return fmt.Errorf("yaml marshal workspace failed: %w", err) + } + + input := &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(s.prefix + "/" + ws.Name + yamlSuffix), + Body: bytes.NewReader(content), + } + if _, err = s.s3.PutObject(input); err != nil { + return fmt.Errorf("put workspace to s3 failed: %w", err) + } + return nil +} diff --git a/pkg/workspace/storages/s3_test.go b/pkg/workspace/storages/s3_test.go new file mode 100644 index 00000000..e2fed52d --- /dev/null +++ b/pkg/workspace/storages/s3_test.go @@ -0,0 +1,266 @@ +package storages + +import ( + "bytes" + "io" + "testing" + + "github.com/aws/aws-sdk-go/service/s3" + "github.com/bytedance/mockey" + "github.com/stretchr/testify/assert" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +func mockS3Storage() *S3Storage { + return &S3Storage{s3: &s3.S3{}} +} + +func mockS3StorageReadMeta(meta *workspacesMetaData) { + mockey.Mock((*S3Storage).readMeta).Return(meta, nil).Build() +} + +func mockS3StorageWriteMeta() { + mockey.Mock((*S3Storage).writeMeta).Return(nil).Build() +} + +func mockS3StorageWriteWorkspace() { + mockey.Mock((*S3Storage).writeWorkspace).Return(nil).Build() +} + +func TestS3Storage_Get(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + content []byte + expectedWorkspace *v1.Workspace + }{ + { + name: "get workspace successfully", + success: true, + wsName: "dev", + content: []byte(mockWorkspaceContent()), + expectedWorkspace: mockWorkspace("dev"), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock s3 operation", t, func() { + mockey.Mock((*s3.S3).GetObject).Return(&s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader([]byte(""))), + }, nil).Build() + mockey.Mock(io.ReadAll).Return(tc.content, nil).Build() + mockS3StorageReadMeta(mockWorkspacesMetaData()) + workspace, err := mockS3Storage().Get(tc.wsName) + assert.Equal(t, tc.success, err == nil) + assert.Equal(t, tc.expectedWorkspace, workspace) + }) + }) + } +} + +func TestS3Storage_Create(t *testing.T) { + testcases := []struct { + name string + success bool + workspace *v1.Workspace + }{ + { + name: "create workspace successfully", + success: true, + workspace: mockWorkspace("pre"), + }, + { + name: "failed to create workspace already exist", + success: false, + workspace: mockWorkspace("dev"), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock s3 operation", t, func() { + mockS3StorageReadMeta(mockWorkspacesMetaData()) + mockS3StorageWriteMeta() + mockS3StorageWriteWorkspace() + err := mockS3Storage().Create(tc.workspace) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} + +func TestS3Storage_Update(t *testing.T) { + testcases := []struct { + name string + success bool + workspace *v1.Workspace + }{ + { + name: "update workspace successfully", + success: true, + workspace: mockWorkspace("dev"), + }, + { + name: "failed to update workspace not exist", + success: false, + workspace: mockWorkspace("pre"), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock s3 operation", t, func() { + mockS3StorageReadMeta(mockWorkspacesMetaData()) + mockS3StorageWriteWorkspace() + err := mockS3Storage().Update(tc.workspace) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} + +func TestS3Storage_Delete(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + }{ + { + name: "delete workspace successfully", + success: true, + wsName: "dev", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock s3 operation", t, func() { + mockey.Mock((*s3.S3).DeleteObject).Return(nil, nil).Build() + mockS3StorageReadMeta(mockWorkspacesMetaData()) + mockS3StorageWriteMeta() + err := mockS3Storage().Delete(tc.wsName) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} + +func TestS3Storage_Exist(t *testing.T) { + testcases := []struct { + name string + success bool + wsName string + expectedExist bool + }{ + { + name: "exist workspace", + success: true, + wsName: "dev", + expectedExist: true, + }, + { + name: "not exist workspace", + success: true, + wsName: "pre", + expectedExist: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock s3 operation", t, func() { + mockS3StorageReadMeta(mockWorkspacesMetaData()) + exist, err := mockS3Storage().Exist(tc.wsName) + assert.Equal(t, tc.success, err == nil) + assert.Equal(t, tc.expectedExist, exist) + }) + }) + } +} + +func TestS3Storage_GetNames(t *testing.T) { + testcases := []struct { + name string + success bool + expectedNames []string + }{ + { + name: "get all workspace names successfully", + success: true, + expectedNames: []string{"default", "dev", "prod"}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock s3 operation", t, func() { + mockS3StorageReadMeta(mockWorkspacesMetaData()) + wsNames, err := mockS3Storage().GetNames() + assert.Equal(t, tc.success, err == nil) + if tc.success { + assert.Equal(t, tc.expectedNames, wsNames) + } + }) + }) + } +} + +func TestS3Storage_GetCurrent(t *testing.T) { + testcases := []struct { + name string + success bool + expectedCurrent string + }{ + { + name: "get current workspace successfully", + success: true, + expectedCurrent: "dev", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock s3 operation", t, func() { + mockS3StorageReadMeta(mockWorkspacesMetaData()) + current, err := mockS3Storage().GetCurrent() + assert.Equal(t, tc.success, err == nil) + if tc.success { + assert.Equal(t, tc.expectedCurrent, current) + } + }) + }) + } +} + +func TestS3Storage_SetCurrent(t *testing.T) { + testcases := []struct { + name string + success bool + current string + }{ + { + name: "set current workspace successfully", + success: true, + current: "prod", + }, + { + name: "failed to set current workspace not exist", + success: false, + current: "pre", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockey.PatchConvey("mock s3 operation", t, func() { + mockS3StorageReadMeta(mockWorkspacesMetaData()) + mockS3StorageWriteMeta() + err := mockS3Storage().SetCurrent(tc.current) + assert.Equal(t, tc.success, err == nil) + }) + }) + } +} diff --git a/pkg/workspace/storages/testdata/for_create_workspaces/.metadata.yml b/pkg/workspace/storages/testdata/for_create_workspaces/.metadata.yml new file mode 100644 index 00000000..86aac9a4 --- /dev/null +++ b/pkg/workspace/storages/testdata/for_create_workspaces/.metadata.yml @@ -0,0 +1,3 @@ +current: default +availableWorkspaces: + - default diff --git a/pkg/workspace/storages/testdata/for_create_workspaces/default.yaml b/pkg/workspace/storages/testdata/for_create_workspaces/default.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/pkg/workspace/storages/testdata/for_create_workspaces/default.yaml @@ -0,0 +1 @@ +{} diff --git a/pkg/workspace/storages/testdata/for_delete_workspaces/.metadata.yml b/pkg/workspace/storages/testdata/for_delete_workspaces/.metadata.yml new file mode 100644 index 00000000..1dae0158 --- /dev/null +++ b/pkg/workspace/storages/testdata/for_delete_workspaces/.metadata.yml @@ -0,0 +1,4 @@ +current: default +availableWorkspaces: + - default + - dev diff --git a/pkg/workspace/storages/testdata/for_delete_workspaces/default.yaml b/pkg/workspace/storages/testdata/for_delete_workspaces/default.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/pkg/workspace/storages/testdata/for_delete_workspaces/default.yaml @@ -0,0 +1 @@ +{} diff --git a/pkg/workspace/storages/testdata/for_delete_workspaces/dev.yaml b/pkg/workspace/storages/testdata/for_delete_workspaces/dev.yaml new file mode 100755 index 00000000..98ebd305 --- /dev/null +++ b/pkg/workspace/storages/testdata/for_delete_workspaces/dev.yaml @@ -0,0 +1,22 @@ +modules: + database: + default: + instanceType: db.t3.micro + type: aws + version: "5.7" + smallClass: + projectSelector: + - foo + - bar + instanceType: db.t3.small + port: + default: + type: aws +runtimes: + kubernetes: + kubeConfig: /etc/kubeconfig.yaml + terraform: + aws: + source: hashicorp/aws + version: 1.0.4 + region: us-east-1 diff --git a/pkg/workspace/storages/testdata/for_set_current_workspaces/.metadata.yml b/pkg/workspace/storages/testdata/for_set_current_workspaces/.metadata.yml new file mode 100644 index 00000000..1dae0158 --- /dev/null +++ b/pkg/workspace/storages/testdata/for_set_current_workspaces/.metadata.yml @@ -0,0 +1,4 @@ +current: default +availableWorkspaces: + - default + - dev diff --git a/pkg/workspace/storages/testdata/for_set_current_workspaces/default.yaml b/pkg/workspace/storages/testdata/for_set_current_workspaces/default.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/pkg/workspace/storages/testdata/for_set_current_workspaces/default.yaml @@ -0,0 +1 @@ +{} diff --git a/pkg/workspace/storages/testdata/for_set_current_workspaces/dev.yaml b/pkg/workspace/storages/testdata/for_set_current_workspaces/dev.yaml new file mode 100644 index 00000000..98ebd305 --- /dev/null +++ b/pkg/workspace/storages/testdata/for_set_current_workspaces/dev.yaml @@ -0,0 +1,22 @@ +modules: + database: + default: + instanceType: db.t3.micro + type: aws + version: "5.7" + smallClass: + projectSelector: + - foo + - bar + instanceType: db.t3.small + port: + default: + type: aws +runtimes: + kubernetes: + kubeConfig: /etc/kubeconfig.yaml + terraform: + aws: + source: hashicorp/aws + version: 1.0.4 + region: us-east-1 diff --git a/pkg/workspace/storages/testdata/invalid_metadata_workspaces/.metadata.yml b/pkg/workspace/storages/testdata/invalid_metadata_workspaces/.metadata.yml new file mode 100644 index 00000000..b5e51dc7 --- /dev/null +++ b/pkg/workspace/storages/testdata/invalid_metadata_workspaces/.metadata.yml @@ -0,0 +1 @@ +not correct yaml \ No newline at end of file diff --git a/pkg/workspace/storages/testdata/workspaces/.metadata.yml b/pkg/workspace/storages/testdata/workspaces/.metadata.yml new file mode 100644 index 00000000..bdd5b4d2 --- /dev/null +++ b/pkg/workspace/storages/testdata/workspaces/.metadata.yml @@ -0,0 +1,5 @@ +current: dev +availableWorkspaces: + - default + - dev + - prod diff --git a/pkg/workspace/storages/testdata/workspaces/default.yaml b/pkg/workspace/storages/testdata/workspaces/default.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/pkg/workspace/storages/testdata/workspaces/default.yaml @@ -0,0 +1 @@ +{} diff --git a/pkg/workspace/storages/testdata/workspaces/dev.yaml b/pkg/workspace/storages/testdata/workspaces/dev.yaml new file mode 100644 index 00000000..98ebd305 --- /dev/null +++ b/pkg/workspace/storages/testdata/workspaces/dev.yaml @@ -0,0 +1,22 @@ +modules: + database: + default: + instanceType: db.t3.micro + type: aws + version: "5.7" + smallClass: + projectSelector: + - foo + - bar + instanceType: db.t3.small + port: + default: + type: aws +runtimes: + kubernetes: + kubeConfig: /etc/kubeconfig.yaml + terraform: + aws: + source: hashicorp/aws + version: 1.0.4 + region: us-east-1 diff --git a/pkg/workspace/storages/testdata/workspaces/prod.yaml b/pkg/workspace/storages/testdata/workspaces/prod.yaml new file mode 100644 index 00000000..98ebd305 --- /dev/null +++ b/pkg/workspace/storages/testdata/workspaces/prod.yaml @@ -0,0 +1,22 @@ +modules: + database: + default: + instanceType: db.t3.micro + type: aws + version: "5.7" + smallClass: + projectSelector: + - foo + - bar + instanceType: db.t3.small + port: + default: + type: aws +runtimes: + kubernetes: + kubeConfig: /etc/kubeconfig.yaml + terraform: + aws: + source: hashicorp/aws + version: 1.0.4 + region: us-east-1 diff --git a/pkg/workspace/storages/util.go b/pkg/workspace/storages/util.go new file mode 100644 index 00000000..1f219799 --- /dev/null +++ b/pkg/workspace/storages/util.go @@ -0,0 +1,58 @@ +package storages + +import ( + "errors" +) + +const ( + metadataFile = ".metadata.yml" + workspaceTable = "workspace" + yamlSuffix = ".yaml" + defaultWorkspace = "default" +) + +var ( + ErrWorkspaceNotExist = errors.New("workspace does not exist") + ErrWorkspaceAlreadyExist = errors.New("workspace has already existed") +) + +// workspacesMetaData contains the name of current workspace and all workspaces, whose serialization +// result contains in the metadataFile for LocalStorage, OssStorage and S3Storage. +type workspacesMetaData struct { + // The name of Current workspace. + Current string `yaml:"current,omitempty" json:"current,omitempty"` + + // AvailableWorkspaces is the name list of all the existing workspaces. + AvailableWorkspaces []string `yaml:"availableWorkspaces,omitempty" json:"availableWorkspaces,omitempty"` +} + +// checkWorkspaceExistence returns the workspace exists or not. +func checkWorkspaceExistence(meta *workspacesMetaData, name string) bool { + for _, ws := range meta.AvailableWorkspaces { + if name == ws { + return true + } + } + return false +} + +// addAvailableWorkspaces adds the workspace name to the available list, should be called if checkWorkspaceExistence +// returns false. +func addAvailableWorkspaces(meta *workspacesMetaData, name string) { + meta.AvailableWorkspaces = append(meta.AvailableWorkspaces, name) +} + +// removeAvailableWorkspaces deletes the workspace name from the available list. +func removeAvailableWorkspaces(meta *workspacesMetaData, name string) { + for i, ws := range meta.AvailableWorkspaces { + if name == ws { + meta.AvailableWorkspaces = append(meta.AvailableWorkspaces[:i], meta.AvailableWorkspaces[i+1:]...) + } + } + + // if the current workspace is the removing one, set current to default. + if meta.Current == name { + meta.Current = defaultWorkspace + } + return +} diff --git a/pkg/workspace/storages/util_test.go b/pkg/workspace/storages/util_test.go new file mode 100644 index 00000000..0b614d26 --- /dev/null +++ b/pkg/workspace/storages/util_test.go @@ -0,0 +1,146 @@ +package storages + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func mockWorkspacesMetaData() *workspacesMetaData { + return &workspacesMetaData{ + Current: "dev", + AvailableWorkspaces: []string{ + "default", + "dev", + "prod", + }, + } +} + +func TestCheckWorkspaceExistence(t *testing.T) { + testcases := []struct { + name string + meta *workspacesMetaData + wsName string + exist bool + }{ + { + name: "empty workspaces meta data", + meta: &workspacesMetaData{}, + wsName: "dev", + exist: false, + }, + { + name: "exist workspace", + meta: mockWorkspacesMetaData(), + wsName: "dev", + exist: true, + }, + { + name: "not exist workspace", + meta: mockWorkspacesMetaData(), + wsName: "pre", + exist: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + exist := checkWorkspaceExistence(tc.meta, tc.wsName) + assert.Equal(t, tc.exist, exist) + }) + } +} + +func TestAddAvailableWorkspaces(t *testing.T) { + testcases := []struct { + name string + meta *workspacesMetaData + wsName string + expectedMeta *workspacesMetaData + }{ + { + name: "empty workspaces meta data add workspace", + meta: &workspacesMetaData{}, + wsName: "default", + expectedMeta: &workspacesMetaData{ + AvailableWorkspaces: []string{"default"}, + }, + }, + { + name: "non-empty workspaces meta data add workspace", + meta: mockWorkspacesMetaData(), + wsName: "pre", + expectedMeta: &workspacesMetaData{ + Current: "dev", + AvailableWorkspaces: []string{ + "default", + "dev", + "prod", + "pre", + }, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + addAvailableWorkspaces(tc.meta, tc.wsName) + assert.Equal(t, tc.expectedMeta, tc.expectedMeta) + }) + } +} + +func TestRemoveAvailableWorkspaces(t *testing.T) { + testcases := []struct { + name string + meta *workspacesMetaData + wsName string + expectedMeta *workspacesMetaData + }{ + { + name: "remove not exist workspace", + meta: mockWorkspacesMetaData(), + wsName: "pre", + expectedMeta: &workspacesMetaData{ + Current: "dev", + AvailableWorkspaces: []string{ + "default", + "dev", + "prod", + }, + }, + }, + { + name: "remove exist workspace", + meta: mockWorkspacesMetaData(), + wsName: "prod", + expectedMeta: &workspacesMetaData{ + Current: "dev", + AvailableWorkspaces: []string{ + "default", + "dev", + }, + }, + }, + { + name: "set current workspace", + meta: mockWorkspacesMetaData(), + wsName: "dev", + expectedMeta: &workspacesMetaData{ + Current: "default", + AvailableWorkspaces: []string{ + "default", + "prod", + }, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + removeAvailableWorkspaces(tc.meta, tc.wsName) + assert.Equal(t, tc.expectedMeta, tc.expectedMeta) + }) + } +}