feat: add mysql/oss/s3 as new backends (#662)

* feat: add mysql/oss/s3 as new backends

* feat: update the default workspace dir from .workspaces to workspaces

pkg/apis/workspace/types.go

@@ -3,6 +3,19 @@ package workspace
 const (
 	DefaultBlock         = "default"
 	ProjectSelectorField = "projectSelector"
+
+	BackendLocal            = "local"
+	BackendMysql            = "mysql"
+	BackendOss              = "oss"
+	BackendS3               = "s3"
+	EnvBackendMysqlPassword = "KUSION_BACKEND_MYSQL_PASSWORD"
+	EnvAwsAccessKeyID       = "AWS_ACCESS_KEY_ID"
+	EnvAwsSecretAccessKey   = "AWS_SECRET_ACCESS_KEY"
+	EnvAwsDefaultRegion     = "AWS_DEFAULT_REGION"
+	EnvAwsRegion            = "AWS_REGION"
+	EnvOssAccessKeyID       = "OSS_ACCESS_KEY_ID"
+	EnvOssAccessKeySecret   = "OSS_ACCESS_KEY_SECRET"
+	DefaultMysqlPort        = 3306
 )
 
 // Workspace is a logical concept representing a target that stacks will be deployed to.
@@ -72,16 +85,68 @@ type TerraformConfig map[string]GenericConfig
 
 // BackendConfigs contains config of the backend, which is used to store state, etc. Only one kind
 // backend can be configured.
-// todo: add more backends declared in pkg/engine/backend
 type BackendConfigs struct {
-	// Local is backend to use local file system.
+	// Local is the backend using local file system.
 	Local *LocalFileConfig `yaml:"local,omitempty" json:"local,omitempty"`
+
+	// Mysql is the backend using mysql database.
+	Mysql *MysqlConfig `yaml:"mysql,omitempty" json:"mysql,omitempty"`
+
+	// Oss is the backend using OSS.
+	Oss *OssConfig `yaml:"oss,omitempty" json:"oss,omitempty"`
+
+	// S3 is the backend using S3.
+	S3 *S3Config `yaml:"s3,omitempty" json:"s3,omitempty"`
 }
 
-// LocalFileConfig contains the config of using local file system as backend.
-type LocalFileConfig struct {
-	// Path is place to store state, etc.
-	Path string `yaml:"path" json:"path"`
+// LocalFileConfig contains the config of using local file system as backend. Now there is no configuration
+// item for local file.
+type LocalFileConfig struct{}
+
+// MysqlConfig contains the config of using mysql database as backend.
+type MysqlConfig struct {
+	// DBName is the database name.
+	DBName string `yaml:"dbName" json:"dbName"`
+
+	// User of the database.
+	User string `yaml:"user" json:"user"`
+
+	// Password of the database.
+	Password string `yaml:"password,omitempty" json:"password,omitempty"`
+
+	// Host of the database.
+	Host string `yaml:"host" json:"host"`
+
+	// Port of the database. If not set, then it will be set to DefaultMysqlPort.
+	Port *int `yaml:"port,omitempty" json:"port,omitempty"`
+}
+
+// OssConfig contains the config of using OSS as backend.
+type OssConfig struct {
+	GenericObjectStorageConfig // OSS asks for non-empty endpoint
+}
+
+// S3Config contains the config of using S3 as backend.
+type S3Config struct {
+	GenericObjectStorageConfig
+
+	// Region of S3.
+	Region string `yaml:"region,omitempty" json:"region,omitempty"`
+}
+
+// GenericObjectStorageConfig contains generic configs which can be reused by OssConfig and S3Config.
+type GenericObjectStorageConfig struct {
+	// Endpoint of the object storage service.
+	Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"`
+
+	// AccessKeyID of the object storage service.
+	AccessKeyID string `yaml:"accessKeyID,omitempty" json:"accessKeyID,omitempty"`
+
+	// AccessKeySecret of the object storage service.
+	AccessKeySecret string `yaml:"accessKeySecret,omitempty" json:"accessKeySecret,omitempty"`
+
+	// Bucket of the object storage service.
+	Bucket string `yaml:"bucket" json:"bucket"`
 }
 
 // GenericConfig is a generic model to describe config which shields the difference among multiple concrete

pkg/cmd/build/builders/appconfig_builder_test.go

@@ -77,9 +77,7 @@ func buildMockWorkspace() *workspace.Workspace {
 			},
 		},
 		Backends: &workspace.BackendConfigs{
-			Local: &workspace.LocalFileConfig{
-				Path: "/etc/.kusion",
-			},
+			Local: &workspace.LocalFileConfig{},
 		},
 	}
 }

pkg/cmd/build/util_test.go

@@ -145,9 +145,7 @@ resources:
 			},
 		},
 		Backends: &workspace.BackendConfigs{
-			Local: &workspace.LocalFileConfig{
-				Path: "/etc/.kusion",
-			},
+			Local: &workspace.LocalFileConfig{},
 		},
 	}
 )

pkg/modules/generators/app_configurations_generator_test.go

@@ -122,9 +122,7 @@ func buildMockWorkspace() *workspace.Workspace {
 			},
 		},
 		Backends: &workspace.BackendConfigs{
-			Local: &workspace.LocalFileConfig{
-				Path: "/etc/.kusion",
-			},
+			Local: &workspace.LocalFileConfig{},
 		},
 	}
 }

pkg/workspace/operator.go

@@ -14,7 +14,7 @@ import (
 )
 
 const (
-	defaultRelativeStoragePath = ".workspaces"
+	defaultRelativeStoragePath = "workspaces"
 	suffixYAML                 = ".yaml"
 )
 

pkg/workspace/operator_test.go

@@ -55,14 +55,14 @@ func TestOperator_Validate(t *testing.T) {
 			name:    "invalid operator not yaml workspace",
 			success: false,
 			operator: &Operator{
-				storagePath: path.Join(testDataFolder(), ".invalid_workspace_not_yaml"),
+				storagePath: path.Join(testDataFolder(), "invalid_workspaces_not_yaml"),
 			},
 		},
 		{
 			name:    "invalid operator dir workspace",
 			success: false,
 			operator: &Operator{
-				storagePath: path.Join(testDataFolder(), ".invalid_workspace_dir"),
+				storagePath: path.Join(testDataFolder(), "invalid_workspaces_dir"),
 			},
 		},
 	}
@@ -76,7 +76,7 @@ func TestOperator_Validate(t *testing.T) {
 }
 
 func TestOperator_GetWorkspaceNames(t *testing.T) {
-	operator, _ := NewOperator(path.Join(testDataFolder(), ".workspace_for_list"))
+	operator, _ := NewOperator(path.Join(testDataFolder(), "workspaces_for_list"))
 	testcases := []struct {
 		name     string
 		success  bool

pkg/workspace/testdata/.workspaces/for_create_failure_ws.yaml → pkg/workspace/testdata/workspaces/for_create_failure_ws.yaml

@@ -21,5 +21,4 @@ runtimes:
             source: hashicorp/aws
             version: 1.0.4
 backends:
-    local:
-        path: /etc/.kusion
+    local: {}

pkg/workspace/testdata/.workspaces/for_delete_ws.yaml → pkg/workspace/testdata/workspaces/for_delete_ws.yaml

@@ -21,5 +21,4 @@ runtimes:
             source: hashicorp/aws
             version: 1.0.4
 backends:
-    local:
-        path: /etc/.kusion
+    local: {}

pkg/workspace/testdata/.workspaces/for_get_ws.yaml → pkg/workspace/testdata/workspaces/for_get_ws.yaml

@@ -21,5 +21,4 @@ runtimes:
             source: hashicorp/aws
             version: 1.0.4
 backends:
-    local:
-        path: /etc/.kusion
+    local: {}

pkg/workspace/testdata/.workspaces/for_update_ws.yaml → pkg/workspace/testdata/workspaces/for_update_ws.yaml

@@ -21,5 +21,4 @@ runtimes:
             source: hashicorp/aws
             version: 1.0.4
 backends:
-    local:
-        path: /etc/.kusion
+    local: {}

pkg/workspace/testdata/.workspace_for_list/dev.yaml → pkg/workspace/testdata/workspaces_for_list/dev.yaml

@@ -21,5 +21,4 @@ runtimes:
             source: hashicorp/aws
             version: 1.0.4
 backends:
-    local:
-        path: /etc/.kusion
+    local: {}

pkg/workspace/testdata/.workspace_for_list/pre.yaml → pkg/workspace/testdata/workspaces_for_list/pre.yaml

@@ -21,5 +21,4 @@ runtimes:
             source: hashicorp/aws
             version: 1.0.4
 backends:
-    local:
-        path: /etc/.kusion
+    local: {}

pkg/workspace/testdata/.workspace_for_list/prod.yaml → pkg/workspace/testdata/workspaces_for_list/prod.yaml

@@ -21,5 +21,4 @@ runtimes:
             source: hashicorp/aws
             version: 1.0.4
 backends:
-    local:
-        path: /etc/.kusion
+    local: {}

pkg/workspace/util.go

@@ -3,6 +3,7 @@ package workspace
 import (
 	"errors"
 	"fmt"
+	"os"
 
 	"gopkg.in/yaml.v3"
 
@@ -20,6 +21,17 @@ var (
 	ErrEmptyTerraformConfig  = errors.New("empty terraform config")
 )
 
+// CompleteWorkspace sets the workspace name and default value of unset item, should be called after ValidateWorkspace.
+// The config items set as environment variables are not got by CompleteWorkspace.
+func CompleteWorkspace(ws *workspace.Workspace, name string) {
+	if ws.Name != "" {
+		ws.Name = name
+	}
+	if ws.Backends != nil && GetBackendName(ws.Backends) == workspace.BackendMysql {
+		CompleteMysqlConfig(ws.Backends.Mysql)
+	}
+}
+
 // GetProjectModuleConfigs returns the module configs of a specified project, whose key is the module name,
 // should be called after ValidateModuleConfigs.
 // If got empty module configs, ErrEmptyProjectModuleConfigs will get returned.
@@ -166,3 +178,84 @@ func GetTerraformProviderConfig(configs *workspace.RuntimeConfigs, providerName
 	}
 	return cfg, nil
 }
+
+// GetBackendName returns the backend name that is configured in BackendConfigs, should be called after
+// ValidateBackendConfigs.
+func GetBackendName(configs *workspace.BackendConfigs) string {
+	if configs == nil {
+		return workspace.BackendLocal
+	}
+	if configs.Local != nil {
+		return workspace.BackendLocal
+	}
+	if configs.Mysql != nil {
+		return workspace.BackendMysql
+	}
+	if configs.Oss != nil {
+		return workspace.BackendOss
+	}
+	if configs.S3 != nil {
+		return workspace.BackendS3
+	}
+	return workspace.BackendLocal
+}
+
+// GetMysqlPasswordFromEnv returns mysql password set by environment variables.
+func GetMysqlPasswordFromEnv() string {
+	return os.Getenv(workspace.EnvBackendMysqlPassword)
+}
+
+// GetOssSensitiveDataFromEnv returns oss accessKeyID, accessKeySecret set by environment variables.
+func GetOssSensitiveDataFromEnv() (string, string) {
+	return os.Getenv(workspace.EnvOssAccessKeyID), os.Getenv(workspace.EnvOssAccessKeySecret)
+}
+
+// GetS3SensitiveDataFromEnv returns s3 accessKeyID, accessKeySecret, region set by environment variables.
+func GetS3SensitiveDataFromEnv() (string, string, string) {
+	region := os.Getenv(workspace.EnvAwsRegion)
+	if region == "" {
+		region = os.Getenv(workspace.EnvAwsDefaultRegion)
+	}
+	return os.Getenv(workspace.EnvAwsAccessKeyID), os.Getenv(workspace.EnvAwsSecretAccessKey), region
+}
+
+// CompleteMysqlConfig sets default value of mysql config if not set.
+func CompleteMysqlConfig(config *workspace.MysqlConfig) {
+	if config.Port == nil {
+		port := workspace.DefaultMysqlPort
+		config.Port = &port
+	}
+}
+
+// CompleteWholeMysqlConfig constructs the whole mysql config by environment variables if set.
+func CompleteWholeMysqlConfig(config *workspace.MysqlConfig) {
+	password := GetMysqlPasswordFromEnv()
+	if password != "" {
+		config.Password = password
+	}
+}
+
+// CompleteWholeOssConfig constructs the whole oss config by environment variables if set.
+func CompleteWholeOssConfig(config *workspace.OssConfig) {
+	accessKeyID, accessKeySecret := GetOssSensitiveDataFromEnv()
+	if accessKeyID != "" {
+		config.AccessKeyID = accessKeyID
+	}
+	if accessKeySecret != "" {
+		config.AccessKeySecret = accessKeySecret
+	}
+}
+
+// CompleteWholeS3Config constructs the whole s3 config by environment variables if set.
+func CompleteWholeS3Config(config *workspace.S3Config) {
+	accessKeyID, accessKeySecret, region := GetS3SensitiveDataFromEnv()
+	if accessKeyID != "" {
+		config.AccessKeyID = accessKeyID
+	}
+	if accessKeySecret != "" {
+		config.AccessKeySecret = accessKeySecret
+	}
+	if region != "" {
+		config.Region = region
+	}
+}