From 07ebf501c5319c4fcec3e92f85f2f5388de10b27 Mon Sep 17 00:00:00 2001 From: healthjyk Date: Thu, 7 Mar 2024 15:43:46 +0800 Subject: [PATCH] feat: add backend config completion and validation funcs --- pkg/apis/core/v1/config.go | 10 + pkg/backend/storages/completion.go | 51 +++++ pkg/backend/storages/completion_test.go | 132 ++++++++++++ pkg/backend/storages/validation.go | 115 +++++++++++ pkg/backend/storages/validation_test.go | 261 ++++++++++++++++++++++++ 5 files changed, 569 insertions(+) create mode 100644 pkg/backend/storages/completion.go create mode 100644 pkg/backend/storages/completion_test.go create mode 100644 pkg/backend/storages/validation.go create mode 100644 pkg/backend/storages/validation_test.go diff --git a/pkg/apis/core/v1/config.go b/pkg/apis/core/v1/config.go index 01c9aa7b..f0ed159f 100644 --- a/pkg/apis/core/v1/config.go +++ b/pkg/apis/core/v1/config.go @@ -29,6 +29,16 @@ const ( BackendTypeMysql = "mysql" BackendTypeOss = "oss" BackendTypeS3 = "s3" + + EnvBackendMysqlPassword = "KUSION_BACKEND_MYSQL_PASSWORD" + EnvOssAccessKeyID = "OSS_ACCESS_KEY_ID" + EnvOssAccessKeySecret = "OSS_ACCESS_KEY_SECRET" + EnvAwsAccessKeyID = "AWS_ACCESS_KEY_ID" + EnvAwsSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + EnvAwsDefaultRegion = "AWS_DEFAULT_REGION" + EnvAwsRegion = "AWS_REGION" + + DefaultMysqlPort = 3306 ) // BackendConfigs contains the configuration of multiple backends and the current backend. diff --git a/pkg/backend/storages/completion.go b/pkg/backend/storages/completion.go new file mode 100644 index 00000000..d1118049 --- /dev/null +++ b/pkg/backend/storages/completion.go @@ -0,0 +1,51 @@ +package storages + +import ( + "os" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +// CompleteMysqlConfig sets default value of mysql config if not set. +func CompleteMysqlConfig(config *v1.BackendMysqlConfig) { + if config.Port == 0 { + config.Port = v1.DefaultMysqlPort + } + password := os.Getenv(v1.EnvBackendMysqlPassword) + if password != "" { + config.Password = password + } +} + +// CompleteOssConfig constructs the whole oss config by environment variables if set. +func CompleteOssConfig(config *v1.BackendOssConfig) { + accessKeyID := os.Getenv(v1.EnvOssAccessKeyID) + accessKeySecret := os.Getenv(v1.EnvOssAccessKeySecret) + + if accessKeyID != "" { + config.AccessKeyID = accessKeyID + } + if accessKeySecret != "" { + config.AccessKeySecret = accessKeySecret + } +} + +// CompleteS3Config constructs the whole s3 config by environment variables if set. +func CompleteS3Config(config *v1.BackendS3Config) { + accessKeyID := os.Getenv(v1.EnvAwsAccessKeyID) + accessKeySecret := os.Getenv(v1.EnvAwsSecretAccessKey) + region := os.Getenv(v1.EnvAwsRegion) + if region == "" { + region = os.Getenv(v1.EnvAwsDefaultRegion) + } + + if accessKeyID != "" { + config.AccessKeyID = accessKeyID + } + if accessKeySecret != "" { + config.AccessKeySecret = accessKeySecret + } + if region != "" { + config.Region = region + } +} diff --git a/pkg/backend/storages/completion_test.go b/pkg/backend/storages/completion_test.go new file mode 100644 index 00000000..9e637f63 --- /dev/null +++ b/pkg/backend/storages/completion_test.go @@ -0,0 +1,132 @@ +package storages + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +func TestCompleteMysqlConfig(t *testing.T) { + testcases := []struct { + name string + config *v1.BackendMysqlConfig + envs map[string]string + completeConfig *v1.BackendMysqlConfig + }{ + { + name: "complete mysql config", + config: &v1.BackendMysqlConfig{ + DBName: "kusion", + User: "kk", + Host: "127.0.0.1", + }, + envs: map[string]string{ + v1.EnvBackendMysqlPassword: "fake-password", + }, + completeConfig: &v1.BackendMysqlConfig{ + DBName: "kusion", + User: "kk", + Host: "127.0.0.1", + Port: 3306, + Password: "fake-password", + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envs { + _ = os.Setenv(k, v) + } + CompleteMysqlConfig(tc.config) + assert.Equal(t, tc.completeConfig, tc.config) + }) + } +} + +func TestCompleteOssConfig(t *testing.T) { + testcases := []struct { + name string + config *v1.BackendOssConfig + envs map[string]string + completeConfig *v1.BackendOssConfig + }{ + { + name: "complete oss config", + config: &v1.BackendOssConfig{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Endpoint: "http://oss-cn-hangzhou.aliyuncs.com", + Bucket: "kusion", + }, + }, + envs: map[string]string{ + v1.EnvOssAccessKeyID: "fake-ak", + v1.EnvOssAccessKeySecret: "fake-sk", + }, + completeConfig: &v1.BackendOssConfig{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Endpoint: "http://oss-cn-hangzhou.aliyuncs.com", + Bucket: "kusion", + AccessKeyID: "fake-ak", + AccessKeySecret: "fake-sk", + }, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envs { + _ = os.Setenv(k, v) + } + CompleteOssConfig(tc.config) + assert.Equal(t, tc.completeConfig, tc.config) + }) + } +} + +func TestCompleteS3Config(t *testing.T) { + testcases := []struct { + name string + config *v1.BackendS3Config + envs map[string]string + completeConfig *v1.BackendS3Config + }{ + { + name: "complete s3 config", + config: &v1.BackendS3Config{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Endpoint: "fake-endpoint", + Bucket: "kusion", + }, + }, + envs: map[string]string{ + v1.EnvAwsRegion: "us-east-1", + v1.EnvAwsAccessKeyID: "fake-ak", + v1.EnvAwsSecretAccessKey: "fake-sk", + }, + completeConfig: &v1.BackendS3Config{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Endpoint: "fake-endpoint", + Bucket: "kusion", + AccessKeyID: "fake-ak", + AccessKeySecret: "fake-sk", + }, + Region: "us-east-1", + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envs { + _ = os.Setenv(k, v) + } + CompleteS3Config(tc.config) + assert.Equal(t, tc.completeConfig, tc.config) + }) + } +} diff --git a/pkg/backend/storages/validation.go b/pkg/backend/storages/validation.go new file mode 100644 index 00000000..7a0bd456 --- /dev/null +++ b/pkg/backend/storages/validation.go @@ -0,0 +1,115 @@ +package storages + +import ( + "errors" + "fmt" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +var ( + ErrEmptyMysqlDBName = errors.New("empty db name") + ErrEmptyMysqlUser = errors.New("empty mysql db user") + ErrEmptyMysqlHost = errors.New("empty mysql host") + ErrInvalidMysqlPort = errors.New("mysql port must be between 1 and 65535") + ErrEmptyBucket = errors.New("empty bucket") + ErrEmptyAccessKeyID = errors.New("empty access key id") + ErrEmptyAccessKeySecret = errors.New("empty access key secret") + ErrEmptyOssEndpoint = errors.New("empty oss endpoint") + ErrEmptyS3Region = errors.New("empty s3 region") +) + +// ValidateMysqlConfig is used to validate the v1.BackendMysqlConfig is valid or not. +// If valid, the config contains all valid items to new a mysql DB. +func ValidateMysqlConfig(config *v1.BackendMysqlConfig) error { + if err := ValidateMysqlConfigFromFile(config); err != nil { + return err + } + if config.Port < 1 || config.Port > 65535 { + return ErrInvalidMysqlPort + } + return nil +} + +// ValidateMysqlConfigFromFile is used to validate the v1.BackendMysqlConfig parsed from config file is valid +// or not, where the sensitive data items set as environment variables are not included. +func ValidateMysqlConfigFromFile(config *v1.BackendMysqlConfig) error { + if config.DBName == "" { + return ErrEmptyMysqlDBName + } + if config.User == "" { + return ErrEmptyMysqlUser + } + if config.Host == "" { + return ErrEmptyMysqlHost + } + if config.Port != 0 && (config.Port < 1 || config.Port > 65535) { + return ErrInvalidMysqlPort + } + return nil +} + +// ValidateOssConfig is used to validate v1.BackendOssConfig is valid or not, where all the items are included. +// If valid, the config contains all valid items to new an oss client. +func ValidateOssConfig(config *v1.BackendOssConfig) error { + if err := ValidateOssConfigFromFile(config); err != nil { + return err + } + if err := validateGenericObjectStorageSecret(config.AccessKeyID, config.AccessKeySecret); err != nil { + return fmt.Errorf("%w of %s", err, v1.BackendTypeOss) + } + return nil +} + +// ValidateOssConfigFromFile is used to validate the v1.BackendOssConfig parsed from config file is valid or not, +// where the sensitive data items set as environment variables are not included. +func ValidateOssConfigFromFile(config *v1.BackendOssConfig) error { + if err := validateGenericObjectStorageBucket(config.Bucket); err != nil { + return fmt.Errorf("%w of %s", err, v1.BackendTypeOss) + } + if config.Endpoint == "" { + return ErrEmptyOssEndpoint + } + return nil +} + +// ValidateS3Config is used to validate s3Config is valid or not, where all the items are included. +// If valid, the config contains all valid items to new a s3 client. +func ValidateS3Config(config *v1.BackendS3Config) error { + if err := ValidateS3ConfigFromFile(config); err != nil { + return err + } + if err := validateGenericObjectStorageSecret(config.AccessKeyID, config.AccessKeySecret); err != nil { + return fmt.Errorf("%w of %s", err, v1.BackendTypeS3) + } + if config.Region == "" { + return ErrEmptyS3Region + } + return nil +} + +// ValidateS3ConfigFromFile is used to validate the v1.BackendS3Config parsed from config file is valid or not, +// where the sensitive data items set as environment variables are not included. +func ValidateS3ConfigFromFile(config *v1.BackendS3Config) error { + if err := validateGenericObjectStorageBucket(config.Bucket); err != nil { + return fmt.Errorf("%w of %s", err, v1.BackendTypeS3) + } + return nil +} + +func validateGenericObjectStorageBucket(bucket string) error { + if bucket == "" { + return ErrEmptyBucket + } + return nil +} + +func validateGenericObjectStorageSecret(ak, sk string) error { + if ak == "" { + return ErrEmptyAccessKeyID + } + if sk == "" { + return ErrEmptyAccessKeySecret + } + return nil +} diff --git a/pkg/backend/storages/validation_test.go b/pkg/backend/storages/validation_test.go new file mode 100644 index 00000000..f60c653d --- /dev/null +++ b/pkg/backend/storages/validation_test.go @@ -0,0 +1,261 @@ +package storages + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + v1 "kusionstack.io/kusion/pkg/apis/core/v1" +) + +func TestValidateMysqlConfig(t *testing.T) { + testcases := []struct { + name string + success bool + config *v1.BackendMysqlConfig + }{ + { + name: "valid mysql config", + success: true, + config: &v1.BackendMysqlConfig{ + DBName: "kusion", + User: "kk", + Host: "127.0.0.1", + Port: 3306, + }, + }, + { + name: "invalid mysql config empty port", + success: false, + config: &v1.BackendMysqlConfig{ + DBName: "kusion", + User: "kk", + Host: "127.0.0.1", + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateMysqlConfig(tc.config) + assert.Equal(t, tc.success, err == nil) + }) + } +} + +func TestValidateMysqlConfigFromFile(t *testing.T) { + testcases := []struct { + name string + success bool + config *v1.BackendMysqlConfig + }{ + { + name: "valid mysql config", + success: true, + config: &v1.BackendMysqlConfig{ + DBName: "kusion", + User: "kk", + Host: "127.0.0.1", + }, + }, + { + name: "invalid mysql config empty dbName", + success: false, + config: &v1.BackendMysqlConfig{ + DBName: "", + User: "kk", + Host: "127.0.0.1", + }, + }, + { + name: "invalid mysql config empty user", + success: false, + config: &v1.BackendMysqlConfig{ + DBName: "kusion", + User: "", + Host: "127.0.0.1", + }, + }, + { + name: "invalid mysql config empty host", + success: false, + config: &v1.BackendMysqlConfig{ + DBName: "kusion", + User: "kk", + Host: "", + }, + }, + { + name: "invalid mysql config invalid port", + success: false, + config: &v1.BackendMysqlConfig{ + DBName: "kusion", + User: "kk", + Host: "127.0.0.1", + Port: -1, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateMysqlConfigFromFile(tc.config) + assert.Equal(t, tc.success, err == nil) + }) + } +} + +func TestValidateOssConfig(t *testing.T) { + testcases := []struct { + name string + success bool + config *v1.BackendOssConfig + }{ + { + name: "valid oss config", + success: true, + config: &v1.BackendOssConfig{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Endpoint: "http://oss-cn-hangzhou.aliyuncs.com", + AccessKeyID: "fake-access-key-id", + AccessKeySecret: "fake-access-key-secret", + Bucket: "kusion_bucket", + }, + }, + }, + { + name: "invalid oss config empty endpoint", + success: false, + config: &v1.BackendOssConfig{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Endpoint: "", + AccessKeyID: "fake-access-key-id", + AccessKeySecret: "fake-access-key-secret", + Bucket: "kusion_bucket", + }, + }, + }, + { + name: "invalid oss config empty access key id", + success: false, + config: &v1.BackendOssConfig{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Endpoint: "http://oss-cn-hangzhou.aliyuncs.com", + AccessKeyID: "", + AccessKeySecret: "fake-access-key-secret", + Bucket: "kusion_bucket", + }, + }, + }, + { + name: "invalid oss config empty access key secret", + success: false, + config: &v1.BackendOssConfig{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Endpoint: "http://oss-cn-hangzhou.aliyuncs.com", + AccessKeyID: "fake-access-key-id", + AccessKeySecret: "", + Bucket: "kusion_bucket", + }, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateOssConfig(tc.config) + assert.Equal(t, tc.success, err == nil) + }) + } +} + +func TestValidateOssConfigFromFile(t *testing.T) { + testcases := []struct { + name string + success bool + config *v1.BackendOssConfig + }{ + { + name: "valid oss config from file", + success: true, + config: &v1.BackendOssConfig{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Endpoint: "http://oss-cn-hangzhou.aliyuncs.com", + Bucket: "kusion_bucket", + }, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateOssConfigFromFile(tc.config) + assert.Equal(t, tc.success, err == nil) + }) + } +} + +func TestValidateS3Config(t *testing.T) { + testcases := []struct { + name string + success bool + config *v1.BackendS3Config + }{ + { + name: "valid s3 config", + success: true, + config: &v1.BackendS3Config{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + AccessKeyID: "fake-access-key-id", + AccessKeySecret: "fake-access-key-secret", + Bucket: "kusion_bucket", + }, + Region: "us-east-1", + }, + }, + { + name: "invalid s3 config empty region", + success: false, + config: &v1.BackendS3Config{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + AccessKeyID: "fake-access-key-id", + AccessKeySecret: "fake-access-key-secret", + Bucket: "kusion_bucket", + }, + Region: "", + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateS3Config(tc.config) + assert.Equal(t, tc.success, err == nil) + }) + } +} + +func TestValidateS3ConfigFromFile(t *testing.T) { + testcases := []struct { + name string + success bool + config *v1.BackendS3Config + }{ + { + name: "valid s3 config from file", + success: true, + config: &v1.BackendS3Config{ + GenericBackendObjectStorageConfig: &v1.GenericBackendObjectStorageConfig{ + Bucket: "kusion_bucket", + }, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateS3ConfigFromFile(tc.config) + assert.Equal(t, tc.success, err == nil) + }) + } +}