diff --git a/cmd/devstream/apply.go b/cmd/devstream/apply.go index 84c33c201..11aa45a7e 100644 --- a/cmd/devstream/apply.go +++ b/cmd/devstream/apply.go @@ -21,7 +21,7 @@ DevStream will generate and execute a new plan based on the config file and the func applyCMDFunc(cmd *cobra.Command, args []string) { log.Info("Apply started.") - if err := pluginengine.Apply(configFile, continueDirectly); err != nil { + if err := pluginengine.Apply(configFilePath, continueDirectly); err != nil { log.Errorf("Apply failed => %s.", err) os.Exit(1) } @@ -29,7 +29,7 @@ func applyCMDFunc(cmd *cobra.Command, args []string) { } func init() { - applyCMD.Flags().StringVarP(&configFile, configFlagName, "f", "config.yaml", "config file") + applyCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file") applyCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", "", "plugins directory") applyCMD.Flags().BoolVarP(&continueDirectly, "yes", "y", false, "apply directly without confirmation") diff --git a/cmd/devstream/common.go b/cmd/devstream/common.go index 6b8ef47eb..afe748e4d 100644 --- a/cmd/devstream/common.go +++ b/cmd/devstream/common.go @@ -1,7 +1,7 @@ package main var ( - configFile string + configFilePath string pluginDir string continueDirectly bool ) diff --git a/cmd/devstream/delete.go b/cmd/devstream/delete.go index 0cc539db4..e41e6ac73 100644 --- a/cmd/devstream/delete.go +++ b/cmd/devstream/delete.go @@ -22,7 +22,7 @@ DevStream will delete everything defined in the config file, regardless of the s func deleteCMDFunc(cmd *cobra.Command, args []string) { log.Info("Delete started.") - if err := pluginengine.Remove(configFile, continueDirectly, isForceDelete); err != nil { + if err := pluginengine.Remove(configFilePath, continueDirectly, isForceDelete); err != nil { log.Errorf("Delete error: %s.", err) os.Exit(1) } @@ -32,7 +32,7 @@ func deleteCMDFunc(cmd *cobra.Command, args []string) { func init() { deleteCMD.Flags().BoolVarP(&isForceDelete, "force", "", false, "force delete by config") - deleteCMD.Flags().StringVarP(&configFile, configFlagName, "f", "config.yaml", "config file") + deleteCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file") deleteCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", "", "plugins directory") deleteCMD.Flags().BoolVarP(&continueDirectly, "yes", "y", false, "delete directly without confirmation") diff --git a/cmd/devstream/destroy.go b/cmd/devstream/destroy.go index f704b7873..3ccff50de 100644 --- a/cmd/devstream/destroy.go +++ b/cmd/devstream/destroy.go @@ -21,7 +21,7 @@ var destroyCMD = &cobra.Command{ func destroyCMDFunc(cmd *cobra.Command, args []string) { log.Info("Destroy started.") - if err := pluginengine.Destroy(configFile, continueDirectly, isForceDestroy); err != nil { + if err := pluginengine.Destroy(configFilePath, continueDirectly, isForceDestroy); err != nil { log.Errorf("Destroy failed => %s.", err) os.Exit(1) } @@ -30,7 +30,7 @@ func destroyCMDFunc(cmd *cobra.Command, args []string) { func init() { destroyCMD.Flags().BoolVarP(&isForceDestroy, "force", "", false, "force destroy by config") - destroyCMD.Flags().StringVarP(&configFile, configFlagName, "f", "config.yaml", "config file") + destroyCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file") destroyCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", "", "plugins directory") destroyCMD.Flags().BoolVarP(&continueDirectly, "yes", "y", false, "destroy directly without confirmation") diff --git a/cmd/devstream/init.go b/cmd/devstream/init.go index 9732fd90f..5eb637aa5 100644 --- a/cmd/devstream/init.go +++ b/cmd/devstream/init.go @@ -65,7 +65,7 @@ func initCMDFunc(_ *cobra.Command, _ []string) { } func GetPluginsAndPluginDirFromConfig() (tools []configmanager.Tool, pluginDir string, err error) { - cfg, err := configmanager.NewManager(configFile).LoadConfig() + cfg, err := configmanager.NewManager(configFilePath).LoadConfig() if err != nil { return nil, "", err } @@ -125,7 +125,7 @@ func GetPluginsAndPluginDirFromFlags() (tools []configmanager.Tool, pluginDir st func init() { // flags for init from config file - initCMD.Flags().StringVarP(&configFile, configFlagName, "f", "config.yaml", "config file") + initCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file") initCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", "", "plugins directory") // downloading specific plugins from flags diff --git a/cmd/devstream/show.go b/cmd/devstream/show.go index 3101d41a1..a016ae505 100644 --- a/cmd/devstream/show.go +++ b/cmd/devstream/show.go @@ -51,7 +51,7 @@ func showConfigCMDFunc(_ *cobra.Command, _ []string) { func showStatusCMDFunc(_ *cobra.Command, _ []string) { log.Debug("Show status information.") - if err := status.Show(configFile); err != nil { + if err := status.Show(configFilePath); err != nil { log.Fatal(err) } } @@ -68,6 +68,6 @@ func init() { showStatusCMD.Flags().StringVarP(&instanceID, "id", "i", "", "specify id with the plugin instance") showStatusCMD.Flags().BoolVarP(&statusAllFlag, "all", "a", false, "show all instances of all plugins status") showStatusCMD.Flags().StringVarP(&pluginDir, "plugin-dir", "d", "", "plugins directory") - showStatusCMD.Flags().StringVarP(&configFile, "config-file", "f", "config.yaml", "config file") + showStatusCMD.Flags().StringVarP(&configFilePath, "config-file", "f", "config.yaml", "config file") completion.FlagPluginsCompletion(showStatusCMD, "plugin") } diff --git a/cmd/devstream/verify.go b/cmd/devstream/verify.go index 946dd02b0..eeaa71afa 100644 --- a/cmd/devstream/verify.go +++ b/cmd/devstream/verify.go @@ -17,7 +17,7 @@ var verifyCMD = &cobra.Command{ func verifyCMDFunc(cmd *cobra.Command, args []string) { log.Info("Verify started.") - if pluginengine.Verify(configFile) { + if pluginengine.Verify(configFilePath) { log.Success("Verify succeeded.") } else { log.Info("Verify finished.") @@ -25,7 +25,7 @@ func verifyCMDFunc(cmd *cobra.Command, args []string) { } func init() { - verifyCMD.Flags().StringVarP(&configFile, configFlagName, "f", "config.yaml", "config file") + verifyCMD.Flags().StringVarP(&configFilePath, configFlagName, "f", "config.yaml", "config file") verifyCMD.Flags().StringVarP(&pluginDir, pluginDirFlagName, "d", "", "plugins directory") completion.FlagFilenameCompletion(verifyCMD, configFlagName) diff --git a/internal/pkg/configmanager/app.go b/internal/pkg/configmanager/app.go index 1d782ebf7..b9669e492 100644 --- a/internal/pkg/configmanager/app.go +++ b/internal/pkg/configmanager/app.go @@ -14,16 +14,14 @@ const ( repoScaffoldingPluginName = "repo-scaffolding" ) -type ( - appRaw struct { - Name string `yaml:"name" mapstructure:"name"` - Spec map[string]any `yaml:"spec" mapstructure:"spec"` - Repo *scm.SCMInfo `yaml:"repo" mapstructure:"repo"` - RepoTemplate *scm.SCMInfo `yaml:"repoTemplate" mapstructure:"repoTemplate"` - CIRawConfigs []pipelineRaw `yaml:"ci" mapstructure:"ci"` - CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"` - } -) +type RawApp struct { + Name string `yaml:"name" mapstructure:"name"` + Spec map[string]any `yaml:"spec" mapstructure:"spec"` + Repo *scm.SCMInfo `yaml:"repo" mapstructure:"repo"` + RepoTemplate *scm.SCMInfo `yaml:"repoTemplate" mapstructure:"repoTemplate"` + CIRawConfigs []pipelineRaw `yaml:"ci" mapstructure:"ci"` + CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"` +} // getToolsFromApp return app tools func getToolsFromApp(appStr string, globalVars map[string]any, templateMap map[string]string) (Tools, error) { @@ -33,8 +31,8 @@ func getToolsFromApp(appStr string, globalVars map[string]any, templateMap map[s log.Debugf("configmanager/app %s render globalVars %+v failed", appRenderStr, globalVars) return nil, fmt.Errorf("app render globalVars failed: %w", err) } - // 2. unmarshal appRaw config for render pipelineTemplate - var rawData appRaw + // 2. unmarshal RawApp config for render pipelineTemplate + var rawData RawApp if err := yaml.Unmarshal([]byte(appRenderStr), &rawData); err != nil { return nil, fmt.Errorf("app parse yaml failed: %w", err) } @@ -57,7 +55,7 @@ func getToolsFromApp(appStr string, globalVars map[string]any, templateMap map[s } // getAppPipelineTool generate ci/cd tools from app config -func (a *appRaw) generateCICDToolsFromAppConfig(templateMap map[string]string, appVars map[string]any) (Tools, error) { +func (a *RawApp) generateCICDToolsFromAppConfig(templateMap map[string]string, appVars map[string]any) (Tools, error) { allPipelineRaw := append(a.CIRawConfigs, a.CDRawConfigs...) var tools Tools for _, p := range allPipelineRaw { @@ -76,7 +74,7 @@ func (a *appRaw) generateCICDToolsFromAppConfig(templateMap map[string]string, a } // getRepoTemplateTool will use repo-scaffolding plugin for app -func (a *appRaw) getRepoTemplateTool(appVars map[string]any) (*Tool, error) { +func (a *RawApp) getRepoTemplateTool(appVars map[string]any) (*Tool, error) { if a.Repo == nil { return nil, fmt.Errorf("app.repo field can't be empty") } @@ -101,14 +99,14 @@ func (a *appRaw) getRepoTemplateTool(appVars map[string]any) (*Tool, error) { } // setDefault will set repoName to appName if repo.name field is empty -func (a *appRaw) setDefault() { +func (a *RawApp) setDefault() { if a.Repo != nil && a.Repo.Name == "" { a.Repo.Name = a.Name } } // since all plugin depends on code is deployed, get dependsOn for repoTemplate -func (a *appRaw) getRepoTemplateDependants() []string { +func (a *RawApp) getRepoTemplateDependants() []string { var dependsOn []string // if a.RepoTemplate is configured, pipeline need to wait reposcaffolding finished if a.RepoTemplate != nil { diff --git a/internal/pkg/configmanager/app_test.go b/internal/pkg/configmanager/app_test.go index 254381171..e1f77e34a 100644 --- a/internal/pkg/configmanager/app_test.go +++ b/internal/pkg/configmanager/app_test.go @@ -195,9 +195,9 @@ options: }) }) -var _ = Describe("appRaw struct", func() { +var _ = Describe("RawApp struct", func() { var ( - a *appRaw + a *RawApp appName string rawConfig []pipelineRaw templateMap map[string]string @@ -211,7 +211,7 @@ var _ = Describe("appRaw struct", func() { When("repoInfo is not config", func() { BeforeEach(func() { appName = "test" - a = &appRaw{ + a = &RawApp{ Repo: &scm.SCMInfo{}, Name: appName, } diff --git a/internal/pkg/configmanager/config.go b/internal/pkg/configmanager/config.go index 9dca3392a..2f32479d7 100644 --- a/internal/pkg/configmanager/config.go +++ b/internal/pkg/configmanager/config.go @@ -1,67 +1,17 @@ package configmanager import ( - "bytes" - "errors" "fmt" - "io" - "os" - - "github.com/mitchellh/mapstructure" - "gopkg.in/yaml.v3" - - "github.com/devstream-io/devstream/pkg/util/file" - "github.com/devstream-io/devstream/pkg/util/log" - - "github.com/imdario/mergo" ) -type ( - // configFileLoc is configFile location - configFileLoc string - // Config records rendered config values and is used as a general config in DevStream. - Config struct { - // Command line flag have a higher priority than the config file. - // If you used the `--plugin-dir` flag with `dtm`, then the "pluginDir" in the config file will be ignored. - PluginDir string - Tools Tools `yaml:"tools"` - State *State - } - // RawConfig is used to describe original raw configs read from files - RawConfig struct { - VarFile configFileLoc `yaml:"varFile" mapstructure:"varFile"` - ToolFile configFileLoc `yaml:"toolFile" mapstructure:"toolFile"` - AppFile configFileLoc `yaml:"appFile" mapstructure:"appFile"` - TemplateFile configFileLoc `yaml:"templateFile" mapstructure:"templateFile"` - PluginDir string `yaml:"pluginDir" mapstructure:"pluginDir"` - State *State `yaml:"state" mapstructure:"state"` - Apps []RawOptions `yaml:"apps"` - Tools []RawOptions `yaml:"tools"` - PipelineTemplates []RawOptions `yaml:"pipelineTemplates"` - - GlobalVars map[string]any `yaml:"-" mapstructure:",remain"` - configFileBaseDir string `mapstructure:"-"` - globalBytes []byte `mapstructure:"-"` - } - // State is the struct for reading the state configuration in the config file. - // It defines how the state is stored, specifies the type of backend and related options. - State struct { - Backend string `yaml:"backend"` - Options StateConfigOptions `yaml:"options"` - } - // StateConfigOptions is the struct for reading the options of the state backend. - StateConfigOptions struct { - // for s3 backend - Bucket string `yaml:"bucket"` - Region string `yaml:"region"` - Key string `yaml:"key"` - // for local backend - StateFile string `yaml:"stateFile"` - // for ConfigMap backend - Namespace string `yaml:"namespace"` - ConfigMap string `yaml:"configmap"` - } -) +// Config records rendered config values and is used as a general config in DevStream. +type Config struct { + // Command line flag have a higher priority than the config file. + // If you used the `--plugin-dir` flag with `dtm`, then the "pluginDir" in the config file will be ignored. + PluginDir string + Tools Tools `yaml:"tools"` + State *State +} func (c *Config) validate() error { if c.State == nil { @@ -70,186 +20,22 @@ func (c *Config) validate() error { return nil } -// newRawConfig generate new RawConfig options -func newRawConfig(configFileLocation string) (*RawConfig, error) { - // 1. get baseDir from configFile - baseDir, err := file.GetFileAbsDirPath(configFileLocation) - if err != nil { - return nil, err - } - - // 2. read the original main config file - configBytes, err := loadConfigFile(configFileLocation) - if err != nil { - return nil, err - } - // replace all "---" - // otherwise yaml.Unmarshal can only read the content before the first "---" - configBytes = bytes.Replace(configBytes, []byte("---"), []byte("\n"), -1) - - // 3. decode total yaml files to get RawConfig - var RawConfig RawConfig - if err := yaml.Unmarshal(configBytes, &RawConfig); err != nil && !errors.Is(err, io.EOF) { - log.Errorf("Please verify the format of your config. Error: %s.", err) - return nil, err - } - RawConfig.configFileBaseDir = baseDir - RawConfig.globalBytes = configBytes - return &RawConfig, nil -} - -// loadConfigFile get config file content by location -func loadConfigFile(fileLoc string) ([]byte, error) { - configBytes, err := os.ReadFile(fileLoc) - if err != nil { - log.Errorf("Failed to read the config file. Error: %s", err) - log.Info(`Maybe the default file (config.yaml) doesn't exist or you forgot to pass your config file to the "-f" option?`) - log.Info(`See "dtm help" for more information."`) - return nil, err - } - log.Debugf("Original config: \n%s\n", string(configBytes)) - return configBytes, err -} - -// getGlobalVars will get global variables from GlobalVars field and varFile content -func (r *RawConfig) getGlobalVars() (map[string]any, error) { - valueContent, err := r.VarFile.getContentBytes(r.configFileBaseDir) - if err != nil { - return nil, err - } - globalVars := make(map[string]any) - if err := yaml.Unmarshal(valueContent, globalVars); err != nil { - return nil, err - } - if err := mergo.Merge(&globalVars, r.GlobalVars); err != nil { - return nil, err - } - return globalVars, nil -} - -// UnmarshalYAML is used for RawConfig -// it will put variables fields in globalVars field -func (r *RawConfig) UnmarshalYAML(value *yaml.Node) error { - configMap := make(map[string]any) - if err := value.Decode(configMap); err != nil { - return err - } - return mapstructure.Decode(configMap, r) -} - -// getApps will get Apps from appStr config -func (r *RawConfig) getAppsTools(globalVars map[string]any) (Tools, error) { - // 1. get tools config str - yamlPath := "$.apps[*]" - fileBytes, err := r.AppFile.getContentBytes(r.configFileBaseDir) - if err != nil { - return nil, err - } - - _, appArray, err := getMergedNodeConfig(fileBytes, r.globalBytes, yamlPath) - if err != nil { - return nil, err - } - // 2. get pipelineTemplates config map - templateMap, err := r.getPipelineTemplatesMap() - if err != nil { - return nil, err - } - // 3. render app with pipelineTemplate and globalVars - var tools Tools - for _, appConfigStr := range appArray { - appTools, err := getToolsFromApp(appConfigStr, globalVars, templateMap) - if err != nil { - return nil, err - } - tools = append(tools, appTools...) - } - return tools, nil -} - -// getTools get Tools from tool config -func (r *RawConfig) getTools(globalVars map[string]any) (Tools, error) { - // 1. get tools config str - yamlPath := "$.tools[*]" - fileBytes, err := r.ToolFile.getContentBytes(r.configFileBaseDir) - if err != nil { - return nil, err - } - toolStr, _, err := getMergedNodeConfig(fileBytes, r.globalBytes, yamlPath) - if err != nil { - return nil, err - } - // 2. render config str with global variables - toolsWithRenderdStr, err := renderConfigWithVariables(toolStr, globalVars) - if err != nil { - return nil, err - } - //3.unmarshal config str to Tools - var tools Tools - if err := yaml.Unmarshal([]byte(toolsWithRenderdStr), &tools); err != nil { - return nil, err - } - //4. validate tools is valid - if err := tools.validateAll(); err != nil { - return nil, err - } - return tools, nil -} - -// getPipelineTemplatesMap generate template name/rawString map -func (r *RawConfig) getPipelineTemplatesMap() (map[string]string, error) { - yamlPath := "$.pipelineTemplates[*]" - fileBytes, err := r.TemplateFile.getContentBytes(r.configFileBaseDir) - if err != nil { - return nil, err - } - _, templateArray, err := getMergedNodeConfig(fileBytes, r.globalBytes, yamlPath) - if err != nil { - return nil, err - } - templateMap := make(map[string]string) - for _, templateStr := range templateArray { - templateName, err := file.GetYamlNodeStrByPath([]byte(templateStr), "$.name") - if err != nil { - return nil, err - } - templateMap[templateName] = templateStr - } - return templateMap, nil -} - -// getContentBytes get file content with abs path for configFile -func (f configFileLoc) getContentBytes(baseDir string) ([]byte, error) { - // if configFile is not setted, return empty content - if string(f) == "" { - return []byte{}, nil - } - // refer other config file path by directory of main config file - fileAbs, err := file.GenerateAbsFilePath(baseDir, string(f)) - if err != nil { - return nil, err - } - bytes, err := os.ReadFile(fileAbs) - if err != nil { - return nil, err - } - return bytes, err -} - -// getMergedNodeConfig will use yamlPath to config from configFile content and global content -// then merge these content -func getMergedNodeConfig(fileBytes []byte, globalBytes []byte, yamlPath string) (string, []string, error) { - fileNode, err := file.GetYamlNodeArrayByPath(fileBytes, yamlPath) - if err != nil { - return "", nil, err - } - globalNode, err := file.GetYamlNodeArrayByPath(globalBytes, yamlPath) - if err != nil { - return "", nil, err - } - mergedNode := file.MergeYamlNode(fileNode, globalNode) - if mergedNode == nil { - return "", nil, err - } - return mergedNode.StrOrigin, mergedNode.StrArray, nil +// State is the struct for reading the state configuration in the config file. +// It defines how the state is stored, specifies the type of backend and related options. +type State struct { + Backend string `yaml:"backend"` + Options StateConfigOptions `yaml:"options"` +} + +// StateConfigOptions is the struct for reading the options of the state backend. +type StateConfigOptions struct { + // for s3 backend + Bucket string `yaml:"bucket"` + Region string `yaml:"region"` + Key string `yaml:"key"` + // for local backend + StateFile string `yaml:"stateFile"` + // for k8s backend + Namespace string `yaml:"namespace"` + ConfigMap string `yaml:"configmap"` } diff --git a/internal/pkg/configmanager/config_test.go b/internal/pkg/configmanager/config_test.go index a348bef3e..7e4d81d9c 100644 --- a/internal/pkg/configmanager/config_test.go +++ b/internal/pkg/configmanager/config_test.go @@ -22,7 +22,7 @@ var _ = Describe("Config struct", func() { }) }) -var _ = Describe("newRawConfig func", func() { +var _ = Describe("GetRawConfigFromFile func", func() { var ( fLoc string baseDir string @@ -38,7 +38,7 @@ var _ = Describe("newRawConfig func", func() { fLoc = "not_exist" }) It("should return err", func() { - _, err := newRawConfig(fLoc) + _, err := GetRawConfigFromFile(fLoc) Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("no such file or directory")) }) @@ -49,7 +49,7 @@ var _ = Describe("newRawConfig func", func() { Expect(err).Error().ShouldNot(HaveOccurred()) }) It("should return err", func() { - _, err := newRawConfig(fLoc) + _, err := GetRawConfigFromFile(fLoc) Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("cannot unmarshal")) }) @@ -71,13 +71,13 @@ var _ = Describe("RawConfig struct", func() { fLoc = f.Name() globalVars = map[string]any{} }) - Context("getGlobalVars method", func() { + Context("GetGlobalVars method", func() { When("varFile get content failed", func() { BeforeEach(func() { r.VarFile = "not_exist" }) It("should return err", func() { - _, err := r.getGlobalVars() + _, err := r.GetGlobalVars() Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("not exists")) }) @@ -89,7 +89,7 @@ var _ = Describe("RawConfig struct", func() { Expect(err).Error().ShouldNot(HaveOccurred()) }) It("should return err", func() { - _, err := r.getGlobalVars() + _, err := r.GetGlobalVars() Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("cannot unmarshal")) }) @@ -102,7 +102,7 @@ var _ = Describe("RawConfig struct", func() { r.AppFile = "not_exist" }) It("should return err", func() { - _, err := r.getAppsTools(globalVars) + _, err := r.GetToolsFromApps(globalVars) Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("not exists")) }) @@ -114,20 +114,20 @@ var _ = Describe("RawConfig struct", func() { r.AppFile = configFileLoc(fLoc) }) It("should return err", func() { - _, err := r.getAppsTools(globalVars) + _, err := r.GetToolsFromApps(globalVars) Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("yaml parse path[$.apps[*]] failed")) }) }) }) - Context("getTools method", func() { + Context("GetTools method", func() { When("toolsFile get content failed", func() { BeforeEach(func() { r.ToolFile = "not_exist" }) It("should return err", func() { - _, err := r.getTools(globalVars) + _, err := r.GetTools(globalVars) Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("not exists")) }) @@ -139,7 +139,7 @@ var _ = Describe("RawConfig struct", func() { r.ToolFile = configFileLoc(fLoc) }) It("should return err", func() { - _, err := r.getTools(globalVars) + _, err := r.GetTools(globalVars) Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("yaml parse path[$.tools[*]] failed")) }) @@ -147,7 +147,7 @@ var _ = Describe("RawConfig struct", func() { When("render failed", func() { BeforeEach(func() { r.ToolFile = "" - r.globalBytes = []byte(` + r.totalConfigBytes = []byte(` tools: - name: plugin1 instanceID: default @@ -155,14 +155,14 @@ tools: key1: [[ var1 ]]`) }) It("should return err", func() { - _, err := r.getTools(globalVars) + _, err := r.GetTools(globalVars) Expect(err).Error().Should(HaveOccurred()) }) }) When("yaml render failed", func() { BeforeEach(func() { r.ToolFile = "" - r.globalBytes = []byte(` + r.totalConfigBytes = []byte(` tools: - name: plugin1 instanceID: default @@ -170,7 +170,7 @@ tools: key1: {{}}`) }) It("should return err", func() { - _, err := r.getTools(globalVars) + _, err := r.GetTools(globalVars) Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("unexpected mapping key")) }) @@ -178,7 +178,7 @@ tools: When("tool validate failed", func() { BeforeEach(func() { r.ToolFile = "" - r.globalBytes = []byte(` + r.totalConfigBytes = []byte(` tools: - name: plugin1 instanceID: default @@ -190,7 +190,7 @@ tools: } }) It("should return err", func() { - _, err := r.getTools(globalVars) + _, err := r.GetTools(globalVars) Expect(err).Error().Should(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring("tool default's dependency not_exist doesn't exist")) }) @@ -212,7 +212,7 @@ tools: When("getMergedNodeConfig failed", func() { BeforeEach(func() { r.TemplateFile = "" - r.globalBytes = []byte(` + r.totalConfigBytes = []byte(` pipelineTemplates: - name: ci-pipeline-1 diff --git a/internal/pkg/configmanager/configmanager.go b/internal/pkg/configmanager/configmanager.go new file mode 100644 index 000000000..b4c19ee75 --- /dev/null +++ b/internal/pkg/configmanager/configmanager.go @@ -0,0 +1,53 @@ +package configmanager + +type Manager struct { + ConfigFilePath string +} + +func NewManager(configFilePath string) *Manager { + return &Manager{ + ConfigFilePath: configFilePath, + } +} + +// LoadConfig reads an input file as a general config. +// It will return "non-nil, nil" or "nil, err". +func (m *Manager) LoadConfig() (*Config, error) { + // 1. get rawConfig from config.yaml file + rawConfig, err := GetRawConfigFromFile(m.ConfigFilePath) + if err != nil { + return nil, err + } + + // 2. get all globals vars + globalVars, err := rawConfig.GetGlobalVars() + if err != nil { + return nil, err + } + + // 3. get Tools from Apps + appTools, err := rawConfig.GetToolsFromApps(globalVars) + if err != nil { + return nil, err + } + + // 4. get Tools from rawConfig + tools, err := rawConfig.GetTools(globalVars) + if err != nil { + return nil, err + } + + tools = append(tools, appTools...) + + config := &Config{ + PluginDir: rawConfig.PluginDir, + State: rawConfig.State, + Tools: tools, + } + + //5. validate config + if err := config.validate(); err != nil { + return nil, err + } + return config, nil +} diff --git a/internal/pkg/configmanager/manager_test.go b/internal/pkg/configmanager/configmanager_test.go similarity index 99% rename from internal/pkg/configmanager/manager_test.go rename to internal/pkg/configmanager/configmanager_test.go index 64dcffa16..85dd06484 100644 --- a/internal/pkg/configmanager/manager_test.go +++ b/internal/pkg/configmanager/configmanager_test.go @@ -420,13 +420,13 @@ tools: key1: test `), 0666) Expect(err).Error().ShouldNot(HaveOccurred()) - m.ConfigFile = fLoc + m.ConfigFilePath = fLoc }) Context("LoadConfig method", func() { When("get RawConfig failed", func() { BeforeEach(func() { - m.ConfigFile = "not_exist" + m.ConfigFilePath = "not_exist" }) It("should return error", func() { _, err := m.LoadConfig() @@ -434,7 +434,7 @@ tools: Expect(err.Error()).Should(ContainSubstring("no such file or directory")) }) }) - When("getGlobalVars failed", func() { + When("GetGlobalVars failed", func() { BeforeEach(func() { err := os.WriteFile(fLoc, []byte(` varFile: not_exist @@ -449,7 +449,7 @@ state: Expect(err).Error().Should(HaveOccurred()) }) }) - When("getTools failed", func() { + When("GetTools failed", func() { BeforeEach(func() { err := os.WriteFile(fLoc, []byte(` toolFile: not_exist diff --git a/internal/pkg/configmanager/manager.go b/internal/pkg/configmanager/manager.go deleted file mode 100644 index c208b58b2..000000000 --- a/internal/pkg/configmanager/manager.go +++ /dev/null @@ -1,48 +0,0 @@ -package configmanager - -type Manager struct { - ConfigFile string -} - -func NewManager(configFileName string) *Manager { - return &Manager{ - ConfigFile: configFileName, - } -} - -// LoadConfig reads an input file as a general config. -// It will return "non-nil, err" or "nil, err". -func (m *Manager) LoadConfig() (*Config, error) { - // 1. get new RawConfig from ConfigFile - RawConfig, err := newRawConfig(m.ConfigFile) - if err != nil { - return nil, err - } - // 2. get all globals vars - globalVars, err := RawConfig.getGlobalVars() - if err != nil { - return nil, err - } - // 3. get Apps from RawConfig - appTools, err := RawConfig.getAppsTools(globalVars) - if err != nil { - return nil, err - } - - // 4. get Tools from RawConfig - tools, err := RawConfig.getTools(globalVars) - if err != nil { - return nil, err - } - tools = append(tools, appTools...) - config := &Config{ - PluginDir: RawConfig.PluginDir, - State: RawConfig.State, - Tools: tools, - } - //5. validate config is valid - if err := config.validate(); err != nil { - return nil, err - } - return config, nil -} diff --git a/internal/pkg/configmanager/rawconfig.go b/internal/pkg/configmanager/rawconfig.go new file mode 100644 index 000000000..3338a2041 --- /dev/null +++ b/internal/pkg/configmanager/rawconfig.go @@ -0,0 +1,215 @@ +package configmanager + +import ( + "bytes" + "errors" + "io" + "os" + + "github.com/imdario/mergo" + "github.com/mitchellh/mapstructure" + "gopkg.in/yaml.v3" + + "github.com/devstream-io/devstream/pkg/util/file" + "github.com/devstream-io/devstream/pkg/util/log" +) + +// RawConfig is used to describe original raw configs read from files +type RawConfig struct { + VarFile configFileLoc `yaml:"varFile" mapstructure:"varFile"` + ToolFile configFileLoc `yaml:"toolFile" mapstructure:"toolFile"` + AppFile configFileLoc `yaml:"appFile" mapstructure:"appFile"` + TemplateFile configFileLoc `yaml:"templateFile" mapstructure:"templateFile"` + PluginDir string `yaml:"pluginDir" mapstructure:"pluginDir"` + State *State `yaml:"state" mapstructure:"state"` + Apps []RawOptions `yaml:"apps"` + Tools []RawOptions `yaml:"tools"` + PipelineTemplates []RawOptions `yaml:"pipelineTemplates"` + + GlobalVars map[string]any `yaml:"-" mapstructure:",remain"` + configFileBaseDir string `mapstructure:"-"` + totalConfigBytes []byte `mapstructure:"-"` +} + +// GetRawConfigFromFile generate new RawConfig options +func GetRawConfigFromFile(configFilePath string) (*RawConfig, error) { + // 1. get baseDir from configFile + baseDir, err := file.GetFileAbsDirPath(configFilePath) + if err != nil { + return nil, err + } + + // 2. read the original main config file + configBytes, err := os.ReadFile(configFilePath) + if err != nil { + return nil, err + } + + // TODO(daniel-hutao): We should change the documents to delete all "---" with config file. After a while, delete the following line of code, or prompt the user with "This is the wrong way" when "---" be detected + + // replace all "---", otherwise yaml.Unmarshal can only read the content before the first "---" + configBytes = bytes.Replace(configBytes, []byte("---"), []byte("\n"), -1) + + // 3. decode total yaml files to get rawConfig + var rawConfig RawConfig + if err := yaml.Unmarshal(configBytes, &rawConfig); err != nil && !errors.Is(err, io.EOF) { + log.Errorf("Please verify the format of your config. Error: %s.", err) + return nil, err + } + rawConfig.configFileBaseDir = baseDir + rawConfig.totalConfigBytes = configBytes + return &rawConfig, nil +} + +// GetGlobalVars will get global variables from GlobalVars field and varFile content +func (r *RawConfig) GetGlobalVars() (map[string]any, error) { + valueContent, err := r.VarFile.getContentBytes(r.configFileBaseDir) + if err != nil { + return nil, err + } + globalVars := make(map[string]any) + if err := yaml.Unmarshal(valueContent, globalVars); err != nil { + return nil, err + } + if err := mergo.Merge(&globalVars, r.GlobalVars); err != nil { + return nil, err + } + return globalVars, nil +} + +// UnmarshalYAML is used for RawConfig +// it will put variables fields in globalVars field +func (r *RawConfig) UnmarshalYAML(value *yaml.Node) error { + configMap := make(map[string]any) + if err := value.Decode(configMap); err != nil { + return err + } + return mapstructure.Decode(configMap, r) +} + +// GetToolsFromApps will get Tools from RawConfig.totalConfigBytes config +func (r *RawConfig) GetToolsFromApps(globalVars map[string]any) (Tools, error) { + // 1. get tools config str + yamlPath := "$.apps[*]" + appFileBytes, err := r.AppFile.getContentBytes(r.configFileBaseDir) + if err != nil { + return nil, err + } + + _, appArray, err := getMergedNodeConfig(appFileBytes, r.totalConfigBytes, yamlPath) + if err != nil { + return nil, err + } + + // 2. get pipelineTemplates config map + templateMap, err := r.getPipelineTemplatesMap() + if err != nil { + return nil, err + } + + // 3. render app with pipelineTemplate and globalVars + var tools Tools + for _, appConfigStr := range appArray { + appTools, err := getToolsFromApp(appConfigStr, globalVars, templateMap) + if err != nil { + return nil, err + } + tools = append(tools, appTools...) + } + return tools, nil +} + +// GetTools get Tools from tool config +func (r *RawConfig) GetTools(globalVars map[string]any) (Tools, error) { + // 1. get tools config str + yamlPath := "$.tools[*]" + fileBytes, err := r.ToolFile.getContentBytes(r.configFileBaseDir) + if err != nil { + return nil, err + } + toolStr, _, err := getMergedNodeConfig(fileBytes, r.totalConfigBytes, yamlPath) + if err != nil { + return nil, err + } + + // 2. render config str with global variables + toolsWithRenderdStr, err := renderConfigWithVariables(toolStr, globalVars) + if err != nil { + return nil, err + } + + //3. unmarshal config str to Tools + var tools Tools + if err := yaml.Unmarshal([]byte(toolsWithRenderdStr), &tools); err != nil { + return nil, err + } + + //4. validate tools is valid + if err := tools.validateAll(); err != nil { + return nil, err + } + return tools, nil +} + +// getPipelineTemplatesMap generate template name/rawString map +func (r *RawConfig) getPipelineTemplatesMap() (map[string]string, error) { + yamlPath := "$.pipelineTemplates[*]" + fileBytes, err := r.TemplateFile.getContentBytes(r.configFileBaseDir) + if err != nil { + return nil, err + } + _, templateArray, err := getMergedNodeConfig(fileBytes, r.totalConfigBytes, yamlPath) + if err != nil { + return nil, err + } + templateMap := make(map[string]string) + for _, templateStr := range templateArray { + templateName, err := file.GetYamlNodeStrByPath([]byte(templateStr), "$.name") + if err != nil { + return nil, err + } + templateMap[templateName] = templateStr + } + return templateMap, nil +} + +// configFileLoc is configFile location +type configFileLoc string + +// getContentBytes get file content with abs path for configFile +func (f configFileLoc) getContentBytes(baseDir string) ([]byte, error) { + // if configFile is not setted, return empty content + if string(f) == "" { + return []byte{}, nil + } + + // refer other config file path by directory of main config file + fileAbs, err := file.GenerateAbsFilePath(baseDir, string(f)) + if err != nil { + return nil, err + } + + bytes, err := os.ReadFile(fileAbs) + if err != nil { + return nil, err + } + return bytes, err +} + +// getMergedNodeConfig will use yamlPath to config from configFile content and global content +// then merge these content +func getMergedNodeConfig(fileBytes []byte, globalBytes []byte, yamlPath string) (string, []string, error) { + fileNode, err := file.GetYamlNodeArrayByPath(fileBytes, yamlPath) + if err != nil { + return "", nil, err + } + globalNode, err := file.GetYamlNodeArrayByPath(globalBytes, yamlPath) + if err != nil { + return "", nil, err + } + mergedNode := file.MergeYamlNode(fileNode, globalNode) + if mergedNode == nil { + return "", nil, err + } + return mergedNode.StrOrigin, mergedNode.StrArray, nil +}