Merge pull request #1234 from merico-dev/feat-github-action-languages

refactor: use more general ci config

internal/pkg/plugin/githubactions/general/create.go

@@ -12,7 +12,7 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) {
 	// Initialize Operator with Operations
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
-			ci.SetSCMDefault,
+			ci.SetDefault(ciType),
 			validate,
 		},
 		ExecuteOperations: plugininstaller.ExecuteOperations{

internal/pkg/plugin/githubactions/general/delete.go

@@ -11,7 +11,7 @@ func Delete(options map[string]interface{}) (bool, error) {
 	// Initialize Operator with Operations
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
-			ci.SetSCMDefault,
+			ci.SetDefault(ciType),
 			validate,
 		},
 		ExecuteOperations: plugininstaller.ExecuteOperations{

internal/pkg/plugin/githubactions/general/githubactions.go

@@ -3,11 +3,14 @@ package general
 import (
 	"github.com/devstream-io/devstream/internal/pkg/configmanager"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci/cifile/server"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci/step"
 	"github.com/devstream-io/devstream/pkg/util/log"
 	"github.com/devstream-io/devstream/pkg/util/scm"
 )
 
+var ciType = server.CIGithubType
+
 func preConfigGithub(options configmanager.RawOptions) error {
 	opts, err := ci.NewCIOptions(options)
 	if err != nil {

internal/pkg/plugin/githubactions/general/read.go

@@ -12,7 +12,7 @@ func Read(options map[string]interface{}) (map[string]interface{}, error) {
 	// Initialize Operator with Operations
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
-			ci.SetSCMDefault,
+			ci.SetDefault(ciType),
 			validate,
 		},
 		GetStatusOperation: cifile.GetCIFileStatus,

internal/pkg/plugin/gitlabci/generic/create.go

@@ -12,7 +12,7 @@ import (
 func Create(options configmanager.RawOptions) (statemanager.ResourceStatus, error) {
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
-			ci.SetSCMDefault,
+			ci.SetDefault(ciType),
 			validate,
 		},
 		ExecuteOperations: plugininstaller.ExecuteOperations{

internal/pkg/plugin/gitlabci/generic/delete.go

@@ -10,7 +10,7 @@ import (
 func Delete(options configmanager.RawOptions) (bool, error) {
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
-			ci.SetSCMDefault,
+			ci.SetDefault(ciType),
 			validate,
 		},
 		ExecuteOperations: plugininstaller.ExecuteOperations{

internal/pkg/plugin/gitlabci/generic/generic.go

@@ -7,6 +7,7 @@ import (
 	"github.com/devstream-io/devstream/internal/pkg/configmanager"
 	"github.com/devstream-io/devstream/internal/pkg/plugin/helminstaller"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci/cifile/server"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci/step"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/reposcaffolding"
 	"github.com/devstream-io/devstream/pkg/util/log"
@@ -19,6 +20,8 @@ import (
 //go:embed tpl/helmValue.tpl.yaml
 var helmValueTpl string
 
+var ciType = server.CIGitLabType
+
 func preConfigGitlab(options configmanager.RawOptions) error {
 	opts, err := ci.NewCIOptions(options)
 	if err != nil {

internal/pkg/plugin/gitlabci/generic/read.go

@@ -12,7 +12,7 @@ import (
 func Read(options configmanager.RawOptions) (statemanager.ResourceStatus, error) {
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
-			ci.SetSCMDefault,
+			ci.SetDefault(ciType),
 			validate,
 		},
 		GetStatusOperation: cifile.GetCIFileStatus,

internal/pkg/plugin/jenkinspipeline/create.go

@@ -3,6 +3,7 @@ package jenkinspipeline
 import (
 	"github.com/devstream-io/devstream/internal/pkg/configmanager"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci/cifile"
 	"github.com/devstream-io/devstream/internal/pkg/statemanager"
 	"github.com/devstream-io/devstream/pkg/util/log"
@@ -12,8 +13,10 @@ func Create(options configmanager.RawOptions) (statemanager.ResourceStatus, erro
 	// Initialize Operator with Operations
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
-			setDefault,
-			validate,
+			ci.SetDefault(ciType),
+			setJenkinsDefault,
+			ci.Validate,
+			validateJenkins,
 		},
 		ExecuteOperations: plugininstaller.ExecuteOperations{
 			installPipeline,

internal/pkg/plugin/jenkinspipeline/delete.go

@@ -3,15 +3,18 @@ package jenkinspipeline
 import (
 	"github.com/devstream-io/devstream/internal/pkg/configmanager"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci/cifile"
 )
 
 func Delete(options configmanager.RawOptions) (bool, error) {
 	// Initialize Operator with Operations
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
-			setDefault,
-			validate,
+			ci.SetDefault(ciType),
+			setJenkinsDefault,
+			ci.Validate,
+			validateJenkins,
 		},
 		ExecuteOperations: plugininstaller.ExecuteOperations{
 			// TODO(daniel-hutao): delete secret: docker-config

internal/pkg/plugin/jenkinspipeline/read.go

@@ -3,15 +3,18 @@ package jenkinspipeline
 import (
 	"github.com/devstream-io/devstream/internal/pkg/configmanager"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci"
 	"github.com/devstream-io/devstream/internal/pkg/statemanager"
 	"github.com/devstream-io/devstream/pkg/util/log"
 )
 
 func Read(options configmanager.RawOptions) (statemanager.ResourceStatus, error) {
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
-			setDefault,
-			validate,
+			ci.SetDefault(ciType),
+			setJenkinsDefault,
+			ci.Validate,
+			validateJenkins,
 		},
 		GetStatusOperation: getStatus,
 	}

internal/pkg/plugin/jenkinspipeline/validate.go

@@ -7,33 +7,24 @@ import (
 	"github.com/devstream-io/devstream/pkg/util/validator"
 )
 
-// setDefault config default fields for usage
-func setDefault(options configmanager.RawOptions) (configmanager.RawOptions, error) {
+// setJenkinsDefault config default fields for usage
+func setJenkinsDefault(options configmanager.RawOptions) (configmanager.RawOptions, error) {
 	opts, err := newJobOptions(options)
 	if err != nil {
 		return nil, err
 	}
-
-	// config scm and projectRepo values
-	projectRepo, err := opts.SCM.BuildRepoInfo()
-	if err != nil {
-		return nil, err
-	}
-	opts.ProjectRepo = projectRepo
-
 	// set field value if empty
 	if opts.Jenkins.Namespace == "" {
 		opts.Jenkins.Namespace = "jenkins"
 	}
-	if opts.JobName == "" {
-		opts.JobName = jenkinsJobName(projectRepo.Repo)
+	if opts.JobName == "" && opts.ProjectRepo != nil {
+		opts.JobName = jenkinsJobName(opts.ProjectRepo.Repo)
 	}
-
-	opts.CIFileConfig = opts.Pipeline.BuildCIFileConfig(ciType, projectRepo)
 	return types.EncodeStruct(opts)
 }
 
-func validate(options configmanager.RawOptions) (configmanager.RawOptions, error) {
+// validateJenkins will validate jenkins jobName field and jenkins field
+func validateJenkins(options configmanager.RawOptions) (configmanager.RawOptions, error) {
 	opts, err := newJobOptions(options)
 	if err != nil {
 		return nil, err
@@ -43,11 +34,6 @@ func validate(options configmanager.RawOptions) (configmanager.RawOptions, error
 		return nil, err
 	}
 
-	// check repo is valid
-	if err := opts.ProjectRepo.CheckValid(); err != nil {
-		log.Debugf("jenkins validate repo invalid: %+v", err)
-		return nil, err
-	}
 	// check jenkins job name
 	if err := opts.JobName.checkValid(); err != nil {
 		log.Debugf("jenkins validate pipeline invalid: %+v", err)

internal/pkg/plugin/jenkinspipeline/validate_test.go

@@ -37,33 +37,23 @@ var _ = Describe("setDefault func", func() {
 			},
 		}
 	})
-	When("repo url is not valie", func() {
-		BeforeEach(func() {
-			options["scm"] = map[string]interface{}{
-				"cloneURL": "not_valid_url/gg",
-			}
-		})
-		It("should return err", func() {
-			_, err := setDefault(options)
-			Expect(err).Error().Should(HaveOccurred())
-		})
-	})
 	When("all input is valid", func() {
 		BeforeEach(func() {
 			options["scm"] = map[string]interface{}{
 				"cloneURL": "git@44.33.22.11:30022:root/spring-demo.git",
 				"apiURL":   "http://www.app.com",
 			}
+			options["projectRepo"] = map[string]interface{}{
+				"repo": "testRepo",
+			}
 		})
 		It("should set default value", func() {
-			newOptions, err := setDefault(options)
+			newOptions, err := setJenkinsDefault(options)
 			Expect(err).Error().ShouldNot(HaveOccurred())
 			opts, err := newJobOptions(newOptions)
 			Expect(err).Error().ShouldNot(HaveOccurred())
-			Expect(opts.CIFileConfig).ShouldNot(BeNil())
-			Expect(string(opts.JobName)).Should(Equal("spring-demo"))
-			Expect(opts.ProjectRepo).ShouldNot(BeNil())
-			Expect(opts.ProjectRepo.Repo).Should(Equal("spring-demo"))
+			Expect(opts.CIFileConfig).Should(BeNil())
+			Expect(string(opts.JobName)).Should(Equal("testRepo"))
 		})
 	})
 	AfterEach(func() {
@@ -134,34 +124,11 @@ var _ = Describe("validate func", func() {
 			}
 		})
 		It("should return error", func() {
-			_, err := validate(options)
-			Expect(err).Error().Should(HaveOccurred())
-		})
-	})
-	When("repo type is gitlab and gitlab env is not configured", func() {
-		BeforeEach(func() {
-			repoType = "gitlab"
-			projectRepo["repoType"] = repoType
-			options["projectRepo"] = projectRepo
-		})
-		It("should return error", func() {
-			_, err := validate(options)
+			_, err := validateJenkins(options)
 			Expect(err).Error().Should(HaveOccurred())
-			Expect(err.Error()).Should(Equal(fmt.Sprintf("jenkins-pipeline gitlab should set env %s", gitlab.TokenEnvKey)))
-		})
-	})
-	When("repo type is github and github env is not configured", func() {
-		BeforeEach(func() {
-			repoType = "github"
-			projectRepo["repoType"] = repoType
-			options["projectRepo"] = projectRepo
-		})
-		It("should return error", func() {
-			_, err := validate(options)
-			Expect(err).Error().Should(HaveOccurred())
-			Expect(err.Error()).Should(Equal(fmt.Sprintf("jenkins-pipeline github should set env %s", github.TokenEnvKey)))
 		})
 	})
+
 	When("jobName is not valid", func() {
 		BeforeEach(func() {
 			options["jobName"] = "folder/not_exist/jobName"
@@ -172,7 +139,7 @@ var _ = Describe("validate func", func() {
 			os.Setenv(github.TokenEnvKey, "test_env")
 		})
 		It("should return error", func() {
-			_, err := validate(options)
+			_, err := validateJenkins(options)
 			Expect(err).Error().Should(HaveOccurred())
 			Expect(err.Error()).Should(Equal(fmt.Sprintf("jenkins jobName illegal: %s", options["jobName"])))
 		})
@@ -186,7 +153,7 @@ var _ = Describe("validate func", func() {
 			os.Setenv(github.TokenEnvKey, "test_env")
 		})
 		It("should return nil error", func() {
-			_, err := validate(options)
+			_, err := validateJenkins(options)
 			Expect(err).Error().ShouldNot(HaveOccurred())
 		})
 	})

internal/pkg/plugininstaller/ci/ci.go

@@ -1,13 +1,15 @@
 package ci
 
 import (
+	"github.com/imdario/mergo"
 	"github.com/mitchellh/mapstructure"
 
 	"github.com/devstream-io/devstream/internal/pkg/configmanager"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci/cifile"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci/cifile/server"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci/step"
 	"github.com/devstream-io/devstream/pkg/util/log"
+	"github.com/devstream-io/devstream/pkg/util/mapz"
 	"github.com/devstream-io/devstream/pkg/util/scm"
 	"github.com/devstream-io/devstream/pkg/util/scm/git"
 )
@@ -21,8 +23,8 @@ type PipelineConfig struct {
 }
 
 type CIConfig struct {
-	SCM      scm.SCMInfo    `mapstructure:"scm"`
-	Pipeline PipelineConfig `mapstructure:"pipeline"`
+	SCM      scm.SCMInfo    `mapstructure:"scm"  validate:"required"`
+	Pipeline PipelineConfig `mapstructure:"pipeline"  validate:"required"`
 
 	// used in package
 	CIFileConfig *cifile.CIFileConfig `mapstructure:"ci"`
@@ -43,9 +45,25 @@ func (p *PipelineConfig) BuildCIFileConfig(ciType server.CIServerType, repoInfo
 		ConfigLocation: p.ConfigLocation,
 	}
 	// update ci render variables by plugins
-	rawConfigVars := step.GenerateCIFileVars(p, repoInfo)
+	rawConfigVars := p.generateCIFileVars(repoInfo)
 	rawConfigVars.Set("AppName", repoInfo.Repo)
 	CIFileConfig.Vars = rawConfigVars
 	log.Debugf("gitlab-ci pipeline get render vars: %+v", CIFileConfig)
 	return CIFileConfig
 }
+
+func (p *PipelineConfig) generateCIFileVars(repoInfo *git.RepoInfo) cifile.CIFileVarsMap {
+	// set default command for language
+	if p.General != nil {
+		p.General.SetDefault()
+	}
+	varMap, _ := mapz.DecodeStructToMap(p)
+	globalVarsMap, _ := mapz.DecodeStructToMap(
+		step.GetStepGlobalVars(repoInfo),
+	)
+	err := mergo.Merge(&varMap, globalVarsMap)
+	if err != nil {
+		log.Warnf("cifile merge CIFileVarsMap failed: %+v", err)
+	}
+	return varMap
+}