Skip to content

Commit

Permalink
Merge pull request #1259 from merico-dev/feat-app-spec-option
Browse files Browse the repository at this point in the history
feat: app support ci/cd default config for app.spec
  • Loading branch information
daniel-hutao authored Nov 26, 2022
2 parents 8101e44 + 9319603 commit 350f65e
Show file tree
Hide file tree
Showing 23 changed files with 542 additions and 399 deletions.
25 changes: 15 additions & 10 deletions internal/pkg/configmanager/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,30 @@ const (
repoScaffoldingPluginName = "repo-scaffolding"
)

type repoTemplate struct {
*scm.SCMInfo `yaml:",inline"`
Vars map[string]any `yaml:"vars"`
}

type app struct {
Name string `yaml:"name" mapstructure:"name"`
Spec map[string]any `yaml:"spec" mapstructure:"spec"`
Repo *scm.SCMInfo `yaml:"repo" mapstructure:"repo"`
RepoTemplate *scm.SCMInfo `yaml:"repoTemplate" mapstructure:"repoTemplate"`
CIRawConfigs []pipelineRaw `yaml:"ci" mapstructure:"ci"`
CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"`
Name string `yaml:"name" mapstructure:"name"`
Spec *appSpec `yaml:"spec" mapstructure:"spec"`
Repo *scm.SCMInfo `yaml:"repo" mapstructure:"repo"`
RepoTemplate *repoTemplate `yaml:"repoTemplate" mapstructure:"repoTemplate"`
CIRawConfigs []pipelineRaw `yaml:"ci" mapstructure:"ci"`
CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"`
}

// getAppPipelineTool generate ci/cd tools from app config
func (a *app) generateCICDToolsFromAppConfig(templateMap map[string]string, appVars map[string]any) (Tools, error) {
allPipelineRaw := append(a.CIRawConfigs, a.CDRawConfigs...)
var tools Tools
for _, p := range allPipelineRaw {
t, err := p.newPipeline(a.Repo, templateMap, appVars)
t, err := p.getPipelineTemplate(templateMap, appVars)
if err != nil {
return nil, err
}
pipelineTool, err := t.getPipelineTool(a.Name)
pipelineTool, err := t.generatePipelineTool(a)
if err != nil {
return nil, err
}
Expand All @@ -39,7 +44,7 @@ func (a *app) generateCICDToolsFromAppConfig(templateMap map[string]string, appV
}

// getRepoTemplateTool will use repo-scaffolding plugin for app
func (a *app) getRepoTemplateTool(appVars map[string]any) (*Tool, error) {
func (a *app) getRepoTemplateTool() (*Tool, error) {
if a.Repo == nil {
return nil, fmt.Errorf("app.repo field can't be empty")
}
Expand All @@ -56,7 +61,7 @@ func (a *app) getRepoTemplateTool(appVars map[string]any) (*Tool, error) {
repoScaffoldingPluginName, a.Name, RawOptions{
"destinationRepo": RawOptions(appRepo.Encode()),
"sourceRepo": RawOptions(templateRepo.Encode()),
"vars": RawOptions(appVars),
"vars": RawOptions(a.RepoTemplate.Vars),
},
), nil
}
Expand Down
42 changes: 42 additions & 0 deletions internal/pkg/configmanager/appspec.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package configmanager

import (
"github.com/imdario/mergo"

"github.com/devstream-io/devstream/pkg/util/log"
"github.com/devstream-io/devstream/pkg/util/mapz"
)

// appSpec is app special options
type appSpec struct {
// language config
Language string `yaml:"language" mapstructure:"language"`
FrameWork string `yaml:"framework" mapstructure:"framework"`
}

// merge will merge vars and appSpec
func (s *appSpec) merge(vars map[string]any) map[string]any {
specMap, err := mapz.DecodeStructToMap(s)
if err != nil {
log.Warnf("appspec %+v decode failed: %+v", s, err)
return map[string]any{}
}
if err := mergo.Merge(&specMap, vars); err != nil {
log.Warnf("appSpec %+v merge map failed: %+v", s, err)
return vars
}
return specMap
}

func (s *appSpec) updatePiplineOption(options RawOptions) {
if _, exist := options["language"]; !exist && s.hasLanguageConfig() {
options["language"] = RawOptions{
"name": s.Language,
"framework": s.FrameWork,
}
}
}

func (s *appSpec) hasLanguageConfig() bool {
return s.Language != "" || s.FrameWork != ""
}
8 changes: 3 additions & 5 deletions internal/pkg/configmanager/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"gopkg.in/yaml.v3"

"github.com/devstream-io/devstream/pkg/util/log"
"github.com/devstream-io/devstream/pkg/util/mapz"
)

// Config is a general config in DevStream.
Expand Down Expand Up @@ -55,16 +54,15 @@ func (c *Config) getToolsWithVarsFromApp(a app) (Tools, error) {
return nil, fmt.Errorf("app parse yaml failed: %w", err)
}

rawApp.setDefault()
appVars := mapz.Merge(c.Vars, rawApp.Spec)

// 3. generate app repo and template repo from scmInfo
repoScaffoldingTool, err := rawApp.getRepoTemplateTool(appVars)
rawApp.setDefault()
repoScaffoldingTool, err := rawApp.getRepoTemplateTool()
if err != nil {
return nil, fmt.Errorf("app[%s] get repo failed: %w", rawApp.Name, err)
}

// 4. get ci/cd pipelineTemplates
appVars := rawApp.Spec.merge(c.Vars)
tools, err := rawApp.generateCICDToolsFromAppConfig(c.pipelineTemplateMap, appVars)
if err != nil {
return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", rawApp.Name, err)
Expand Down
2 changes: 1 addition & 1 deletion internal/pkg/configmanager/configmanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,6 @@ func (m *Manager) getConfigFromFile() (*Config, error) {

// escapeBrackets is used to escape []byte(": [[xxx]]xxx\n") to []byte(": \"[[xxx]]\"xxx\n")
func escapeBrackets(param []byte) []byte {
re := regexp.MustCompile(`([^:]+:)(\s*)(\[\[[^\]]+\]\][^\s]*)`)
re := regexp.MustCompile(`([^:]+:)(\s*)((\[\[[^\]]+\]\][^\s\[]*)+)[^\s#\n]*`)
return re.ReplaceAll(param, []byte("$1$2\"$3\""))
}
69 changes: 29 additions & 40 deletions internal/pkg/configmanager/configmanager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ pipelineTemplates:
namespace: [[ argocdNamespace ]] # you can use global vars in templates
destination:
server: https://kubernetes.default.svc
namespace: default
namespace: devstream-io
source:
valuefile: values.yaml
path: helm/[[ app ]]
Expand Down Expand Up @@ -128,6 +128,10 @@ var _ = Describe("LoadConfig", func() {
Options: RawOptions{
"instanceID": "service-a",
"pipeline": RawOptions{
"language": RawOptions{
"name": "python",
"framework": "django",
},
"docker": RawOptions{
"registry": RawOptions{
"repository": "service-a",
Expand Down Expand Up @@ -159,28 +163,17 @@ var _ = Describe("LoadConfig", func() {
},
Options: RawOptions{
"instanceID": "service-a",
"pipeline": RawOptions{
"destination": RawOptions{
"namespace": "devstream-io",
"server": "https://kubernetes.default.svc",
},
"app": RawOptions{
"namespace": "argocd",
},
"source": RawOptions{
"valuefile": "values.yaml",
"path": "helm/service-a",
"repoURL": "${{repo-scaffolding.myapp.outputs.repoURL}}",
},
"configLocation": "",
"destination": RawOptions{
"namespace": "devstream-io",
"server": "https://kubernetes.default.svc",
},
"scm": RawOptions{
"url": "https://github.com/devstream-io/service-a",
"apiURL": "gitlab.com/some/path/to/your/api",
"owner": "devstream-io",
"org": "devstream-io",
"name": "service-a",
"scmType": "github",
"app": RawOptions{
"namespace": "argocd",
},
"source": RawOptions{
"valuefile": "values.yaml",
"path": "helm/service-a",
"repoURL": "${{repo-scaffolding.myapp.outputs.repoURL}}",
},
},
}
Expand Down Expand Up @@ -210,14 +203,7 @@ var _ = Describe("LoadConfig", func() {
"repo": "dtm-scaffolding-golang",
"branch": "main",
},
"vars": RawOptions{
"foo1": "bar1",
"foo2": "bar2",
"registryType": "dockerhub",
"framework": "django",
"language": "python",
"argocdNamespace": "argocd",
},
"vars": RawOptions{},
},
}

Expand Down Expand Up @@ -273,17 +259,20 @@ var _ = Describe("LoadConfig", func() {
var _ = Describe("escapeBrackets", func() {
When("escape brackets", func() {
It("should works right", func() {
testStrBytes1 := "foo: [[ foo ]]\n"
testStr2 := "foo: xx[[ foo ]]\n"
testStr3 := "foo: [[ foo ]]xx\n"

retStr1 := escapeBrackets([]byte(testStrBytes1))
retStr2 := escapeBrackets([]byte(testStr2))
retStr3 := escapeBrackets([]byte(testStr3))
testMap := map[string]string{
"foo: [[ foo ]]": "foo: \"[[ foo ]]\"",
"foo: [[ foo ]] #comment": "foo: \"[[ foo ]]\" #comment",
"foo: xx[[ foo ]]": "foo: xx[[ foo ]]",
"foo: [[ foo ]]xx": "foo: \"[[ foo ]]xx\"",
"foo: [[ foo ]]/[[ poo ]]": "foo: \"[[ foo ]]/[[ poo ]]\"",
`foo: [[ test ]]
poo: [[ gg ]]`: "foo: \"[[ test ]]\"\npoo: \"[[ gg ]]\"",
}

Expect(string(retStr1)).To(Equal("foo: \"[[ foo ]]\"\n"))
Expect(string(retStr2)).To(Equal("foo: xx[[ foo ]]\n"))
Expect(string(retStr3)).To(Equal("foo: \"[[ foo ]]xx\"\n"))
for testStr, expectStr := range testMap {
retStr1 := escapeBrackets([]byte(testStr))
Expect(string(retStr1)).Should(Equal(expectStr))
}
})
})
})
96 changes: 96 additions & 0 deletions internal/pkg/configmanager/pipelineDefault.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package configmanager

import (
"fmt"

"github.com/devstream-io/devstream/pkg/util/log"
)

var optionConfiguratorMap = map[string]pipelineOption{
"github-actions": githubGeneral,
"gitlab-ci": gitlabGeneral,
"jenkins-pipeline": jenkinsGeneral,
"argocdapp": argocdApp,
}

type pipelineOptionGenerator func(originOption RawOptions, app *app) RawOptions

type pipelineOption struct {
defaultConfigLocation string
optionGeneratorFunc pipelineOptionGenerator
}

// TODO(steinliber) unify all ci/cd config to same config options
var (
// github actions pipeline options
githubGeneral = pipelineOption{
defaultConfigLocation: "git@github.com:devstream-io/ci-template.git//github-actions",
optionGeneratorFunc: pipelineGeneralGenerator,
}
gitlabGeneral = pipelineOption{
defaultConfigLocation: "https://raw.githubusercontent.com/devstream-io/ci-template/main/gitlab-ci/.gitlab-ci.yml",
optionGeneratorFunc: pipelineGeneralGenerator,
}
jenkinsGeneral = pipelineOption{
defaultConfigLocation: "https://raw.githubusercontent.com/devstream-io/ci-template/main/jenkins-pipeline/general/Jenkinsfile",
optionGeneratorFunc: pipelineGeneralGenerator,
}
argocdApp = pipelineOption{
optionGeneratorFunc: pipelineArgocdAppGenerator,
}
)

// pipelineGeneralGenerator generate pipeline general options from RawOptions
func pipelineGeneralGenerator(options RawOptions, app *app) RawOptions {
if app.Spec != nil {
app.Spec.updatePiplineOption(options)
}
// update image related config
newOption := make(RawOptions)
newOption["pipeline"] = options
newOption["scm"] = RawOptions(app.Repo.Encode())
return newOption
}

// pipelineArgocdAppGenerator generate argocdApp options from RawOptions
func pipelineArgocdAppGenerator(options RawOptions, app *app) RawOptions {
// config app default options
if _, exist := options["app"]; !exist {
options["app"] = RawOptions{
"name": app.Name,
"namespace": "argocd",
}
}
// config destination options
if _, exist := options["destination"]; !exist {
options["destination"] = RawOptions{
"server": "https://kubernetes.default.svc",
"namespace": "default",
}
}
// config source default options
repoInfo, err := app.Repo.BuildRepoInfo()
if err != nil {
log.Errorf("parse argocd repoInfo failed: %+v", err)
return options
}
if source, sourceExist := options["source"]; sourceExist {
sourceMap := source.(RawOptions)
if _, repoURLExist := sourceMap["repoURL"]; !repoURLExist {
sourceMap["repoURL"] = repoInfo.CloneURL
}
options["source"] = sourceMap
} else {
options["source"] = RawOptions{
"valuefile": "values.yaml",
"path": fmt.Sprintf("helm/%s", app.Name),
"repoURL": repoInfo.CloneURL,
}
}
return options
}

// hasDefaultConfig check whether
func (o *pipelineOption) hasDefaultConfig() bool {
return o.defaultConfigLocation != ""
}
Loading

0 comments on commit 350f65e

Please sign in to comment.