diff --git a/pkg/parameters/expression.go b/pkg/parameters/expression.go new file mode 100644 index 0000000..794189b --- /dev/null +++ b/pkg/parameters/expression.go @@ -0,0 +1,57 @@ +package parameters + +import ( + "os" + "regexp" + + consolelogger "github.com/semaphoreci/spc/pkg/consolelogger" +) + +// revive:disable:add-constant + +func findRegex() *regexp.Regexp { + return regexp.MustCompile(`\$\{\{\s*parameters\.([a-zA-Z0-9_]+)\s*\}\}`) +} + +func updateRegex(envName string) *regexp.Regexp { + return regexp.MustCompile(`\$\{\{\s*parameters\.` + envName + `\s*\}\}`) +} + +type ParametersExpression struct { + Expression string + Path []string + YamlPath string + Value string +} + +func ContainsParametersExpression(value string) bool { + return findRegex().MatchString(value) +} + +func (exp *ParametersExpression) Substitute() error { + consolelogger.EmptyLine() + + exp.Value = exp.Expression + + allExpressions := findRegex().FindAllStringSubmatch(exp.Expression, -1) + + for _, matchGroup := range allExpressions { + envName := matchGroup[1] + consolelogger.Infof("Fetching the value for: %s\n", envName) + + envVal := os.Getenv(envName) + if envVal == "" { + consolelogger.Infof("\t** WARNING *** Environment variable %s not found.\n", envName) + consolelogger.Infof("\tThe name of the environment variable will be used instead.\n") + + envVal = envName + } + + consolelogger.Infof("Value: %s\n", envVal) + consolelogger.EmptyLine() + + exp.Value = updateRegex(envName).ReplaceAllString(exp.Value, envVal) + } + + return nil +} diff --git a/pkg/parameters/expression_test.go b/pkg/parameters/expression_test.go new file mode 100644 index 0000000..0704d38 --- /dev/null +++ b/pkg/parameters/expression_test.go @@ -0,0 +1,73 @@ +package parameters + +import ( + "os" + "testing" + + assert "github.com/stretchr/testify/assert" +) + +func Test__Substitute(t *testing.T) { + os.Setenv("TEST_VAL_1", "Foo") + os.Setenv("TEST_VAL_2", "Bar") + os.Setenv("TEST_VAL_3", "Baz") + + exp := ParametersExpression{ + Expression: "", + Path: []string{"semaphore.yml"}, + YamlPath: "name", + } + + // Only params expression with various number of whitespaces + + exp.Expression = "${{parameters.TEST_VAL_1}}" + err := exp.Substitute() + assert.Nil(t, err) + assert.Equal(t, "Foo", exp.Value) + + exp.Expression = "${{ parameters.TEST_VAL_1}}" + err = exp.Substitute() + assert.Nil(t, err) + assert.Equal(t, "Foo", exp.Value) + + exp.Expression = "${{ parameters.TEST_VAL_1 }}" + err = exp.Substitute() + assert.Nil(t, err) + assert.Equal(t, "Foo", exp.Value) + + // Text before and after params expression + + exp.Expression = "Hello ${{parameters.TEST_VAL_3}}" + err = exp.Substitute() + assert.Nil(t, err) + assert.Equal(t, "Hello Baz", exp.Value) + + exp.Expression = "${{parameters.TEST_VAL_3}} world" + err = exp.Substitute() + assert.Nil(t, err) + assert.Equal(t, "Baz world", exp.Value) + + exp.Expression = "Hello ${{parameters.TEST_VAL_3}} world" + err = exp.Substitute() + assert.Nil(t, err) + assert.Equal(t, "Hello Baz world", exp.Value) + + // Multiple params expressions + + exp.Expression = "Hello ${{parameters.TEST_VAL_1}} ${{parameters.TEST_VAL_2}}" + err = exp.Substitute() + assert.Nil(t, err) + assert.Equal(t, "Hello Foo Bar", exp.Value) + + exp.Expression = "My name is ${{parameters.TEST_VAL_2}}, ${{parameters.TEST_VAL_1}} ${{parameters.TEST_VAL_2}}" + err = exp.Substitute() + assert.Nil(t, err) + assert.Equal(t, "My name is Bar, Foo Bar", exp.Value) + + // If the env var is not present, the env var name is used + + exp.Expression = "Missing ${{parameters.THE_POINT}}" + err = exp.Substitute() + assert.Nil(t, err) + assert.Equal(t, "Missing THE_POINT", exp.Value) +} diff --git a/pkg/pipelines/model.go b/pkg/pipelines/model.go index 9465877..cbac979 100644 --- a/pkg/pipelines/model.go +++ b/pkg/pipelines/model.go @@ -17,7 +17,7 @@ func n() int64 { return time.Now().UnixNano() / int64(time.Millisecond) } -func (p *Pipeline) UpdateWhenExpression(path []string, value string) error { +func (p *Pipeline) UpdateField(path []string, value string) error { _, err := p.raw.Set(value, path...) return err @@ -43,6 +43,10 @@ func (p *Pipeline) GlobalPriorityRules() []*gabs.Container { return p.raw.Search("global_job_config", "priority").Children() } +func (p *Pipeline) GlobalSecrets() []*gabs.Container { + return p.raw.Search("global_job_config", "secrets").Children() +} + func (p *Pipeline) QueueRules() []*gabs.Container { return p.raw.Search("queue").Children() } diff --git a/pkg/pipelines/parameters_evaluator.go b/pkg/pipelines/parameters_evaluator.go new file mode 100644 index 0000000..250c5ce --- /dev/null +++ b/pkg/pipelines/parameters_evaluator.go @@ -0,0 +1,153 @@ +package pipelines + +import ( + "fmt" + "strconv" + + consolelogger "github.com/semaphoreci/spc/pkg/consolelogger" + parameters "github.com/semaphoreci/spc/pkg/parameters" +) + +// revive:disable:add-constant + +type parametersEvaluator struct { + pipeline *Pipeline + + list []parameters.ParametersExpression +} + +func newParametersEvaluator(p *Pipeline) *parametersEvaluator { + return ¶metersEvaluator{pipeline: p} +} + +func (e *parametersEvaluator) Run() error { + var err error + + e.ExtractAll() + + e.displayFound() + + err = e.substituteValues() + if err != nil { + return err + } + + err = e.updatePipeline() + if err != nil { + return err + } + + return nil +} + +func (e *parametersEvaluator) ExtractAll() { + e.ExtractPipelineName() + e.ExtractFromQueue() + e.ExtractFromGlobalSecrets() + e.ExtractFromSecrets() +} + +func (e *parametersEvaluator) ExtractPipelineName() { + e.tryExtractingFromPath([]string{"name"}) +} + +func (e *parametersEvaluator) ExtractFromSecrets() { + for blockIndex, block := range e.pipeline.Blocks() { + secrets := block.Search("task", "secrets").Children() + + for secretIndex := range secrets { + e.tryExtractingFromPath([]string{ + "blocks", + strconv.Itoa(blockIndex), + "task", + "secrets", + strconv.Itoa(secretIndex), + "name", + }) + } + } +} + +func (e *parametersEvaluator) ExtractFromGlobalSecrets() { + for index := range e.pipeline.GlobalSecrets() { + e.tryExtractingFromPath([]string{"global_job_config", "secrets", strconv.Itoa(index), "name"}) + } +} + +func (e *parametersEvaluator) ExtractFromQueue() { + e.tryExtractingFromPath([]string{"queue", "name"}) + + for index := range e.pipeline.QueueRules() { + e.tryExtractingFromPath([]string{"queue", strconv.Itoa(index), "name"}) + } +} + +func (e *parametersEvaluator) tryExtractingFromPath(path []string) { + if !e.pipeline.PathExists(path) { + return + } + + value, ok := e.pipeline.raw.Search(path...).Data().(string) + if !ok { + return + } + + if !parameters.ContainsParametersExpression(value) { + return + } + + expression := parameters.ParametersExpression{ + Expression: value, + Path: path, + YamlPath: e.pipeline.yamlPath, + } + + e.list = append(e.list, expression) +} + +func (e *parametersEvaluator) displayFound() { + consolelogger.Infof("Found parameters expressions at %d locations.\n", len(e.list)) + consolelogger.EmptyLine() + + for index, item := range e.list { + consolelogger.IncrementNesting() + consolelogger.InfoNumberListLn(index+1, fmt.Sprintf("Location: %+v", item.Path)) + consolelogger.Infof("File: %s\n", item.YamlPath) + consolelogger.Infof("Expression: %s\n", item.Expression) + consolelogger.DecreaseNesting() + consolelogger.EmptyLine() + } +} + +func (e *parametersEvaluator) substituteValues() error { + consolelogger.Infof("Substituting parameters with their values .\n") + consolelogger.EmptyLine() + + for index, item := range e.list { + consolelogger.IncrementNesting() + consolelogger.InfoNumberListLn(index+1, "Parameters Expression: "+item.Expression) + + err := e.list[index].Substitute() + if err != nil { + return err + } + + consolelogger.Infof("Result: %s\n", e.list[index].Value) + consolelogger.DecreaseNesting() + consolelogger.EmptyLine() + } + + return nil +} + +func (e *parametersEvaluator) updatePipeline() error { + for index := range e.list { + err := e.pipeline.UpdateField(e.list[index].Path, e.list[index].Value) + + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/pipelines/parameters_evaluator_test.go b/pkg/pipelines/parameters_evaluator_test.go new file mode 100644 index 0000000..191eec9 --- /dev/null +++ b/pkg/pipelines/parameters_evaluator_test.go @@ -0,0 +1,111 @@ +package pipelines + +// revive:disable:add-constant + +import ( + "fmt" + "os" + "testing" + + parameters "github.com/semaphoreci/spc/pkg/parameters" + assert "github.com/stretchr/testify/assert" +) + +func Test__ParametersEvaluatorExtractAll(t *testing.T) { + pipeline, err := LoadFromFile("../../test/fixtures/all_parameters_locations.yml") + assert.Nil(t, err) + + e := newParametersEvaluator(pipeline) + e.ExtractAll() + + for _, e1 := range e.list { + fmt.Printf("%+v\n", e1) + } + + assert.Equal(t, 8, len(e.list)) + assert.Equal(t, e.list, []parameters.ParametersExpression{ + { + Expression: "Deploy to ${{parameters.DEPLOY_ENV}} on ${{parameters.SERVER}}", + Path: []string{"name"}, + YamlPath: "../../test/fixtures/all_parameters_locations.yml", + Value: "", + }, + { + Expression: "${{parameters.DEPLOY_ENV}}_deployment_queue", + Path: []string{"queue", "0", "name"}, + YamlPath: "../../test/fixtures/all_parameters_locations.yml", + Value: "", + }, + { + Expression: "${{parameters.MISSING}}_queue", + Path: []string{"queue", "1", "name"}, + YamlPath: "../../test/fixtures/all_parameters_locations.yml", + Value: "", + }, + { + Expression: "${{parameters.DEPLOY_ENV}}_deploy_key", + Path: []string{"global_job_config", "secrets", "0", "name"}, + YamlPath: "../../test/fixtures/all_parameters_locations.yml", + Value: "", + }, + { + Expression: "${{parameters.DEPLOY_ENV}}_dockerhub", + Path: []string{"blocks", "0", "task", "secrets", "0", "name"}, + YamlPath: "../../test/fixtures/all_parameters_locations.yml", + Value: "", + }, + { + Expression: "${{parameters.DEPLOY_ENV}}_ecr", + Path: []string{"blocks", "0", "task", "secrets", "1", "name"}, + YamlPath: "../../test/fixtures/all_parameters_locations.yml", + Value: "", + }, + { + Expression: "${{parameters.DEPLOY_ENV}}_deploy_key", + Path: []string{"blocks", "1", "task", "secrets", "0", "name"}, + YamlPath: "../../test/fixtures/all_parameters_locations.yml", + Value: "", + }, + { + Expression: "${{parameters.DEPLOY_ENV}}_aws_creds", + Path: []string{"blocks", "1", "task", "secrets", "1", "name"}, + YamlPath: "../../test/fixtures/all_parameters_locations.yml", + Value: "", + }, + }) +} + +func Test__Run(t *testing.T) { + pipeline, err := LoadFromFile("../../test/fixtures/all_parameters_locations.yml") + assert.Nil(t, err) + + e := newParametersEvaluator(pipeline) + + os.Setenv("DEPLOY_ENV", "prod") + os.Setenv("SERVER", "server_1") + + err = e.Run() + assert.Nil(t, err) + + yaml_result, er := e.pipeline.ToYAML() + assert.Nil(t, er) + fmt.Printf("%s\n", yaml_result) + + assert_value_on_path(t, e, []string{"name"}, "Deploy to prod on server_1") + assert_value_on_path(t, e, []string{"queue", "0", "name"}, "prod_deployment_queue") + assert_value_on_path(t, e, []string{"queue", "1", "name"}, "MISSING_queue") + assert_value_on_path(t, e, []string{"global_job_config", "secrets", "0", "name"}, "prod_deploy_key") + assert_value_on_path(t, e, []string{"blocks", "0", "task", "secrets", "0", "name"}, "prod_dockerhub") + assert_value_on_path(t, e, []string{"blocks", "0", "task", "secrets", "1", "name"}, "prod_ecr") + assert_value_on_path(t, e, []string{"blocks", "1", "task", "secrets", "0", "name"}, "prod_deploy_key") + assert_value_on_path(t, e, []string{"blocks", "1", "task", "secrets", "1", "name"}, "prod_aws_creds") +} + +func assert_value_on_path(t *testing.T, e *parametersEvaluator, path []string, value string) { + field, ok := e.pipeline.raw.Search(path...).Data().(string) + if !ok { + assert.Equal(t, "Invalid value after parsing at", path) + } + + assert.Equal(t, field, value) +} diff --git a/pkg/pipelines/when_evaluator.go b/pkg/pipelines/when_evaluator.go index 90609b4..2055540 100644 --- a/pkg/pipelines/when_evaluator.go +++ b/pkg/pipelines/when_evaluator.go @@ -55,7 +55,7 @@ func (e *whenEvaluator) Run() error { func (e *whenEvaluator) updatePipeline() error { for index := range e.results { - err := e.pipeline.UpdateWhenExpression(e.list[index].Path, e.results[index]) + err := e.pipeline.UpdateField(e.list[index].Path, e.results[index]) if err != nil { return err diff --git a/test/fixtures/all_parameters_locations.yml b/test/fixtures/all_parameters_locations.yml new file mode 100644 index 0000000..2b6ce9f --- /dev/null +++ b/test/fixtures/all_parameters_locations.yml @@ -0,0 +1,27 @@ +version: v1.0 + +name: "Deploy to ${{parameters.DEPLOY_ENV}} on ${{parameters.SERVER}}" + +global_job_config: + secrets: + - name: "${{parameters.DEPLOY_ENV}}_deploy_key" + - name: "github_key" + + +queue: + - name: "${{parameters.DEPLOY_ENV}}_deployment_queue" + - name: "${{parameters.MISSING}}_queue" + - name: "default_queue" + +blocks: + - name: Build and push image + task: + secrets: + - name: ${{parameters.DEPLOY_ENV}}_dockerhub + - name: ${{parameters.DEPLOY_ENV}}_ecr + + - name: Deploy image + task: + secrets: + - name: ${{parameters.DEPLOY_ENV}}_deploy_key + - name: ${{parameters.DEPLOY_ENV}}_aws_creds