Replace parameters in YAML with their values

pkg/parameters/expression.go

@@ -0,0 +1,57 @@
+package parameters
+
+import (
+	"os"
+	"regexp"
+
+	consolelogger "github.com/semaphoreci/spc/pkg/consolelogger"
+)
+
+// revive:disable:add-constant
+
+func findRegex() *regexp.Regexp {
+	return regexp.MustCompile(`\$\{\{\s*parameters\.([a-zA-Z0-9_]+)\s*\}\}`)
+}
+
+func updateRegex(envName string) *regexp.Regexp {
+	return regexp.MustCompile(`\$\{\{\s*parameters\.` + envName + `\s*\}\}`)
+}
+
+type ParametersExpression struct {
+	Expression string
+	Path       []string
+	YamlPath   string
+	Value      string
+}
+
+func ContainsParametersExpression(value string) bool {
+	return findRegex().MatchString(value)
+}
+
+func (exp *ParametersExpression) Substitute() error {
+	consolelogger.EmptyLine()
+
+	exp.Value = exp.Expression
+
+	allExpressions := findRegex().FindAllStringSubmatch(exp.Expression, -1)
+
+	for _, matchGroup := range allExpressions {
+		envName := matchGroup[1]
+		consolelogger.Infof("Fetching the value for: %s\n", envName)
+
+		envVal := os.Getenv(envName)
+		if envVal == "" {
+			consolelogger.Infof("\t** WARNING *** Environment variable %s not found.\n", envName)
+			consolelogger.Infof("\tThe name of the environment variable will be used instead.\n")
+
+			envVal = envName
+		}
+
+		consolelogger.Infof("Value: %s\n", envVal)
+		consolelogger.EmptyLine()
+
+		exp.Value = updateRegex(envName).ReplaceAllString(exp.Value, envVal)
+	}
+
+	return nil
+}

pkg/parameters/expression_test.go

@@ -0,0 +1,73 @@
+package parameters
+
+import (
+	"os"
+	"testing"
+
+	assert "github.com/stretchr/testify/assert"
+)
+
+func Test__Substitute(t *testing.T) {
+	os.Setenv("TEST_VAL_1", "Foo")
+	os.Setenv("TEST_VAL_2", "Bar")
+	os.Setenv("TEST_VAL_3", "Baz")
+
+	exp := ParametersExpression{
+		Expression: "",
+		Path:       []string{"semaphore.yml"},
+		YamlPath:   "name",
+	}
+
+	// Only params expression with various number of whitespaces
+
+	exp.Expression = "${{parameters.TEST_VAL_1}}"
+	err := exp.Substitute()
+	assert.Nil(t, err)
+	assert.Equal(t, "Foo", exp.Value)
+
+	exp.Expression = "${{  parameters.TEST_VAL_1}}"
+	err = exp.Substitute()
+	assert.Nil(t, err)
+	assert.Equal(t, "Foo", exp.Value)
+
+	exp.Expression = "${{  parameters.TEST_VAL_1  }}"
+	err = exp.Substitute()
+	assert.Nil(t, err)
+	assert.Equal(t, "Foo", exp.Value)
+
+	// Text before and after  params expression
+
+	exp.Expression = "Hello ${{parameters.TEST_VAL_3}}"
+	err = exp.Substitute()
+	assert.Nil(t, err)
+	assert.Equal(t, "Hello Baz", exp.Value)
+
+	exp.Expression = "${{parameters.TEST_VAL_3}} world"
+	err = exp.Substitute()
+	assert.Nil(t, err)
+	assert.Equal(t, "Baz world", exp.Value)
+
+	exp.Expression = "Hello ${{parameters.TEST_VAL_3}} world"
+	err = exp.Substitute()
+	assert.Nil(t, err)
+	assert.Equal(t, "Hello Baz world", exp.Value)
+
+	// Multiple params expressions
+
+	exp.Expression = "Hello ${{parameters.TEST_VAL_1}} ${{parameters.TEST_VAL_2}}"
+	err = exp.Substitute()
+	assert.Nil(t, err)
+	assert.Equal(t, "Hello Foo Bar", exp.Value)
+
+	exp.Expression = "My name is ${{parameters.TEST_VAL_2}}, ${{parameters.TEST_VAL_1}} ${{parameters.TEST_VAL_2}}"
+	err = exp.Substitute()
+	assert.Nil(t, err)
+	assert.Equal(t, "My name is Bar, Foo Bar", exp.Value)
+
+	// If the env var is not present, the env var name is used
+
+	exp.Expression = "Missing ${{parameters.THE_POINT}}"
+	err = exp.Substitute()
+	assert.Nil(t, err)
+	assert.Equal(t, "Missing THE_POINT", exp.Value)
+}

pkg/pipelines/model.go

@@ -17,7 +17,7 @@ func n() int64 {
 	return time.Now().UnixNano() / int64(time.Millisecond)
 }
 
-func (p *Pipeline) UpdateWhenExpression(path []string, value string) error {
+func (p *Pipeline) UpdateField(path []string, value string) error {
 	_, err := p.raw.Set(value, path...)
 
 	return err
@@ -43,6 +43,10 @@ func (p *Pipeline) GlobalPriorityRules() []*gabs.Container {
 	return p.raw.Search("global_job_config", "priority").Children()
 }
 
+func (p *Pipeline) GlobalSecrets() []*gabs.Container {
+	return p.raw.Search("global_job_config", "secrets").Children()
+}
+
 func (p *Pipeline) QueueRules() []*gabs.Container {
 	return p.raw.Search("queue").Children()
 }

pkg/pipelines/parameters_evaluator.go

@@ -0,0 +1,153 @@
+package pipelines
+
+import (
+	"fmt"
+	"strconv"
+
+	consolelogger "github.com/semaphoreci/spc/pkg/consolelogger"
+	parameters "github.com/semaphoreci/spc/pkg/parameters"
+)
+
+// revive:disable:add-constant
+
+type parametersEvaluator struct {
+	pipeline *Pipeline
+
+	list []parameters.ParametersExpression
+}
+
+func newParametersEvaluator(p *Pipeline) *parametersEvaluator {
+	return &parametersEvaluator{pipeline: p}
+}
+
+func (e *parametersEvaluator) Run() error {
+	var err error
+
+	e.ExtractAll()
+
+	e.displayFound()
+
+	err = e.substituteValues()
+	if err != nil {
+		return err
+	}
+
+	err = e.updatePipeline()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *parametersEvaluator) ExtractAll() {
+	e.ExtractPipelineName()
+	e.ExtractFromQueue()
+	e.ExtractFromGlobalSecrets()
+	e.ExtractFromSecrets()
+}
+
+func (e *parametersEvaluator) ExtractPipelineName() {
+	e.tryExtractingFromPath([]string{"name"})
+}
+
+func (e *parametersEvaluator) ExtractFromSecrets() {
+	for blockIndex, block := range e.pipeline.Blocks() {
+		secrets := block.Search("task", "secrets").Children()
+
+		for secretIndex := range secrets {
+			e.tryExtractingFromPath([]string{
+				"blocks",
+				strconv.Itoa(blockIndex),
+				"task",
+				"secrets",
+				strconv.Itoa(secretIndex),
+				"name",
+			})
+		}
+	}
+}
+
+func (e *parametersEvaluator) ExtractFromGlobalSecrets() {
+	for index := range e.pipeline.GlobalSecrets() {
+		e.tryExtractingFromPath([]string{"global_job_config", "secrets", strconv.Itoa(index), "name"})
+	}
+}
+
+func (e *parametersEvaluator) ExtractFromQueue() {
+	e.tryExtractingFromPath([]string{"queue", "name"})
+
+	for index := range e.pipeline.QueueRules() {
+		e.tryExtractingFromPath([]string{"queue", strconv.Itoa(index), "name"})
+	}
+}
+
+func (e *parametersEvaluator) tryExtractingFromPath(path []string) {
+	if !e.pipeline.PathExists(path) {
+		return
+	}
+
+	value, ok := e.pipeline.raw.Search(path...).Data().(string)
+	if !ok {
+		return
+	}
+
+	if !parameters.ContainsParametersExpression(value) {
+		return
+	}
+
+	expression := parameters.ParametersExpression{
+		Expression: value,
+		Path:       path,
+		YamlPath:   e.pipeline.yamlPath,
+	}
+
+	e.list = append(e.list, expression)
+}
+
+func (e *parametersEvaluator) displayFound() {
+	consolelogger.Infof("Found parameters expressions at %d locations.\n", len(e.list))
+	consolelogger.EmptyLine()
+
+	for index, item := range e.list {
+		consolelogger.IncrementNesting()
+		consolelogger.InfoNumberListLn(index+1, fmt.Sprintf("Location: %+v", item.Path))
+		consolelogger.Infof("File: %s\n", item.YamlPath)
+		consolelogger.Infof("Expression: %s\n", item.Expression)
+		consolelogger.DecreaseNesting()
+		consolelogger.EmptyLine()
+	}
+}
+
+func (e *parametersEvaluator) substituteValues() error {
+	consolelogger.Infof("Substituting parameters with their values .\n")
+	consolelogger.EmptyLine()
+
+	for index, item := range e.list {
+		consolelogger.IncrementNesting()
+		consolelogger.InfoNumberListLn(index+1, "Parameters Expression: "+item.Expression)
+
+		err := e.list[index].Substitute()
+		if err != nil {
+			return err
+		}
+
+		consolelogger.Infof("Result: %s\n", e.list[index].Value)
+		consolelogger.DecreaseNesting()
+		consolelogger.EmptyLine()
+	}
+
+	return nil
+}
+
+func (e *parametersEvaluator) updatePipeline() error {
+	for index := range e.list {
+		err := e.pipeline.UpdateField(e.list[index].Path, e.list[index].Value)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}