Replace commands_files with commands

.semaphore/semaphore.yml

@@ -104,6 +104,7 @@ blocks:
                 - test/e2e/list_diff_with_default_branch.rb
                 - test/e2e/when_conditions_without_change_in.rb
                 - test/e2e/parameters_and_change_in.rb
+                - test/e2e/cmd_files_all_possible_locations.rb
 
           commands:
             - make build

pkg/cli/compile.go

@@ -20,14 +20,19 @@ var compileCmd = &cobra.Command{
 		output := fetchRequiredStringFlag(cmd, "output")
 		logsPath := fetchRequiredStringFlag(cmd, "logs")
 
-		fmt.Printf("Evaluating template expressions in %s.\n\n", input)
-
+		fmt.Printf("Extracting commands from commands_files in %s.\n\n", input)
+		
 		logs.Open(logsPath)
 		logs.SetCurrentPipelineFilePath(input)
-
+		
 		ppl, err := pipelines.LoadFromFile(input)
 		check(err)
 
+		err = ppl.ExtractCommandsFromCommandsFiles()
+		check(err)
+
+		fmt.Printf("Evaluating template expressions in %s.\n\n", input)
+
 		err = ppl.EvaluateTemplates()
 		check(err)
 

pkg/commands/file.go

@@ -0,0 +1,46 @@
+package commands
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+type File struct {
+	FilePath    string
+	ParentPath  []string
+	YamlPath    string
+	Commands    []string
+}
+
+func (f *File) Extract() error {
+	// Resolve FilePath relative to YamlPath
+	absoluteFilePath := filepath.Join(filepath.Dir(f.YamlPath), f.FilePath)
+
+	// Open the file
+	file, err := os.Open(absoluteFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to open the commands_file at %s, error: %w", absoluteFilePath, err)
+	}
+	defer file.Close()
+
+	// Read the file line by line
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		f.Commands = append(f.Commands, line)
+	}
+
+	// Check for scanning errors
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("error reading file: %w", err)
+	}
+
+	// If no commands were read, return an error indicating that the file is empty
+	if len(f.Commands) == 0 {
+		return fmt.Errorf("the commands_file at location %s is empty.", absoluteFilePath)
+	}
+
+	return nil
+}

pkg/commands/file_test.go

@@ -0,0 +1,41 @@
+package commands
+
+import (
+	"testing"
+
+	assert "github.com/stretchr/testify/assert"
+)
+
+func Test__Extract(t *testing.T) {
+	// If commands file does not exist, it retruns the error
+	file := File{
+		FilePath:   "non_existing_file.txt",
+		ParentPath: []string{},
+		YamlPath:   "../../test/fixtures/all_commands_file_locations.yml",
+		Commands:   []string{},
+	}
+	err := file.Extract()
+
+	assert.Error(t, err)
+
+	expectedErrorMessage := "failed to open the commands_file at ../../test/fixtures/non_existing_file.txt"
+	assert.Contains(t, err.Error(), expectedErrorMessage)
+
+	// If commands file is empty, it retruns the error
+	file.FilePath = "empty_file.txt"
+	err = file.Extract()
+
+	assert.Error(t, err)
+
+	expectedErrorMessage = "the commands_file at location ../../test/fixtures/empty_file.txt is empty."
+	assert.Contains(t, err.Error(), expectedErrorMessage)
+
+	// Commands are read successfully from the valid file.
+	file.FilePath = "valid_commands_file.txt"
+	err = file.Extract()
+
+	assert.Nil(t, err)
+
+	expectedCommands := []string{"echo 1", "echo 12", "echo 123"}
+	assert.Equal(t, file.Commands, expectedCommands)
+}

pkg/pipelines/commands_extractor.go

@@ -0,0 +1,148 @@
+package pipelines
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/Jeffail/gabs/v2"
+	consolelogger "github.com/semaphoreci/spc/pkg/consolelogger"
+	commands "github.com/semaphoreci/spc/pkg/commands"
+)
+
+// revive:disable:add-constant
+
+type commandsExtractor struct {
+	pipeline *Pipeline
+
+	files []commands.File
+}
+
+func newCommandsExtractor(p *Pipeline) *commandsExtractor {
+	return &commandsExtractor{pipeline: p}
+}
+
+func (e *commandsExtractor) Run() error {
+	var err error
+
+	e.findAll()
+
+	e.displayFound()
+
+	err = e.extractCommands()
+	if err != nil {
+		return err
+	}
+
+	err = e.updatePipeline()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *commandsExtractor) findAll() {
+	e.findCommandsFiles(e.pipeline.raw, []string{})
+}
+
+func (e *commandsExtractor) findCommandsFiles(parent *gabs.Container, parentPath []string){
+	path := []string{}
+
+	switch parent.Data().(type) {
+
+		case []interface{}:
+			for childIndex, child := range parent.Children() {
+				path = concatPaths(parentPath, []string{strconv.Itoa(childIndex)})
+				e.findCommandsFiles(child, path)
+			}
+
+		case map[string]interface{}:
+			for key, child := range parent.ChildrenMap() {
+				if key == "commands_file" {
+					e.gatherCommandsFileData(child, parentPath)
+				} else {
+					path = concatPaths(parentPath, []string{key})
+					e.findCommandsFiles(child, path)
+				}
+			}
+	}
+}
+
+func (e *commandsExtractor) gatherCommandsFileData(element *gabs.Container, path []string) {
+	file := commands.File{
+		FilePath:   element.Data().(string),
+		ParentPath: path,
+		YamlPath:   e.pipeline.yamlPath,
+		Commands:   []string{},
+	}
+
+	e.files = append(e.files, file)
+}
+
+func (e *commandsExtractor) displayFound() {
+	consolelogger.Infof("Found commands_file fields at %d locations.\n", len(e.files))
+	consolelogger.EmptyLine()
+
+	for index, item := range e.files {
+		itemPath := concatPaths(item.ParentPath, []string{"commands_file"})
+
+		consolelogger.IncrementNesting()
+		consolelogger.InfoNumberListLn(index+1, fmt.Sprintf("Location: %+v", itemPath))
+		consolelogger.Infof("File: %s\n", item.YamlPath)
+		consolelogger.Infof("The commands_file path: %s\n", item.FilePath)
+		consolelogger.DecreaseNesting()
+		consolelogger.EmptyLine()
+	}
+}
+
+func (e *commandsExtractor) extractCommands() error {
+	consolelogger.Infof("Extracting commands from commands_files.\n")
+	consolelogger.EmptyLine()
+
+	for index, item := range e.files {
+		consolelogger.IncrementNesting()
+		consolelogger.InfoNumberListLn(index+1, "The commands_file path: "+item.FilePath)
+
+		err := e.files[index].Extract()
+		if err != nil {
+			return err
+		}
+
+		consolelogger.Infof("Extracted %d commands.\n", len(e.files[index].Commands))
+		consolelogger.DecreaseNesting()
+		consolelogger.EmptyLine()
+	}
+
+	return nil
+}
+
+func (e *commandsExtractor) updatePipeline() error {
+	for _, item  := range e.files {
+
+		cmdFilePath := concatPaths(item.ParentPath, []string{"commands_file"})
+
+		err := e.pipeline.raw.Delete(cmdFilePath...)
+
+		if err != nil {
+			return err
+		}
+
+		cmdPath := concatPaths(item.ParentPath, []string{"commands"})
+
+		_, err = e.pipeline.raw.Array(cmdPath...)
+
+		if err != nil {
+			return err
+		}
+
+		for _, command := range item.Commands{
+			e.pipeline.raw.ArrayAppend(command, cmdPath...)
+
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}