From b6d1302d8f8d2b74dc9eaa8fe19d3f12ebaa859f Mon Sep 17 00:00:00 2001 From: Damjan Becirovic Date: Thu, 10 Oct 2024 18:48:38 +0200 Subject: [PATCH] Replace commands_files with commands --- .semaphore/semaphore.yml | 1 + pkg/cli/compile.go | 11 +- pkg/commands/file.go | 46 +++++ pkg/commands/file_test.go | 41 +++++ pkg/pipelines/commands_extractor.go | 148 ++++++++++++++++ pkg/pipelines/commands_extractor_test.go | 164 ++++++++++++++++++ pkg/pipelines/model.go | 4 + test/e2e/cmd_files_all_possible_locations.rb | 163 +++++++++++++++++ test/fixtures/all_commands_file_locations.yml | 50 ++++++ test/fixtures/empty_file.txt | 0 test/fixtures/valid_commands_file.txt | 3 + 11 files changed, 628 insertions(+), 3 deletions(-) create mode 100644 pkg/commands/file.go create mode 100644 pkg/commands/file_test.go create mode 100644 pkg/pipelines/commands_extractor.go create mode 100644 pkg/pipelines/commands_extractor_test.go create mode 100644 test/e2e/cmd_files_all_possible_locations.rb create mode 100644 test/fixtures/all_commands_file_locations.yml create mode 100644 test/fixtures/empty_file.txt create mode 100644 test/fixtures/valid_commands_file.txt diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 988f9e5..c68bc6c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -104,6 +104,7 @@ blocks: - test/e2e/list_diff_with_default_branch.rb - test/e2e/when_conditions_without_change_in.rb - test/e2e/parameters_and_change_in.rb + - test/e2e/cmd_files_all_possible_locations.rb commands: - make build diff --git a/pkg/cli/compile.go b/pkg/cli/compile.go index 3fef4d2..2d93cec 100644 --- a/pkg/cli/compile.go +++ b/pkg/cli/compile.go @@ -20,14 +20,19 @@ var compileCmd = &cobra.Command{ output := fetchRequiredStringFlag(cmd, "output") logsPath := fetchRequiredStringFlag(cmd, "logs") - fmt.Printf("Evaluating template expressions in %s.\n\n", input) - + fmt.Printf("Extracting commands from commands_files in %s.\n\n", input) + logs.Open(logsPath) logs.SetCurrentPipelineFilePath(input) - + ppl, err := pipelines.LoadFromFile(input) check(err) + err = ppl.ExtractCommandsFromCommandsFiles() + check(err) + + fmt.Printf("Evaluating template expressions in %s.\n\n", input) + err = ppl.EvaluateTemplates() check(err) diff --git a/pkg/commands/file.go b/pkg/commands/file.go new file mode 100644 index 0000000..6327846 --- /dev/null +++ b/pkg/commands/file.go @@ -0,0 +1,46 @@ +package commands + +import ( + "bufio" + "fmt" + "os" + "path/filepath" +) + +type File struct { + FilePath string + ParentPath []string + YamlPath string + Commands []string +} + +func (f *File) Extract() error { + // Resolve FilePath relative to YamlPath + absoluteFilePath := filepath.Join(filepath.Dir(f.YamlPath), f.FilePath) + + // Open the file + file, err := os.Open(absoluteFilePath) + if err != nil { + return fmt.Errorf("failed to open the commands_file at %s, error: %w", absoluteFilePath, err) + } + defer file.Close() + + // Read the file line by line + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + f.Commands = append(f.Commands, line) + } + + // Check for scanning errors + if err := scanner.Err(); err != nil { + return fmt.Errorf("error reading file: %w", err) + } + + // If no commands were read, return an error indicating that the file is empty + if len(f.Commands) == 0 { + return fmt.Errorf("the commands_file at location %s is empty.", absoluteFilePath) + } + + return nil +} \ No newline at end of file diff --git a/pkg/commands/file_test.go b/pkg/commands/file_test.go new file mode 100644 index 0000000..0cb74fa --- /dev/null +++ b/pkg/commands/file_test.go @@ -0,0 +1,41 @@ +package commands + +import ( + "testing" + + assert "github.com/stretchr/testify/assert" +) + +func Test__Extract(t *testing.T) { + // If commands file does not exist, it retruns the error + file := File{ + FilePath: "non_existing_file.txt", + ParentPath: []string{}, + YamlPath: "../../test/fixtures/all_commands_file_locations.yml", + Commands: []string{}, + } + err := file.Extract() + + assert.Error(t, err) + + expectedErrorMessage := "failed to open the commands_file at ../../test/fixtures/non_existing_file.txt" + assert.Contains(t, err.Error(), expectedErrorMessage) + + // If commands file is empty, it retruns the error + file.FilePath = "empty_file.txt" + err = file.Extract() + + assert.Error(t, err) + + expectedErrorMessage = "the commands_file at location ../../test/fixtures/empty_file.txt is empty." + assert.Contains(t, err.Error(), expectedErrorMessage) + + // Commands are read successfully from the valid file. + file.FilePath = "valid_commands_file.txt" + err = file.Extract() + + assert.Nil(t, err) + + expectedCommands := []string{"echo 1", "echo 12", "echo 123"} + assert.Equal(t, file.Commands, expectedCommands) +} \ No newline at end of file diff --git a/pkg/pipelines/commands_extractor.go b/pkg/pipelines/commands_extractor.go new file mode 100644 index 0000000..c71c5a7 --- /dev/null +++ b/pkg/pipelines/commands_extractor.go @@ -0,0 +1,148 @@ +package pipelines + +import ( + "fmt" + "strconv" + + "github.com/Jeffail/gabs/v2" + consolelogger "github.com/semaphoreci/spc/pkg/consolelogger" + commands "github.com/semaphoreci/spc/pkg/commands" +) + +// revive:disable:add-constant + +type commandsExtractor struct { + pipeline *Pipeline + + files []commands.File +} + +func newCommandsExtractor(p *Pipeline) *commandsExtractor { + return &commandsExtractor{pipeline: p} +} + +func (e *commandsExtractor) Run() error { + var err error + + e.findAll() + + e.displayFound() + + err = e.extractCommands() + if err != nil { + return err + } + + err = e.updatePipeline() + if err != nil { + return err + } + + return nil +} + +func (e *commandsExtractor) findAll() { + e.findCommandsFiles(e.pipeline.raw, []string{}) +} + +func (e *commandsExtractor) findCommandsFiles(parent *gabs.Container, parentPath []string){ + path := []string{} + + switch parent.Data().(type) { + + case []interface{}: + for childIndex, child := range parent.Children() { + path = concatPaths(parentPath, []string{strconv.Itoa(childIndex)}) + e.findCommandsFiles(child, path) + } + + case map[string]interface{}: + for key, child := range parent.ChildrenMap() { + if key == "commands_file" { + e.gatherCommandsFileData(child, parentPath) + } else { + path = concatPaths(parentPath, []string{key}) + e.findCommandsFiles(child, path) + } + } + } +} + +func (e *commandsExtractor) gatherCommandsFileData(element *gabs.Container, path []string) { + file := commands.File{ + FilePath: element.Data().(string), + ParentPath: path, + YamlPath: e.pipeline.yamlPath, + Commands: []string{}, + } + + e.files = append(e.files, file) +} + +func (e *commandsExtractor) displayFound() { + consolelogger.Infof("Found commands_file fields at %d locations.\n", len(e.files)) + consolelogger.EmptyLine() + + for index, item := range e.files { + itemPath := concatPaths(item.ParentPath, []string{"commands_file"}) + + consolelogger.IncrementNesting() + consolelogger.InfoNumberListLn(index+1, fmt.Sprintf("Location: %+v", itemPath)) + consolelogger.Infof("File: %s\n", item.YamlPath) + consolelogger.Infof("The commands_file path: %s\n", item.FilePath) + consolelogger.DecreaseNesting() + consolelogger.EmptyLine() + } +} + +func (e *commandsExtractor) extractCommands() error { + consolelogger.Infof("Extracting commands from commands_files.\n") + consolelogger.EmptyLine() + + for index, item := range e.files { + consolelogger.IncrementNesting() + consolelogger.InfoNumberListLn(index+1, "The commands_file path: "+item.FilePath) + + err := e.files[index].Extract() + if err != nil { + return err + } + + consolelogger.Infof("Extracted %d commands.\n", len(e.files[index].Commands)) + consolelogger.DecreaseNesting() + consolelogger.EmptyLine() + } + + return nil +} + +func (e *commandsExtractor) updatePipeline() error { + for _, item := range e.files { + + cmdFilePath := concatPaths(item.ParentPath, []string{"commands_file"}) + + err := e.pipeline.raw.Delete(cmdFilePath...) + + if err != nil { + return err + } + + cmdPath := concatPaths(item.ParentPath, []string{"commands"}) + + _, err = e.pipeline.raw.Array(cmdPath...) + + if err != nil { + return err + } + + for _, command := range item.Commands{ + e.pipeline.raw.ArrayAppend(command, cmdPath...) + + if err != nil { + return err + } + } + } + + return nil +} \ No newline at end of file diff --git a/pkg/pipelines/commands_extractor_test.go b/pkg/pipelines/commands_extractor_test.go new file mode 100644 index 0000000..461859e --- /dev/null +++ b/pkg/pipelines/commands_extractor_test.go @@ -0,0 +1,164 @@ +package pipelines + +import ( + "testing" + "fmt" + "reflect" + + assert "github.com/stretchr/testify/assert" + commands "github.com/semaphoreci/spc/pkg/commands" +) + +func Test__findAll(t *testing.T) { + yamlPath := "../../test/fixtures/all_commands_file_locations.yml" + pipeline, err := LoadFromFile(yamlPath) + assert.Nil(t, err) + + e := newCommandsExtractor(pipeline) + e.findAll() + + for _, f := range e.files { + fmt.Printf("%+v\n", f) + } + + expectedFiles := []commands.File{ + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"after_pipeline", "task", "epilogue", "always",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"after_pipeline", "task", "epilogue", "on_fail",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"after_pipeline", "task", "epilogue", "on_pass",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"after_pipeline", "task", "jobs", "0",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"after_pipeline", "task", "prologue",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"blocks", "0", "task", "prologue",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"blocks", "0", "task", "epilogue", "always",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"blocks", "0", "task", "epilogue", "on_fail",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"blocks", "0", "task", "epilogue", "on_pass",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"blocks", "0", "task", "jobs", "0",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"global_job_config", "prologue",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"global_job_config", "epilogue", "always",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"global_job_config", "epilogue", "on_fail",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + { + FilePath: "valid_commands_file.txt", + ParentPath: []string{"global_job_config", "epilogue", "on_pass",}, + YamlPath: yamlPath, + Commands: []string{}, + }, + } + + assert.Equal(t, len(expectedFiles), len(e.files)) + for _, f1 := range e.files { + expectedFile := findFile(f1, expectedFiles) + + assert.Equal(t, expectedFile.FilePath, f1.FilePath) + assert.Equal(t, expectedFile.ParentPath, f1.ParentPath) + assert.Equal(t, expectedFile.YamlPath, f1.YamlPath) + } +} + +func findFile(file commands.File, expectedFiles []commands.File) commands.File { + for _, e := range expectedFiles { + if reflect.DeepEqual(e.ParentPath, file.ParentPath) { + return e + } + } + return commands.File{} +} + +func Test__CommandsExtractorRun(t *testing.T) { + yamlPath := "../../test/fixtures/all_commands_file_locations.yml" + pipeline, err := LoadFromFile(yamlPath) + assert.Nil(t, err) + + e := newCommandsExtractor(pipeline) + err = e.Run() + assert.Nil(t, err) + + yamlResult, er := e.pipeline.ToYAML() + assert.Nil(t, er) + fmt.Printf("%s\n", yamlResult) + + expectedCommands := []interface{}{"echo 1", "echo 12", "echo 123"} + + assertCommandsOnPath(t, e, []string{"after_pipeline", "task", "epilogue", "always", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"after_pipeline", "task", "epilogue", "on_fail", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"after_pipeline", "task", "epilogue", "on_pass", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"after_pipeline", "task", "jobs", "0", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"after_pipeline", "task", "prologue", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"blocks", "0", "task", "prologue", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"blocks", "0", "task", "epilogue", "always", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"blocks", "0", "task", "epilogue", "on_fail", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"blocks", "0", "task", "epilogue", "on_pass", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"blocks", "0", "task", "jobs", "0", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"global_job_config", "prologue", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"global_job_config", "epilogue", "always", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"global_job_config", "epilogue", "on_fail", "commands"}, expectedCommands) + assertCommandsOnPath(t, e, []string{"global_job_config", "epilogue", "on_pass", "commands"}, expectedCommands) +} + +func assertCommandsOnPath(t *testing.T, e *commandsExtractor, path []string, value interface{}) { + field := e.pipeline.raw.Search(path...).Data() + assert.Equal(t, value, field) +} \ No newline at end of file diff --git a/pkg/pipelines/model.go b/pkg/pipelines/model.go index e7e27eb..b331440 100644 --- a/pkg/pipelines/model.go +++ b/pkg/pipelines/model.go @@ -26,6 +26,10 @@ func (p *Pipeline) EvaluateTemplates() error { return newTemplateEvaluator(p).Run() } +func (p *Pipeline) ExtractCommandsFromCommandsFiles() error { + return newCommandsExtractor(p).Run() +} + func (p *Pipeline) Blocks() []*gabs.Container { return p.raw.Search("blocks").Children() } diff --git a/test/e2e/cmd_files_all_possible_locations.rb b/test/e2e/cmd_files_all_possible_locations.rb new file mode 100644 index 0000000..6224816 --- /dev/null +++ b/test/e2e/cmd_files_all_possible_locations.rb @@ -0,0 +1,163 @@ +# rubocop:disable all + +# +# This test verifies if the compiler is able to recognize and +# process all locations where a commands_file expression can appear. +# + +require_relative "../e2e" +require 'yaml' + +pipeline = %{ +version: v1.0 +name: "Tests" +agent: + machine: + type: "e1-standard-2" + os_image: "ubuntu2004" +global_job_config: + prologue: + commands_file: "valid_commands_file.txt" + epilogue: + always: + commands_file: "valid_commands_file.txt" + on_pass: + commands_file: "valid_commands_file.txt" + on_fail: + commands_file: "valid_commands_file.txt" +blocks: + - name: Run tests + task: + prologue: + commands_file: "valid_commands_file.txt" + epilogue: + always: + commands_file: "valid_commands_file.txt" + on_pass: + commands_file: "valid_commands_file.txt" + on_fail: + commands_file: "valid_commands_file.txt" + jobs: + - name: Run tests + commands_file: "valid_commands_file.txt" +after_pipeline: + task: + prologue: + commands_file: "valid_commands_file.txt" + epilogue: + always: + commands_file: "valid_commands_file.txt" + on_pass: + commands_file: "valid_commands_file.txt" + on_fail: + commands_file: "valid_commands_file.txt" + jobs: + - name: "Notify on Slack" + commands_file: "valid_commands_file.txt" +} + +commands_file = %{ +echo 1 +echo 12 +echo 123 +} + +origin = TestRepoForChangeIn.setup() + +origin.add_file('.semaphore/semaphore.yml', pipeline) +origin.add_file('.semaphore/valid_commands_file.txt', commands_file) +origin.commit!("Bootstrap") + +repo = origin.clone_local_copy(branch: "master") +repo.run("#{spc} compile --input .semaphore/semaphore.yml --output /tmp/output.yml --logs /tmp/logs.yml") + +assert_eq(YAML.load_file('/tmp/output.yml'), YAML.load(%{ + version: v1.0 + name: "Tests" + agent: + machine: + type: "e1-standard-2" + os_image: "ubuntu2004" + global_job_config: + prologue: + commands: + - echo 1 + - echo 12 + - echo 123 + epilogue: + always: + commands: + - echo 1 + - echo 12 + - echo 123 + on_pass: + commands: + - echo 1 + - echo 12 + - echo 123 + on_fail: + commands: + - echo 1 + - echo 12 + - echo 123 + blocks: + - name: Run tests + task: + prologue: + commands: + - echo 1 + - echo 12 + - echo 123 + epilogue: + always: + commands: + - echo 1 + - echo 12 + - echo 123 + on_pass: + commands: + - echo 1 + - echo 12 + - echo 123 + on_fail: + commands: + - echo 1 + - echo 12 + - echo 123 + jobs: + - name: Run tests + commands: + - echo 1 + - echo 12 + - echo 123 + after_pipeline: + task: + prologue: + commands: + - echo 1 + - echo 12 + - echo 123 + epilogue: + always: + commands: + - echo 1 + - echo 12 + - echo 123 + on_pass: + commands: + - echo 1 + - echo 12 + - echo 123 + on_fail: + commands: + - echo 1 + - echo 12 + - echo 123 + jobs: + - name: "Notify on Slack" + commands: + - echo 1 + - echo 12 + - echo 123 +})) + diff --git a/test/fixtures/all_commands_file_locations.yml b/test/fixtures/all_commands_file_locations.yml new file mode 100644 index 0000000..637e7ea --- /dev/null +++ b/test/fixtures/all_commands_file_locations.yml @@ -0,0 +1,50 @@ +version: v1.0 + +name: "Tests" + +agent: + machine: + type: "e1-standard-2" + os_image: "ubuntu2004" + +global_job_config: + prologue: + commands_file: "valid_commands_file.txt" + epilogue: + always: + commands_file: "valid_commands_file.txt" + on_pass: + commands_file: "valid_commands_file.txt" + on_fail: + commands_file: "valid_commands_file.txt" + +blocks: + - name: Run tests + task: + prologue: + commands_file: "valid_commands_file.txt" + epilogue: + always: + commands_file: "valid_commands_file.txt" + on_pass: + commands_file: "valid_commands_file.txt" + on_fail: + commands_file: "valid_commands_file.txt" + jobs: + - name: Run tests + commands_file: "valid_commands_file.txt" + +after_pipeline: + task: + prologue: + commands_file: "valid_commands_file.txt" + epilogue: + always: + commands_file: "valid_commands_file.txt" + on_pass: + commands_file: "valid_commands_file.txt" + on_fail: + commands_file: "valid_commands_file.txt" + jobs: + - name: "Notify on Slack" + commands_file: "valid_commands_file.txt" diff --git a/test/fixtures/empty_file.txt b/test/fixtures/empty_file.txt new file mode 100644 index 0000000..e69de29 diff --git a/test/fixtures/valid_commands_file.txt b/test/fixtures/valid_commands_file.txt new file mode 100644 index 0000000..5ee5ccf --- /dev/null +++ b/test/fixtures/valid_commands_file.txt @@ -0,0 +1,3 @@ +echo 1 +echo 12 +echo 123 \ No newline at end of file