From c44eafc8e26aa7a54f86c2a9cd2d483f176d4c00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Torres=20Cogollo?= Date: Tue, 25 Jul 2023 01:44:24 +0200 Subject: [PATCH] Refactor and improved asciinema --- .gitignore | 4 + README.md | 2 +- assets/asciinema.sh | 24 ++++ cmd/cascade.go | 10 +- cmd/dependencies.go | 17 +++ cmd/root.go | 21 ++- cmd/terraform.go | 129 +++++++----------- internal/controller/cascade_controller.go | 26 ++++ internal/controller/terraform_controller.go | 45 ++++++ internal/orchestration/dependency_resolve.go | 85 ------------ internal/orchestration/project_executor.go | 61 +++++++++ .../orchestration/project_order_resolver.go | 80 +++++++++++ internal/project/project.go | 13 ++ internal/shared/utils/exec.go | 24 ++++ internal/terraform/exec.go | 3 +- .../usecases/run_raw_terraform_usecase.go | 15 ++ .../run_recursive_terraform_usecase.go | 25 ++++ 17 files changed, 412 insertions(+), 172 deletions(-) create mode 100644 assets/asciinema.sh create mode 100644 cmd/dependencies.go create mode 100644 internal/controller/cascade_controller.go create mode 100644 internal/controller/terraform_controller.go delete mode 100644 internal/orchestration/dependency_resolve.go create mode 100644 internal/orchestration/project_executor.go create mode 100644 internal/orchestration/project_order_resolver.go create mode 100644 internal/project/project.go create mode 100644 internal/shared/utils/exec.go create mode 100644 internal/usecases/run_raw_terraform_usecase.go create mode 100644 internal/usecases/run_recursive_terraform_usecase.go diff --git a/.gitignore b/.gitignore index 3e64fab..ccdd312 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,7 @@ terraform.rc # Samples /samples/**/tmp/* /samples/**/.terraform.lock.hcl + +# Asciinema +/.asciinema/ +/assets/asciinema.sh.cast diff --git a/README.md b/README.md index 1c16f2d..0ca222d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Demo -[![asciicast](https://asciinema.org/a/598560.svg)](https://asciinema.org/a/598560) +[![asciicast](https://asciinema.org/a/JPYlivXxoZvB5PvjNvOxVQb7O.svg)](https://asciinema.org/a/JPYlivXxoZvB5PvjNvOxVQb7O) # Overview **Terraform Cascade** is a terraform-like tool that allows you to manage multiple terraform projects. diff --git a/assets/asciinema.sh b/assets/asciinema.sh new file mode 100644 index 0000000..ae19508 --- /dev/null +++ b/assets/asciinema.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash -e + +# This is the code structure (from a basic example) +cd samples/basic +tree -P backend.tf . + +# Each depth level has a dependency on the previous one. For example: +cat dev/base/data.tf +cat dev/base/vpc.tf + +# In this case it simulates that the VPC needs first the account where it has to be created +cat base/accounts.tf + +# Let's run terraform init through cascade recursively +go run ../.. init --cascade-recursive + +# Now, let's run apply +go run ../.. apply --cascade-recursive --auto-approve + +# Let's see the created "infra" +tree -a -I .terraform /tmp/cascade + +# And the state files +tree -a /tmp/cascade/.terraform/ diff --git a/cmd/cascade.go b/cmd/cascade.go index 109da38..e1d3a9f 100644 --- a/cmd/cascade.go +++ b/cmd/cascade.go @@ -20,6 +20,10 @@ import ( "github.com/spf13/cobra" ) +//var cascadeController = controller.NewCascadeController( +// *runRecursiveTerraformUseCase, +//) + // cascadeCmd represents the cascade command var cascadeCmd = &cobra.Command{ Use: "cascade", @@ -29,7 +33,11 @@ Specific commands for cascade orchestration. WORK IN PROGRESS `, - //Run: func(cmd *cobra.Command, args []string) {}, + //Run: func(cmd *cobra.Command, args []string) { + // utils.ExitWithErr( + // cascadeController.HandleCascade(), + // ) + //}, } func init() { diff --git a/cmd/dependencies.go b/cmd/dependencies.go new file mode 100644 index 0000000..5da97da --- /dev/null +++ b/cmd/dependencies.go @@ -0,0 +1,17 @@ +package cmd + +import ( + "github.com/atorrescogollo/terraform-cascade/internal/orchestration" + "github.com/atorrescogollo/terraform-cascade/internal/usecases" +) + +// Orchestration +var projectExecutor = orchestration.NewTerraformProjectExecutorWithOS() +var projectOrderResolver = orchestration.NewProjectOrderResolver() + +// Use cases +var runRawTerraformUseCase = usecases.NewRunRawTerraformUseCase() +var runRecursiveTerraformUseCase = usecases.NewRunRecursiveTerraformUseCase( + *projectExecutor, + *projectOrderResolver, +) diff --git a/cmd/root.go b/cmd/root.go index a3c34b9..46e5685 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,13 +27,22 @@ import ( // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "terraform-cascade", - Short: "A brief description of your application", - Long: `A longer description that spans multiple lines and likely contains -examples and usage of using your application. For example: + Short: "An opinionated terraform project orchestrator", + Long: ` +Terraform Cascade is a terraform-like tool that allows you to manage multiple terraform projects. -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, +It's made to be fully compatible with terraform, so you can use it as a drop-in replacement. However, it requires the terraform binary to be available in the PATH. + +== Design == +It works with a very opinionated design: + +* Every project is inside a deep directory structure. +* To define a project, you only need to place a backend.tf file in that directory. + * In each layer, will be executed in the following order: + * Current directory (only when it has a backend.tf file) + * Whole base directory (with its layer) + * Other directories (with its layer) +`, // Uncomment the following line if your bare application // has an action associated with it: //Run: func(cmd *cobra.Command, args []string) {}, diff --git a/cmd/terraform.go b/cmd/terraform.go index d589bb2..01e7dd7 100644 --- a/cmd/terraform.go +++ b/cmd/terraform.go @@ -19,15 +19,18 @@ package cmd import ( "fmt" "os" - "os/exec" "strings" - "github.com/atorrescogollo/terraform-cascade/internal/orchestration" + "github.com/atorrescogollo/terraform-cascade/internal/controller" "github.com/atorrescogollo/terraform-cascade/internal/shared/utils" - "github.com/atorrescogollo/terraform-cascade/internal/terraform" "github.com/spf13/cobra" ) +var terraformController = controller.NewTerraformController( + *runRawTerraformUseCase, + *runRecursiveTerraformUseCase, +) + // terraformCmd represents the terraform command var terraformCmd = &cobra.Command{ Use: "terraform", @@ -35,54 +38,12 @@ var terraformCmd = &cobra.Command{ Long: `Terraform commands through the cascade cli.`, Run: func(cmd *cobra.Command, args []string) { cmd.Flags().Parse(args) - help := false - if len(args) == 0 || - (len(args) == 1 && (args[0] == "-h" || - args[0] == "--help" || - args[0] == "-help" || - args[0] == "help")) { - help = true - } - terraformArgs := utils.ExtractUnknownArgs(cmd.Flags(), args) - if !help { - /* - * Help command is actually there since cobra adds it behind the scenes. - * We add it when it's not the first argument so that it executes - * help as a terraform command instead of the cascade help command. - * - * For example: - * - * cascade terraform --help # Executes cascade help + terraform help - * - * cascade terraform plan --help # Executes terraform-plan help - * - */ - trailingHelp, _ := cmd.Flags().GetBool("help") - if trailingHelp { - terraformArgs = append(terraformArgs, "--help") - } - } - showUsage := false - for _, tfArg := range terraformArgs { - if strings.HasPrefix(tfArg, "--cascade-") { - // This is not a terraform flag. We need to handle - // it here since UnknownFlags is whitelisted - fmt.Println("Error: unknown flag:", tfArg) - showUsage = true - continue - } - } - if help || showUsage { - cmd.Usage() - return - } - - exitErr := Run(cmd, terraformArgs) - if exitErr != nil && exitErr.ExitCode() != 0 { - fmt.Println("Error: Terraform exited with code", exitErr.ExitCode()) - os.Exit(exitErr.ExitCode()) - } + terraformArgs := retrieveTerraformArgsOrExit(cmd, args) + recursive, _ := cmd.Flags().GetBool("cascade-recursive") + utils.ExitWithErr( + terraformController.Handle(recursive, terraformArgs), + ) }, } @@ -112,41 +73,53 @@ Global Flags: Terraform Args: `) - defaultUsageFunc := terraformCmd.UsageFunc() - terraformCmd.SetUsageFunc(func(cmd *cobra.Command) error { - terraformUsage := terraform.TerraformUsage() - err := defaultUsageFunc(cmd) - if err != nil { - return err - } - - fmt.Println() - fmt.Println(terraformUsage) - return nil - }) + terraformCmd.SetUsageFunc(terraformController.Usage) terraformCmd.Flags().Bool("cascade-recursive", false, "Execute terraform projects recursively in order") } -func Run(cmd *cobra.Command, args []string) *exec.ExitError { - cwd, err := os.Getwd() - if err != nil { - fmt.Println(err) - os.Exit(1) +func retrieveTerraformArgsOrExit(cmd *cobra.Command, args []string) []string { + help := false + if len(args) == 0 || + (len(args) == 1 && (args[0] == "-h" || + args[0] == "--help" || + args[0] == "-help" || + args[0] == "help")) { + help = true } - recursive, _ := cmd.Flags().GetBool("cascade-recursive") - if recursive { - orchestrateDir, err := orchestration.OrchestrateProjectDirectory(cwd) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - return orchestration.RunTerraformRecursively(*orchestrateDir, args, 0) - } else { + terraformArgs := utils.ExtractUnknownArgs(cmd.Flags(), args) + if !help { /* - * Simply run terraform in the current directory + * Help command is actually there since cobra adds it behind the scenes. + * We add it when it's not the first argument so that it executes + * help as a terraform command instead of the cascade help command. + * + * For example: + * + * cascade terraform --help # Executes cascade help + terraform help + * + * cascade terraform plan --help # Executes terraform-plan help + * */ - return terraform.TerraformExecWithOS(cwd, args) + trailingHelp, _ := cmd.Flags().GetBool("help") + if trailingHelp { + terraformArgs = append(terraformArgs, "--help") + } + } + showUsage := false + for _, tfArg := range terraformArgs { + if strings.HasPrefix(tfArg, "--cascade-") { + // This is not a terraform flag. We need to handle + // it here since UnknownFlags is whitelisted + fmt.Println("Error: unknown flag:", tfArg) + showUsage = true + continue + } + } + if help || showUsage { + cmd.Usage() + os.Exit(1) } + return terraformArgs } diff --git a/internal/controller/cascade_controller.go b/internal/controller/cascade_controller.go new file mode 100644 index 0000000..0a7f6cf --- /dev/null +++ b/internal/controller/cascade_controller.go @@ -0,0 +1,26 @@ +package controller + +import ( + "fmt" + + "github.com/atorrescogollo/terraform-cascade/internal/usecases" +) + +type CascadeController struct { + RunRecursiveTerraformUseCase usecases.RunRecursiveTerraformUseCase +} + +func NewCascadeController(runRecursiveTerraformUseCase usecases.RunRecursiveTerraformUseCase) *CascadeController { + return &CascadeController{ + runRecursiveTerraformUseCase, + } +} + +func (c CascadeController) HandleCascade() error { + // TODO: Implement cascade logic + //cwd, _ := os.Getwd() + //tfargs := []string{"init"} + //chdir := cwd + "/samples/basic" + //return c.RunRecursiveTerraformUseCase.Execute(chdir, tfargs) + return fmt.Errorf("not implemented") +} diff --git a/internal/controller/terraform_controller.go b/internal/controller/terraform_controller.go new file mode 100644 index 0000000..340408f --- /dev/null +++ b/internal/controller/terraform_controller.go @@ -0,0 +1,45 @@ +package controller + +import ( + "fmt" + "os" + + "github.com/atorrescogollo/terraform-cascade/internal/terraform" + "github.com/atorrescogollo/terraform-cascade/internal/usecases" + "github.com/spf13/cobra" +) + +type TerraformController struct { + RunRawTerraformUseCase usecases.RunRawTerraformUseCase + RunRecursiveTerraformUseCase usecases.RunRecursiveTerraformUseCase +} + +func NewTerraformController(runRawTerraformUseCase usecases.RunRawTerraformUseCase, runRecursiveTerraformUseCase usecases.RunRecursiveTerraformUseCase) *TerraformController { + return &TerraformController{ + RunRawTerraformUseCase: runRawTerraformUseCase, + RunRecursiveTerraformUseCase: runRecursiveTerraformUseCase, + } +} + +func (c TerraformController) Handle(recursive bool, tfargs []string) error { + cwd, _ := os.Getwd() + if !recursive { + /* + * Simply run terraform in the current directory + */ + return c.RunRawTerraformUseCase.Execute(cwd, tfargs) + } + return c.RunRecursiveTerraformUseCase.Execute(cwd, tfargs) +} + +func (c TerraformController) Usage(cmd *cobra.Command) error { + terraformUsage := terraform.TerraformUsage() + err := cmd.UsageFunc()(cmd) + if err != nil { + return err + } + + fmt.Println() + fmt.Println(terraformUsage) + return nil +} diff --git a/internal/orchestration/dependency_resolve.go b/internal/orchestration/dependency_resolve.go deleted file mode 100644 index f223afa..0000000 --- a/internal/orchestration/dependency_resolve.go +++ /dev/null @@ -1,85 +0,0 @@ -package orchestration - -import ( - "fmt" - "os" - "os/exec" - "strings" - - "github.com/atorrescogollo/terraform-cascade/internal/terraform" -) - -type OrchestrateDir struct { - IsProject bool `json:"is_project"` - ProjectPath string `json:"project_path"` - Base []OrchestrateDir `json:"base"` - Others []OrchestrateDir `json:"others"` -} - -func NewOrchestrateDir(projectPath string) *OrchestrateDir { - return &OrchestrateDir{ - IsProject: false, - ProjectPath: projectPath, - Base: make([]OrchestrateDir, 0), - Others: make([]OrchestrateDir, 0), - } -} - -func OrchestrateProjectDirectory(baseDir string) (*OrchestrateDir, error) { - entries, err := os.ReadDir(baseDir) - if err != nil { - return nil, err - } - - result := NewOrchestrateDir(baseDir) - - for _, entry := range entries { - if strings.HasPrefix(entry.Name(), ".") { - continue - } - if entry.IsDir() { - orchestrateDir, err := OrchestrateProjectDirectory(baseDir + "/" + entry.Name()) - if err != nil { - return nil, err - } - if entry.Name() == "base" { - result.Base = append(result.Base, *orchestrateDir) - } else { - result.Others = append(result.Others, *orchestrateDir) - } - } else if entry.Name() == "backend.tf" { - result.IsProject = true - } - } - return result, nil -} - -func RunTerraformRecursively(orchestrateDir OrchestrateDir, args []string, recursiveLevel int) *exec.ExitError { - if orchestrateDir.IsProject { - //utils.WaitForConfirmation("Run terraform in "+orchestrateDir.ProjectPath+"? (y/n)") - fmt.Println(` - - -=============== Running terraform in ` + orchestrateDir.ProjectPath + ` =============== - - `) - exitErr := terraform.TerraformExecWithOS(orchestrateDir.ProjectPath, args) - if exitErr != nil && exitErr.ExitCode() != 0 { - fmt.Println("Error: Terraform exited with code", exitErr.ExitCode()) - return exitErr - } - } - for _, base := range orchestrateDir.Base { - exitErr := RunTerraformRecursively(base, args, recursiveLevel+1) - if exitErr != nil && exitErr.ExitCode() != 0 { - return exitErr - } - } - for _, other := range orchestrateDir.Others { - exitErr := RunTerraformRecursively(other, args, recursiveLevel+1) - if exitErr != nil && exitErr.ExitCode() != 0 { - return exitErr - } - } - return nil -} diff --git a/internal/orchestration/project_executor.go b/internal/orchestration/project_executor.go new file mode 100644 index 0000000..b858cf5 --- /dev/null +++ b/internal/orchestration/project_executor.go @@ -0,0 +1,61 @@ +package orchestration + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + "github.com/atorrescogollo/terraform-cascade/internal/project" + "github.com/atorrescogollo/terraform-cascade/internal/terraform" +) + +type terraformExecutorFn = func(execDir string, args []string, stdout io.Writer, sterr io.Writer, stdin io.Reader) *exec.ExitError + +type ProjectExecutor struct { + terraformExecFn terraformExecutorFn + stdout io.Writer + stderr io.Writer + stdin io.Reader +} + +func NewTerraformProjectExecutorWithOS() *ProjectExecutor { + return NewProjectExecutor(terraform.TerraformExec, os.Stdout, os.Stderr, os.Stdin) +} + +func NewTerraformProjectExecutor(stdout io.Writer, stderr io.Writer, stdin io.Reader) *ProjectExecutor { + return NewProjectExecutor(terraform.TerraformExec, stdout, stderr, stdin) +} + +func NewProjectExecutor(terraformExecFn terraformExecutorFn, stdout io.Writer, stderr io.Writer, stdin io.Reader) *ProjectExecutor { + return &ProjectExecutor{ + terraformExecFn, + stdout, + stderr, + stdin, + } +} + +func (p ProjectExecutor) Execute(projects []project.TerraformProject, args []string) error { + for _, project := range projects { + fmt.Println(` + + +===============[ Running terraform in ` + project.RelativePath + ` ]=============== + + `) + exitErr := p.terraformExecFn( + filepath.Join(project.BaseDir, project.RelativePath), + args, + p.stdout, + p.stderr, + p.stdin, + ) + if exitErr != nil && exitErr.ExitCode() != 0 { + fmt.Println("Error running terraform in", project.BaseDir, ":", exitErr) + return exitErr + } + } + return nil +} diff --git a/internal/orchestration/project_order_resolver.go b/internal/orchestration/project_order_resolver.go new file mode 100644 index 0000000..e1ef6ed --- /dev/null +++ b/internal/orchestration/project_order_resolver.go @@ -0,0 +1,80 @@ +package orchestration + +import ( + "os" + "path/filepath" + "strings" + + "github.com/atorrescogollo/terraform-cascade/internal/project" +) + +type ProjectOrderResolver struct { +} + +func NewProjectOrderResolver() *ProjectOrderResolver { + return &ProjectOrderResolver{} +} + +func (r ProjectOrderResolver) Resolve(cwd string) ([]project.TerraformProject, error) { + resolution, err := r.resolveDir(cwd, ".") + if err != nil { + return nil, err + } + /* + * Flatten the map so that it follows the following order: + * - current directory + * - base projects + * - other projects + */ + result := make([]project.TerraformProject, 0) + for i := 1; i <= len(resolution); i++ { + bases := make([]project.TerraformProject, 0) + other := make([]project.TerraformProject, 0) + for _, p := range resolution[i] { + if strings.HasSuffix("/"+p.RelativePath, "/base") { + bases = append(bases, p) + } else { + other = append(other, p) + } + } + result = append(result, bases...) + result = append(result, other...) + } + return result, nil +} + +func (r ProjectOrderResolver) resolveDir(workDir string, dir string) (map[int][]project.TerraformProject, error) { + files, err := os.ReadDir(workDir + "/" + dir) + if err != nil { + return nil, err + } + + // The key is the depth of the directory + result := make(map[int][]project.TerraformProject, 0) + + for _, file := range files { + if strings.HasPrefix(file.Name(), ".") { + continue + } + if file.IsDir() { + partial, err := r.resolveDir( + workDir, + filepath.Join(dir, file.Name()), + ) + if err != nil { + return nil, err + } + for k, v := range partial { + // Recursion happened, so the real depth is k+1 + result[k+1] = append(result[k+1], v...) + } + } else if file.Name() == "backend.tf" { + result[0] = append(result[0], project.NewTerraformProject( + workDir, + dir, + )) + } + } + + return result, nil +} diff --git a/internal/project/project.go b/internal/project/project.go new file mode 100644 index 0000000..91fba2c --- /dev/null +++ b/internal/project/project.go @@ -0,0 +1,13 @@ +package project + +type TerraformProject struct { + BaseDir string + RelativePath string +} + +func NewTerraformProject(baseDir string, relativePath string) TerraformProject { + return TerraformProject{ + BaseDir: baseDir, + RelativePath: relativePath, + } +} diff --git a/internal/shared/utils/exec.go b/internal/shared/utils/exec.go new file mode 100644 index 0000000..7be9aed --- /dev/null +++ b/internal/shared/utils/exec.go @@ -0,0 +1,24 @@ +package utils + +import ( + "fmt" + "os" + "os/exec" +) + +func ExitWithErr(err error) { + var exitErr *exec.ExitError + if err != nil { + errCast, ok := err.(*exec.ExitError) + if !ok { + fmt.Println("Unexpected error:", err) + os.Exit(1) + } + exitErr = errCast + + } + if exitErr != nil && exitErr.ExitCode() != 0 { + os.Exit(exitErr.ExitCode()) + } + os.Exit(0) +} diff --git a/internal/terraform/exec.go b/internal/terraform/exec.go index 1754cfe..ac239d9 100644 --- a/internal/terraform/exec.go +++ b/internal/terraform/exec.go @@ -21,7 +21,8 @@ func TerraformExec(execDir string, args []string, stdout io.Writer, sterr io.Wri cmd.Stderr = sterr cmd.Stdin = stdin - fmt.Println("Running terraform:", cmd.String(), "[dir=", execDir, "]") + fmt.Printf("%s\nRunning terraform %s\n", execDir, args) + if err := cmd.Run(); err != nil { if exiterr, ok := err.(*exec.ExitError); ok { return exiterr diff --git a/internal/usecases/run_raw_terraform_usecase.go b/internal/usecases/run_raw_terraform_usecase.go new file mode 100644 index 0000000..7e52591 --- /dev/null +++ b/internal/usecases/run_raw_terraform_usecase.go @@ -0,0 +1,15 @@ +package usecases + +import ( + "github.com/atorrescogollo/terraform-cascade/internal/terraform" +) + +type RunRawTerraformUseCase struct{} + +func NewRunRawTerraformUseCase() *RunRawTerraformUseCase { + return &RunRawTerraformUseCase{} +} + +func (u RunRawTerraformUseCase) Execute(execDir string, terraformArgs []string) error { + return terraform.TerraformExecWithOS(execDir, terraformArgs) +} diff --git a/internal/usecases/run_recursive_terraform_usecase.go b/internal/usecases/run_recursive_terraform_usecase.go new file mode 100644 index 0000000..06b6838 --- /dev/null +++ b/internal/usecases/run_recursive_terraform_usecase.go @@ -0,0 +1,25 @@ +package usecases + +import ( + "github.com/atorrescogollo/terraform-cascade/internal/orchestration" +) + +type RunRecursiveTerraformUseCase struct { + projectExecutor orchestration.ProjectExecutor + projectObjectResolver orchestration.ProjectOrderResolver +} + +func NewRunRecursiveTerraformUseCase(projectExecutor orchestration.ProjectExecutor, projectObjectResolver orchestration.ProjectOrderResolver) *RunRecursiveTerraformUseCase { + return &RunRecursiveTerraformUseCase{ + projectExecutor, + projectObjectResolver, + } +} + +func (u RunRecursiveTerraformUseCase) Execute(execDir string, terraformArgs []string) error { + projects, err := u.projectObjectResolver.Resolve(execDir) + if err != nil { + return err + } + return u.projectExecutor.Execute(projects, terraformArgs) +}