From 2c1083272b6c063ac6da6d526c7d63b8966f7e15 Mon Sep 17 00:00:00 2001 From: ivan-kostko Date: Wed, 15 Feb 2023 15:28:09 +0300 Subject: [PATCH] Custom destinations for modules (#13) * Added destination option to configuration, so each module could be added to one or more specific destination folder --------- Co-authored-by: Rafal Przybyla Co-authored-by: Nathaniel Ritholtz --- .gitignore | 4 +- README.md | 70 ++++++++++++++++++++++++++++---- main.go | 86 +++++++++++++++++++++++++++++++++------- main_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 242 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index ab05a59..1a8e6ab 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,10 @@ # Terrafile output vendor/modules +testdata terrafile #IDEs .idea/ -dist/ \ No newline at end of file +dist/ +.DS_Store diff --git a/README.md b/README.md index 2adc5b2..e819742 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,12 @@ curl -L https://github.com/coretech/terrafile/releases/download/v{VERSION}/terra ## How to use Terrafile expects a file named `Terrafile` which will contain your terraform module dependencies in a yaml like format. -An example Terrafile: +There are two approaches that can be used for managing your modules depending on the structure of your terraform code: +1. The default approach: `Terrafile` is located directly in the directory where terraform is run +2. Centrally managed: `Terrafile` is located in "root" directory of your terraform code, managing modules in all subfolders / stacks + +### Default Approach +An example of default approach (#1) to `Terrafile` ``` tf-aws-vpc: source: "git@github.com:terraform-aws-modules/terraform-aws-vpc" @@ -34,24 +39,75 @@ tf-aws-vpc-experimental: Terrafile config file in current directory and modules exported to ./vendor/modules ```sh $ terrafile -INFO[0000] [*] Checking out v1.46.0 of git@github.com:terraform-aws-modules/terraform-aws-vpc -INFO[0000] [*] Checking out master of git@github.com:terraform-aws-modules/terraform-aws-vpc +INFO[0000] [*] Checking out v1.46.0 of git@github.com:terraform-aws-modules/terraform-aws-vpc +INFO[0000] [*] Checking out master of git@github.com:terraform-aws-modules/terraform-aws-vpc ``` Terrafile config file in custom directory ```sh $ terrafile -f config/Terrafile -INFO[0000] [*] Checking out v1.46.0 of git@github.com:terraform-aws-modules/terraform-aws-vpc -INFO[0000] [*] Checking out master of git@github.com:terraform-aws-modules/terraform-aws-vpc +INFO[0000] [*] Checking out v1.46.0 of git@github.com:terraform-aws-modules/terraform-aws-vpc +INFO[0000] [*] Checking out master of git@github.com:terraform-aws-modules/terraform-aws-vpc ``` Terraform modules exported to custom directory ```sh $ terrafile -p custom_directory -INFO[0000] [*] Checking out master of git@github.com:terraform-aws-modules/terraform-aws-vpc -INFO[0001] [*] Checking out v1.46.0 of git@github.com:terraform-aws-modules/terraform-aws-vpc +INFO[0000] [*] Checking out master of git@github.com:terraform-aws-modules/terraform-aws-vpc +INFO[0001] [*] Checking out v1.46.0 of git@github.com:terraform-aws-modules/terraform-aws-vpc +``` + +### Centrally Managed Approach +An example of using `Terrafile` in a root directory (#2): + +Let's assume the following directory structure: + +``` +. +├── iam +│   ├── main.tf +│   └── .....tf +├── networking +│   ├── main.tf +│   └── .....tf +├── onboarding +. +. +. +├── some-other-stack +└── Terrafile +``` + +In the above scenario, Terrafile is not in every single folder but in the "root" of terraform code. + +An example usage of centrally managed modules: + +``` +tf-aws-vpc: + source: "git@github.com:terraform-aws-modules/terraform-aws-vpc" + version: "v1.46.0" + destination: + - networking +tf-aws-iam: + source: "git@github.com:terraform-aws-modules/terraform-aws-iam" + version: "v5.11.1" + destination: + - iam +tf-aws-s3-bucket: + source: "git@github.com:terraform-aws-modules/terraform-aws-s3-bucket" + version: "v3.6.1" + destination: + - networking + - onboarding + - some-other-stack ``` +The `destination` of module is an array of directories (stacks) where the module should be used. +The module itself is fetched once and copied over to designated destinations. +Final destination of the module is handled in a similar way as in first approach: `$destination/$module_path/$module_key`. + +The output of the run is exactly the same in both options. + ## TODO * Break out the main logic into seperate commands (e.g. version, help, run) * Update tests to include unit tests for broken out commands diff --git a/main.go b/main.go index bfa08b4..e403d63 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,6 @@ package main import ( "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -31,12 +30,13 @@ import ( ) type module struct { - Source string `yaml:"source"` - Version string `yaml:"version"` + Source string `yaml:"source"` + Version string `yaml:"version"` + Destinations []string `yaml:"destinations"` } var opts struct { - ModulePath string `short:"p" long:"module_path" default:"./vendor/modules" description:"File path to install generated terraform modules"` + ModulePath string `short:"p" long:"module_path" default:"./vendor/modules" description:"File path to install generated terraform modules, if not overridden by 'destinations:' field"` TerrafilePath string `short:"f" long:"terrafile_file" default:"./Terrafile" description:"File path to the Terrafile file"` } @@ -53,47 +53,105 @@ func init() { log.AddHook(stdemuxerhook.New(log.StandardLogger())) } -func gitClone(repository string, version string, moduleName string) { +func gitClone(repository string, version string, moduleName string, destinationDir string) { + cleanupPath := filepath.Join(destinationDir, moduleName) + log.Printf("[*] Removing previously cloned artifacts at %s", cleanupPath) + os.RemoveAll(cleanupPath) log.Printf("[*] Checking out %s of %s \n", version, repository) cmd := exec.Command("git", "clone", "--single-branch", "--depth=1", "-b", version, repository, moduleName) - cmd.Dir = opts.ModulePath - err := cmd.Run() - if err != nil { - log.Fatalln(err) + cmd.Dir = destinationDir + if err := cmd.Run(); err != nil { + log.Fatalf("failed to clone repository %s due to error: %s", cmd.String(), err) } } func main() { + fmt.Printf("Terrafile: version %v, commit %v, built at %v \n", version, commit, date) _, err := flags.Parse(&opts) // Invalid choice if err != nil { + log.Errorf("failed to parse flags due to: %s", err) os.Exit(1) } + workDirAbsolutePath, err := os.Getwd() + if err != nil { + log.Errorf("failed to get working directory absolute path due to: %s", err) + } + // Read File - yamlFile, err := ioutil.ReadFile(opts.TerrafilePath) + yamlFile, err := os.ReadFile(opts.TerrafilePath) if err != nil { - log.Fatalln(err) + log.Fatalf("failed to read configuration in file %s due to error: %s", opts.TerrafilePath, err) } // Parse File var config map[string]module if err := yaml.Unmarshal(yamlFile, &config); err != nil { - log.Fatalln(err) + log.Fatalf("failed to parse yaml file due to error: %s", err) } // Clone modules var wg sync.WaitGroup _ = os.RemoveAll(opts.ModulePath) _ = os.MkdirAll(opts.ModulePath, os.ModePerm) + for key, mod := range config { wg.Add(1) go func(m module, key string) { defer wg.Done() - gitClone(m.Source, m.Version, key) - _ = os.RemoveAll(filepath.Join(opts.ModulePath, key, ".git")) + + // path to clone module + cloneDestination := opts.ModulePath + // list of paths to link module to. empty, unless Destinations are more than 1 location + var linkDestinations []string + + if m.Destinations != nil && len(m.Destinations) > 0 { + // set first in Destinations as location to clone to + cloneDestination = filepath.Join(m.Destinations[0], opts.ModulePath) + // the rest of Destinations are locations to link module to + linkDestinations = m.Destinations[1:] + + } + + // create folder to clone into + if err := os.MkdirAll(cloneDestination, os.ModePerm); err != nil { + log.Errorf("failed to create folder %s due to error: %s", cloneDestination, err) + + // no reason to continue as failed to create folder + return + } + + // clone repository + gitClone(m.Source, m.Version, key, cloneDestination) + + for _, d := range linkDestinations { + // the source location as folder where module was cloned and module folder name + moduleSrc := filepath.Join(workDirAbsolutePath, cloneDestination, key) + // append destination path with module path + dst := filepath.Join(d, opts.ModulePath) + + log.Infof("[*] Creating folder %s", dst) + if err := os.MkdirAll(dst, os.ModePerm); err != nil { + log.Errorf("failed to create folder %s due to error: %s", dst, err) + return + } + + dst = filepath.Join(dst, key) + + log.Infof("[*] Remove existing artifacts at %s", dst) + if err := os.RemoveAll(dst); err != nil { + log.Errorf("failed to remove location %s due to error: %s", dst, err) + return + } + + log.Infof("[*] Link %s to %s", moduleSrc, dst) + if err := os.Symlink(moduleSrc, dst); err != nil { + log.Errorf("failed to link module from %s to %s due to error: %s", moduleSrc, dst, err) + } + } }(mod, key) } diff --git a/main_test.go b/main_test.go index 1bbac55..fee7542 100644 --- a/main_test.go +++ b/main_test.go @@ -18,7 +18,6 @@ package main import ( "fmt" - "io/ioutil" "os" "path" "testing" @@ -44,17 +43,27 @@ func TestTerraformWithTerrafilePath(t *testing.T) { testcli.Run(terrafileBinaryPath, "-f", fmt.Sprint(folder, "/Terrafile")) + defer println(testcli.Stdout()) + defer println(testcli.Stderr()) + + defer func() { + assert.NoError(t, os.RemoveAll("testdata/")) + }() + if !testcli.Success() { t.Fatalf("Expected to succeed, but failed: %q with message: %q", testcli.Error(), testcli.Stderr()) } // Assert output for _, output := range []string{ - "Checking out v1.46.0 of git@github.com:terraform-aws-modules/terraform-aws-vpc", "Checking out master of git@github.com:terraform-aws-modules/terraform-aws-vpc", + "Checking out v1.46.0 of git@github.com:terraform-aws-modules/terraform-aws-vpc", + "Checking out v3.2.0 of git@github.com:terraform-aws-modules/terraform-aws-vpn-gateway", + "Checking out v3.6.1 of git@github.com:terraform-aws-modules/terraform-aws-s3-bucket", + "Checking out v5.11.1 of git@github.com:terraform-aws-modules/terraform-aws-iam", } { assert.Contains(t, testcli.Stdout(), output) } - // Assert folder exist + // Assert folder exist with default destination for _, moduleName := range []string{ "tf-aws-vpc", "tf-aws-vpc-experimental", @@ -62,7 +71,20 @@ func TestTerraformWithTerrafilePath(t *testing.T) { assert.DirExists(t, path.Join(workingDirectory, "vendor/modules", moduleName)) } - // Assert files exist + // Assert folder exist with non-default destinations + for _, moduleName := range []string{ + "testdata/networking/vendor/modules/tf-aws-vpn-gateway/", + "testdata/networking/vendor/modules/tf-aws-s3-bucket/", + "testdata/iam/vendor/modules/tf-aws-iam/", + + // Symlinks are not Dirs. But contents will be tested later on + // "testdata/onboarding/vendor/modules/tf-aws-s3-bucket/", + // "testdata/some-other-stack/vendor/modules/tf-aws-s3-bucket/", + } { + assert.DirExists(t, path.Join(workingDirectory, moduleName)) + } + + // Assert files exist with default destination for _, moduleName := range []string{ "tf-aws-vpc/main.tf", "tf-aws-vpc-experimental/main.tf", @@ -70,6 +92,20 @@ func TestTerraformWithTerrafilePath(t *testing.T) { assert.FileExists(t, path.Join(workingDirectory, "vendor/modules", moduleName)) } + // Assert files exist with non-default destinations + for _, moduleName := range []string{ + "testdata/networking/vendor/modules/tf-aws-vpn-gateway/main.tf", + "testdata/networking/vendor/modules/tf-aws-s3-bucket/main.tf", + + // terraform-aws-modules/terraform-aws-iam doesn't have main.tf, as it represents set of modules + // However, some terraform-aws-modules/terraform-aws-iam/modules have, e.g.: + "testdata/iam/vendor/modules/tf-aws-iam/modules/iam-account/main.tf", + "testdata/onboarding/vendor/modules/tf-aws-s3-bucket/main.tf", + "testdata/some-other-stack/vendor/modules/tf-aws-s3-bucket/main.tf", + } { + assert.FileExists(t, path.Join(workingDirectory, moduleName)) + } + // Assert checked out correct version for moduleName, cloneOptions := range map[string]map[string]string{ "tf-aws-vpc": map[string]string{ @@ -91,10 +127,55 @@ func TestTerraformWithTerrafilePath(t *testing.T) { t.Fatalf("File difference found for %q, with failure: %q with message: %q", moduleName, testcli.Error(), testcli.Stderr()) } } + + // Assert checked out correct version to non-default destinations + for dst, checkout := range map[string]map[string]map[string]string{ + "testdata/networking/vendor/modules": { + "tf-aws-s3-bucket": { + "repository": "git@github.com:terraform-aws-modules/terraform-aws-s3-bucket", + "version": "v3.6.1", + }, + "tf-aws-vpn-gateway": { + "repository": "git@github.com:terraform-aws-modules/terraform-aws-vpn-gateway", + "version": "v3.2.0", + }, + }, + "testdata/iam/vendor/modules": { + "tf-aws-iam": { + "repository": "git@github.com:terraform-aws-modules/terraform-aws-iam", + "version": "v5.11.1", + }, + }, + "testdata/onboarding/vendor/modules": { + "tf-aws-s3-bucket": { + "repository": "git@github.com:terraform-aws-modules/terraform-aws-s3-bucket", + "version": "v3.6.1", + }, + }, + "testdata/some-other-stack/vendor/modules": { + "tf-aws-s3-bucket": { + "repository": "git@github.com:terraform-aws-modules/terraform-aws-s3-bucket", + "version": "v3.6.1", + }, + }, + } { + for moduleName, cloneOptions := range checkout { + testModuleLocation := path.Join(workingDirectory, dst, moduleName+"__test") + _ = os.RemoveAll(testModuleLocation) + testcli.Run("git", "clone", "-b", cloneOptions["version"], cloneOptions["repository"], testModuleLocation) + if !testcli.Success() { + t.Fatalf("Expected to succeed, but failed: %q with message: %q", testcli.Error(), testcli.Stderr()) + } + testcli.Run("diff", "--exclude=.git", "-r", path.Join(workingDirectory, dst, moduleName), testModuleLocation) + if !testcli.Success() { + t.Fatalf("File difference found for %q, with failure: %q with message: %q", moduleName, testcli.Error(), testcli.Stderr()) + } + } + } } func setup(t *testing.T) (current string, back func()) { - folder, err := ioutil.TempDir("", "") + folder, err := os.MkdirTemp("", "") assert.NoError(t, err) createTerrafile(t, folder) return folder, func() { @@ -103,7 +184,7 @@ func setup(t *testing.T) (current string, back func()) { } func createFile(t *testing.T, filename string, contents string) { - assert.NoError(t, ioutil.WriteFile(filename, []byte(contents), 0644)) + assert.NoError(t, os.WriteFile(filename, []byte(contents), 0644)) } func createTerrafile(t *testing.T, folder string) { @@ -113,6 +194,23 @@ func createTerrafile(t *testing.T, folder string) { tf-aws-vpc-experimental: source: "git@github.com:terraform-aws-modules/terraform-aws-vpc" version: "master" +tf-aws-vpn-gateway: + source: "git@github.com:terraform-aws-modules/terraform-aws-vpn-gateway" + version: "v3.2.0" + destinations: + - testdata/networking +tf-aws-iam: + source: "git@github.com:terraform-aws-modules/terraform-aws-iam" + version: "v5.11.1" + destinations: + - testdata/iam +tf-aws-s3-bucket: + source: "git@github.com:terraform-aws-modules/terraform-aws-s3-bucket" + version: "v3.6.1" + destinations: + - testdata/networking + - testdata/onboarding + - testdata/some-other-stack ` createFile(t, path.Join(folder, "Terrafile"), yaml) }