diff --git a/cli/cli_app.go b/cli/cli_app.go index d2d7f25d9..dc482c8c8 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -13,7 +13,6 @@ import ( "github.com/urfave/cli" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/spin" - "io/ioutil" ) const OPT_TERRAGRUNT_CONFIG = "terragrunt-config" @@ -128,7 +127,7 @@ func runTerragrunt(terragruntOptions *options.TerragruntOptions) error { } if sourceUrl, hasSourceUrl := getTerraformSourceUrl(terragruntOptions, conf); hasSourceUrl { - if err := checkoutTerraformSource(sourceUrl, terragruntOptions); err != nil { + if err := downloadTerraformSource(sourceUrl, terragruntOptions); err != nil { return err } } @@ -151,19 +150,6 @@ func runTerragrunt(terragruntOptions *options.TerragruntOptions) error { return runTerraformCommandWithLock(conf.Lock, terragruntOptions) } -// There are two ways a user can tell Terragrunt that it needs to download Terraform configurations from a specific -// URL: via a command-line option or via an entry in the .terragrunt config file. If the user used one of these, this -// method returns the source URL and the boolean true; if not, this method returns an empty string and false. -func getTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) (string, bool) { - if terragruntOptions.Source != "" { - return terragruntOptions.Source, true - } else if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.Source != "" { - return terragruntConfig.Terraform.Source, true - } else { - return "", false - } -} - // Returns true if the command the user wants to execute is supposed to affect multiple Terraform modules, such as the // spin-up or tear-down command. func isMultiModuleCommand(command string) bool { @@ -233,39 +219,6 @@ func configureRemoteState(remoteState *remote.RemoteState, terragruntOptions *op return nil } -// 1. Check out the given source URL, which should use Terraform's module source syntax, into a temporary folder -// 2. Copy the contents of terragruntOptions.WorkingDir into the temporary folder. -// 3. Set terragruntOptions.WorkingDir to the temporary folder. -func checkoutTerraformSource(source string, terragruntOptions *options.TerragruntOptions) error { - tmpFolder, err := ioutil.TempDir("", "terragrunt-tmp-checkout") - if err != nil { - return errors.WithStackTrace(err) - } - - terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", source, tmpFolder) - if err := terraformInit(source, tmpFolder, terragruntOptions); err != nil { - return err - } - - terragruntOptions.Logger.Printf("Copying files from %s into %s", terragruntOptions.WorkingDir, tmpFolder) - if err := util.CopyFolderContents(terragruntOptions.WorkingDir, tmpFolder); err != nil { - return err - } - - terragruntOptions.Logger.Printf("Setting working directory to %s", tmpFolder) - terragruntOptions.WorkingDir = tmpFolder - - return nil -} - -// Download the code from source into dest using the terraform init command -func terraformInit(source string, dest string, terragruntOptions *options.TerragruntOptions) error { - terragruntInitOptions := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath) - terragruntInitOptions.TerraformCliArgs = []string{"init", source, dest} - - return runTerraformCommand(terragruntInitOptions) -} - // Run the given Terraform command with the given lock (if the command requires locking) func runTerraformCommandWithLock(lock locks.Lock, terragruntOptions *options.TerragruntOptions) error { switch firstArg(terragruntOptions.TerraformCliArgs) { diff --git a/cli/download_source.go b/cli/download_source.go new file mode 100644 index 000000000..fcb1793b2 --- /dev/null +++ b/cli/download_source.go @@ -0,0 +1,260 @@ +package cli + +import ( + "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/util" + "github.com/gruntwork-io/terragrunt/config" + "os" + "github.com/gruntwork-io/terragrunt/errors" + "path/filepath" + "github.com/hashicorp/go-getter" + urlhelper "github.com/hashicorp/go-getter/helper/url" + "io/ioutil" + "net/url" + "fmt" +) + +// This struct represents information about Terraform source code that needs to be downloaded +type TerraformSource struct { + // A canonical version of RawSource, in URL format + CanonicalSourceURL *url.URL + + // The folder where we should download the source to + DownloadDir string + + // The path to a file in DownloadDir that stores the version number of the code + VersionFile string +} + +func (src *TerraformSource) String() string { + return fmt.Sprintf("TerraformSource{CanonicalSourceURL = %v, DownloadDir = %v, VersionFile = %v}", src.CanonicalSourceURL, src.DownloadDir, src.VersionFile) +} + +// 1. Download the given source URL, which should use Terraform's module source syntax, into a temporary folder +// 2. Copy the contents of terragruntOptions.WorkingDir into the temporary folder. +// 3. Set terragruntOptions.WorkingDir to the temporary folder. +// +// See the processTerraformSource method for how we determine the temporary folder so we can reuse it across multiple +// runs of Terragrunt to avoid downloading everything from scratch every time. +func downloadTerraformSource(source string, terragruntOptions *options.TerragruntOptions) error { + terraformSource, err := processTerraformSource(source, terragruntOptions) + if err != nil { + return err + } + + if err := downloadTerraformSourceIfNecessary(terraformSource, terragruntOptions); err != nil { + return err + } + + terragruntOptions.Logger.Printf("Copying files from %s into %s", terragruntOptions.WorkingDir, terraformSource.DownloadDir) + if err := util.CopyFolderContents(terragruntOptions.WorkingDir, terraformSource.DownloadDir); err != nil { + return err + } + + terragruntOptions.Logger.Printf("Setting working directory to %s", terraformSource.DownloadDir) + terragruntOptions.WorkingDir = terraformSource.DownloadDir + + return nil +} + +// Download the specified TerraformSource if the latest code hasn't already been downloaded. +func downloadTerraformSourceIfNecessary(terraformSource *TerraformSource, terragruntOptions *options.TerragruntOptions) error { + alreadyLatest, err := alreadyHaveLatestCode(terraformSource) + if err != nil { + return err + } + + if alreadyLatest { + terragruntOptions.Logger.Printf("Terraform files in %s are up to date. Will not download again.", terraformSource.DownloadDir) + return nil + } + + if err := cleanupTerraformFiles(terraformSource.DownloadDir, terragruntOptions); err != nil { + return err + } + + if err := terraformInit(terraformSource, terragruntOptions); err != nil { + return err + } + + if err := writeVersionFile(terraformSource); err != nil { + return err + } + + return nil +} + +// Returns true if the specified TerraformSource, of the exact same version, has already been downloaded into the +// DownloadFolder. This helps avoid downloading the same code multiple times. Note that if the TerraformSource points +// to a local file path, we assume the user is doing local development and always return false to ensure the latest +// code is downloaded (or rather, copied) every single time. See the processTerraformSource method for more info. +func alreadyHaveLatestCode(terraformSource *TerraformSource) (bool, error) { + if isLocalSource(terraformSource.CanonicalSourceURL) || + !util.FileExists(terraformSource.DownloadDir) || + !util.FileExists(terraformSource.VersionFile) { + + return false, nil + } + + currentVersion := encodeSourceVersion(terraformSource.CanonicalSourceURL) + previousVersion, err := readVersionFile(terraformSource) + + if err != nil { + return false, err + } + + return previousVersion == currentVersion, nil +} + +// Return the version number stored in the DownloadDir. This version number can be used to check if the Terraform code +// that has already been downloaded is the same as the version the user is currently requesting. The version number is +// calculated using the encodeSourceVersion method. +func readVersionFile(terraformSource *TerraformSource) (string, error) { + return util.ReadFileAsString(terraformSource.VersionFile) +} + +// Write a file into the DownloadDir that contains the version number of this source code. The version number is +// calculated using the encodeSourceVersion method. +func writeVersionFile(terraformSource *TerraformSource) error { + version := encodeSourceVersion(terraformSource.CanonicalSourceURL) + return errors.WithStackTrace(ioutil.WriteFile(terraformSource.VersionFile, []byte(version), 0640)) +} + +// Take the given source path and create a TerraformSource struct from it, including the folder where the source should +// be downloaded to. Our goal is to reuse the download folder for the same source URL between Terragrunt runs. +// Otherwise, for every Terragrunt command, you'd have to wait for Terragrunt to download your Terraform code, download +// that code's dependencies (terraform get), and configure remote state (terraform remote config), which is very slow. +// +// To maximize reuse, given a working directory w and a source URL s, we download the code into the folder /T/W/S where: +// +// 1. T is the OS temp dir (e.g. /tmp). +// 2. W is the base 64 encoded sha1 hash of w. This ensures that if you are running Terragrunt concurrently in +// multiple folders (e.g. during automated tests), then even if those folders are using the same source URL s, they +// do not overwrite each other. +// 3. S is the base 64 encoded sha1 of s without its query string. For remote source URLs (e.g. Git +// URLs), this is based on the assumption that the scheme/host/path of the URL +// (e.g. git::github.com/foo/bar//some-module) identifies the module name, and we always want to download the same +// module name into the same folder (see the encodeSourceName method). We also assume the version of the module is +// stored in the query string (e.g. ref=v0.0.3), so we store the base 64 encoded sha1 of the query string in a +// file called .terragrunt-source-version within S. +// +// The downloadTerraformSourceIfNecessary decides when we should download the Terraform code and when not to. It uses +// the following rules: +// +// 1. Always download source URLs pointing to local file paths. +// 2. Only download source URLs pointing to remote paths if /T/W/S doesn't already exist or, if it does exist, if the +// version number in /T/W/S/.terragrunt-source-version doesn't match the current version. +func processTerraformSource(source string, terragruntOptions *options.TerragruntOptions) (*TerraformSource, error) { + canonicalWorkingDir, err := util.CanonicalPath(terragruntOptions.WorkingDir, "") + if err != nil { + return nil, err + } + + rawSourceUrl, err := getter.Detect(source, canonicalWorkingDir, getter.Detectors) + if err != nil { + return nil, errors.WithStackTrace(err) + } + + canonicalSourceUrl, err := urlhelper.Parse(rawSourceUrl) + if err != nil { + return nil, errors.WithStackTrace(err) + } + + if isLocalSource(canonicalSourceUrl) { + // Always use canonical file paths for local source folders, rather than relative paths, to ensure + // that the same local folder always maps to the same download folder, no matter how the local folder + // path is specified + canonicalFilePath, err := util.CanonicalPath(canonicalSourceUrl.Path, "") + if err != nil { + return nil, err + } + canonicalSourceUrl.Path = canonicalFilePath + } + + moduleName, err := encodeSourceName(canonicalSourceUrl) + if err != nil { + return nil, err + } + + encodedWorkingDir := util.EncodeBase64Sha1(canonicalWorkingDir) + downloadDir := filepath.Join(os.TempDir(), "terragrunt-download", encodedWorkingDir, moduleName) + versionFile := filepath.Join(downloadDir, ".terragrunt-source-version") + + return &TerraformSource{ + CanonicalSourceURL: canonicalSourceUrl, + DownloadDir: downloadDir, + VersionFile: versionFile, + }, nil +} + +// Encode a version number for the given source URL. When calculating a version number, we simply take the query +// string of the source URL, calculate its sha1, and base 64 encode it. For remote URLs (e.g. Git URLs), this is +// based on the assumption that the scheme/host/path of the URL (e.g. git::github.com/foo/bar//some-module) identifies +// the module name and the query string (e.g. ?ref=v0.0.3) identifies the version. For local file paths, there is no +// query string, so the same file path (/foo/bar) is always considered the same version. See also the encodeSourceName +// and processTerraformSource methods. +func encodeSourceVersion(sourceUrl *url.URL) string { + return util.EncodeBase64Sha1(sourceUrl.Query().Encode()) +} + +// Encode a the module name for the given source URL. When calculating a module name, we calculate the base 64 encoded +// sha1 of the entire source URL without the query string. For remote URLs (e.g. Git URLs), this is based on the +// assumption that the scheme/host/path of the URL (e.g. git::github.com/foo/bar//some-module) identifies +// the module name and the query string (e.g. ?ref=v0.0.3) identifies the version. For local file paths, there is no +// query string, so the same file path (/foo/bar) is always considered the same version. See also the encodeSourceVersion +// and processTerraformSource methods. +func encodeSourceName(sourceUrl *url.URL) (string, error) { + sourceUrlNoQuery, err := urlhelper.Parse(sourceUrl.String()) + if err != nil { + return "", errors.WithStackTrace(err) + } + + sourceUrlNoQuery.RawQuery = "" + + return util.EncodeBase64Sha1(sourceUrlNoQuery.String()), nil +} + +// Returns true if the given URL refers to a path on the local file system +func isLocalSource(sourceUrl *url.URL) bool { + return sourceUrl.Scheme == "file" +} + +// If this temp folder already exists, simply delete all the Terraform configurations (*.tf) within it +// (the terraform init command will redownload the latest ones), but leave all the other files, such +// as the .terraform folder with the downloaded modules and remote state settings. +func cleanupTerraformFiles(path string, terragruntOptions *options.TerragruntOptions) error { + if !util.FileExists(path) { + return nil + } + + terragruntOptions.Logger.Printf("Cleaning up existing *.tf files in %s", path) + + files, err := filepath.Glob(filepath.Join(path, "*.tf")) + if err != nil { + return errors.WithStackTrace(err) + } + return util.DeleteFiles(files) +} + +// There are two ways a user can tell Terragrunt that it needs to download Terraform configurations from a specific +// URL: via a command-line option or via an entry in the .terragrunt config file. If the user used one of these, this +// method returns the source URL and the boolean true; if not, this method returns an empty string and false. +func getTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) (string, bool) { + if terragruntOptions.Source != "" { + return terragruntOptions.Source, true + } else if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.Source != "" { + return terragruntConfig.Terraform.Source, true + } else { + return "", false + } +} + +// Download the code from the Canonical Source URL into the Download Folder using the terraform init command +func terraformInit(terraformSource *TerraformSource, terragruntOptions *options.TerragruntOptions) error { + terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", terraformSource.CanonicalSourceURL, terraformSource.DownloadDir) + + terragruntInitOptions := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath) + terragruntInitOptions.TerraformCliArgs = []string{"init", terraformSource.CanonicalSourceURL.String(), terraformSource.DownloadDir} + + return runTerraformCommand(terragruntInitOptions) +} diff --git a/cli/download_source_test.go b/cli/download_source_test.go new file mode 100644 index 000000000..3601885c9 --- /dev/null +++ b/cli/download_source_test.go @@ -0,0 +1,208 @@ +package cli + +import ( + "testing" + "fmt" + "path/filepath" + "github.com/stretchr/testify/assert" + "net/url" + "github.com/gruntwork-io/terragrunt/options" + "io/ioutil" + "github.com/gruntwork-io/terragrunt/util" + "os" +) + +func TestAlreadyHaveLatestCodeLocalFilePath(t *testing.T) { + t.Parallel() + + canonicalUrl := fmt.Sprintf("file://%s", absPath(t, "../test/fixture-download-source/hello-world")) + downloadDir := "does-not-exist" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, false) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirDoesNotExist(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com" + downloadDir := "does-not-exist" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, false) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirExistsNoVersionNoVersionFile(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com" + downloadDir := "../test/fixture-download-source/download-dir-empty" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, false) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirExistsNoVersionWithVersionFile(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com" + downloadDir := "../test/fixture-download-source/download-dir-version-file-no-query" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, true) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirExistsWithVersionNoVersionFile(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com?ref=v0.0.1" + downloadDir := "../test/fixture-download-source/download-dir-empty" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, false) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirExistsWithVersionAndVersionFile(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com?ref=v0.0.1" + downloadDir := "../test/fixture-download-source/download-dir-version-file" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, true) +} + +func TestDownloadTerraformSourceIfNecessaryLocalDirToEmptyDir(t *testing.T) { + t.Parallel() + + canonicalUrl := fmt.Sprintf("file://%s", absPath(t, "../test/fixture-download-source/hello-world")) + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +} + +func TestDownloadTerraformSourceIfNecessaryLocalDirToAlreadyDownloadedDir(t *testing.T) { + t.Parallel() + + canonicalUrl := fmt.Sprintf("file://%s", absPath(t, "../test/fixture-download-source/hello-world")) + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + copyFolder(t, "../test/fixture-download-source/hello-world-2", downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +} + +// TODO re-enable these two tests once this is merged into master, since they are trying to download code from master +//func TestDownloadTerraformSourceIfNecessaryRemoteUrlToEmptyDir(t *testing.T) { +// t.Parallel() +// +// // TODO: update URL to master branch once this is merged +// canonicalUrl := "github.com/gruntwork-io/terragrunt//test/fixture-download-source/hello-world" +// downloadDir := tmpDir(t) +// defer os.Remove(downloadDir) +// +// testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +//} +// +//func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDir(t *testing.T) { +// t.Parallel() +// +// // TODO: update URL to master branch once this is merged +// canonicalUrl := "github.com/gruntwork-io/terragrunt//test/fixture-download-source/hello-world" +// downloadDir := tmpDir(t) +// defer os.Remove(downloadDir) +// +// copyFolder(t, "../test/fixture-download-source/hello-world-2", downloadDir) +// +// testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World 2") +//} + +func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDirDifferentVersion(t *testing.T) { + t.Parallel() + + // TODO: update URL to master branch once this is merged + canonicalUrl := "github.com/gruntwork-io/terragrunt//test/fixture-download-source/hello-world?ref=cache-tmp-folder" + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + copyFolder(t, "../test/fixture-download-source/hello-world-2", downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +} + +func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDirSameVersion(t *testing.T) { + t.Parallel() + + // TODO: update URL to master branch once this is merged + canonicalUrl := "github.com/gruntwork-io/terragrunt//test/fixture-download-source/hello-world?ref=cache-tmp-folder" + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + copyFolder(t, "../test/fixture-download-source/hello-world-version-remote", downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World version remote") +} + +func testDownloadTerraformSourceIfNecessary(t *testing.T, canonicalUrl string, downloadDir string, expectedFileContents string) { + terraformSource := &TerraformSource{ + CanonicalSourceURL: parseUrl(t, canonicalUrl), + DownloadDir: downloadDir, + VersionFile: filepath.Join(downloadDir, "version-file.txt"), + } + + err := downloadTerraformSourceIfNecessary(terraformSource, options.NewTerragruntOptionsForTest("./should-not-be-used")) + assert.Nil(t, err, "For terraform source %v: %v", terraformSource, err) + + expectedFilePath := filepath.Join(downloadDir, "main.tf") + if assert.True(t, util.FileExists(expectedFilePath), "For terraform source %v", terraformSource) { + actualFileContents := readFile(t, expectedFilePath) + assert.Equal(t, expectedFileContents, actualFileContents, "For terraform source %v", terraformSource) + } +} + +func testAlreadyHaveLatestCode(t *testing.T, canonicalUrl string, downloadDir string, expected bool) { + terraformSource := &TerraformSource{ + CanonicalSourceURL: parseUrl(t, canonicalUrl), + DownloadDir: downloadDir, + VersionFile: filepath.Join(downloadDir, "version-file.txt"), + } + + actual, err := alreadyHaveLatestCode(terraformSource) + assert.Nil(t, err, "Unexpected error for terraform source %v: %v", terraformSource, err) + assert.Equal(t, expected, actual, "For terraform source %v", terraformSource) +} + +func tmpDir(t *testing.T) string { + dir, err := ioutil.TempDir("", "download-source-test") + if err != nil { + t.Fatal(err) + } + return dir +} + +func absPath(t *testing.T, path string) string { + abs, err := filepath.Abs(path) + if err != nil { + t.Fatal(err) + } + return abs +} + +func parseUrl(t *testing.T, rawUrl string) *url.URL { + parsed, err := url.Parse(rawUrl) + if err != nil { + t.Fatal(err) + } + return parsed +} + +func readFile(t *testing.T, path string) string { + contents, err := util.ReadFileAsString(path) + if err != nil { + t.Fatal(err) + } + return contents +} + +func copyFolder(t *testing.T, src string, dest string) { + err := util.CopyFolderContents(src, dest) + if err != nil { + t.Fatal(err) + } +} \ No newline at end of file diff --git a/glide.lock b/glide.lock index f65a7a9cd..fc877b7d5 100644 --- a/glide.lock +++ b/glide.lock @@ -1,45 +1,43 @@ -hash: 86932a90d9bfe69d9f4a0bbf6e103c3b0670d47e8e1e2176fdd0b2637257d3a9 -updated: 2017-01-05T10:16:18.659229152-08:00 +hash: b50c74fff3d496405d1bc9dadfb044d56f1557317f22e34c186b8744af9acaa0 +updated: 2017-02-01T00:01:23.320944332Z imports: - name: github.com/aws/aws-sdk-go - version: 8649d278323ebf6bd20c9cd56ecb152b1c617375 + version: 7524cb911daddd6e5c9195def8e59ae892bef8d9 subpackages: - aws + - aws/defaults + - aws/session - aws/awserr - - aws/awsutil - - aws/client - - aws/client/metadata - - aws/corehandlers + - aws/service/dynamodb + - aws/service/s3 + - service/dynamodb + - service/sts + - service/s3 - aws/credentials + - aws/endpoints + - aws/corehandlers - aws/credentials/ec2rolecreds - aws/credentials/endpointcreds - - aws/credentials/stscreds - - aws/defaults - aws/ec2metadata - - aws/endpoints - aws/request - - aws/service/dynamodb - - aws/service/s3 - - aws/session + - aws/client + - aws/credentials/stscreds + - aws/awsutil + - aws/client/metadata - aws/signer/v4 - private/protocol - - private/protocol/json/jsonutil - private/protocol/jsonrpc + - private/waiter - private/protocol/query - - private/protocol/query/queryutil - - private/protocol/rest - private/protocol/restxml + - private/protocol/rest + - private/protocol/json/jsonutil + - private/protocol/query/queryutil - private/protocol/xml/xmlutil - - private/waiter - - service/dynamodb - - service/kms - - service/kms/kmsiface - - service/s3 - - service/s3/s3iface - - service/s3/s3manager - - service/sts -- name: github.com/BurntSushi/toml - version: 99064174e013895bbd9b025c31100bd1d9b590ca +- name: github.com/bgentry/go-netrc + version: 9fd32a8b3d3d3f9d43c341bfe098430e07609480 + subpackages: + - netrc - name: github.com/davecgh/go-spew version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 subpackages: @@ -47,37 +45,42 @@ imports: - name: github.com/go-errors/errors version: 8fa88b06e5974e97fbf9899a7f86a344bfd1f105 - name: github.com/go-ini/ini - version: 6f66b0e091edb3c7b380f7c4f0f884274d550b67 + version: e3c2d47c61e5333f9aa2974695dd94396eb69c75 +- name: github.com/hashicorp/go-getter + version: cc80f38c726badeae53775d179755e1c4953d6cf + subpackages: + - helper/url +- name: github.com/hashicorp/go-version + version: e96d3840402619007766590ecea8dd7af1292276 - name: github.com/hashicorp/hcl - version: 80e628d796135357b3d2e33a985c666b9f35eee1 + version: 88e9565e9965f4054f86c72ff1ad1bb50e560d6f subpackages: - hcl/ast - hcl/parser - - hcl/printer - - hcl/scanner - - hcl/strconv - hcl/token - json/parser + - hcl/scanner + - hcl/strconv - json/scanner - json/token - name: github.com/jmespath/go-jmespath version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d - name: github.com/mattn/go-zglob - version: 2dbd7f37a45e993d5180a251b4bdd314d6333b70 + version: 95345c4e1c0ebc9d16a3284177f09360f4d20fab + subpackages: + - fastwalk +- name: github.com/mitchellh/go-homedir + version: b8bc1bf767474819792c23f32d8286a45736f1c6 - name: github.com/mitchellh/mapstructure - version: bfdb1a85537d60bc7e954e600c250219ea497417 + version: db1efb556f84b25a0a13a04aad883943538ad2e0 - name: github.com/pmezard/go-difflib version: d8ed2627bdf02c080bf22230dbb337003b7aba2d subpackages: - difflib - name: github.com/stretchr/testify - version: 2402e8e7a02fc811447d11f881aa9746cdc57983 + version: 4d4bfba8f1d1027c4fdbe371823030df51419987 subpackages: - assert - name: github.com/urfave/cli - version: 0bdeddeeb0f650497d603c4ad7b20cfe685682f6 -- name: gopkg.in/urfave/cli.v1 - version: 0bdeddeeb0f650497d603c4ad7b20cfe685682f6 -- name: gopkg.in/yaml.v2 - version: a5b47d31c556af34a302ce5d659e6fea44d90de0 -testImports: [] + version: 347a9884a87374d000eec7e6445a34487c1f4a2b +devImports: [] diff --git a/glide.yaml b/glide.yaml index 319062de5..d1bda812b 100644 --- a/glide.yaml +++ b/glide.yaml @@ -5,6 +5,7 @@ owners: import: - package: github.com/urfave/cli - package: github.com/hashicorp/hcl +- package: github.com/hashicorp/go-getter - package: github.com/stretchr/testify/assert - package: github.com/go-errors/errors - package: github.com/mitchellh/mapstructure diff --git a/test/fixture-download-source/download-dir-version-file-no-query/version-file.txt b/test/fixture-download-source/download-dir-version-file-no-query/version-file.txt new file mode 100644 index 000000000..c39d1108c --- /dev/null +++ b/test/fixture-download-source/download-dir-version-file-no-query/version-file.txt @@ -0,0 +1 @@ +2jmj7l5rSw0yVb_vlWAYkK_YBwk= \ No newline at end of file diff --git a/test/fixture-download-source/download-dir-version-file/version-file.txt b/test/fixture-download-source/download-dir-version-file/version-file.txt new file mode 100644 index 000000000..1e75d115f --- /dev/null +++ b/test/fixture-download-source/download-dir-version-file/version-file.txt @@ -0,0 +1 @@ +zqg_v-R2bnqTbCe-ZO3mHRTRKX0= \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-2/main.tf b/test/fixture-download-source/hello-world-2/main.tf new file mode 100644 index 000000000..e202673c2 --- /dev/null +++ b/test/fixture-download-source/hello-world-2/main.tf @@ -0,0 +1 @@ +# Hello, World 2 \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-2/version-file.txt b/test/fixture-download-source/hello-world-2/version-file.txt new file mode 100644 index 000000000..c39d1108c --- /dev/null +++ b/test/fixture-download-source/hello-world-2/version-file.txt @@ -0,0 +1 @@ +2jmj7l5rSw0yVb_vlWAYkK_YBwk= \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-version-remote/main.tf b/test/fixture-download-source/hello-world-version-remote/main.tf new file mode 100644 index 000000000..2972d5df9 --- /dev/null +++ b/test/fixture-download-source/hello-world-version-remote/main.tf @@ -0,0 +1 @@ +# Hello, World version remote \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-version-remote/version-file.txt b/test/fixture-download-source/hello-world-version-remote/version-file.txt new file mode 100644 index 000000000..a6e2ccf16 --- /dev/null +++ b/test/fixture-download-source/hello-world-version-remote/version-file.txt @@ -0,0 +1 @@ +s7cjjCUtiLre6qopc6Z_GIdMmJ8= \ No newline at end of file diff --git a/test/fixture-download-source/hello-world/main.tf b/test/fixture-download-source/hello-world/main.tf new file mode 100644 index 000000000..3a372db9e --- /dev/null +++ b/test/fixture-download-source/hello-world/main.tf @@ -0,0 +1 @@ +# Hello, World \ No newline at end of file diff --git a/test/integration_test.go b/test/integration_test.go index 4f5f1bdbe..42793d2fa 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -133,6 +133,9 @@ func TestLocalCheckout(t *testing.T) { cleanupTerraformFolder(t, TEST_FIXTURE_LOCAL_CHECKOUT_PATH) runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", TEST_FIXTURE_LOCAL_CHECKOUT_PATH)) + + // Run a second time to make sure the temporary folder can be reused without errors + runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", TEST_FIXTURE_LOCAL_CHECKOUT_PATH)) } func TestRemoteCheckout(t *testing.T) { @@ -141,6 +144,9 @@ func TestRemoteCheckout(t *testing.T) { cleanupTerraformFolder(t, TEST_FIXTURE_REMOTE_CHECKOUT_PATH) runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", TEST_FIXTURE_REMOTE_CHECKOUT_PATH)) + + // Run a second time to make sure the temporary folder can be reused without errors + runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", TEST_FIXTURE_REMOTE_CHECKOUT_PATH)) } func TestRemoteCheckoutOverride(t *testing.T) { @@ -149,6 +155,9 @@ func TestRemoteCheckoutOverride(t *testing.T) { cleanupTerraformFolder(t, TEST_FIXTURE_OVERRIDE_CHECKOUT_PATH) runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-source %s", TEST_FIXTURE_OVERRIDE_CHECKOUT_PATH, "../hello-world")) + + // Run a second time to make sure the temporary folder can be reused without errors + runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-source %s", TEST_FIXTURE_OVERRIDE_CHECKOUT_PATH, "../hello-world")) } func cleanupTerraformFolder(t *testing.T, templatesPath string) { diff --git a/util/file.go b/util/file.go index 74105ed32..fe2481ba4 100644 --- a/util/file.go +++ b/util/file.go @@ -46,6 +46,17 @@ func CanonicalPaths(paths []string, basePath string) ([]string, error) { return canonicalPaths, nil } +// Delete the given list of files. Note: this function ONLY deletes files and will return an error if you pass in a +// folder path. +func DeleteFiles(files []string) error { + for _, file := range files { + if err := os.Remove(file); err != nil { + return errors.WithStackTrace(err) + } + } + return nil +} + // Returns true if the given regex can be found in any of the files matched by the given glob func Grep(regex *regexp.Regexp, glob string) (bool, error) { matches, err := filepath.Glob(glob) diff --git a/util/hash.go b/util/hash.go new file mode 100644 index 000000000..c61df02c2 --- /dev/null +++ b/util/hash.go @@ -0,0 +1,12 @@ +package util + +import ( + "crypto/sha1" + "encoding/base64" +) + +// Returns the base 64 encoded sha1 hash of the given string +func EncodeBase64Sha1(str string) string { + hash := sha1.Sum([]byte(str)) + return base64.URLEncoding.EncodeToString(hash[:]) +} \ No newline at end of file