From 0e5b9074a452e1c4d501d51aca33162ad0c7b32d Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Fri, 27 Jan 2017 17:47:17 +0000 Subject: [PATCH 1/9] Re-use temporary folder for source downloads --- cli/cli_app.go | 49 +-------------------- cli/download_source.go | 95 ++++++++++++++++++++++++++++++++++++++++ test/integration_test.go | 9 ++++ util/file.go | 10 +++++ 4 files changed, 115 insertions(+), 48 deletions(-) create mode 100644 cli/download_source.go diff --git a/cli/cli_app.go b/cli/cli_app.go index d2d7f25d9..dc482c8c8 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -13,7 +13,6 @@ import ( "github.com/urfave/cli" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/spin" - "io/ioutil" ) const OPT_TERRAGRUNT_CONFIG = "terragrunt-config" @@ -128,7 +127,7 @@ func runTerragrunt(terragruntOptions *options.TerragruntOptions) error { } if sourceUrl, hasSourceUrl := getTerraformSourceUrl(terragruntOptions, conf); hasSourceUrl { - if err := checkoutTerraformSource(sourceUrl, terragruntOptions); err != nil { + if err := downloadTerraformSource(sourceUrl, terragruntOptions); err != nil { return err } } @@ -151,19 +150,6 @@ func runTerragrunt(terragruntOptions *options.TerragruntOptions) error { return runTerraformCommandWithLock(conf.Lock, terragruntOptions) } -// There are two ways a user can tell Terragrunt that it needs to download Terraform configurations from a specific -// URL: via a command-line option or via an entry in the .terragrunt config file. If the user used one of these, this -// method returns the source URL and the boolean true; if not, this method returns an empty string and false. -func getTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) (string, bool) { - if terragruntOptions.Source != "" { - return terragruntOptions.Source, true - } else if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.Source != "" { - return terragruntConfig.Terraform.Source, true - } else { - return "", false - } -} - // Returns true if the command the user wants to execute is supposed to affect multiple Terraform modules, such as the // spin-up or tear-down command. func isMultiModuleCommand(command string) bool { @@ -233,39 +219,6 @@ func configureRemoteState(remoteState *remote.RemoteState, terragruntOptions *op return nil } -// 1. Check out the given source URL, which should use Terraform's module source syntax, into a temporary folder -// 2. Copy the contents of terragruntOptions.WorkingDir into the temporary folder. -// 3. Set terragruntOptions.WorkingDir to the temporary folder. -func checkoutTerraformSource(source string, terragruntOptions *options.TerragruntOptions) error { - tmpFolder, err := ioutil.TempDir("", "terragrunt-tmp-checkout") - if err != nil { - return errors.WithStackTrace(err) - } - - terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", source, tmpFolder) - if err := terraformInit(source, tmpFolder, terragruntOptions); err != nil { - return err - } - - terragruntOptions.Logger.Printf("Copying files from %s into %s", terragruntOptions.WorkingDir, tmpFolder) - if err := util.CopyFolderContents(terragruntOptions.WorkingDir, tmpFolder); err != nil { - return err - } - - terragruntOptions.Logger.Printf("Setting working directory to %s", tmpFolder) - terragruntOptions.WorkingDir = tmpFolder - - return nil -} - -// Download the code from source into dest using the terraform init command -func terraformInit(source string, dest string, terragruntOptions *options.TerragruntOptions) error { - terragruntInitOptions := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath) - terragruntInitOptions.TerraformCliArgs = []string{"init", source, dest} - - return runTerraformCommand(terragruntInitOptions) -} - // Run the given Terraform command with the given lock (if the command requires locking) func runTerraformCommandWithLock(lock locks.Lock, terragruntOptions *options.TerragruntOptions) error { switch firstArg(terragruntOptions.TerraformCliArgs) { diff --git a/cli/download_source.go b/cli/download_source.go new file mode 100644 index 000000000..65d3c3d01 --- /dev/null +++ b/cli/download_source.go @@ -0,0 +1,95 @@ +package cli + +import ( + "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/util" + "github.com/gruntwork-io/terragrunt/config" + "os" + "github.com/gruntwork-io/terragrunt/errors" + "path/filepath" +) + +// 1. Check out the given source URL, which should use Terraform's module source syntax, into a temporary folder +// 2. Copy the contents of terragruntOptions.WorkingDir into the temporary folder. +// 3. Set terragruntOptions.WorkingDir to the temporary folder. +func downloadTerraformSource(source string, terragruntOptions *options.TerragruntOptions) error { + tmpFolder, err := prepareTempDir(terragruntOptions) + if err != nil { + return err + } + + terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", source, tmpFolder) + if err := terraformInit(source, tmpFolder, terragruntOptions); err != nil { + return err + } + + terragruntOptions.Logger.Printf("Copying files from %s into %s", terragruntOptions.WorkingDir, tmpFolder) + if err := util.CopyFolderContents(terragruntOptions.WorkingDir, tmpFolder); err != nil { + return err + } + + terragruntOptions.Logger.Printf("Setting working directory to %s", tmpFolder) + terragruntOptions.WorkingDir = tmpFolder + + return nil +} + +// Prepare a temp folder into which Terragrunt can download Terraform code. We take the canonical path of the current +// working directory and create a folder under the system temporary folder with the same path. This allows us to reuse +// the same temp folder for a given working directory so that if you run Terragrunt from the same folder multiple +// times, we only have to download modules and configure remote state once (however, we download the source code every +// time). +func prepareTempDir(terragruntOptions *options.TerragruntOptions) (string, error) { + canonicalPath, err := util.CanonicalPath(terragruntOptions.WorkingDir, "") + if err != nil { + return "", err + } + + tmpFolder := filepath.Join(os.TempDir(), "terragrunt-downloads", canonicalPath) + + if util.FileExists(tmpFolder) { + terragruntOptions.Logger.Printf("Temp folder %s already exists. Will delete Terraform configurations within it before downloading the latest ones.", tmpFolder) + if err := cleanupTerraformFiles(tmpFolder); err != nil { + return "", err + } + } else { + terragruntOptions.Logger.Printf("Creating temp folder %s to store downloaded Terraform configurations.", tmpFolder) + if err := os.MkdirAll(tmpFolder, 0777); err != nil { + return "", errors.WithStackTrace(err) + } + } + + return tmpFolder, nil +} + +// If this temp folder already exists, simply delete all the Terraform configurations (*.tf) within it +// (the terraform init command will redownload the latest ones), but leave all the other files, such +// as the .terraform folder with the downloaded modules and remote state settings. +func cleanupTerraformFiles(path string) error { + files, err := filepath.Glob(filepath.Join(path, "*.tf")) + if err != nil { + return errors.WithStackTrace(err) + } + return util.DeleteFiles(files) +} + +// There are two ways a user can tell Terragrunt that it needs to download Terraform configurations from a specific +// URL: via a command-line option or via an entry in the .terragrunt config file. If the user used one of these, this +// method returns the source URL and the boolean true; if not, this method returns an empty string and false. +func getTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) (string, bool) { + if terragruntOptions.Source != "" { + return terragruntOptions.Source, true + } else if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.Source != "" { + return terragruntConfig.Terraform.Source, true + } else { + return "", false + } +} + +// Download the code from source into dest using the terraform init command +func terraformInit(source string, dest string, terragruntOptions *options.TerragruntOptions) error { + terragruntInitOptions := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath) + terragruntInitOptions.TerraformCliArgs = []string{"init", source, dest} + + return runTerraformCommand(terragruntInitOptions) +} diff --git a/test/integration_test.go b/test/integration_test.go index 4f5f1bdbe..42793d2fa 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -133,6 +133,9 @@ func TestLocalCheckout(t *testing.T) { cleanupTerraformFolder(t, TEST_FIXTURE_LOCAL_CHECKOUT_PATH) runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", TEST_FIXTURE_LOCAL_CHECKOUT_PATH)) + + // Run a second time to make sure the temporary folder can be reused without errors + runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", TEST_FIXTURE_LOCAL_CHECKOUT_PATH)) } func TestRemoteCheckout(t *testing.T) { @@ -141,6 +144,9 @@ func TestRemoteCheckout(t *testing.T) { cleanupTerraformFolder(t, TEST_FIXTURE_REMOTE_CHECKOUT_PATH) runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", TEST_FIXTURE_REMOTE_CHECKOUT_PATH)) + + // Run a second time to make sure the temporary folder can be reused without errors + runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", TEST_FIXTURE_REMOTE_CHECKOUT_PATH)) } func TestRemoteCheckoutOverride(t *testing.T) { @@ -149,6 +155,9 @@ func TestRemoteCheckoutOverride(t *testing.T) { cleanupTerraformFolder(t, TEST_FIXTURE_OVERRIDE_CHECKOUT_PATH) runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-source %s", TEST_FIXTURE_OVERRIDE_CHECKOUT_PATH, "../hello-world")) + + // Run a second time to make sure the temporary folder can be reused without errors + runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-source %s", TEST_FIXTURE_OVERRIDE_CHECKOUT_PATH, "../hello-world")) } func cleanupTerraformFolder(t *testing.T, templatesPath string) { diff --git a/util/file.go b/util/file.go index 74105ed32..0133f2861 100644 --- a/util/file.go +++ b/util/file.go @@ -46,6 +46,16 @@ func CanonicalPaths(paths []string, basePath string) ([]string, error) { return canonicalPaths, nil } +// Delete the given list of files +func DeleteFiles(files []string) error { + for _, file := range files { + if err := os.Remove(file); err != nil { + return errors.WithStackTrace(err) + } + } + return nil +} + // Returns true if the given regex can be found in any of the files matched by the given glob func Grep(regex *regexp.Regexp, glob string) (bool, error) { matches, err := filepath.Glob(glob) From 354c2db829dbe6edf57394540323d40824f51dc6 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Sun, 29 Jan 2017 18:15:36 +0000 Subject: [PATCH 2/9] Only download files if the source path has changed or is a local file path --- cli/download_source.go | 108 +++++++++++++++++------- glide.lock | 181 ++++++++++++++++++++++++++++++++--------- glide.yaml | 1 + 3 files changed, 223 insertions(+), 67 deletions(-) diff --git a/cli/download_source.go b/cli/download_source.go index 65d3c3d01..624f97b16 100644 --- a/cli/download_source.go +++ b/cli/download_source.go @@ -7,65 +7,111 @@ import ( "os" "github.com/gruntwork-io/terragrunt/errors" "path/filepath" + "github.com/hashicorp/go-getter" + urlhelper "github.com/hashicorp/go-getter/helper/url" + "encoding/base64" ) -// 1. Check out the given source URL, which should use Terraform's module source syntax, into a temporary folder +type TerraformSource struct { + // The source path specified by the user + RawSource string + + // The canonical URL we compute from the source path + CanonicalSourceUrl string + + // The folder where we should download the source to + DownloadFolder string + + // True if the source path points to a local file path and false otherwise + IsLocalSource bool +} + +// 1. Download the given source URL, which should use Terraform's module source syntax, into a temporary folder // 2. Copy the contents of terragruntOptions.WorkingDir into the temporary folder. // 3. Set terragruntOptions.WorkingDir to the temporary folder. func downloadTerraformSource(source string, terragruntOptions *options.TerragruntOptions) error { - tmpFolder, err := prepareTempDir(terragruntOptions) + terraformSource, err := processTerraformSource(source, terragruntOptions) if err != nil { return err } - terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", source, tmpFolder) - if err := terraformInit(source, tmpFolder, terragruntOptions); err != nil { - return err - } + terragruntOptions.Logger.Printf("Using temporary folder %s for Terraform files from source %s", terraformSource.DownloadFolder, terraformSource.CanonicalSourceUrl) - terragruntOptions.Logger.Printf("Copying files from %s into %s", terragruntOptions.WorkingDir, tmpFolder) - if err := util.CopyFolderContents(terragruntOptions.WorkingDir, tmpFolder); err != nil { + // If the temp folder already exists, we assume we've already downloaded the Terraform files and don't need + // to spend time doing it again. The only exception is if the source URL points to a local file path, in which + // case the user is probably doing local, iterative development and we should assume the Terraform files have + // changed and need to be recopied every time (which is very fast anyway). + if !util.FileExists(terraformSource.DownloadFolder) { + if err := terraformInit(source, terraformSource.DownloadFolder, terragruntOptions); err != nil { + return err + } + } else if terraformSource.IsLocalSource { + if err := cleanupTerraformFiles(terraformSource.DownloadFolder, terragruntOptions); err != nil { + return err + } + if err := terraformInit(source, terraformSource.DownloadFolder, terragruntOptions); err != nil { + return err + } + } else { + terragruntOptions.Logger.Printf("Terraform files in %s are already up to date. Will not download again.", terraformSource.DownloadFolder) + } + + terragruntOptions.Logger.Printf("Copying files from %s into %s", terragruntOptions.WorkingDir, terraformSource.DownloadFolder) + if err := util.CopyFolderContents(terragruntOptions.WorkingDir, terraformSource.DownloadFolder); err != nil { return err } - terragruntOptions.Logger.Printf("Setting working directory to %s", tmpFolder) - terragruntOptions.WorkingDir = tmpFolder + terragruntOptions.Logger.Printf("Setting working directory to %s", terraformSource.DownloadFolder) + terragruntOptions.WorkingDir = terraformSource.DownloadFolder return nil } -// Prepare a temp folder into which Terragrunt can download Terraform code. We take the canonical path of the current -// working directory and create a folder under the system temporary folder with the same path. This allows us to reuse -// the same temp folder for a given working directory so that if you run Terragrunt from the same folder multiple -// times, we only have to download modules and configure remote state once (however, we download the source code every -// time). -func prepareTempDir(terragruntOptions *options.TerragruntOptions) (string, error) { - canonicalPath, err := util.CanonicalPath(terragruntOptions.WorkingDir, "") +// Take the given source path and create a TerraformSource struct from it, including the folder where the source should +// be downloaded to. We try to use the same download folder for a given source path by converting the source path to +// a canonical form (e.g. converting relative file paths to absolute file paths) and using the base 64 encoded version +// of the canonical path, within the OS tmp folder, as the download path. This way, if the source path doesn't change, +// we don't have to unnecessarily re-download the source code, modules, and Terraform state. +func processTerraformSource(source string, terragruntOptions *options.TerragruntOptions) (*TerraformSource, error) { + canonicalUrl, err := getter.Detect(source, terragruntOptions.WorkingDir, getter.Detectors) if err != nil { - return "", err + return nil, errors.WithStackTrace(err) } - tmpFolder := filepath.Join(os.TempDir(), "terragrunt-downloads", canonicalPath) + sourceUrl, err := urlhelper.Parse(canonicalUrl) + if err != nil { + return nil, errors.WithStackTrace(err) + } - if util.FileExists(tmpFolder) { - terragruntOptions.Logger.Printf("Temp folder %s already exists. Will delete Terraform configurations within it before downloading the latest ones.", tmpFolder) - if err := cleanupTerraformFiles(tmpFolder); err != nil { - return "", err - } - } else { - terragruntOptions.Logger.Printf("Creating temp folder %s to store downloaded Terraform configurations.", tmpFolder) - if err := os.MkdirAll(tmpFolder, 0777); err != nil { - return "", errors.WithStackTrace(err) + isLocalSource := sourceUrl.Scheme == "file" + + if isLocalSource { + // Always use the canonical file path to ensure that a given path on the local file system is always + // represented the same way (i.e. doesn't differ just because the user provided a different relative + // path) + canonicalUrl, err = util.CanonicalPath(sourceUrl.Path, "") + if err != nil { + return nil, err } } - return tmpFolder, nil + canonicalUrlHash := base64.StdEncoding.EncodeToString([]byte(canonicalUrl)) + downloadFolder := filepath.Join(os.TempDir(), "terragrunt-download", canonicalUrlHash) + + return &TerraformSource{ + RawSource: source, + CanonicalSourceUrl: canonicalUrl, + DownloadFolder: downloadFolder, + IsLocalSource: isLocalSource, + }, nil } // If this temp folder already exists, simply delete all the Terraform configurations (*.tf) within it // (the terraform init command will redownload the latest ones), but leave all the other files, such // as the .terraform folder with the downloaded modules and remote state settings. -func cleanupTerraformFiles(path string) error { +func cleanupTerraformFiles(path string, terragruntOptions *options.TerragruntOptions) error { + terragruntOptions.Logger.Printf("Cleaning up existing *.tf files in %s", path) + files, err := filepath.Glob(filepath.Join(path, "*.tf")) if err != nil { return errors.WithStackTrace(err) @@ -88,6 +134,8 @@ func getTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragr // Download the code from source into dest using the terraform init command func terraformInit(source string, dest string, terragruntOptions *options.TerragruntOptions) error { + terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", source, dest) + terragruntInitOptions := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath) terragruntInitOptions.TerraformCliArgs = []string{"init", source, dest} diff --git a/glide.lock b/glide.lock index f65a7a9cd..3947305bb 100644 --- a/glide.lock +++ b/glide.lock @@ -1,43 +1,102 @@ -hash: 86932a90d9bfe69d9f4a0bbf6e103c3b0670d47e8e1e2176fdd0b2637257d3a9 -updated: 2017-01-05T10:16:18.659229152-08:00 +hash: b50c74fff3d496405d1bc9dadfb044d56f1557317f22e34c186b8744af9acaa0 +updated: 2017-01-29T17:21:51.671174219Z imports: - name: github.com/aws/aws-sdk-go - version: 8649d278323ebf6bd20c9cd56ecb152b1c617375 + version: 2ae1f45d01cdc7316a4d6c518c21a6d9d81a4970 subpackages: - aws + - aws/defaults + - aws/session - aws/awserr - - aws/awsutil - - aws/client - - aws/client/metadata - - aws/corehandlers + - aws/service/dynamodb + - aws/service/s3 + - service/dynamodb + - service/sts + - service/s3 - aws/credentials - aws/credentials/ec2rolecreds - - aws/credentials/endpointcreds - - aws/credentials/stscreds - - aws/defaults - aws/ec2metadata - aws/endpoints + - aws/client/metadata - aws/request - - aws/service/dynamodb - - aws/service/s3 - - aws/session - - aws/signer/v4 - - private/protocol - - private/protocol/json/jsonutil - - private/protocol/jsonrpc - - private/protocol/query - - private/protocol/query/queryutil - private/protocol/rest - - private/protocol/restxml - - private/protocol/xml/xmlutil - - private/waiter - - service/dynamodb + - awsmigrate/awsmigrate-renamer/rename + - private/model/api + - private/util - service/kms - - service/kms/kmsiface - - service/s3 + - service/s3/s3crypto + - service/acm + - service/apigateway + - service/applicationdiscoveryservice + - service/autoscaling + - service/cloudformation + - service/cloudfront + - service/cloudhsm + - service/cloudsearch + - service/cloudtrail + - service/cloudwatch + - service/cloudwatchlogs + - service/codecommit + - service/codedeploy + - service/codepipeline + - service/cognitoidentity + - service/cognitosync + - service/configservice + - service/datapipeline + - service/devicefarm + - service/directconnect + - service/directoryservice + - service/dynamodbstreams + - service/ec2 + - service/ecs + - service/efs + - service/elasticache + - service/elasticbeanstalk + - service/elb + - service/elastictranscoder + - service/emr + - service/elasticsearchservice + - service/glacier + - service/iam + - service/iot + - service/iotdataplane + - service/kinesis + - service/lambda + - service/machinelearning + - service/opsworks + - service/rds + - service/redshift + - service/route53 + - service/route53domains + - service/ses + - service/simpledb + - service/sns + - service/sqs + - service/ssm + - service/storagegateway + - service/support + - service/swf + - service/waf + - service/workspaces + - awstesting/unit + - service/cloudsearchdomain + - service/cloudwatchevents + - service/dynamodb/dynamodbattribute + - service/ecr + - service/firehose + - service/inspector + - service/marketplacecommerceanalytics + - service/mobileanalytics + - service/cloudfront/sign + - service/dynamodb/dynamodbiface + - service/sqs/sqsiface + - private/protocol/query/queryutil + - private/protocol/xml/xmlutil - service/s3/s3iface - - service/s3/s3manager - - service/sts +- name: github.com/bgentry/go-netrc + version: 9fd32a8b3d3d3f9d43c341bfe098430e07609480 + subpackages: + - netrc - name: github.com/BurntSushi/toml version: 99064174e013895bbd9b025c31100bd1d9b590ca - name: github.com/davecgh/go-spew @@ -47,37 +106,85 @@ imports: - name: github.com/go-errors/errors version: 8fa88b06e5974e97fbf9899a7f86a344bfd1f105 - name: github.com/go-ini/ini - version: 6f66b0e091edb3c7b380f7c4f0f884274d550b67 + version: e3c2d47c61e5333f9aa2974695dd94396eb69c75 +- name: github.com/gucumber/gucumber + version: 71608e2f6e76fd4da5b09a376aeec7a5c0b5edbc + subpackages: + - gherkin +- name: github.com/hashicorp/go-getter + version: cc80f38c726badeae53775d179755e1c4953d6cf + subpackages: + - helper/url +- name: github.com/hashicorp/go-version + version: e96d3840402619007766590ecea8dd7af1292276 - name: github.com/hashicorp/hcl - version: 80e628d796135357b3d2e33a985c666b9f35eee1 + version: db4f0768927a665a06c32855186e1bc762bcc8f5 subpackages: - hcl/ast - hcl/parser - - hcl/printer - - hcl/scanner - - hcl/strconv - hcl/token - json/parser + - hcl/printer + - hcl/scanner - json/scanner - json/token - name: github.com/jmespath/go-jmespath version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d - name: github.com/mattn/go-zglob - version: 2dbd7f37a45e993d5180a251b4bdd314d6333b70 + version: 95345c4e1c0ebc9d16a3284177f09360f4d20fab + subpackages: + - fastwalk +- name: github.com/mitchellh/go-homedir + version: b8bc1bf767474819792c23f32d8286a45736f1c6 - name: github.com/mitchellh/mapstructure - version: bfdb1a85537d60bc7e954e600c250219ea497417 + version: db1efb556f84b25a0a13a04aad883943538ad2e0 - name: github.com/pmezard/go-difflib version: d8ed2627bdf02c080bf22230dbb337003b7aba2d subpackages: - difflib +- name: github.com/shiena/ansicolor + version: a422bbe96644373c5753384a59d678f7d261ff10 +- name: github.com/stretchr/objx + version: cbeaeb16a013161a98496fad62933b1d21786672 - name: github.com/stretchr/testify version: 2402e8e7a02fc811447d11f881aa9746cdc57983 subpackages: - assert + - http + - mock - name: github.com/urfave/cli - version: 0bdeddeeb0f650497d603c4ad7b20cfe685682f6 + version: 347a9884a87374d000eec7e6445a34487c1f4a2b +- name: golang.org/x/net + version: f2499483f923065a842d38eb4c7f1927e6fc6e6d + subpackages: + - html + - html/atom +- name: golang.org/x/text + version: ece019dcfd29abcf65d0d1dfe145e8faad097640 + subpackages: + - encoding + - encoding/charmap + - encoding/htmlindex + - transform + - encoding/internal/identifier + - encoding/internal + - encoding/japanese + - encoding/korean + - encoding/simplifiedchinese + - encoding/traditionalchinese + - encoding/unicode + - language + - internal/utf8internal + - runes + - internal/tag +- name: golang.org/x/tools + version: f8ed2e405fdcbb42c55233531ad98721e6d1cb3e + subpackages: + - go/loader + - go/ast/astutil + - go/buildutil - name: gopkg.in/urfave/cli.v1 version: 0bdeddeeb0f650497d603c4ad7b20cfe685682f6 - name: gopkg.in/yaml.v2 - version: a5b47d31c556af34a302ce5d659e6fea44d90de0 -testImports: [] + version: 4c78c975fe7c825c6d1466c42be594d1d6f3aba6 +devImports: [] diff --git a/glide.yaml b/glide.yaml index 319062de5..d1bda812b 100644 --- a/glide.yaml +++ b/glide.yaml @@ -5,6 +5,7 @@ owners: import: - package: github.com/urfave/cli - package: github.com/hashicorp/hcl +- package: github.com/hashicorp/go-getter - package: github.com/stretchr/testify/assert - package: github.com/go-errors/errors - package: github.com/mitchellh/mapstructure From ccf316e5be5a4601f29b698598e11f9f8041d71f Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Sun, 29 Jan 2017 18:40:49 +0000 Subject: [PATCH 3/9] Use sha1 hash of file path --- cli/download_source.go | 36 ++++++++++++++++++++++-------------- util/hash.go | 13 +++++++++++++ 2 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 util/hash.go diff --git a/cli/download_source.go b/cli/download_source.go index 624f97b16..06fe3ff4e 100644 --- a/cli/download_source.go +++ b/cli/download_source.go @@ -9,7 +9,6 @@ import ( "path/filepath" "github.com/hashicorp/go-getter" urlhelper "github.com/hashicorp/go-getter/helper/url" - "encoding/base64" ) type TerraformSource struct { @@ -42,14 +41,14 @@ func downloadTerraformSource(source string, terragruntOptions *options.Terragrun // case the user is probably doing local, iterative development and we should assume the Terraform files have // changed and need to be recopied every time (which is very fast anyway). if !util.FileExists(terraformSource.DownloadFolder) { - if err := terraformInit(source, terraformSource.DownloadFolder, terragruntOptions); err != nil { + if err := terraformInit(terraformSource, terragruntOptions); err != nil { return err } } else if terraformSource.IsLocalSource { if err := cleanupTerraformFiles(terraformSource.DownloadFolder, terragruntOptions); err != nil { return err } - if err := terraformInit(source, terraformSource.DownloadFolder, terragruntOptions); err != nil { + if err := terraformInit(terraformSource, terragruntOptions); err != nil { return err } } else { @@ -68,12 +67,21 @@ func downloadTerraformSource(source string, terragruntOptions *options.Terragrun } // Take the given source path and create a TerraformSource struct from it, including the folder where the source should -// be downloaded to. We try to use the same download folder for a given source path by converting the source path to -// a canonical form (e.g. converting relative file paths to absolute file paths) and using the base 64 encoded version -// of the canonical path, within the OS tmp folder, as the download path. This way, if the source path doesn't change, -// we don't have to unnecessarily re-download the source code, modules, and Terraform state. +// be downloaded to. We try to use the same download folder, D, for a given source path by: +// +// 1. C = Convert the source path to a canonical form (e.g. converting relative file paths to absolute file paths) +// 2. H = Compute the base 64 encoded sha1 hash of C +// 3. D = TMP/terragrunt-download/H (where TMP is the temp folder for the current OS) +// +// By reusing the same folder, we only have to download the Terraform code, modules, and Terraform state for each +// source path once. func processTerraformSource(source string, terragruntOptions *options.TerragruntOptions) (*TerraformSource, error) { - canonicalUrl, err := getter.Detect(source, terragruntOptions.WorkingDir, getter.Detectors) + workingDirAbs, err := filepath.Abs(terragruntOptions.WorkingDir) + if err != nil { + return nil, errors.WithStackTrace(err) + } + + canonicalUrl, err := getter.Detect(source, workingDirAbs, getter.Detectors) if err != nil { return nil, errors.WithStackTrace(err) } @@ -95,8 +103,7 @@ func processTerraformSource(source string, terragruntOptions *options.Terragrunt } } - canonicalUrlHash := base64.StdEncoding.EncodeToString([]byte(canonicalUrl)) - downloadFolder := filepath.Join(os.TempDir(), "terragrunt-download", canonicalUrlHash) + downloadFolder := filepath.Join(os.TempDir(), "terragrunt-download", util.Base64EncodedSha1(canonicalUrl)) return &TerraformSource{ RawSource: source, @@ -106,6 +113,7 @@ func processTerraformSource(source string, terragruntOptions *options.Terragrunt }, nil } + // If this temp folder already exists, simply delete all the Terraform configurations (*.tf) within it // (the terraform init command will redownload the latest ones), but leave all the other files, such // as the .terraform folder with the downloaded modules and remote state settings. @@ -132,12 +140,12 @@ func getTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragr } } -// Download the code from source into dest using the terraform init command -func terraformInit(source string, dest string, terragruntOptions *options.TerragruntOptions) error { - terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", source, dest) +// Download the code from the Canonical Source URL into the Download Folder using the terraform init command +func terraformInit(terraformSource *TerraformSource, terragruntOptions *options.TerragruntOptions) error { + terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", terraformSource.CanonicalSourceUrl, terraformSource.DownloadFolder) terragruntInitOptions := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath) - terragruntInitOptions.TerraformCliArgs = []string{"init", source, dest} + terragruntInitOptions.TerraformCliArgs = []string{"init", terraformSource.CanonicalSourceUrl, terraformSource.DownloadFolder} return runTerraformCommand(terragruntInitOptions) } diff --git a/util/hash.go b/util/hash.go new file mode 100644 index 000000000..6ade1c972 --- /dev/null +++ b/util/hash.go @@ -0,0 +1,13 @@ +package util + +import ( + "crypto/sha1" + "encoding/base64" +) + +// Returns the base 64 encoded sha1 hash of the given string +func Base64EncodedSha1(str string) string { + hash := sha1.Sum([]byte(str)) + return base64.StdEncoding.EncodeToString(hash[:]) +} + From d60f47c0b8ba97861494ae9fa228034e36d78939 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Mon, 30 Jan 2017 01:07:14 +0000 Subject: [PATCH 4/9] Cache bust only if version changes. Add unit tests. --- cli/download_source.go | 218 +++++++++++++----- cli/download_source_test.go | 207 +++++++++++++++++ .../version-file.txt | 1 + .../version-file.txt | 1 + .../hello-world-2/main.tf | 1 + .../hello-world-2/version-file.txt | 1 + .../hello-world-ref-v0.0.3/main.tf | 1 + .../hello-world-ref-v0.0.3/version-file.txt | 1 + .../hello-world/main.tf | 1 + util/hash.go | 7 +- 10 files changed, 380 insertions(+), 59 deletions(-) create mode 100644 cli/download_source_test.go create mode 100644 test/fixture-download-source/download-dir-version-file-no-query/version-file.txt create mode 100644 test/fixture-download-source/download-dir-version-file/version-file.txt create mode 100644 test/fixture-download-source/hello-world-2/main.tf create mode 100644 test/fixture-download-source/hello-world-2/version-file.txt create mode 100644 test/fixture-download-source/hello-world-ref-v0.0.3/main.tf create mode 100644 test/fixture-download-source/hello-world-ref-v0.0.3/version-file.txt create mode 100644 test/fixture-download-source/hello-world/main.tf diff --git a/cli/download_source.go b/cli/download_source.go index 06fe3ff4e..035893e6c 100644 --- a/cli/download_source.go +++ b/cli/download_source.go @@ -9,115 +9,223 @@ import ( "path/filepath" "github.com/hashicorp/go-getter" urlhelper "github.com/hashicorp/go-getter/helper/url" + "io/ioutil" + "net/url" + "fmt" ) type TerraformSource struct { - // The source path specified by the user - RawSource string - - // The canonical URL we compute from the source path - CanonicalSourceUrl string + // A canonical version of RawSource, in URL format + CanonicalSourceURL *url.URL // The folder where we should download the source to - DownloadFolder string + DownloadDir string + + // The path to a file in DownloadDir that stores the version number of the code + VersionFile string +} - // True if the source path points to a local file path and false otherwise - IsLocalSource bool +func (src *TerraformSource) String() string { + return fmt.Sprintf("TerraformSource{CanonicalSourceURL = %v, DownloadDir = %v, VersionFile = %v}", src.CanonicalSourceURL, src.DownloadDir, src.VersionFile) } // 1. Download the given source URL, which should use Terraform's module source syntax, into a temporary folder // 2. Copy the contents of terragruntOptions.WorkingDir into the temporary folder. // 3. Set terragruntOptions.WorkingDir to the temporary folder. +// +// See the processTerraformSource method for how we determine the temporary folder so we can reuse it across multiple +// runs of Terragrunt to avoid downloading everything from scratch every time. func downloadTerraformSource(source string, terragruntOptions *options.TerragruntOptions) error { terraformSource, err := processTerraformSource(source, terragruntOptions) if err != nil { return err } - terragruntOptions.Logger.Printf("Using temporary folder %s for Terraform files from source %s", terraformSource.DownloadFolder, terraformSource.CanonicalSourceUrl) - - // If the temp folder already exists, we assume we've already downloaded the Terraform files and don't need - // to spend time doing it again. The only exception is if the source URL points to a local file path, in which - // case the user is probably doing local, iterative development and we should assume the Terraform files have - // changed and need to be recopied every time (which is very fast anyway). - if !util.FileExists(terraformSource.DownloadFolder) { - if err := terraformInit(terraformSource, terragruntOptions); err != nil { - return err - } - } else if terraformSource.IsLocalSource { - if err := cleanupTerraformFiles(terraformSource.DownloadFolder, terragruntOptions); err != nil { - return err - } - if err := terraformInit(terraformSource, terragruntOptions); err != nil { - return err - } - } else { - terragruntOptions.Logger.Printf("Terraform files in %s are already up to date. Will not download again.", terraformSource.DownloadFolder) + if err := downloadTerraformSourceIfNecessary(terraformSource, terragruntOptions); err != nil { + return err } - terragruntOptions.Logger.Printf("Copying files from %s into %s", terragruntOptions.WorkingDir, terraformSource.DownloadFolder) - if err := util.CopyFolderContents(terragruntOptions.WorkingDir, terraformSource.DownloadFolder); err != nil { + terragruntOptions.Logger.Printf("Copying files from %s into %s", terragruntOptions.WorkingDir, terraformSource.DownloadDir) + if err := util.CopyFolderContents(terragruntOptions.WorkingDir, terraformSource.DownloadDir); err != nil { return err } - terragruntOptions.Logger.Printf("Setting working directory to %s", terraformSource.DownloadFolder) - terragruntOptions.WorkingDir = terraformSource.DownloadFolder + terragruntOptions.Logger.Printf("Setting working directory to %s", terraformSource.DownloadDir) + terragruntOptions.WorkingDir = terraformSource.DownloadDir return nil } +// Download the specified TerraformSource if the latest code hasn't already been downloaded. +func downloadTerraformSourceIfNecessary(terraformSource *TerraformSource, terragruntOptions *options.TerragruntOptions) error { + alreadyLatest, err := alreadyHaveLatestCode(terraformSource) + if err != nil { + return err + } + + if alreadyLatest { + terragruntOptions.Logger.Printf("Terraform files in %s are up to date. Will not download again.", terraformSource.DownloadDir) + return nil + } + + if err := cleanupTerraformFiles(terraformSource.DownloadDir, terragruntOptions); err != nil { + return err + } + + if err := terraformInit(terraformSource, terragruntOptions); err != nil { + return err + } + + if err := writeVersionFile(terraformSource); err != nil { + return err + } + + return nil +} + +// Returns true if the specified TerraformSource, of the exact same version, has already been downloaded into the +// DownloadFolder. This helps avoid downloading the same code multiple times. Note that if the TerraformSource points +// to a local file path, we assume the user is doing local development and always return false to ensure the latest +// code is downloaded (or rather, copied) every single time. See the processTerraformSource method for more info. +func alreadyHaveLatestCode(terraformSource *TerraformSource) (bool, error) { + if isLocalSource(terraformSource.CanonicalSourceURL) || + !util.FileExists(terraformSource.DownloadDir) || + !util.FileExists(terraformSource.VersionFile) { + + return false, nil + } + + currentVersion := encodeSourceVersion(terraformSource.CanonicalSourceURL) + previousVersion, err := readVersionFile(terraformSource) + + if err != nil { + return false, err + } + + return previousVersion == currentVersion, nil +} + +// Return the version number stored in the DownloadDir. This version number can be used to check if the Terraform code +// that has already been downloaded is the same as the version the user is currently requesting. The version number is +// calculated using the encodeSourceVersion method. +func readVersionFile(terraformSource *TerraformSource) (string, error) { + return util.ReadFileAsString(terraformSource.VersionFile) +} + +// Write a file into the DownloadDir that contains the version number of this source code. The version number is +// calculated using the encodeSourceVersion method. +func writeVersionFile(terraformSource *TerraformSource) error { + version := encodeSourceVersion(terraformSource.CanonicalSourceURL) + return errors.WithStackTrace(ioutil.WriteFile(terraformSource.VersionFile, []byte(version), 0640)) +} + // Take the given source path and create a TerraformSource struct from it, including the folder where the source should -// be downloaded to. We try to use the same download folder, D, for a given source path by: +// be downloaded to. Our goal is to reuse the download folder for the same source URL between Terragrunt runs. +// Otherwise, for every Terragrunt command, you'd have to wait for Terragrunt to download your Terraform code, download +// that code's dependencies (terraform get), and configure remote state (terraform remote config), which is very slow. +// +// To maximize reuse, given a working directory w and a source URL s, we download the code into the folder /T/W/S where: // -// 1. C = Convert the source path to a canonical form (e.g. converting relative file paths to absolute file paths) -// 2. H = Compute the base 64 encoded sha1 hash of C -// 3. D = TMP/terragrunt-download/H (where TMP is the temp folder for the current OS) +// 1. T is the OS temp dir (e.g. /tmp). +// 2. W is the base 64 encoded sha1 hash of w. This ensures that if you are running Terragrunt concurrently in +// multiple folders (e.g. during automated tests), then even if those folders are using the same source URL s, they +// do not overwrite each other. +// 3. S is the base 64 encoded sha1 has of s without its query string. For remote source URLs (e.g. Git +// URLs), this is based on the assumption that the scheme/host/path of the URL +// (e.g. git::github.com/foo/bar//some-module) identifies the module name, and we always want to download the same +// module name into the same folder (see the encodeSourceName method). We also assume the version of the module is +// stored in the query string (e.g. ref=v0.0.3), so we store the base 64 encoded sha1 of the query string in a +// file called .terragrunt-source-version within S. // -// By reusing the same folder, we only have to download the Terraform code, modules, and Terraform state for each -// source path once. +// The downloadTerraformSourceIfNecessary decides when we should download the Terraform code and when not to. It uses +// the following rules: +// +// 1. Always download source URLs pointing to local file paths. +// 2. Only download source URLs pointing to remote paths if /T/W/S doesn't already exist or, if it does exist, if the +// version number in /T/W/S/.terragrunt-source-version doesn't match the current version. func processTerraformSource(source string, terragruntOptions *options.TerragruntOptions) (*TerraformSource, error) { - workingDirAbs, err := filepath.Abs(terragruntOptions.WorkingDir) + canonicalWorkingDir, err := util.CanonicalPath(terragruntOptions.WorkingDir, "") if err != nil { - return nil, errors.WithStackTrace(err) + return nil, err } - canonicalUrl, err := getter.Detect(source, workingDirAbs, getter.Detectors) + rawSourceUrl, err := getter.Detect(source, canonicalWorkingDir, getter.Detectors) if err != nil { return nil, errors.WithStackTrace(err) } - sourceUrl, err := urlhelper.Parse(canonicalUrl) + canonicalSourceUrl, err := urlhelper.Parse(rawSourceUrl) if err != nil { return nil, errors.WithStackTrace(err) } - isLocalSource := sourceUrl.Scheme == "file" - - if isLocalSource { - // Always use the canonical file path to ensure that a given path on the local file system is always - // represented the same way (i.e. doesn't differ just because the user provided a different relative - // path) - canonicalUrl, err = util.CanonicalPath(sourceUrl.Path, "") + if isLocalSource(canonicalSourceUrl) { + // Always use canonical file paths for local source folders, rather than relative paths, to ensure + // that the same local folder always maps to the same download folder, no matter how the local folder + // path is specified + canonicalFilePath, err := util.CanonicalPath(canonicalSourceUrl.Path, "") if err != nil { return nil, err } + canonicalSourceUrl.Path = canonicalFilePath } - downloadFolder := filepath.Join(os.TempDir(), "terragrunt-download", util.Base64EncodedSha1(canonicalUrl)) + moduleName, err := encodeSourceName(canonicalSourceUrl) + if err != nil { + return nil, err + } + + encodedWorkingDir := util.EncodeBase64Sha1(canonicalWorkingDir) + downloadDir := filepath.Join(os.TempDir(), "terragrunt-download", encodedWorkingDir, moduleName) + versionFile := filepath.Join(downloadDir, ".terragrunt-source-version") return &TerraformSource{ - RawSource: source, - CanonicalSourceUrl: canonicalUrl, - DownloadFolder: downloadFolder, - IsLocalSource: isLocalSource, + CanonicalSourceURL: canonicalSourceUrl, + DownloadDir: downloadDir, + VersionFile: versionFile, }, nil } +// Encode a version number for the given source URL. When calculating a version number, we simply take the query +// string of the source URL, calculate its sha1, and base 64 encode it. For remote URLs (e.g. Git URLs), this is +// based on the assumption that the scheme/host/path of the URL (e.g. git::github.com/foo/bar//some-module) identifies +// the module name and the query string (e.g. ?ref=v0.0.3) identifies the version. For local file paths, there is no +// query string, so the same file path (/foo/bar) is always considered the same version. See also the encodeSourceName +// and processTerraformSource methods. +func encodeSourceVersion(sourceUrl *url.URL) string { + return util.EncodeBase64Sha1(sourceUrl.Query().Encode()) +} + +// Encode a the module name for the given source URL. When calculating a module name, we calculate the base 64 encoded +// sha1 of the entire source URL without the query string. For remote URLs (e.g. Git URLs), this is based on the +// assumption that the scheme/host/path of the URL (e.g. git::github.com/foo/bar//some-module) identifies +// the module name and the query string (e.g. ?ref=v0.0.3) identifies the version. For local file paths, there is no +// query string, so the same file path (/foo/bar) is always considered the same version. See also the encodeSourceVersion +// and processTerraformSource methods. +func encodeSourceName(sourceUrl *url.URL) (string, error) { + sourceUrlNoQuery, err := urlhelper.Parse(sourceUrl.String()) + if err != nil { + return "", errors.WithStackTrace(err) + } + + sourceUrlNoQuery.RawQuery = "" + + return util.EncodeBase64Sha1(sourceUrlNoQuery.String()), nil +} + +// Returns true if the given URL refers to a path on the local file system +func isLocalSource(sourceUrl *url.URL) bool { + return sourceUrl.Scheme == "file" +} // If this temp folder already exists, simply delete all the Terraform configurations (*.tf) within it // (the terraform init command will redownload the latest ones), but leave all the other files, such // as the .terraform folder with the downloaded modules and remote state settings. func cleanupTerraformFiles(path string, terragruntOptions *options.TerragruntOptions) error { + if !util.FileExists(path) { + return nil + } + terragruntOptions.Logger.Printf("Cleaning up existing *.tf files in %s", path) files, err := filepath.Glob(filepath.Join(path, "*.tf")) @@ -142,10 +250,10 @@ func getTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragr // Download the code from the Canonical Source URL into the Download Folder using the terraform init command func terraformInit(terraformSource *TerraformSource, terragruntOptions *options.TerragruntOptions) error { - terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", terraformSource.CanonicalSourceUrl, terraformSource.DownloadFolder) + terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", terraformSource.CanonicalSourceURL, terraformSource.DownloadDir) terragruntInitOptions := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath) - terragruntInitOptions.TerraformCliArgs = []string{"init", terraformSource.CanonicalSourceUrl, terraformSource.DownloadFolder} + terragruntInitOptions.TerraformCliArgs = []string{"init", terraformSource.CanonicalSourceURL.String(), terraformSource.DownloadDir} return runTerraformCommand(terragruntInitOptions) } diff --git a/cli/download_source_test.go b/cli/download_source_test.go new file mode 100644 index 000000000..9b08bb1cf --- /dev/null +++ b/cli/download_source_test.go @@ -0,0 +1,207 @@ +package cli + +import ( + "testing" + "fmt" + "path/filepath" + "github.com/stretchr/testify/assert" + "net/url" + "github.com/gruntwork-io/terragrunt/options" + "io/ioutil" + "github.com/gruntwork-io/terragrunt/util" + "os" +) + +func TestAlreadyHaveLatestCodeLocalFilePath(t *testing.T) { + t.Parallel() + + canonicalUrl := fmt.Sprintf("file://%s", absPath(t, "../test/fixture-download-source/hello-world")) + downloadDir := "does-not-exist" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, false) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirDoesNotExist(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com" + downloadDir := "does-not-exist" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, false) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirExistsNoVersionNoVersionFile(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com" + downloadDir := "../test/fixture-download-source/download-dir-empty" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, false) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirExistsNoVersionWithVersionFile(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com" + downloadDir := "../test/fixture-download-source/download-dir-version-file-no-query" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, true) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirExistsWithVersionNoVersionFile(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com?ref=v0.0.1" + downloadDir := "../test/fixture-download-source/download-dir-empty" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, false) +} + +func TestAlreadyHaveLatestCodeRemoteFilePathDownloadDirExistsWithVersionAndVersionFile(t *testing.T) { + t.Parallel() + + canonicalUrl := "http://www.some-url.com?ref=v0.0.1" + downloadDir := "../test/fixture-download-source/download-dir-version-file" + + testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, true) +} + +func TestDownloadTerraformSourceIfNecessaryLocalDirToEmptyDir(t *testing.T) { + t.Parallel() + + canonicalUrl := fmt.Sprintf("file://%s", absPath(t, "../test/fixture-download-source/hello-world")) + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +} + +func TestDownloadTerraformSourceIfNecessaryLocalDirToAlreadyDownloadedDir(t *testing.T) { + t.Parallel() + + canonicalUrl := fmt.Sprintf("file://%s", absPath(t, "../test/fixture-download-source/hello-world")) + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + copyFolder(t, "../test/fixture-downlooad-source/hello-world-2", downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +} + +func TestDownloadTerraformSourceIfNecessaryRemoteUrlToEmptyDir(t *testing.T) { + t.Parallel() + + // TODO: update URL to master branch once this is merged + canonicalUrl := "https://github.com/gruntwork-io/terragrunt/tree/cache-tmp-folder/test/fixture-download-source/hello-world" + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +} + +func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDir(t *testing.T) { + t.Parallel() + + // TODO: update URL to master branch once this is merged + canonicalUrl := "https://github.com/gruntwork-io/terragrunt/tree/cache-tmp-folder/test/fixture-download-source/hello-world-2" + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + copyFolder(t, "../test/fixture-downlooad-source/hello-world-2", downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +} + +func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDirDifferentVersion(t *testing.T) { + t.Parallel() + + // TODO: update URL to master branch once this is merged + canonicalUrl := "https://github.com/gruntwork-io/terragrunt/tree/cache-tmp-folder/test/fixture-download-source/hello-world-2?ref=v0.0.3" + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + copyFolder(t, "../test/fixture-downlooad-source/hello-world-2", downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World 2") +} + +func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDirSameVersion(t *testing.T) { + t.Parallel() + + // TODO: update URL to master branch once this is merged + canonicalUrl := "https://github.com/gruntwork-io/terragrunt/tree/cache-tmp-folder/test/fixture-download-source/hello-world-2?ref=v0.0.3" + downloadDir := tmpDir(t) + defer os.Remove(downloadDir) + + copyFolder(t, "../test/fixture-downlooad-source/hello-world-ref-v0.0.3", downloadDir) + + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +} + +func testDownloadTerraformSourceIfNecessary(t *testing.T, canonicalUrl string, downloadDir string, expectedFileContents string) { + terraformSource := &TerraformSource{ + CanonicalSourceURL: canonicalUrl, + DownloadDir: downloadDir, + VersionFile: filepath.Join(downloadDir, "version-file.txt"), + } + + err := downloadTerraformSourceIfNecessary(terraformSource, options.NewTerragruntOptionsForTest("./should-not-be-used")) + assert.Nil(t, err, "For terraform source %v: %v", terraformSource, err) + + expectedFilePath := filepath.Join(downloadDir, "main.tf") + if assert.True(t, util.FileExists(expectedFilePath), "For terraform source %v", terraformSource) { + actualFileContents := readFile(t, expectedFilePath) + assert.Equal(t, expectedFileContents, actualFileContents, "For terraform source %v", terraformSource) + } +} + +func testAlreadyHaveLatestCode(t *testing.T, canonicalUrl string, downloadDir string, expected bool) { + terraformSource := &TerraformSource{ + CanonicalSourceURL: parseUrl(t, canonicalUrl), + DownloadDir: downloadDir, + VersionFile: filepath.Join(downloadDir, "version-file.txt"), + } + + actual, err := alreadyHaveLatestCode(terraformSource) + assert.Nil(t, err, "Unexpected error for terraform source %v: %v", err, terraformSource) + assert.Equal(t, expected, actual, "For terraform source %v", terraformSource) +} + +func tmpDir(t *testing.T) string { + dir, err := ioutil.TempDir("", "download-source-test") + if err != nil { + t.Fatal(err) + } + return dir +} + +func absPath(t *testing.T, path string) string { + abs, err := filepath.Abs(path) + if err != nil { + t.Fatal(err) + } + return abs +} + +func parseUrl(t *testing.T, rawUrl string) *url.URL { + parsed, err := url.Parse(rawUrl) + if err != nil { + t.Fatal(err) + } + return parsed +} + +func readFile(t *testing.T, path string) string { + contents, err := util.ReadFileAsString(path) + if err != nil { + t.Fatal(err) + } + return contents +} + +func copyFolder(t *testing.T, src string, dest string) { + err := util.CopyFolderContents(src, dest) + if err != nil { + t.Fatal(err) + } +} \ No newline at end of file diff --git a/test/fixture-download-source/download-dir-version-file-no-query/version-file.txt b/test/fixture-download-source/download-dir-version-file-no-query/version-file.txt new file mode 100644 index 000000000..c39d1108c --- /dev/null +++ b/test/fixture-download-source/download-dir-version-file-no-query/version-file.txt @@ -0,0 +1 @@ +2jmj7l5rSw0yVb_vlWAYkK_YBwk= \ No newline at end of file diff --git a/test/fixture-download-source/download-dir-version-file/version-file.txt b/test/fixture-download-source/download-dir-version-file/version-file.txt new file mode 100644 index 000000000..1e75d115f --- /dev/null +++ b/test/fixture-download-source/download-dir-version-file/version-file.txt @@ -0,0 +1 @@ +zqg_v-R2bnqTbCe-ZO3mHRTRKX0= \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-2/main.tf b/test/fixture-download-source/hello-world-2/main.tf new file mode 100644 index 000000000..e202673c2 --- /dev/null +++ b/test/fixture-download-source/hello-world-2/main.tf @@ -0,0 +1 @@ +# Hello, World 2 \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-2/version-file.txt b/test/fixture-download-source/hello-world-2/version-file.txt new file mode 100644 index 000000000..c39d1108c --- /dev/null +++ b/test/fixture-download-source/hello-world-2/version-file.txt @@ -0,0 +1 @@ +2jmj7l5rSw0yVb_vlWAYkK_YBwk= \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-ref-v0.0.3/main.tf b/test/fixture-download-source/hello-world-ref-v0.0.3/main.tf new file mode 100644 index 000000000..3a372db9e --- /dev/null +++ b/test/fixture-download-source/hello-world-ref-v0.0.3/main.tf @@ -0,0 +1 @@ +# Hello, World \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-ref-v0.0.3/version-file.txt b/test/fixture-download-source/hello-world-ref-v0.0.3/version-file.txt new file mode 100644 index 000000000..897be4706 --- /dev/null +++ b/test/fixture-download-source/hello-world-ref-v0.0.3/version-file.txt @@ -0,0 +1 @@ +Yj43LYqskfxodW8GOrKG9oMCwws= \ No newline at end of file diff --git a/test/fixture-download-source/hello-world/main.tf b/test/fixture-download-source/hello-world/main.tf new file mode 100644 index 000000000..3a372db9e --- /dev/null +++ b/test/fixture-download-source/hello-world/main.tf @@ -0,0 +1 @@ +# Hello, World \ No newline at end of file diff --git a/util/hash.go b/util/hash.go index 6ade1c972..c61df02c2 100644 --- a/util/hash.go +++ b/util/hash.go @@ -6,8 +6,7 @@ import ( ) // Returns the base 64 encoded sha1 hash of the given string -func Base64EncodedSha1(str string) string { +func EncodeBase64Sha1(str string) string { hash := sha1.Sum([]byte(str)) - return base64.StdEncoding.EncodeToString(hash[:]) -} - + return base64.URLEncoding.EncodeToString(hash[:]) +} \ No newline at end of file From ab12052ca656bdbcaf69cd3b39042f636370ed46 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Mon, 30 Jan 2017 01:27:15 +0000 Subject: [PATCH 5/9] Fix download-source test cases --- cli/download_source_test.go | 65 ++++++++++--------- .../hello-world-ref-v0.0.3/main.tf | 1 - .../hello-world-ref-v0.0.3/version-file.txt | 1 - .../hello-world-version-remote/main.tf | 1 + .../version-file.txt | 1 + 5 files changed, 35 insertions(+), 34 deletions(-) delete mode 100644 test/fixture-download-source/hello-world-ref-v0.0.3/main.tf delete mode 100644 test/fixture-download-source/hello-world-ref-v0.0.3/version-file.txt create mode 100644 test/fixture-download-source/hello-world-version-remote/main.tf create mode 100644 test/fixture-download-source/hello-world-version-remote/version-file.txt diff --git a/cli/download_source_test.go b/cli/download_source_test.go index 9b08bb1cf..3601885c9 100644 --- a/cli/download_source_test.go +++ b/cli/download_source_test.go @@ -83,64 +83,65 @@ func TestDownloadTerraformSourceIfNecessaryLocalDirToAlreadyDownloadedDir(t *tes downloadDir := tmpDir(t) defer os.Remove(downloadDir) - copyFolder(t, "../test/fixture-downlooad-source/hello-world-2", downloadDir) + copyFolder(t, "../test/fixture-download-source/hello-world-2", downloadDir) testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") } -func TestDownloadTerraformSourceIfNecessaryRemoteUrlToEmptyDir(t *testing.T) { - t.Parallel() - - // TODO: update URL to master branch once this is merged - canonicalUrl := "https://github.com/gruntwork-io/terragrunt/tree/cache-tmp-folder/test/fixture-download-source/hello-world" - downloadDir := tmpDir(t) - defer os.Remove(downloadDir) - - testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") -} - -func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDir(t *testing.T) { - t.Parallel() - - // TODO: update URL to master branch once this is merged - canonicalUrl := "https://github.com/gruntwork-io/terragrunt/tree/cache-tmp-folder/test/fixture-download-source/hello-world-2" - downloadDir := tmpDir(t) - defer os.Remove(downloadDir) - - copyFolder(t, "../test/fixture-downlooad-source/hello-world-2", downloadDir) - - testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") -} +// TODO re-enable these two tests once this is merged into master, since they are trying to download code from master +//func TestDownloadTerraformSourceIfNecessaryRemoteUrlToEmptyDir(t *testing.T) { +// t.Parallel() +// +// // TODO: update URL to master branch once this is merged +// canonicalUrl := "github.com/gruntwork-io/terragrunt//test/fixture-download-source/hello-world" +// downloadDir := tmpDir(t) +// defer os.Remove(downloadDir) +// +// testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") +//} +// +//func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDir(t *testing.T) { +// t.Parallel() +// +// // TODO: update URL to master branch once this is merged +// canonicalUrl := "github.com/gruntwork-io/terragrunt//test/fixture-download-source/hello-world" +// downloadDir := tmpDir(t) +// defer os.Remove(downloadDir) +// +// copyFolder(t, "../test/fixture-download-source/hello-world-2", downloadDir) +// +// testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World 2") +//} func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDirDifferentVersion(t *testing.T) { t.Parallel() // TODO: update URL to master branch once this is merged - canonicalUrl := "https://github.com/gruntwork-io/terragrunt/tree/cache-tmp-folder/test/fixture-download-source/hello-world-2?ref=v0.0.3" + canonicalUrl := "github.com/gruntwork-io/terragrunt//test/fixture-download-source/hello-world?ref=cache-tmp-folder" downloadDir := tmpDir(t) defer os.Remove(downloadDir) - copyFolder(t, "../test/fixture-downlooad-source/hello-world-2", downloadDir) + copyFolder(t, "../test/fixture-download-source/hello-world-2", downloadDir) - testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World 2") + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") } func TestDownloadTerraformSourceIfNecessaryRemoteUrlToAlreadyDownloadedDirSameVersion(t *testing.T) { t.Parallel() // TODO: update URL to master branch once this is merged - canonicalUrl := "https://github.com/gruntwork-io/terragrunt/tree/cache-tmp-folder/test/fixture-download-source/hello-world-2?ref=v0.0.3" + canonicalUrl := "github.com/gruntwork-io/terragrunt//test/fixture-download-source/hello-world?ref=cache-tmp-folder" downloadDir := tmpDir(t) defer os.Remove(downloadDir) - copyFolder(t, "../test/fixture-downlooad-source/hello-world-ref-v0.0.3", downloadDir) + copyFolder(t, "../test/fixture-download-source/hello-world-version-remote", downloadDir) - testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World") + testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, "# Hello, World version remote") } func testDownloadTerraformSourceIfNecessary(t *testing.T, canonicalUrl string, downloadDir string, expectedFileContents string) { terraformSource := &TerraformSource{ - CanonicalSourceURL: canonicalUrl, + CanonicalSourceURL: parseUrl(t, canonicalUrl), DownloadDir: downloadDir, VersionFile: filepath.Join(downloadDir, "version-file.txt"), } @@ -163,7 +164,7 @@ func testAlreadyHaveLatestCode(t *testing.T, canonicalUrl string, downloadDir st } actual, err := alreadyHaveLatestCode(terraformSource) - assert.Nil(t, err, "Unexpected error for terraform source %v: %v", err, terraformSource) + assert.Nil(t, err, "Unexpected error for terraform source %v: %v", terraformSource, err) assert.Equal(t, expected, actual, "For terraform source %v", terraformSource) } diff --git a/test/fixture-download-source/hello-world-ref-v0.0.3/main.tf b/test/fixture-download-source/hello-world-ref-v0.0.3/main.tf deleted file mode 100644 index 3a372db9e..000000000 --- a/test/fixture-download-source/hello-world-ref-v0.0.3/main.tf +++ /dev/null @@ -1 +0,0 @@ -# Hello, World \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-ref-v0.0.3/version-file.txt b/test/fixture-download-source/hello-world-ref-v0.0.3/version-file.txt deleted file mode 100644 index 897be4706..000000000 --- a/test/fixture-download-source/hello-world-ref-v0.0.3/version-file.txt +++ /dev/null @@ -1 +0,0 @@ -Yj43LYqskfxodW8GOrKG9oMCwws= \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-version-remote/main.tf b/test/fixture-download-source/hello-world-version-remote/main.tf new file mode 100644 index 000000000..2972d5df9 --- /dev/null +++ b/test/fixture-download-source/hello-world-version-remote/main.tf @@ -0,0 +1 @@ +# Hello, World version remote \ No newline at end of file diff --git a/test/fixture-download-source/hello-world-version-remote/version-file.txt b/test/fixture-download-source/hello-world-version-remote/version-file.txt new file mode 100644 index 000000000..a6e2ccf16 --- /dev/null +++ b/test/fixture-download-source/hello-world-version-remote/version-file.txt @@ -0,0 +1 @@ +s7cjjCUtiLre6qopc6Z_GIdMmJ8= \ No newline at end of file From 6de18efb9906293f86a4c5cd0b474807b32d8b1e Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Tue, 31 Jan 2017 23:52:29 +0000 Subject: [PATCH 6/9] Add comments on TerraformSource struct --- cli/download_source.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/download_source.go b/cli/download_source.go index 035893e6c..a9de53e5a 100644 --- a/cli/download_source.go +++ b/cli/download_source.go @@ -14,6 +14,7 @@ import ( "fmt" ) +// This struct represents information about Terraform source code that needs to be downloaded type TerraformSource struct { // A canonical version of RawSource, in URL format CanonicalSourceURL *url.URL From f937e393dfdbb2b18941c3e1dc31fdc54ed4fc4e Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Tue, 31 Jan 2017 23:54:24 +0000 Subject: [PATCH 7/9] Fix typo in download_source comments --- cli/download_source.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/download_source.go b/cli/download_source.go index a9de53e5a..fcb1793b2 100644 --- a/cli/download_source.go +++ b/cli/download_source.go @@ -131,7 +131,7 @@ func writeVersionFile(terraformSource *TerraformSource) error { // 2. W is the base 64 encoded sha1 hash of w. This ensures that if you are running Terragrunt concurrently in // multiple folders (e.g. during automated tests), then even if those folders are using the same source URL s, they // do not overwrite each other. -// 3. S is the base 64 encoded sha1 has of s without its query string. For remote source URLs (e.g. Git +// 3. S is the base 64 encoded sha1 of s without its query string. For remote source URLs (e.g. Git // URLs), this is based on the assumption that the scheme/host/path of the URL // (e.g. git::github.com/foo/bar//some-module) identifies the module name, and we always want to download the same // module name into the same folder (see the encodeSourceName method). We also assume the version of the module is From 15ed21e254a4ef42b63b5d296c5a42ef772e792e Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Wed, 1 Feb 2017 00:02:17 +0000 Subject: [PATCH 8/9] Fix glide.lock --- glide.lock | 142 +++++++---------------------------------------------- 1 file changed, 19 insertions(+), 123 deletions(-) diff --git a/glide.lock b/glide.lock index 3947305bb..fc877b7d5 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,8 @@ hash: b50c74fff3d496405d1bc9dadfb044d56f1557317f22e34c186b8744af9acaa0 -updated: 2017-01-29T17:21:51.671174219Z +updated: 2017-02-01T00:01:23.320944332Z imports: - name: github.com/aws/aws-sdk-go - version: 2ae1f45d01cdc7316a4d6c518c21a6d9d81a4970 + version: 7524cb911daddd6e5c9195def8e59ae892bef8d9 subpackages: - aws - aws/defaults @@ -14,91 +14,30 @@ imports: - service/sts - service/s3 - aws/credentials + - aws/endpoints + - aws/corehandlers - aws/credentials/ec2rolecreds + - aws/credentials/endpointcreds - aws/ec2metadata - - aws/endpoints - - aws/client/metadata - aws/request + - aws/client + - aws/credentials/stscreds + - aws/awsutil + - aws/client/metadata + - aws/signer/v4 + - private/protocol + - private/protocol/jsonrpc + - private/waiter + - private/protocol/query + - private/protocol/restxml - private/protocol/rest - - awsmigrate/awsmigrate-renamer/rename - - private/model/api - - private/util - - service/kms - - service/s3/s3crypto - - service/acm - - service/apigateway - - service/applicationdiscoveryservice - - service/autoscaling - - service/cloudformation - - service/cloudfront - - service/cloudhsm - - service/cloudsearch - - service/cloudtrail - - service/cloudwatch - - service/cloudwatchlogs - - service/codecommit - - service/codedeploy - - service/codepipeline - - service/cognitoidentity - - service/cognitosync - - service/configservice - - service/datapipeline - - service/devicefarm - - service/directconnect - - service/directoryservice - - service/dynamodbstreams - - service/ec2 - - service/ecs - - service/efs - - service/elasticache - - service/elasticbeanstalk - - service/elb - - service/elastictranscoder - - service/emr - - service/elasticsearchservice - - service/glacier - - service/iam - - service/iot - - service/iotdataplane - - service/kinesis - - service/lambda - - service/machinelearning - - service/opsworks - - service/rds - - service/redshift - - service/route53 - - service/route53domains - - service/ses - - service/simpledb - - service/sns - - service/sqs - - service/ssm - - service/storagegateway - - service/support - - service/swf - - service/waf - - service/workspaces - - awstesting/unit - - service/cloudsearchdomain - - service/cloudwatchevents - - service/dynamodb/dynamodbattribute - - service/ecr - - service/firehose - - service/inspector - - service/marketplacecommerceanalytics - - service/mobileanalytics - - service/cloudfront/sign - - service/dynamodb/dynamodbiface - - service/sqs/sqsiface + - private/protocol/json/jsonutil - private/protocol/query/queryutil - private/protocol/xml/xmlutil - - service/s3/s3iface - name: github.com/bgentry/go-netrc version: 9fd32a8b3d3d3f9d43c341bfe098430e07609480 subpackages: - netrc -- name: github.com/BurntSushi/toml - version: 99064174e013895bbd9b025c31100bd1d9b590ca - name: github.com/davecgh/go-spew version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 subpackages: @@ -107,10 +46,6 @@ imports: version: 8fa88b06e5974e97fbf9899a7f86a344bfd1f105 - name: github.com/go-ini/ini version: e3c2d47c61e5333f9aa2974695dd94396eb69c75 -- name: github.com/gucumber/gucumber - version: 71608e2f6e76fd4da5b09a376aeec7a5c0b5edbc - subpackages: - - gherkin - name: github.com/hashicorp/go-getter version: cc80f38c726badeae53775d179755e1c4953d6cf subpackages: @@ -118,14 +53,14 @@ imports: - name: github.com/hashicorp/go-version version: e96d3840402619007766590ecea8dd7af1292276 - name: github.com/hashicorp/hcl - version: db4f0768927a665a06c32855186e1bc762bcc8f5 + version: 88e9565e9965f4054f86c72ff1ad1bb50e560d6f subpackages: - hcl/ast - hcl/parser - hcl/token - json/parser - - hcl/printer - hcl/scanner + - hcl/strconv - json/scanner - json/token - name: github.com/jmespath/go-jmespath @@ -142,49 +77,10 @@ imports: version: d8ed2627bdf02c080bf22230dbb337003b7aba2d subpackages: - difflib -- name: github.com/shiena/ansicolor - version: a422bbe96644373c5753384a59d678f7d261ff10 -- name: github.com/stretchr/objx - version: cbeaeb16a013161a98496fad62933b1d21786672 - name: github.com/stretchr/testify - version: 2402e8e7a02fc811447d11f881aa9746cdc57983 + version: 4d4bfba8f1d1027c4fdbe371823030df51419987 subpackages: - assert - - http - - mock - name: github.com/urfave/cli version: 347a9884a87374d000eec7e6445a34487c1f4a2b -- name: golang.org/x/net - version: f2499483f923065a842d38eb4c7f1927e6fc6e6d - subpackages: - - html - - html/atom -- name: golang.org/x/text - version: ece019dcfd29abcf65d0d1dfe145e8faad097640 - subpackages: - - encoding - - encoding/charmap - - encoding/htmlindex - - transform - - encoding/internal/identifier - - encoding/internal - - encoding/japanese - - encoding/korean - - encoding/simplifiedchinese - - encoding/traditionalchinese - - encoding/unicode - - language - - internal/utf8internal - - runes - - internal/tag -- name: golang.org/x/tools - version: f8ed2e405fdcbb42c55233531ad98721e6d1cb3e - subpackages: - - go/loader - - go/ast/astutil - - go/buildutil -- name: gopkg.in/urfave/cli.v1 - version: 0bdeddeeb0f650497d603c4ad7b20cfe685682f6 -- name: gopkg.in/yaml.v2 - version: 4c78c975fe7c825c6d1466c42be594d1d6f3aba6 devImports: [] From 931215de4b4a61f043ca735767af497ea949a93f Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Wed, 1 Feb 2017 00:04:13 +0000 Subject: [PATCH 9/9] Update comment on DeleteFiles method --- util/file.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/file.go b/util/file.go index 0133f2861..fe2481ba4 100644 --- a/util/file.go +++ b/util/file.go @@ -46,7 +46,8 @@ func CanonicalPaths(paths []string, basePath string) ([]string, error) { return canonicalPaths, nil } -// Delete the given list of files +// Delete the given list of files. Note: this function ONLY deletes files and will return an error if you pass in a +// folder path. func DeleteFiles(files []string) error { for _, file := range files { if err := os.Remove(file); err != nil {