From 81a8735754ff94c5bbd4967e66c0a8e4f7aeae1c Mon Sep 17 00:00:00 2001 From: Luke Kysow <1034429+lkysow@users.noreply.github.com> Date: Thu, 14 Mar 2019 10:40:40 -0500 Subject: [PATCH 1/2] Remove -coverpkg flag to stop OOM issues. CircleCI was getting OOM errors since the go-getter package was included. I'm removing the -coverpkg flag for now. --- .circleci/config.yml | 4 ++-- Makefile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e44e967147..c9ff6fa325 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,11 +7,11 @@ jobs: steps: - checkout - run: make test-coverage - - run: make check-fmt - - run: make check-lint - run: name: post coverage to codecov.io command: bash <(curl -s https://codecov.io/bash) + - run: make check-fmt + - run: make check-lint e2e: working_directory: /go/src/github.com/runatlantis/atlantis docker: diff --git a/Makefile b/Makefile index d2baee6064..512e958031 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ test-all: ## Run tests including integration test-coverage: @mkdir -p .cover - @go test -coverpkg $(PKG_COMMAS) -coverprofile .cover/cover.out $(PKG) + @go test -covermode atomic -coverprofile .cover/cover.out $(PKG) test-coverage-html: @mkdir -p .cover From ca58ebd7e007d3480d6795435c3ab031db49212d Mon Sep 17 00:00:00 2001 From: Luke Kysow <1034429+lkysow@users.noreply.github.com> Date: Wed, 13 Mar 2019 17:53:02 -0500 Subject: [PATCH 2/2] Autodownload tf versions. Add --default-tf-version. This changeset has two features: 1. We now automatically download the version of terraform specified in atlantis.yaml configs if we don't already have that version available locally. 2. Add a new --default-tf-version flag that allows users to set a default version of Terraform that we will also download if it's not on disk. These mean that users don't need to build custom Docker images to just add terraform versions. It also means that upgrading the version of terraform that is packaged with the Atlantis Docker image won't cause issues for existing users because as long as they're running with --default-tf-version, Atlantis will always use that version. --- Makefile | 4 +- cmd/server.go | 15 +- cmd/server_test.go | 14 ++ runatlantis.io/.vuepress/config.js | 5 +- .../docs/atlantis-yaml-reference.md | 2 +- runatlantis.io/docs/deployment.md | 4 +- runatlantis.io/docs/requirements.md | 8 +- runatlantis.io/docs/terraform-versions.md | 20 ++ .../guide/atlantis-yaml-use-cases.md | 11 +- server/events/terraform/terraform_client.go | 219 +++++++++++++++--- .../terraform_client_internal_test.go | 14 +- .../events/terraform/terraform_client_test.go | 208 +++++++++++++++++ server/events_controller_e2e_test.go | 9 +- server/server.go | 11 +- server/user_config.go | 1 + 15 files changed, 473 insertions(+), 72 deletions(-) create mode 100644 runatlantis.io/docs/terraform-versions.md diff --git a/Makefile b/Makefile index 512e958031..8b38ea1e3c 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ go-generate: ## Run go generate in all packages #echo "this doesn't work anymore: go generate \$\$(go list ./... | grep -v e2e | grep -v vendor | grep -v static)" test: ## Run tests - @go test -race -short $(PKG) + @go test -short $(PKG) test-all: ## Run tests including integration @go test $(PKG) @@ -48,7 +48,7 @@ test-coverage: test-coverage-html: @mkdir -p .cover - @go test -coverpkg $(PKG_COMMAS) -coverprofile .cover/cover.out $(PKG) + @go test -covermode atomic -coverprofile .cover/cover.out $(PKG) go tool cover -html .cover/cover.out dist: ## Package up everything in static/ using go-bindata-assetfs so it can be served by a single binary diff --git a/cmd/server.go b/cmd/server.go index fe74365b50..b59f8be428 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -47,6 +47,7 @@ const ( ConfigFlag = "config" CheckoutStrategyFlag = "checkout-strategy" DataDirFlag = "data-dir" + DefaultTFVersionFlag = "default-tf-version" GHHostnameFlag = "gh-hostname" GHTokenFlag = "gh-token" GHUserFlag = "gh-user" @@ -192,6 +193,11 @@ var stringFlags = []stringFlag{ " Only set if using TFE as a backend." + " Should be specified via the ATLANTIS_TFE_TOKEN environment variable for security.", }, + { + name: DefaultTFVersionFlag, + description: "Terraform version to default to (ex. v0.12.0). Will download if not yet on disk." + + " If not set, Atlantis uses the terraform binary in its PATH.", + }, } var boolFlags = []boolFlag{ { @@ -383,10 +389,11 @@ func (s *ServerCmd) run() error { // Config looks good. Start the server. server, err := s.ServerCreator.NewServer(userConfig, server.Config{ - AllowForkPRsFlag: AllowForkPRsFlag, - AllowRepoConfigFlag: AllowRepoConfigFlag, - AtlantisURLFlag: AtlantisURLFlag, - AtlantisVersion: s.AtlantisVersion, + AllowForkPRsFlag: AllowForkPRsFlag, + AllowRepoConfigFlag: AllowRepoConfigFlag, + AtlantisURLFlag: AtlantisURLFlag, + AtlantisVersion: s.AtlantisVersion, + DefaultTFVersionFlag: DefaultTFVersionFlag, }) if err != nil { return errors.Wrap(err, "initializing server") diff --git a/cmd/server_test.go b/cmd/server_test.go index c9396bf373..fb78e641c8 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -341,6 +341,7 @@ func TestExecute_Defaults(t *testing.T) { Equals(t, dataDir, passedConfig.DataDir) Equals(t, "branch", passedConfig.CheckoutStrategy) + Equals(t, "", passedConfig.DefaultTFVersion) Equals(t, "github.com", passedConfig.GithubHostname) Equals(t, "token", passedConfig.GithubToken) Equals(t, "user", passedConfig.GithubUser) @@ -446,6 +447,7 @@ func TestExecute_Flags(t *testing.T) { cmd.BitbucketWebhookSecretFlag: "bitbucket-secret", cmd.CheckoutStrategyFlag: "merge", cmd.DataDirFlag: "/path", + cmd.DefaultTFVersionFlag: "v0.11.0", cmd.GHHostnameFlag: "ghhostname", cmd.GHTokenFlag: "token", cmd.GHUserFlag: "user", @@ -477,6 +479,7 @@ func TestExecute_Flags(t *testing.T) { Equals(t, "bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "merge", passedConfig.CheckoutStrategy) Equals(t, "/path", passedConfig.DataDir) + Equals(t, "v0.11.0", passedConfig.DefaultTFVersion) Equals(t, "ghhostname", passedConfig.GithubHostname) Equals(t, "token", passedConfig.GithubToken) Equals(t, "user", passedConfig.GithubUser) @@ -509,6 +512,7 @@ bitbucket-user: "bitbucket-user" bitbucket-webhook-secret: "bitbucket-secret" checkout-strategy: "merge" data-dir: "/path" +default-tf-version: "v0.11.0" gh-hostname: "ghhostname" gh-token: "token" gh-user: "user" @@ -544,6 +548,7 @@ tfe-token: my-token Equals(t, "bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "merge", passedConfig.CheckoutStrategy) Equals(t, "/path", passedConfig.DataDir) + Equals(t, "v0.11.0", passedConfig.DefaultTFVersion) Equals(t, "ghhostname", passedConfig.GithubHostname) Equals(t, "token", passedConfig.GithubToken) Equals(t, "user", passedConfig.GithubUser) @@ -576,6 +581,7 @@ bitbucket-user: "bitbucket-user" bitbucket-webhook-secret: "bitbucket-secret" checkout-strategy: "merge" data-dir: "/path" +default-tf-version: "v0.11.0" gh-hostname: "ghhostname" gh-token: "token" gh-user: "user" @@ -607,6 +613,7 @@ tfe-token: my-token "BITBUCKET_WEBHOOK_SECRET": "override-bitbucket-secret", "CHECKOUT_STRATEGY": "branch", "DATA_DIR": "/override-path", + "DEFAULT_TF_VERSION": "v0.12.0", "GH_HOSTNAME": "override-gh-hostname", "GH_TOKEN": "override-gh-token", "GH_USER": "override-gh-user", @@ -642,6 +649,7 @@ tfe-token: my-token Equals(t, "override-bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "branch", passedConfig.CheckoutStrategy) Equals(t, "/override-path", passedConfig.DataDir) + Equals(t, "v0.12.0", passedConfig.DefaultTFVersion) Equals(t, "override-gh-hostname", passedConfig.GithubHostname) Equals(t, "override-gh-token", passedConfig.GithubToken) Equals(t, "override-gh-user", passedConfig.GithubUser) @@ -674,6 +682,7 @@ bitbucket-user: "bitbucket-user" bitbucket-webhook-secret: "bitbucket-secret" checkout-strategy: "merge" data-dir: "/path" +default-tf-version: "v0.11.0" gh-hostname: "ghhostname" gh-token: "token" gh-user: "user" @@ -705,6 +714,7 @@ tfe-token: my-token cmd.BitbucketWebhookSecretFlag: "override-bitbucket-secret", cmd.CheckoutStrategyFlag: "branch", cmd.DataDirFlag: "/override-path", + cmd.DefaultTFVersionFlag: "v0.12.0", cmd.GHHostnameFlag: "override-gh-hostname", cmd.GHTokenFlag: "override-gh-token", cmd.GHUserFlag: "override-gh-user", @@ -734,6 +744,7 @@ tfe-token: my-token Equals(t, "override-bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "branch", passedConfig.CheckoutStrategy) Equals(t, "/override-path", passedConfig.DataDir) + Equals(t, "v0.12.0", passedConfig.DefaultTFVersion) Equals(t, "override-gh-hostname", passedConfig.GithubHostname) Equals(t, "override-gh-token", passedConfig.GithubToken) Equals(t, "override-gh-user", passedConfig.GithubUser) @@ -768,6 +779,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) { "BITBUCKET_WEBHOOK_SECRET": "bitbucket-secret", "CHECKOUT_STRATEGY": "merge", "DATA_DIR": "/path", + "DEFAULT_TF_VERSION": "v0.11.0", "GH_HOSTNAME": "gh-hostname", "GH_TOKEN": "gh-token", "GH_USER": "gh-user", @@ -807,6 +819,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) { cmd.BitbucketWebhookSecretFlag: "override-bitbucket-secret", cmd.CheckoutStrategyFlag: "branch", cmd.DataDirFlag: "/override-path", + cmd.DefaultTFVersionFlag: "v0.12.0", cmd.GHHostnameFlag: "override-gh-hostname", cmd.GHTokenFlag: "override-gh-token", cmd.GHUserFlag: "override-gh-user", @@ -838,6 +851,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) { Equals(t, "override-bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "branch", passedConfig.CheckoutStrategy) Equals(t, "/override-path", passedConfig.DataDir) + Equals(t, "v0.12.0", passedConfig.DefaultTFVersion) Equals(t, "override-gh-hostname", passedConfig.GithubHostname) Equals(t, "override-gh-token", passedConfig.GithubToken) Equals(t, "override-gh-user", passedConfig.GithubUser) diff --git a/runatlantis.io/.vuepress/config.js b/runatlantis.io/.vuepress/config.js index cc0a703bb8..377f198840 100644 --- a/runatlantis.io/.vuepress/config.js +++ b/runatlantis.io/.vuepress/config.js @@ -71,7 +71,9 @@ module.exports = { ['customizing-atlantis', 'Overview'], 'atlantis-yaml-reference', 'upgrading-atlantis-yaml-to-version-2', - 'apply-requirements' + 'apply-requirements', + 'checkout-strategy', + 'terraform-versions' ] }, { @@ -82,7 +84,6 @@ module.exports = { 'locking', 'autoplanning', 'automerging', - 'checkout-strategy', 'security' ] } diff --git a/runatlantis.io/docs/atlantis-yaml-reference.md b/runatlantis.io/docs/atlantis-yaml-reference.md index 125a0d0c55..f188c8e0eb 100644 --- a/runatlantis.io/docs/atlantis-yaml-reference.md +++ b/runatlantis.io/docs/atlantis-yaml-reference.md @@ -96,7 +96,7 @@ workflow: myworkflow | dir | string | none | yes | The directory of this project relative to the repo root. Use `.` for the root. For example if the project was under `./project1` then use `project1` | | workspace | string | default | no | The [Terraform workspace](https://www.terraform.io/docs/state/workspaces.html) for this project. Atlantis will switch to this workplace when planning/applying and will create it if it doesn't exist. | | autoplan | [Autoplan](atlantis-yaml-reference.html#autoplan) | none | no | A custom autoplan configuration. If not specified, will use the default algorithm. See [Autoplanning](autoplanning.html). | -| terraform_version | string | none | no | A specific Terraform version to use when running commands for this project. Requires there to be a binary in the Atlantis `PATH` with the name `terraform{VERSION}`, ex. `terraform0.11.0` | +| terraform_version | string | none | no | A specific Terraform version to use when running commands for this project. Must be [Semver compatible](https://semver.org/), ex. `v0.11.0`, `0.12.0-beta1`. | | apply_requirements | array[string] | [] | no | Requirements that must be satisfied before `atlantis apply` can be run. Currently the only supported requirements are `approved` and `mergeable`. See [Apply Requirements](apply-requirements.html) for more details. | | workflow | string | none | no | A custom workflow. If not specified, Atlantis will use its default workflow. | diff --git a/runatlantis.io/docs/deployment.md b/runatlantis.io/docs/deployment.md index 97eb052720..a5852819fa 100644 --- a/runatlantis.io/docs/deployment.md +++ b/runatlantis.io/docs/deployment.md @@ -383,14 +383,14 @@ Once you're done, see [Next Steps](#next-steps). Atlantis has an [official](https://hub.docker.com/r/runatlantis/atlantis/) Docker image: `runatlantis/atlantis`. #### Customization -If you need to modify the Docker image that we provide, for instance to add a specific version of Terraform, you can do something like this: +If you need to modify the Docker image that we provide, for instance to add the terragrunt binary, you can do something like this: 1. Create a custom docker file ```dockerfile FROM runatlantis/atlantis:{latest version} # copy a terraform binary of the version you need - COPY terraform /usr/local/bin/terraform + COPY terragrunt /usr/local/bin/terrgrunt ``` 1. Build your Docker image diff --git a/runatlantis.io/docs/requirements.md b/runatlantis.io/docs/requirements.md index e0de6a4c8f..14a6a1e33d 100644 --- a/runatlantis.io/docs/requirements.md +++ b/runatlantis.io/docs/requirements.md @@ -83,12 +83,8 @@ an `atlantis.yaml` file to tell it to use `-var-file={YOUR_FILE}`. See [atlantis.yaml Use Cases](/guide/atlantis-yaml-use-cases.html#using-tfvars-files) for more details. ## Terraform Versions -By default, Atlantis will use the `terraform` executable that is in its path. -To use a specific version of Terraform: -1. Install the desired version of Terraform into the `$PATH` of where Atlantis is - running and name it `terraform{version}`, ex. `terraform0.8.8`. -2. Create an `atlantis.yaml` file for your repo and set the `terraform_version` key. -See [atlantis.yaml Use Cases](/guide/atlantis-yaml-use-cases.html#terraform-versions) for more details. +Atlantis supports all Terraform versions (including 0.12) and can be configured +to use different versions for different repositories/projects. See [Terraform Versions](/docs/terraform-versions.html)l ## Next Steps * If your Terraform setup meets the Atlantis requirements, head back to our [Installation Guide](installation-guide.html) to get started diff --git a/runatlantis.io/docs/terraform-versions.md b/runatlantis.io/docs/terraform-versions.md new file mode 100644 index 0000000000..bad7a15ab3 --- /dev/null +++ b/runatlantis.io/docs/terraform-versions.md @@ -0,0 +1,20 @@ +# Terraform Versions + +You can customize which version of Terraform Atlantis defaults to by setting +the `--default-tf-version` flag (ex. `--default-tf-version=v0.12.0`). + +If you wish to use a different version than the default for a specific repo or project, you need +to create an `atlantis.yaml` file and set the `terraform_version` key: +```yaml +version: 2 +projects: +- dir: . + terraform_version: v0.10.5 +``` +See [atlantis.yaml Use Cases](/guide/atlantis-yaml-use-cases.html#terraform-versions) for more details. + +::: tip NOTE +Atlantis will automatically download the version specified. +::: + + diff --git a/runatlantis.io/guide/atlantis-yaml-use-cases.md b/runatlantis.io/guide/atlantis-yaml-use-cases.md index 522916d441..f3e96e1072 100644 --- a/runatlantis.io/guide/atlantis-yaml-use-cases.md +++ b/runatlantis.io/guide/atlantis-yaml-use-cases.md @@ -222,6 +222,11 @@ workflows: - run: terragrunt apply -no-color $PLANFILE ``` +::: warning +Atlantis will need to have the `terragrunt` binary in its PATH. +If you're using Docker you can build your own image, see [Customization](/docs/deployment.html#customization). +::: + ## Running custom commands Atlantis supports running custom commands. In this example, we want to run a script after every `apply`: @@ -255,7 +260,7 @@ isn't set, Atlantis will use the default plan workflow which is what we want in ## Terraform Versions If you'd like to use a different version of Terraform than what is in Atlantis' -`PATH` then set the `terraform_version` key: +`PATH` or is set by the `--default-tf-version` flag, then set the `terraform_version` key: ```yaml version: 2 @@ -264,9 +269,7 @@ projects: terraform_version: 0.10.0 ``` -Atlantis will then execute all Terraform commands with `terraform0.10.0` instead -of `terraform`. This requires that the 0.10.0 binary is in Atlantis's `PATH` with the -name `terraform0.10.0`. +Atlantis will automatically download and use this version. ## Requiring Approvals For Production In this example, we only want to require `apply` approvals for the `production` directory. diff --git a/server/events/terraform/terraform_client.go b/server/events/terraform/terraform_client.go index bb01d10fbe..8890935b6d 100644 --- a/server/events/terraform/terraform_client.go +++ b/server/events/terraform/terraform_client.go @@ -17,6 +17,7 @@ package terraform import ( "bufio" "fmt" + "github.com/hashicorp/go-getter" "github.com/hashicorp/go-version" "github.com/mitchellh/go-homedir" "github.com/pkg/errors" @@ -27,6 +28,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "strings" "sync" ) @@ -39,15 +41,42 @@ type Client interface { } type DefaultClient struct { + // defaultVersion is the default version of terraform to use if another + // version isn't specified. defaultVersion *version.Version terraformPluginCacheDir string - // tfExecutableName is the name of the default terraform binary. - // This should always be set to "terraform" by the NewClient constructor - // however it can be overridden during testing. - tfExecutableName string + binDir string + // overrideTF can be used to override the terraform binary during testing + // with another binary, ex. echo. + overrideTF string + // downloader downloads terraform versions. + downloader Downloader + // versions maps from the string representation of a tf version (ex. 0.11.10) + // to the absolute path of that binary on disk (if it exists). + // Use versionsLock to control access. + versions map[string]string + + // versionsLock is used to ensure versions isn't being concurrently written to. + versionsLock *sync.Mutex +} + +//go:generate pegomock generate -m --use-experimental-model-gen --package mocks -o mocks/mock_downloader.go Downloader + +// Downloader is for downloading terraform versions. +type Downloader interface { + GetFile(dst, src string, opts ...getter.ClientOption) error } -const terraformPluginCacheDirName = "plugin-cache" +const ( + // terraformPluginCacheDir is the name of the dir inside our data dir + // where we tell terraform to cache plugins and modules. + terraformPluginCacheDirName = "plugin-cache" + // binDirName is the name of the directory inside our data dir where + // we download terraform binaries. + binDirName = "bin" + // releasesURL is the base url to download terraform from. + releasesURL = "https://releases.hashicorp.com" +) // versionRegex extracts the version from `terraform version` output. // Terraform v0.12.0-alpha4 (2c36829d3265661d8edbd5014de8090ea7e2a076) @@ -57,24 +86,58 @@ const terraformPluginCacheDirName = "plugin-cache" // => 0.11.10 var versionRegex = regexp.MustCompile("Terraform v(.*?)(\\s.*)?\n") -func NewClient(dataDir string, tfeToken string) (*DefaultClient, error) { - _, err := exec.LookPath("terraform") - if err != nil { - return nil, errors.New("terraform not found in $PATH. \n\nDownload terraform from https://www.terraform.io/downloads.html") +// NewClient constructs a terraform client. +// tfeToken is an optional terraform enterprise token. +// defaultVersionStr is an optional default terraform version to use unless +// a specific version is set. +// defaultVersionFlagName is the name of the flag that sets the default terraform +// version. +// tfDownloader is used to download terraform versions. +// Will asynchronously download the required version if it doesn't exist already. +func NewClient(log *logging.SimpleLogger, dataDir string, tfeToken string, defaultVersionStr string, defaultVersionFlagName string, tfDownloader Downloader) (*DefaultClient, error) { + var finalDefaultVersion *version.Version + var localVersion *version.Version + versions := make(map[string]string) + var versionsLock sync.Mutex + + localPath, err := exec.LookPath("terraform") + if err != nil && defaultVersionStr == "" { + return nil, fmt.Errorf("terraform not found in $PATH. Set --%s or download terraform from https://www.terraform.io/downloads.html", defaultVersionFlagName) } - versionOutBytes, err := exec.Command("terraform", "version"). - Output() // #nosec - versionOutput := string(versionOutBytes) - if err != nil { - return nil, errors.Wrapf(err, "running terraform version: %s", versionOutput) + if err == nil { + localVersion, err = getVersion(localPath) + if err != nil { + return nil, err + } + versions[localVersion.String()] = localPath + if defaultVersionStr == "" { + // If they haven't set a default version, then whatever they had + // locally is now the default. + finalDefaultVersion = localVersion + } } - match := versionRegex.FindStringSubmatch(versionOutput) - if len(match) <= 1 { - return nil, fmt.Errorf("could not parse terraform version from %s", versionOutput) + + binDir := filepath.Join(dataDir, binDirName) + if err := os.MkdirAll(binDir, 0700); err != nil { + return nil, errors.Wrapf(err, "unable to create terraform bin dir %q", binDir) } - v, err := version.NewVersion(match[1]) - if err != nil { - return nil, errors.Wrap(err, "parsing terraform version") + + if defaultVersionStr != "" { + defaultVersion, err := version.NewVersion(defaultVersionStr) + if err != nil { + return nil, err + } + finalDefaultVersion = defaultVersion + go func() { + // Since ensureVersion might end up downloading terraform, + // we call it asynchronously so as to not delay server startup. + versionsLock.Lock() + _, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir) + versionsLock.Unlock() + if err != nil { + log.Err("could not download terraform %s", defaultVersion.String()) + } + }() } // If tfeToken is set, we try to create a ~/.terraformrc file. @@ -96,9 +159,12 @@ func NewClient(dataDir string, tfeToken string) (*DefaultClient, error) { } return &DefaultClient{ - defaultVersion: v, + defaultVersion: finalDefaultVersion, terraformPluginCacheDir: cacheDir, - tfExecutableName: "terraform", + binDir: binDir, + downloader: tfDownloader, + versionsLock: &versionsLock, + versions: versions, }, nil } @@ -113,7 +179,10 @@ func (c *DefaultClient) Version() *version.Version { // Workspace is the terraform workspace to run in. We won't switch workspaces, // just set a WORKSPACE environment variable. func (c *DefaultClient) RunCommandWithVersion(log *logging.SimpleLogger, path string, args []string, v *version.Version, workspace string) (string, error) { - tfCmd, cmd := c.prepCmd(v, workspace, path, args) + tfCmd, cmd, err := c.prepCmd(log, v, workspace, path, args) + if err != nil { + return "", err + } out, err := cmd.CombinedOutput() if err != nil { err = errors.Wrapf(err, "running %q in %q", tfCmd, path) @@ -124,14 +193,28 @@ func (c *DefaultClient) RunCommandWithVersion(log *logging.SimpleLogger, path st return string(out), nil } -func (c *DefaultClient) prepCmd(v *version.Version, workspace string, path string, args []string) (string, *exec.Cmd) { - tfExecutable := c.tfExecutableName - tfVersionStr := c.defaultVersion.String() - // if version is the same as the default, don't need to prepend the version name to the executable - if v != nil && !v.Equal(c.defaultVersion) { - tfExecutable = fmt.Sprintf("%s%s", tfExecutable, v.String()) - tfVersionStr = v.String() +// prepCmd builds a ready to execute command based on the version of terraform +// v, and args. It returns a printable representation of the command that will +// be run and the actual command. +func (c *DefaultClient) prepCmd(log *logging.SimpleLogger, v *version.Version, workspace string, path string, args []string) (string, *exec.Cmd, error) { + if v == nil { + v = c.defaultVersion } + + var binPath string + if c.overrideTF != "" { + // This is only set during testing. + binPath = c.overrideTF + } else { + var err error + c.versionsLock.Lock() + binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir) + c.versionsLock.Unlock() + if err != nil { + return "", nil, err + } + } + // We add custom variables so that if `extra_args` is specified with env // vars then they'll be substituted. envVars := []string{ @@ -140,18 +223,17 @@ func (c *DefaultClient) prepCmd(v *version.Version, workspace string, path strin // Cache plugins so terraform init runs faster. fmt.Sprintf("TF_PLUGIN_CACHE_DIR=%s", c.terraformPluginCacheDir), fmt.Sprintf("WORKSPACE=%s", workspace), - fmt.Sprintf("ATLANTIS_TERRAFORM_VERSION=%s", tfVersionStr), + fmt.Sprintf("ATLANTIS_TERRAFORM_VERSION=%s", v.String()), fmt.Sprintf("DIR=%s", path), } - // Append current Atlantis process's environment variables so PATH is - // preserved and any vars that users purposely exec'd Atlantis with. + // Append current Atlantis process's environment variables, ex. + // AWS_ACCESS_KEY. envVars = append(envVars, os.Environ()...) - // append terraform executable name with args - tfCmd := fmt.Sprintf("%s %s", tfExecutable, strings.Join(args, " ")) + tfCmd := fmt.Sprintf("%s %s", binPath, strings.Join(args, " ")) cmd := exec.Command("sh", "-c", tfCmd) cmd.Dir = path cmd.Env = envVars - return tfCmd, cmd + return tfCmd, cmd, nil } // Line represents a line that was output from a terraform command. @@ -182,13 +264,18 @@ func (c *DefaultClient) RunCommandAsync(log *logging.SimpleLogger, path string, close(inCh) }() - tfCmd, cmd := c.prepCmd(v, workspace, path, args) + tfCmd, cmd, err := c.prepCmd(log, v, workspace, path, args) + if err != nil { + log.Err(err.Error()) + outCh <- Line{Err: err} + return + } stdout, _ := cmd.StdoutPipe() stderr, _ := cmd.StderrPipe() stdin, _ := cmd.StdinPipe() log.Debug("starting %q in %q", tfCmd, path) - err := cmd.Start() + err = cmd.Start() if err != nil { err = errors.Wrapf(err, "running %q in %q", tfCmd, path) log.Err(err.Error()) @@ -259,6 +346,43 @@ func MustConstraint(v string) version.Constraints { return c } +// ensureVersion returns the path to a terraform binary of version v. +// It will download this version if we don't have it. +func ensureVersion(log *logging.SimpleLogger, dl Downloader, versions map[string]string, v *version.Version, binDir string) (string, error) { + if binPath, ok := versions[v.String()]; ok { + return binPath, nil + } + + // This tf version might not yet be in the versions map even though it + // exists on disk. This would happen if users have manually added + // terraform{version} binaries. In this case we don't want to re-download. + binFile := "terraform" + v.String() + if binPath, err := exec.LookPath(binFile); err == nil { + versions[v.String()] = binPath + return binPath, nil + } + + // The version might also not be in the versions map if it's in our bin dir. + // This could happen if Atlantis was restarted without losing its disk. + dest := filepath.Join(binDir, binFile) + if _, err := os.Stat(dest); err == nil { + versions[v.String()] = dest + return dest, nil + } + + log.Info("could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, releasesURL) + urlPrefix := fmt.Sprintf("%s/terraform/%s/terraform_%s", releasesURL, v.String(), v.String()) + binURL := fmt.Sprintf("%s_%s_%s.zip", urlPrefix, runtime.GOOS, runtime.GOARCH) + checksumURL := fmt.Sprintf("%s_SHA256SUMS", urlPrefix) + if err := dl.GetFile(dest, fmt.Sprintf("%s?checksum=file:%s", binURL, checksumURL)); err != nil { + return "", errors.Wrapf(err, "downloading terraform version %s", v.String()) + } + + log.Info("downloaded terraform %s to %s", v.String(), dest) + versions[v.String()] = dest + return dest, nil +} + // generateRCFile generates a .terraformrc file containing config for tfeToken. // It will create the file in home/.terraformrc. func generateRCFile(tfeToken string, home string) error { @@ -288,9 +412,28 @@ func generateRCFile(tfeToken string, home string) error { return nil } +func getVersion(tfBinary string) (*version.Version, error) { + versionOutBytes, err := exec.Command(tfBinary, "version").Output() // #nosec + versionOutput := string(versionOutBytes) + if err != nil { + return nil, errors.Wrapf(err, "running terraform version: %s", versionOutput) + } + match := versionRegex.FindStringSubmatch(versionOutput) + if len(match) <= 1 { + return nil, fmt.Errorf("could not parse terraform version from %s", versionOutput) + } + return version.NewVersion(match[1]) +} + // rcFileContents is a format string to be used with Sprintf that can be used // to generate the contents of a ~/.terraformrc file for authenticating with // Terraform Enterprise. var rcFileContents = `credentials "app.terraform.io" { token = %q }` + +type DefaultDownloader struct{} + +func (d *DefaultDownloader) GetFile(dst, src string, opts ...getter.ClientOption) error { + return getter.GetFile(dst, src, opts...) +} diff --git a/server/events/terraform/terraform_client_internal_test.go b/server/events/terraform/terraform_client_internal_test.go index 495c73e5c8..94a14e131d 100644 --- a/server/events/terraform/terraform_client_internal_test.go +++ b/server/events/terraform/terraform_client_internal_test.go @@ -93,7 +93,7 @@ func TestDefaultClient_RunCommandWithVersion_EnvVars(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } args := []string{ @@ -118,7 +118,7 @@ func TestDefaultClient_RunCommandWithVersion_Error(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } args := []string{ @@ -142,7 +142,7 @@ func TestDefaultClient_RunCommandAsync_Success(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } args := []string{ @@ -168,7 +168,7 @@ func TestDefaultClient_RunCommandAsync_BigOutput(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "cat", + overrideTF: "cat", } filename := filepath.Join(tmp, "data") f, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) @@ -196,7 +196,7 @@ func TestDefaultClient_RunCommandAsync_StderrOutput(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } log := logging.NewSimpleLogger("test", false, logging.Debug) _, outCh := client.RunCommandAsync(log, tmp, []string{"stderr", ">&2"}, nil, "workspace") @@ -214,7 +214,7 @@ func TestDefaultClient_RunCommandAsync_ExitOne(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } log := logging.NewSimpleLogger("test", false, logging.Debug) _, outCh := client.RunCommandAsync(log, tmp, []string{"dying", "&&", "exit", "1"}, nil, "workspace") @@ -233,7 +233,7 @@ func TestDefaultClient_RunCommandAsync_Input(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "read", + overrideTF: "read", } log := logging.NewSimpleLogger("test", false, logging.Debug) inCh, outCh := client.RunCommandAsync(log, tmp, []string{"a", "&&", "echo", "$a"}, nil, "workspace") diff --git a/server/events/terraform/terraform_client_test.go b/server/events/terraform/terraform_client_test.go index 00126ad493..6c0696271e 100644 --- a/server/events/terraform/terraform_client_test.go +++ b/server/events/terraform/terraform_client_test.go @@ -14,9 +14,19 @@ package terraform_test import ( + "fmt" + "github.com/petergtz/pegomock" + "github.com/runatlantis/atlantis/cmd" + "github.com/runatlantis/atlantis/server/events/terraform/mocks" + "io/ioutil" + "os" + "path/filepath" + "runtime" "testing" + "time" "github.com/hashicorp/go-version" + . "github.com/petergtz/pegomock" "github.com/runatlantis/atlantis/server/events/terraform" . "github.com/runatlantis/atlantis/testing" ) @@ -39,3 +49,201 @@ func TestMustConstraint(t *testing.T) { Ok(t, err) Equals(t, expectedConstraint.String(), c.String()) } + +// Test that if terraform is in path and we're not setting the default-tf flag, +// that we use that version as our default version. +func TestNewClient_LocalTFOnly(t *testing.T) { + fakeBinOut := `Terraform v0.11.10 + +Your version of Terraform is out of date! The latest version +is 0.11.13. You can update by downloading from www.terraform.io/downloads.html +` + tmp, cleanup := TempDir(t) + defer cleanup() + + // We're testing this by adding our own "fake" terraform binary to path that + // outputs what would normally come from terraform version. + err := ioutil.WriteFile(filepath.Join(tmp, "terraform"), []byte(fmt.Sprintf("#!/bin/sh\necho '%s'", fakeBinOut)), 0755) + Ok(t, err) + defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() + + c, err := terraform.NewClient(nil, tmp, "", "", cmd.DefaultTFVersionFlag, nil) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, fakeBinOut+"\n", output) +} + +// Test that if terraform is in path and the default-tf flag is set to the +// same version that we don't download anything. +func TestNewClient_LocalTFMatchesFlag(t *testing.T) { + fakeBinOut := `Terraform v0.11.10 + +Your version of Terraform is out of date! The latest version +is 0.11.13. You can update by downloading from www.terraform.io/downloads.html +` + tmp, cleanup := TempDir(t) + defer cleanup() + + // We're testing this by adding our own "fake" terraform binary to path that + // outputs what would normally come from terraform version. + err := ioutil.WriteFile(filepath.Join(tmp, "terraform"), []byte(fmt.Sprintf("#!/bin/sh\necho '%s'", fakeBinOut)), 0755) + Ok(t, err) + defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() + + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, nil) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, fakeBinOut+"\n", output) +} + +// Test that if terraform is not in PATH and we didn't set the default-tf flag +// that we error. +func TestNewClient_NoTF(t *testing.T) { + tmp, cleanup := TempDir(t) + defer cleanup() + + // Set PATH to only include our empty directory. + defer tempSetEnv(t, "PATH", tmp)() + + _, err := terraform.NewClient(nil, tmp, "", "", cmd.DefaultTFVersionFlag, nil) + ErrEquals(t, "terraform not found in $PATH. Set --default-tf-version or download terraform from https://www.terraform.io/downloads.html", err) +} + +// Test that if the default-tf flag is set and that binary is in our PATH +// that we use it. +func TestNewClient_DefaultTFFlagInPath(t *testing.T) { + fakeBinOut := "Terraform v0.11.10\n" + tmp, cleanup := TempDir(t) + defer cleanup() + + // We're testing this by adding our own "fake" terraform binary to path that + // outputs what would normally come from terraform version. + err := ioutil.WriteFile(filepath.Join(tmp, "terraform0.11.10"), []byte(fmt.Sprintf("#!/bin/sh\necho '%s'", fakeBinOut)), 0755) + Ok(t, err) + defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() + + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, nil) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, fakeBinOut+"\n", output) +} + +// Test that if the default-tf flag is set and that binary is in our download +// bin dir that we use it. +func TestNewClient_DefaultTFFlagInBinDir(t *testing.T) { + fakeBinOut := "Terraform v0.11.10\n" + tmp, cleanup := TempDir(t) + defer cleanup() + + // Add our fake binary to {datadir}/bin/terraform{version}. + Ok(t, os.Mkdir(filepath.Join(tmp, "bin"), 0700)) + err := ioutil.WriteFile(filepath.Join(tmp, "bin", "terraform0.11.10"), []byte(fmt.Sprintf("#!/bin/sh\necho '%s'", fakeBinOut)), 0755) + Ok(t, err) + defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() + + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, nil) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, fakeBinOut+"\n", output) +} + +// Test that if we don't have that version of TF that we download it. +func TestNewClient_DefaultTFFlagDownload(t *testing.T) { + RegisterMockTestingT(t) + tmp, cleanup := TempDir(t) + defer cleanup() + + // Set PATH to empty so there's no TF available. + orig := os.Getenv("PATH") + defer tempSetEnv(t, "PATH", "")() + + mockDownloader := mocks.NewMockDownloader() + When(mockDownloader.GetFile(AnyString(), AnyString())).Then(func(params []pegomock.Param) pegomock.ReturnValues { + err := ioutil.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v0.11.10\n'"), 0755) + return []pegomock.ReturnValue{err} + }) + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + baseURL := "https://releases.hashicorp.com/terraform/0.11.10" + expURL := fmt.Sprintf("%s/terraform_0.11.10_%s_%s.zip?checksum=file:%s/terraform_0.11.10_SHA256SUMS", + baseURL, + runtime.GOOS, + runtime.GOARCH, + baseURL) + mockDownloader.VerifyWasCalledEventually(Once(), 2*time.Second).GetFile(filepath.Join(tmp, "bin", "terraform0.11.10"), expURL) + + // Reset PATH so that it has sh. + Ok(t, os.Setenv("PATH", orig)) + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, "\nTerraform v0.11.10\n\n", output) +} + +// Test that we get an error if the terraform version flag is malformed. +func TestNewClient_BadVersion(t *testing.T) { + tmp, cleanup := TempDir(t) + defer cleanup() + _, err := terraform.NewClient(nil, tmp, "", "malformed", cmd.DefaultTFVersionFlag, nil) + ErrEquals(t, "Malformed version: malformed", err) +} + +// Test that if we run a command with a version we don't have, we download it. +func TestRunCommandWithVersion_DLsTF(t *testing.T) { + RegisterMockTestingT(t) + tmp, cleanup := TempDir(t) + defer cleanup() + + mockDownloader := mocks.NewMockDownloader() + // Set up our mock downloader to write a fake tf binary when it's called. + baseURL := "https://releases.hashicorp.com/terraform/0.12.0" + expURL := fmt.Sprintf("%s/terraform_0.12.0_%s_%s.zip?checksum=file:%s/terraform_0.12.0_SHA256SUMS", + baseURL, + runtime.GOOS, + runtime.GOARCH, + baseURL) + When(mockDownloader.GetFile(filepath.Join(tmp, "bin", "terraform0.12.0"), expURL)).Then(func(params []pegomock.Param) pegomock.ReturnValues { + err := ioutil.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v0.12.0\n'"), 0755) + return []pegomock.ReturnValue{err} + }) + + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader) + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + v, err := version.NewVersion("0.12.0") + Ok(t, err) + output, err := c.RunCommandWithVersion(nil, tmp, nil, v, "") + Assert(t, err == nil, "err: %s: %s", err, output) + Equals(t, "\nTerraform v0.12.0\n\n", output) +} + +// tempSetEnv sets env var key to value. It returns a function that when called +// will reset the env var to its original value. +func tempSetEnv(t *testing.T, key string, value string) func() { + orig := os.Getenv(key) + Ok(t, os.Setenv(key, value)) + return func() { os.Setenv(key, orig) } +} diff --git a/server/events_controller_e2e_test.go b/server/events_controller_e2e_test.go index df42eb3d95..6403611475 100644 --- a/server/events_controller_e2e_test.go +++ b/server/events_controller_e2e_test.go @@ -3,6 +3,7 @@ package server_test import ( "bytes" "fmt" + "github.com/hashicorp/go-getter" "github.com/runatlantis/atlantis/server/events/db" "io/ioutil" "net/http" @@ -31,6 +32,12 @@ import ( . "github.com/runatlantis/atlantis/testing" ) +type NoopTFDownloader struct{} + +func (m *NoopTFDownloader) GetFile(dst, src string, opts ...getter.ClientOption) error { + return nil +} + func TestGitHubWorkflow(t *testing.T) { if testing.Short() { t.SkipNow() @@ -371,7 +378,7 @@ func setupE2E(t *testing.T) (server.EventsController, *vcsmocks.MockClient, *moc GithubUser: "github-user", GitlabUser: "gitlab-user", } - terraformClient, err := terraform.NewClient(dataDir, "") + terraformClient, err := terraform.NewClient(logger, dataDir, "", "", "default-tf-version", &NoopTFDownloader{}) Ok(t, err) boltdb, err := db.New(dataDir) Ok(t, err) diff --git a/server/server.go b/server/server.go index d0e4ec57e1..686673e104 100644 --- a/server/server.go +++ b/server/server.go @@ -79,10 +79,11 @@ type Server struct { // Config holds config for server that isn't passed in by the user. type Config struct { - AllowForkPRsFlag string - AllowRepoConfigFlag string - AtlantisURLFlag string - AtlantisVersion string + AllowForkPRsFlag string + AllowRepoConfigFlag string + AtlantisURLFlag string + AtlantisVersion string + DefaultTFVersionFlag string } // WebhookConfig is nested within UserConfig. It's used to configure webhooks. @@ -165,7 +166,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { } vcsClient := vcs.NewClientProxy(githubClient, gitlabClient, bitbucketCloudClient, bitbucketServerClient) commitStatusUpdater := &events.DefaultCommitStatusUpdater{Client: vcsClient} - terraformClient, err := terraform.NewClient(userConfig.DataDir, userConfig.TFEToken) + terraformClient, err := terraform.NewClient(logger, userConfig.DataDir, userConfig.TFEToken, userConfig.DefaultTFVersion, config.DefaultTFVersionFlag, &terraform.DefaultDownloader{}) // The flag.Lookup call is to detect if we're running in a unit test. If we // are, then we don't error out because we don't have/want terraform // installed on our CI system where the unit tests run. diff --git a/server/user_config.go b/server/user_config.go index 83d4089e13..b188751746 100644 --- a/server/user_config.go +++ b/server/user_config.go @@ -38,6 +38,7 @@ type UserConfig struct { SSLCertFile string `mapstructure:"ssl-cert-file"` SSLKeyFile string `mapstructure:"ssl-key-file"` TFEToken string `mapstructure:"tfe-token"` + DefaultTFVersion string `mapstructure:"default-tf-version"` Webhooks []WebhookConfig `mapstructure:"webhooks"` }