diff --git a/.circleci/config.yml b/.circleci/config.yml index e44e967147..c9ff6fa325 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,11 +7,11 @@ jobs: steps: - checkout - run: make test-coverage - - run: make check-fmt - - run: make check-lint - run: name: post coverage to codecov.io command: bash <(curl -s https://codecov.io/bash) + - run: make check-fmt + - run: make check-lint e2e: working_directory: /go/src/github.com/runatlantis/atlantis docker: diff --git a/Makefile b/Makefile index d2baee6064..8b38ea1e3c 100644 --- a/Makefile +++ b/Makefile @@ -37,18 +37,18 @@ go-generate: ## Run go generate in all packages #echo "this doesn't work anymore: go generate \$\$(go list ./... | grep -v e2e | grep -v vendor | grep -v static)" test: ## Run tests - @go test -race -short $(PKG) + @go test -short $(PKG) test-all: ## Run tests including integration @go test $(PKG) test-coverage: @mkdir -p .cover - @go test -coverpkg $(PKG_COMMAS) -coverprofile .cover/cover.out $(PKG) + @go test -covermode atomic -coverprofile .cover/cover.out $(PKG) test-coverage-html: @mkdir -p .cover - @go test -coverpkg $(PKG_COMMAS) -coverprofile .cover/cover.out $(PKG) + @go test -covermode atomic -coverprofile .cover/cover.out $(PKG) go tool cover -html .cover/cover.out dist: ## Package up everything in static/ using go-bindata-assetfs so it can be served by a single binary diff --git a/cmd/server.go b/cmd/server.go index fe74365b50..b59f8be428 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -47,6 +47,7 @@ const ( ConfigFlag = "config" CheckoutStrategyFlag = "checkout-strategy" DataDirFlag = "data-dir" + DefaultTFVersionFlag = "default-tf-version" GHHostnameFlag = "gh-hostname" GHTokenFlag = "gh-token" GHUserFlag = "gh-user" @@ -192,6 +193,11 @@ var stringFlags = []stringFlag{ " Only set if using TFE as a backend." + " Should be specified via the ATLANTIS_TFE_TOKEN environment variable for security.", }, + { + name: DefaultTFVersionFlag, + description: "Terraform version to default to (ex. v0.12.0). Will download if not yet on disk." + + " If not set, Atlantis uses the terraform binary in its PATH.", + }, } var boolFlags = []boolFlag{ { @@ -383,10 +389,11 @@ func (s *ServerCmd) run() error { // Config looks good. Start the server. server, err := s.ServerCreator.NewServer(userConfig, server.Config{ - AllowForkPRsFlag: AllowForkPRsFlag, - AllowRepoConfigFlag: AllowRepoConfigFlag, - AtlantisURLFlag: AtlantisURLFlag, - AtlantisVersion: s.AtlantisVersion, + AllowForkPRsFlag: AllowForkPRsFlag, + AllowRepoConfigFlag: AllowRepoConfigFlag, + AtlantisURLFlag: AtlantisURLFlag, + AtlantisVersion: s.AtlantisVersion, + DefaultTFVersionFlag: DefaultTFVersionFlag, }) if err != nil { return errors.Wrap(err, "initializing server") diff --git a/cmd/server_test.go b/cmd/server_test.go index c9396bf373..fb78e641c8 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -341,6 +341,7 @@ func TestExecute_Defaults(t *testing.T) { Equals(t, dataDir, passedConfig.DataDir) Equals(t, "branch", passedConfig.CheckoutStrategy) + Equals(t, "", passedConfig.DefaultTFVersion) Equals(t, "github.com", passedConfig.GithubHostname) Equals(t, "token", passedConfig.GithubToken) Equals(t, "user", passedConfig.GithubUser) @@ -446,6 +447,7 @@ func TestExecute_Flags(t *testing.T) { cmd.BitbucketWebhookSecretFlag: "bitbucket-secret", cmd.CheckoutStrategyFlag: "merge", cmd.DataDirFlag: "/path", + cmd.DefaultTFVersionFlag: "v0.11.0", cmd.GHHostnameFlag: "ghhostname", cmd.GHTokenFlag: "token", cmd.GHUserFlag: "user", @@ -477,6 +479,7 @@ func TestExecute_Flags(t *testing.T) { Equals(t, "bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "merge", passedConfig.CheckoutStrategy) Equals(t, "/path", passedConfig.DataDir) + Equals(t, "v0.11.0", passedConfig.DefaultTFVersion) Equals(t, "ghhostname", passedConfig.GithubHostname) Equals(t, "token", passedConfig.GithubToken) Equals(t, "user", passedConfig.GithubUser) @@ -509,6 +512,7 @@ bitbucket-user: "bitbucket-user" bitbucket-webhook-secret: "bitbucket-secret" checkout-strategy: "merge" data-dir: "/path" +default-tf-version: "v0.11.0" gh-hostname: "ghhostname" gh-token: "token" gh-user: "user" @@ -544,6 +548,7 @@ tfe-token: my-token Equals(t, "bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "merge", passedConfig.CheckoutStrategy) Equals(t, "/path", passedConfig.DataDir) + Equals(t, "v0.11.0", passedConfig.DefaultTFVersion) Equals(t, "ghhostname", passedConfig.GithubHostname) Equals(t, "token", passedConfig.GithubToken) Equals(t, "user", passedConfig.GithubUser) @@ -576,6 +581,7 @@ bitbucket-user: "bitbucket-user" bitbucket-webhook-secret: "bitbucket-secret" checkout-strategy: "merge" data-dir: "/path" +default-tf-version: "v0.11.0" gh-hostname: "ghhostname" gh-token: "token" gh-user: "user" @@ -607,6 +613,7 @@ tfe-token: my-token "BITBUCKET_WEBHOOK_SECRET": "override-bitbucket-secret", "CHECKOUT_STRATEGY": "branch", "DATA_DIR": "/override-path", + "DEFAULT_TF_VERSION": "v0.12.0", "GH_HOSTNAME": "override-gh-hostname", "GH_TOKEN": "override-gh-token", "GH_USER": "override-gh-user", @@ -642,6 +649,7 @@ tfe-token: my-token Equals(t, "override-bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "branch", passedConfig.CheckoutStrategy) Equals(t, "/override-path", passedConfig.DataDir) + Equals(t, "v0.12.0", passedConfig.DefaultTFVersion) Equals(t, "override-gh-hostname", passedConfig.GithubHostname) Equals(t, "override-gh-token", passedConfig.GithubToken) Equals(t, "override-gh-user", passedConfig.GithubUser) @@ -674,6 +682,7 @@ bitbucket-user: "bitbucket-user" bitbucket-webhook-secret: "bitbucket-secret" checkout-strategy: "merge" data-dir: "/path" +default-tf-version: "v0.11.0" gh-hostname: "ghhostname" gh-token: "token" gh-user: "user" @@ -705,6 +714,7 @@ tfe-token: my-token cmd.BitbucketWebhookSecretFlag: "override-bitbucket-secret", cmd.CheckoutStrategyFlag: "branch", cmd.DataDirFlag: "/override-path", + cmd.DefaultTFVersionFlag: "v0.12.0", cmd.GHHostnameFlag: "override-gh-hostname", cmd.GHTokenFlag: "override-gh-token", cmd.GHUserFlag: "override-gh-user", @@ -734,6 +744,7 @@ tfe-token: my-token Equals(t, "override-bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "branch", passedConfig.CheckoutStrategy) Equals(t, "/override-path", passedConfig.DataDir) + Equals(t, "v0.12.0", passedConfig.DefaultTFVersion) Equals(t, "override-gh-hostname", passedConfig.GithubHostname) Equals(t, "override-gh-token", passedConfig.GithubToken) Equals(t, "override-gh-user", passedConfig.GithubUser) @@ -768,6 +779,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) { "BITBUCKET_WEBHOOK_SECRET": "bitbucket-secret", "CHECKOUT_STRATEGY": "merge", "DATA_DIR": "/path", + "DEFAULT_TF_VERSION": "v0.11.0", "GH_HOSTNAME": "gh-hostname", "GH_TOKEN": "gh-token", "GH_USER": "gh-user", @@ -807,6 +819,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) { cmd.BitbucketWebhookSecretFlag: "override-bitbucket-secret", cmd.CheckoutStrategyFlag: "branch", cmd.DataDirFlag: "/override-path", + cmd.DefaultTFVersionFlag: "v0.12.0", cmd.GHHostnameFlag: "override-gh-hostname", cmd.GHTokenFlag: "override-gh-token", cmd.GHUserFlag: "override-gh-user", @@ -838,6 +851,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) { Equals(t, "override-bitbucket-secret", passedConfig.BitbucketWebhookSecret) Equals(t, "branch", passedConfig.CheckoutStrategy) Equals(t, "/override-path", passedConfig.DataDir) + Equals(t, "v0.12.0", passedConfig.DefaultTFVersion) Equals(t, "override-gh-hostname", passedConfig.GithubHostname) Equals(t, "override-gh-token", passedConfig.GithubToken) Equals(t, "override-gh-user", passedConfig.GithubUser) diff --git a/runatlantis.io/.vuepress/config.js b/runatlantis.io/.vuepress/config.js index cc0a703bb8..377f198840 100644 --- a/runatlantis.io/.vuepress/config.js +++ b/runatlantis.io/.vuepress/config.js @@ -71,7 +71,9 @@ module.exports = { ['customizing-atlantis', 'Overview'], 'atlantis-yaml-reference', 'upgrading-atlantis-yaml-to-version-2', - 'apply-requirements' + 'apply-requirements', + 'checkout-strategy', + 'terraform-versions' ] }, { @@ -82,7 +84,6 @@ module.exports = { 'locking', 'autoplanning', 'automerging', - 'checkout-strategy', 'security' ] } diff --git a/runatlantis.io/docs/atlantis-yaml-reference.md b/runatlantis.io/docs/atlantis-yaml-reference.md index 125a0d0c55..f188c8e0eb 100644 --- a/runatlantis.io/docs/atlantis-yaml-reference.md +++ b/runatlantis.io/docs/atlantis-yaml-reference.md @@ -96,7 +96,7 @@ workflow: myworkflow | dir | string | none | yes | The directory of this project relative to the repo root. Use `.` for the root. For example if the project was under `./project1` then use `project1` | | workspace | string | default | no | The [Terraform workspace](https://www.terraform.io/docs/state/workspaces.html) for this project. Atlantis will switch to this workplace when planning/applying and will create it if it doesn't exist. | | autoplan | [Autoplan](atlantis-yaml-reference.html#autoplan) | none | no | A custom autoplan configuration. If not specified, will use the default algorithm. See [Autoplanning](autoplanning.html). | -| terraform_version | string | none | no | A specific Terraform version to use when running commands for this project. Requires there to be a binary in the Atlantis `PATH` with the name `terraform{VERSION}`, ex. `terraform0.11.0` | +| terraform_version | string | none | no | A specific Terraform version to use when running commands for this project. Must be [Semver compatible](https://semver.org/), ex. `v0.11.0`, `0.12.0-beta1`. | | apply_requirements | array[string] | [] | no | Requirements that must be satisfied before `atlantis apply` can be run. Currently the only supported requirements are `approved` and `mergeable`. See [Apply Requirements](apply-requirements.html) for more details. | | workflow | string | none | no | A custom workflow. If not specified, Atlantis will use its default workflow. | diff --git a/runatlantis.io/docs/deployment.md b/runatlantis.io/docs/deployment.md index 97eb052720..a5852819fa 100644 --- a/runatlantis.io/docs/deployment.md +++ b/runatlantis.io/docs/deployment.md @@ -383,14 +383,14 @@ Once you're done, see [Next Steps](#next-steps). Atlantis has an [official](https://hub.docker.com/r/runatlantis/atlantis/) Docker image: `runatlantis/atlantis`. #### Customization -If you need to modify the Docker image that we provide, for instance to add a specific version of Terraform, you can do something like this: +If you need to modify the Docker image that we provide, for instance to add the terragrunt binary, you can do something like this: 1. Create a custom docker file ```dockerfile FROM runatlantis/atlantis:{latest version} # copy a terraform binary of the version you need - COPY terraform /usr/local/bin/terraform + COPY terragrunt /usr/local/bin/terrgrunt ``` 1. Build your Docker image diff --git a/runatlantis.io/docs/requirements.md b/runatlantis.io/docs/requirements.md index e0de6a4c8f..14a6a1e33d 100644 --- a/runatlantis.io/docs/requirements.md +++ b/runatlantis.io/docs/requirements.md @@ -83,12 +83,8 @@ an `atlantis.yaml` file to tell it to use `-var-file={YOUR_FILE}`. See [atlantis.yaml Use Cases](/guide/atlantis-yaml-use-cases.html#using-tfvars-files) for more details. ## Terraform Versions -By default, Atlantis will use the `terraform` executable that is in its path. -To use a specific version of Terraform: -1. Install the desired version of Terraform into the `$PATH` of where Atlantis is - running and name it `terraform{version}`, ex. `terraform0.8.8`. -2. Create an `atlantis.yaml` file for your repo and set the `terraform_version` key. -See [atlantis.yaml Use Cases](/guide/atlantis-yaml-use-cases.html#terraform-versions) for more details. +Atlantis supports all Terraform versions (including 0.12) and can be configured +to use different versions for different repositories/projects. See [Terraform Versions](/docs/terraform-versions.html)l ## Next Steps * If your Terraform setup meets the Atlantis requirements, head back to our [Installation Guide](installation-guide.html) to get started diff --git a/runatlantis.io/docs/terraform-versions.md b/runatlantis.io/docs/terraform-versions.md new file mode 100644 index 0000000000..bad7a15ab3 --- /dev/null +++ b/runatlantis.io/docs/terraform-versions.md @@ -0,0 +1,20 @@ +# Terraform Versions + +You can customize which version of Terraform Atlantis defaults to by setting +the `--default-tf-version` flag (ex. `--default-tf-version=v0.12.0`). + +If you wish to use a different version than the default for a specific repo or project, you need +to create an `atlantis.yaml` file and set the `terraform_version` key: +```yaml +version: 2 +projects: +- dir: . + terraform_version: v0.10.5 +``` +See [atlantis.yaml Use Cases](/guide/atlantis-yaml-use-cases.html#terraform-versions) for more details. + +::: tip NOTE +Atlantis will automatically download the version specified. +::: + + diff --git a/runatlantis.io/guide/atlantis-yaml-use-cases.md b/runatlantis.io/guide/atlantis-yaml-use-cases.md index 522916d441..f3e96e1072 100644 --- a/runatlantis.io/guide/atlantis-yaml-use-cases.md +++ b/runatlantis.io/guide/atlantis-yaml-use-cases.md @@ -222,6 +222,11 @@ workflows: - run: terragrunt apply -no-color $PLANFILE ``` +::: warning +Atlantis will need to have the `terragrunt` binary in its PATH. +If you're using Docker you can build your own image, see [Customization](/docs/deployment.html#customization). +::: + ## Running custom commands Atlantis supports running custom commands. In this example, we want to run a script after every `apply`: @@ -255,7 +260,7 @@ isn't set, Atlantis will use the default plan workflow which is what we want in ## Terraform Versions If you'd like to use a different version of Terraform than what is in Atlantis' -`PATH` then set the `terraform_version` key: +`PATH` or is set by the `--default-tf-version` flag, then set the `terraform_version` key: ```yaml version: 2 @@ -264,9 +269,7 @@ projects: terraform_version: 0.10.0 ``` -Atlantis will then execute all Terraform commands with `terraform0.10.0` instead -of `terraform`. This requires that the 0.10.0 binary is in Atlantis's `PATH` with the -name `terraform0.10.0`. +Atlantis will automatically download and use this version. ## Requiring Approvals For Production In this example, we only want to require `apply` approvals for the `production` directory. diff --git a/server/events/terraform/terraform_client.go b/server/events/terraform/terraform_client.go index bb01d10fbe..8890935b6d 100644 --- a/server/events/terraform/terraform_client.go +++ b/server/events/terraform/terraform_client.go @@ -17,6 +17,7 @@ package terraform import ( "bufio" "fmt" + "github.com/hashicorp/go-getter" "github.com/hashicorp/go-version" "github.com/mitchellh/go-homedir" "github.com/pkg/errors" @@ -27,6 +28,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "strings" "sync" ) @@ -39,15 +41,42 @@ type Client interface { } type DefaultClient struct { + // defaultVersion is the default version of terraform to use if another + // version isn't specified. defaultVersion *version.Version terraformPluginCacheDir string - // tfExecutableName is the name of the default terraform binary. - // This should always be set to "terraform" by the NewClient constructor - // however it can be overridden during testing. - tfExecutableName string + binDir string + // overrideTF can be used to override the terraform binary during testing + // with another binary, ex. echo. + overrideTF string + // downloader downloads terraform versions. + downloader Downloader + // versions maps from the string representation of a tf version (ex. 0.11.10) + // to the absolute path of that binary on disk (if it exists). + // Use versionsLock to control access. + versions map[string]string + + // versionsLock is used to ensure versions isn't being concurrently written to. + versionsLock *sync.Mutex +} + +//go:generate pegomock generate -m --use-experimental-model-gen --package mocks -o mocks/mock_downloader.go Downloader + +// Downloader is for downloading terraform versions. +type Downloader interface { + GetFile(dst, src string, opts ...getter.ClientOption) error } -const terraformPluginCacheDirName = "plugin-cache" +const ( + // terraformPluginCacheDir is the name of the dir inside our data dir + // where we tell terraform to cache plugins and modules. + terraformPluginCacheDirName = "plugin-cache" + // binDirName is the name of the directory inside our data dir where + // we download terraform binaries. + binDirName = "bin" + // releasesURL is the base url to download terraform from. + releasesURL = "https://releases.hashicorp.com" +) // versionRegex extracts the version from `terraform version` output. // Terraform v0.12.0-alpha4 (2c36829d3265661d8edbd5014de8090ea7e2a076) @@ -57,24 +86,58 @@ const terraformPluginCacheDirName = "plugin-cache" // => 0.11.10 var versionRegex = regexp.MustCompile("Terraform v(.*?)(\\s.*)?\n") -func NewClient(dataDir string, tfeToken string) (*DefaultClient, error) { - _, err := exec.LookPath("terraform") - if err != nil { - return nil, errors.New("terraform not found in $PATH. \n\nDownload terraform from https://www.terraform.io/downloads.html") +// NewClient constructs a terraform client. +// tfeToken is an optional terraform enterprise token. +// defaultVersionStr is an optional default terraform version to use unless +// a specific version is set. +// defaultVersionFlagName is the name of the flag that sets the default terraform +// version. +// tfDownloader is used to download terraform versions. +// Will asynchronously download the required version if it doesn't exist already. +func NewClient(log *logging.SimpleLogger, dataDir string, tfeToken string, defaultVersionStr string, defaultVersionFlagName string, tfDownloader Downloader) (*DefaultClient, error) { + var finalDefaultVersion *version.Version + var localVersion *version.Version + versions := make(map[string]string) + var versionsLock sync.Mutex + + localPath, err := exec.LookPath("terraform") + if err != nil && defaultVersionStr == "" { + return nil, fmt.Errorf("terraform not found in $PATH. Set --%s or download terraform from https://www.terraform.io/downloads.html", defaultVersionFlagName) } - versionOutBytes, err := exec.Command("terraform", "version"). - Output() // #nosec - versionOutput := string(versionOutBytes) - if err != nil { - return nil, errors.Wrapf(err, "running terraform version: %s", versionOutput) + if err == nil { + localVersion, err = getVersion(localPath) + if err != nil { + return nil, err + } + versions[localVersion.String()] = localPath + if defaultVersionStr == "" { + // If they haven't set a default version, then whatever they had + // locally is now the default. + finalDefaultVersion = localVersion + } } - match := versionRegex.FindStringSubmatch(versionOutput) - if len(match) <= 1 { - return nil, fmt.Errorf("could not parse terraform version from %s", versionOutput) + + binDir := filepath.Join(dataDir, binDirName) + if err := os.MkdirAll(binDir, 0700); err != nil { + return nil, errors.Wrapf(err, "unable to create terraform bin dir %q", binDir) } - v, err := version.NewVersion(match[1]) - if err != nil { - return nil, errors.Wrap(err, "parsing terraform version") + + if defaultVersionStr != "" { + defaultVersion, err := version.NewVersion(defaultVersionStr) + if err != nil { + return nil, err + } + finalDefaultVersion = defaultVersion + go func() { + // Since ensureVersion might end up downloading terraform, + // we call it asynchronously so as to not delay server startup. + versionsLock.Lock() + _, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir) + versionsLock.Unlock() + if err != nil { + log.Err("could not download terraform %s", defaultVersion.String()) + } + }() } // If tfeToken is set, we try to create a ~/.terraformrc file. @@ -96,9 +159,12 @@ func NewClient(dataDir string, tfeToken string) (*DefaultClient, error) { } return &DefaultClient{ - defaultVersion: v, + defaultVersion: finalDefaultVersion, terraformPluginCacheDir: cacheDir, - tfExecutableName: "terraform", + binDir: binDir, + downloader: tfDownloader, + versionsLock: &versionsLock, + versions: versions, }, nil } @@ -113,7 +179,10 @@ func (c *DefaultClient) Version() *version.Version { // Workspace is the terraform workspace to run in. We won't switch workspaces, // just set a WORKSPACE environment variable. func (c *DefaultClient) RunCommandWithVersion(log *logging.SimpleLogger, path string, args []string, v *version.Version, workspace string) (string, error) { - tfCmd, cmd := c.prepCmd(v, workspace, path, args) + tfCmd, cmd, err := c.prepCmd(log, v, workspace, path, args) + if err != nil { + return "", err + } out, err := cmd.CombinedOutput() if err != nil { err = errors.Wrapf(err, "running %q in %q", tfCmd, path) @@ -124,14 +193,28 @@ func (c *DefaultClient) RunCommandWithVersion(log *logging.SimpleLogger, path st return string(out), nil } -func (c *DefaultClient) prepCmd(v *version.Version, workspace string, path string, args []string) (string, *exec.Cmd) { - tfExecutable := c.tfExecutableName - tfVersionStr := c.defaultVersion.String() - // if version is the same as the default, don't need to prepend the version name to the executable - if v != nil && !v.Equal(c.defaultVersion) { - tfExecutable = fmt.Sprintf("%s%s", tfExecutable, v.String()) - tfVersionStr = v.String() +// prepCmd builds a ready to execute command based on the version of terraform +// v, and args. It returns a printable representation of the command that will +// be run and the actual command. +func (c *DefaultClient) prepCmd(log *logging.SimpleLogger, v *version.Version, workspace string, path string, args []string) (string, *exec.Cmd, error) { + if v == nil { + v = c.defaultVersion } + + var binPath string + if c.overrideTF != "" { + // This is only set during testing. + binPath = c.overrideTF + } else { + var err error + c.versionsLock.Lock() + binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir) + c.versionsLock.Unlock() + if err != nil { + return "", nil, err + } + } + // We add custom variables so that if `extra_args` is specified with env // vars then they'll be substituted. envVars := []string{ @@ -140,18 +223,17 @@ func (c *DefaultClient) prepCmd(v *version.Version, workspace string, path strin // Cache plugins so terraform init runs faster. fmt.Sprintf("TF_PLUGIN_CACHE_DIR=%s", c.terraformPluginCacheDir), fmt.Sprintf("WORKSPACE=%s", workspace), - fmt.Sprintf("ATLANTIS_TERRAFORM_VERSION=%s", tfVersionStr), + fmt.Sprintf("ATLANTIS_TERRAFORM_VERSION=%s", v.String()), fmt.Sprintf("DIR=%s", path), } - // Append current Atlantis process's environment variables so PATH is - // preserved and any vars that users purposely exec'd Atlantis with. + // Append current Atlantis process's environment variables, ex. + // AWS_ACCESS_KEY. envVars = append(envVars, os.Environ()...) - // append terraform executable name with args - tfCmd := fmt.Sprintf("%s %s", tfExecutable, strings.Join(args, " ")) + tfCmd := fmt.Sprintf("%s %s", binPath, strings.Join(args, " ")) cmd := exec.Command("sh", "-c", tfCmd) cmd.Dir = path cmd.Env = envVars - return tfCmd, cmd + return tfCmd, cmd, nil } // Line represents a line that was output from a terraform command. @@ -182,13 +264,18 @@ func (c *DefaultClient) RunCommandAsync(log *logging.SimpleLogger, path string, close(inCh) }() - tfCmd, cmd := c.prepCmd(v, workspace, path, args) + tfCmd, cmd, err := c.prepCmd(log, v, workspace, path, args) + if err != nil { + log.Err(err.Error()) + outCh <- Line{Err: err} + return + } stdout, _ := cmd.StdoutPipe() stderr, _ := cmd.StderrPipe() stdin, _ := cmd.StdinPipe() log.Debug("starting %q in %q", tfCmd, path) - err := cmd.Start() + err = cmd.Start() if err != nil { err = errors.Wrapf(err, "running %q in %q", tfCmd, path) log.Err(err.Error()) @@ -259,6 +346,43 @@ func MustConstraint(v string) version.Constraints { return c } +// ensureVersion returns the path to a terraform binary of version v. +// It will download this version if we don't have it. +func ensureVersion(log *logging.SimpleLogger, dl Downloader, versions map[string]string, v *version.Version, binDir string) (string, error) { + if binPath, ok := versions[v.String()]; ok { + return binPath, nil + } + + // This tf version might not yet be in the versions map even though it + // exists on disk. This would happen if users have manually added + // terraform{version} binaries. In this case we don't want to re-download. + binFile := "terraform" + v.String() + if binPath, err := exec.LookPath(binFile); err == nil { + versions[v.String()] = binPath + return binPath, nil + } + + // The version might also not be in the versions map if it's in our bin dir. + // This could happen if Atlantis was restarted without losing its disk. + dest := filepath.Join(binDir, binFile) + if _, err := os.Stat(dest); err == nil { + versions[v.String()] = dest + return dest, nil + } + + log.Info("could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, releasesURL) + urlPrefix := fmt.Sprintf("%s/terraform/%s/terraform_%s", releasesURL, v.String(), v.String()) + binURL := fmt.Sprintf("%s_%s_%s.zip", urlPrefix, runtime.GOOS, runtime.GOARCH) + checksumURL := fmt.Sprintf("%s_SHA256SUMS", urlPrefix) + if err := dl.GetFile(dest, fmt.Sprintf("%s?checksum=file:%s", binURL, checksumURL)); err != nil { + return "", errors.Wrapf(err, "downloading terraform version %s", v.String()) + } + + log.Info("downloaded terraform %s to %s", v.String(), dest) + versions[v.String()] = dest + return dest, nil +} + // generateRCFile generates a .terraformrc file containing config for tfeToken. // It will create the file in home/.terraformrc. func generateRCFile(tfeToken string, home string) error { @@ -288,9 +412,28 @@ func generateRCFile(tfeToken string, home string) error { return nil } +func getVersion(tfBinary string) (*version.Version, error) { + versionOutBytes, err := exec.Command(tfBinary, "version").Output() // #nosec + versionOutput := string(versionOutBytes) + if err != nil { + return nil, errors.Wrapf(err, "running terraform version: %s", versionOutput) + } + match := versionRegex.FindStringSubmatch(versionOutput) + if len(match) <= 1 { + return nil, fmt.Errorf("could not parse terraform version from %s", versionOutput) + } + return version.NewVersion(match[1]) +} + // rcFileContents is a format string to be used with Sprintf that can be used // to generate the contents of a ~/.terraformrc file for authenticating with // Terraform Enterprise. var rcFileContents = `credentials "app.terraform.io" { token = %q }` + +type DefaultDownloader struct{} + +func (d *DefaultDownloader) GetFile(dst, src string, opts ...getter.ClientOption) error { + return getter.GetFile(dst, src, opts...) +} diff --git a/server/events/terraform/terraform_client_internal_test.go b/server/events/terraform/terraform_client_internal_test.go index 495c73e5c8..94a14e131d 100644 --- a/server/events/terraform/terraform_client_internal_test.go +++ b/server/events/terraform/terraform_client_internal_test.go @@ -93,7 +93,7 @@ func TestDefaultClient_RunCommandWithVersion_EnvVars(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } args := []string{ @@ -118,7 +118,7 @@ func TestDefaultClient_RunCommandWithVersion_Error(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } args := []string{ @@ -142,7 +142,7 @@ func TestDefaultClient_RunCommandAsync_Success(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } args := []string{ @@ -168,7 +168,7 @@ func TestDefaultClient_RunCommandAsync_BigOutput(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "cat", + overrideTF: "cat", } filename := filepath.Join(tmp, "data") f, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) @@ -196,7 +196,7 @@ func TestDefaultClient_RunCommandAsync_StderrOutput(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } log := logging.NewSimpleLogger("test", false, logging.Debug) _, outCh := client.RunCommandAsync(log, tmp, []string{"stderr", ">&2"}, nil, "workspace") @@ -214,7 +214,7 @@ func TestDefaultClient_RunCommandAsync_ExitOne(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "echo", + overrideTF: "echo", } log := logging.NewSimpleLogger("test", false, logging.Debug) _, outCh := client.RunCommandAsync(log, tmp, []string{"dying", "&&", "exit", "1"}, nil, "workspace") @@ -233,7 +233,7 @@ func TestDefaultClient_RunCommandAsync_Input(t *testing.T) { client := &DefaultClient{ defaultVersion: v, terraformPluginCacheDir: tmp, - tfExecutableName: "read", + overrideTF: "read", } log := logging.NewSimpleLogger("test", false, logging.Debug) inCh, outCh := client.RunCommandAsync(log, tmp, []string{"a", "&&", "echo", "$a"}, nil, "workspace") diff --git a/server/events/terraform/terraform_client_test.go b/server/events/terraform/terraform_client_test.go index 00126ad493..6c0696271e 100644 --- a/server/events/terraform/terraform_client_test.go +++ b/server/events/terraform/terraform_client_test.go @@ -14,9 +14,19 @@ package terraform_test import ( + "fmt" + "github.com/petergtz/pegomock" + "github.com/runatlantis/atlantis/cmd" + "github.com/runatlantis/atlantis/server/events/terraform/mocks" + "io/ioutil" + "os" + "path/filepath" + "runtime" "testing" + "time" "github.com/hashicorp/go-version" + . "github.com/petergtz/pegomock" "github.com/runatlantis/atlantis/server/events/terraform" . "github.com/runatlantis/atlantis/testing" ) @@ -39,3 +49,201 @@ func TestMustConstraint(t *testing.T) { Ok(t, err) Equals(t, expectedConstraint.String(), c.String()) } + +// Test that if terraform is in path and we're not setting the default-tf flag, +// that we use that version as our default version. +func TestNewClient_LocalTFOnly(t *testing.T) { + fakeBinOut := `Terraform v0.11.10 + +Your version of Terraform is out of date! The latest version +is 0.11.13. You can update by downloading from www.terraform.io/downloads.html +` + tmp, cleanup := TempDir(t) + defer cleanup() + + // We're testing this by adding our own "fake" terraform binary to path that + // outputs what would normally come from terraform version. + err := ioutil.WriteFile(filepath.Join(tmp, "terraform"), []byte(fmt.Sprintf("#!/bin/sh\necho '%s'", fakeBinOut)), 0755) + Ok(t, err) + defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() + + c, err := terraform.NewClient(nil, tmp, "", "", cmd.DefaultTFVersionFlag, nil) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, fakeBinOut+"\n", output) +} + +// Test that if terraform is in path and the default-tf flag is set to the +// same version that we don't download anything. +func TestNewClient_LocalTFMatchesFlag(t *testing.T) { + fakeBinOut := `Terraform v0.11.10 + +Your version of Terraform is out of date! The latest version +is 0.11.13. You can update by downloading from www.terraform.io/downloads.html +` + tmp, cleanup := TempDir(t) + defer cleanup() + + // We're testing this by adding our own "fake" terraform binary to path that + // outputs what would normally come from terraform version. + err := ioutil.WriteFile(filepath.Join(tmp, "terraform"), []byte(fmt.Sprintf("#!/bin/sh\necho '%s'", fakeBinOut)), 0755) + Ok(t, err) + defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() + + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, nil) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, fakeBinOut+"\n", output) +} + +// Test that if terraform is not in PATH and we didn't set the default-tf flag +// that we error. +func TestNewClient_NoTF(t *testing.T) { + tmp, cleanup := TempDir(t) + defer cleanup() + + // Set PATH to only include our empty directory. + defer tempSetEnv(t, "PATH", tmp)() + + _, err := terraform.NewClient(nil, tmp, "", "", cmd.DefaultTFVersionFlag, nil) + ErrEquals(t, "terraform not found in $PATH. Set --default-tf-version or download terraform from https://www.terraform.io/downloads.html", err) +} + +// Test that if the default-tf flag is set and that binary is in our PATH +// that we use it. +func TestNewClient_DefaultTFFlagInPath(t *testing.T) { + fakeBinOut := "Terraform v0.11.10\n" + tmp, cleanup := TempDir(t) + defer cleanup() + + // We're testing this by adding our own "fake" terraform binary to path that + // outputs what would normally come from terraform version. + err := ioutil.WriteFile(filepath.Join(tmp, "terraform0.11.10"), []byte(fmt.Sprintf("#!/bin/sh\necho '%s'", fakeBinOut)), 0755) + Ok(t, err) + defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() + + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, nil) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, fakeBinOut+"\n", output) +} + +// Test that if the default-tf flag is set and that binary is in our download +// bin dir that we use it. +func TestNewClient_DefaultTFFlagInBinDir(t *testing.T) { + fakeBinOut := "Terraform v0.11.10\n" + tmp, cleanup := TempDir(t) + defer cleanup() + + // Add our fake binary to {datadir}/bin/terraform{version}. + Ok(t, os.Mkdir(filepath.Join(tmp, "bin"), 0700)) + err := ioutil.WriteFile(filepath.Join(tmp, "bin", "terraform0.11.10"), []byte(fmt.Sprintf("#!/bin/sh\necho '%s'", fakeBinOut)), 0755) + Ok(t, err) + defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() + + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, nil) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, fakeBinOut+"\n", output) +} + +// Test that if we don't have that version of TF that we download it. +func TestNewClient_DefaultTFFlagDownload(t *testing.T) { + RegisterMockTestingT(t) + tmp, cleanup := TempDir(t) + defer cleanup() + + // Set PATH to empty so there's no TF available. + orig := os.Getenv("PATH") + defer tempSetEnv(t, "PATH", "")() + + mockDownloader := mocks.NewMockDownloader() + When(mockDownloader.GetFile(AnyString(), AnyString())).Then(func(params []pegomock.Param) pegomock.ReturnValues { + err := ioutil.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v0.11.10\n'"), 0755) + return []pegomock.ReturnValue{err} + }) + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader) + Ok(t, err) + + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + baseURL := "https://releases.hashicorp.com/terraform/0.11.10" + expURL := fmt.Sprintf("%s/terraform_0.11.10_%s_%s.zip?checksum=file:%s/terraform_0.11.10_SHA256SUMS", + baseURL, + runtime.GOOS, + runtime.GOARCH, + baseURL) + mockDownloader.VerifyWasCalledEventually(Once(), 2*time.Second).GetFile(filepath.Join(tmp, "bin", "terraform0.11.10"), expURL) + + // Reset PATH so that it has sh. + Ok(t, os.Setenv("PATH", orig)) + output, err := c.RunCommandWithVersion(nil, tmp, nil, nil, "") + Ok(t, err) + Equals(t, "\nTerraform v0.11.10\n\n", output) +} + +// Test that we get an error if the terraform version flag is malformed. +func TestNewClient_BadVersion(t *testing.T) { + tmp, cleanup := TempDir(t) + defer cleanup() + _, err := terraform.NewClient(nil, tmp, "", "malformed", cmd.DefaultTFVersionFlag, nil) + ErrEquals(t, "Malformed version: malformed", err) +} + +// Test that if we run a command with a version we don't have, we download it. +func TestRunCommandWithVersion_DLsTF(t *testing.T) { + RegisterMockTestingT(t) + tmp, cleanup := TempDir(t) + defer cleanup() + + mockDownloader := mocks.NewMockDownloader() + // Set up our mock downloader to write a fake tf binary when it's called. + baseURL := "https://releases.hashicorp.com/terraform/0.12.0" + expURL := fmt.Sprintf("%s/terraform_0.12.0_%s_%s.zip?checksum=file:%s/terraform_0.12.0_SHA256SUMS", + baseURL, + runtime.GOOS, + runtime.GOARCH, + baseURL) + When(mockDownloader.GetFile(filepath.Join(tmp, "bin", "terraform0.12.0"), expURL)).Then(func(params []pegomock.Param) pegomock.ReturnValues { + err := ioutil.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v0.12.0\n'"), 0755) + return []pegomock.ReturnValue{err} + }) + + c, err := terraform.NewClient(nil, tmp, "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader) + Ok(t, err) + Equals(t, "0.11.10", c.Version().String()) + + v, err := version.NewVersion("0.12.0") + Ok(t, err) + output, err := c.RunCommandWithVersion(nil, tmp, nil, v, "") + Assert(t, err == nil, "err: %s: %s", err, output) + Equals(t, "\nTerraform v0.12.0\n\n", output) +} + +// tempSetEnv sets env var key to value. It returns a function that when called +// will reset the env var to its original value. +func tempSetEnv(t *testing.T, key string, value string) func() { + orig := os.Getenv(key) + Ok(t, os.Setenv(key, value)) + return func() { os.Setenv(key, orig) } +} diff --git a/server/events_controller_e2e_test.go b/server/events_controller_e2e_test.go index df42eb3d95..6403611475 100644 --- a/server/events_controller_e2e_test.go +++ b/server/events_controller_e2e_test.go @@ -3,6 +3,7 @@ package server_test import ( "bytes" "fmt" + "github.com/hashicorp/go-getter" "github.com/runatlantis/atlantis/server/events/db" "io/ioutil" "net/http" @@ -31,6 +32,12 @@ import ( . "github.com/runatlantis/atlantis/testing" ) +type NoopTFDownloader struct{} + +func (m *NoopTFDownloader) GetFile(dst, src string, opts ...getter.ClientOption) error { + return nil +} + func TestGitHubWorkflow(t *testing.T) { if testing.Short() { t.SkipNow() @@ -371,7 +378,7 @@ func setupE2E(t *testing.T) (server.EventsController, *vcsmocks.MockClient, *moc GithubUser: "github-user", GitlabUser: "gitlab-user", } - terraformClient, err := terraform.NewClient(dataDir, "") + terraformClient, err := terraform.NewClient(logger, dataDir, "", "", "default-tf-version", &NoopTFDownloader{}) Ok(t, err) boltdb, err := db.New(dataDir) Ok(t, err) diff --git a/server/server.go b/server/server.go index d0e4ec57e1..686673e104 100644 --- a/server/server.go +++ b/server/server.go @@ -79,10 +79,11 @@ type Server struct { // Config holds config for server that isn't passed in by the user. type Config struct { - AllowForkPRsFlag string - AllowRepoConfigFlag string - AtlantisURLFlag string - AtlantisVersion string + AllowForkPRsFlag string + AllowRepoConfigFlag string + AtlantisURLFlag string + AtlantisVersion string + DefaultTFVersionFlag string } // WebhookConfig is nested within UserConfig. It's used to configure webhooks. @@ -165,7 +166,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { } vcsClient := vcs.NewClientProxy(githubClient, gitlabClient, bitbucketCloudClient, bitbucketServerClient) commitStatusUpdater := &events.DefaultCommitStatusUpdater{Client: vcsClient} - terraformClient, err := terraform.NewClient(userConfig.DataDir, userConfig.TFEToken) + terraformClient, err := terraform.NewClient(logger, userConfig.DataDir, userConfig.TFEToken, userConfig.DefaultTFVersion, config.DefaultTFVersionFlag, &terraform.DefaultDownloader{}) // The flag.Lookup call is to detect if we're running in a unit test. If we // are, then we don't error out because we don't have/want terraform // installed on our CI system where the unit tests run. diff --git a/server/user_config.go b/server/user_config.go index 83d4089e13..b188751746 100644 --- a/server/user_config.go +++ b/server/user_config.go @@ -38,6 +38,7 @@ type UserConfig struct { SSLCertFile string `mapstructure:"ssl-cert-file"` SSLKeyFile string `mapstructure:"ssl-key-file"` TFEToken string `mapstructure:"tfe-token"` + DefaultTFVersion string `mapstructure:"default-tf-version"` Webhooks []WebhookConfig `mapstructure:"webhooks"` }