Merge branch 'master' of https://github.com/hashicorp/terraform

.travis.yml

@@ -4,8 +4,10 @@ language: go
 go:
 - 1.8
 
+# add TF_CONSUL_TEST=1 to run consul tests
+# they were causing timouts in travis
 env:
-  - CONSUL_VERSION=0.7.5 TF_CONSUL_TEST=1 GOMAXPROCS=4
+  - CONSUL_VERSION=0.7.5 GOMAXPROCS=4
 
 # Fetch consul for the backend and provider tests
 before_install:

Makefile

@@ -41,7 +41,7 @@ plugin-dev: generate
 test: fmtcheck errcheck generate
 	go test -i $(TEST) || exit 1
 	echo $(TEST) | \
-		xargs -t -n4 go test $(TESTARGS) -timeout=30s -parallel=4
+		xargs -t -n4 go test $(TESTARGS) -timeout=60s -parallel=4
 
 # testacc runs acceptance tests
 testacc: fmtcheck generate

README.md

@@ -152,3 +152,13 @@ $ tree ./pkg/ -P "terraform|*.zip"
 ```
 
 _Note: Cross-compilation uses [gox](https://github.com/mitchellh/gox), which requires toolchains to be built with versions of Go prior to 1.5. In order to successfully cross-compile with older versions of Go, you will need to run `gox -build-toolchain` before running the commands detailed above._
+
+#### Docker
+
+When using docker you don't need to have any of the Go development tools installed and you can clone terraform to any location on disk (doesn't have to be in your $GOPATH).  This is useful for users who want to build `master` or a specific branch for testing without setting up a proper Go environment.
+
+For example, run the following command to build terraform in a linux-based container for macOS.
+
+```sh
+docker run --rm -v $(pwd):/go/src/github.com/hashicorp/terraform -w /go/src/github.com/hashicorp/terraform -e XC_OS=darwin -e XC_ARCH=amd64 golang:latest bash -c "apt-get update && apt-get install -y zip && make bin"
+```

backend/backend.go

@@ -7,6 +7,7 @@ package backend
 import (
 	"context"
 	"errors"
+	"time"
 
 	"github.com/hashicorp/terraform/config/module"
 	"github.com/hashicorp/terraform/state"
@@ -132,6 +133,9 @@ type Operation struct {
 	// state.Lockers for its duration, and Unlock when complete.
 	LockState bool
 
+	// The duration to retry obtaining a State lock.
+	StateLockTimeout time.Duration
+
 	// Environment is the named state that should be loaded from the Backend.
 	Environment string
 }

backend/init/init.go

@@ -12,6 +12,7 @@ import (
 	backendlocal "github.com/hashicorp/terraform/backend/local"
 	backendconsul "github.com/hashicorp/terraform/backend/remote-state/consul"
 	backendinmem "github.com/hashicorp/terraform/backend/remote-state/inmem"
+	backendS3 "github.com/hashicorp/terraform/backend/remote-state/s3"
 )
 
 // backends is the list of available backends. This is a global variable
@@ -36,6 +37,7 @@ func init() {
 		"local":  func() backend.Backend { return &backendlocal.Local{} },
 		"consul": func() backend.Backend { return backendconsul.New() },
 		"inmem":  func() backend.Backend { return backendinmem.New() },
+		"s3":     func() backend.Backend { return backendS3.New() },
 	}
 
 	// Add the legacy remote backends that haven't yet been convertd to

backend/local/backend_apply.go

@@ -9,7 +9,7 @@ import (
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/terraform/backend"
-	clistate "github.com/hashicorp/terraform/command/state"
+	"github.com/hashicorp/terraform/command/clistate"
 	"github.com/hashicorp/terraform/config/module"
 	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/terraform"
@@ -52,9 +52,12 @@ func (b *Local) opApply(
 	}
 
 	if op.LockState {
+		lockCtx, cancel := context.WithTimeout(ctx, op.StateLockTimeout)
+		defer cancel()
+
 		lockInfo := state.NewLockInfo()
 		lockInfo.Operation = op.Type.String()
-		lockID, err := clistate.Lock(opState, lockInfo, b.CLI, b.Colorize())
+		lockID, err := clistate.Lock(lockCtx, opState, lockInfo, b.CLI, b.Colorize())
 		if err != nil {
 			runningOp.Err = errwrap.Wrapf("Error locking state: {{err}}", err)
 			return

backend/local/backend_plan.go

@@ -10,8 +10,8 @@ import (
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/command/clistate"
 	"github.com/hashicorp/terraform/command/format"
-	clistate "github.com/hashicorp/terraform/command/state"
 	"github.com/hashicorp/terraform/config/module"
 	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/terraform"
@@ -61,9 +61,12 @@ func (b *Local) opPlan(
 	}
 
 	if op.LockState {
+		lockCtx, cancel := context.WithTimeout(ctx, op.StateLockTimeout)
+		defer cancel()
+
 		lockInfo := state.NewLockInfo()
 		lockInfo.Operation = op.Type.String()
-		lockID, err := clistate.Lock(opState, lockInfo, b.CLI, b.Colorize())
+		lockID, err := clistate.Lock(lockCtx, opState, lockInfo, b.CLI, b.Colorize())
 		if err != nil {
 			runningOp.Err = errwrap.Wrapf("Error locking state: {{err}}", err)
 			return
@@ -110,6 +113,12 @@ func (b *Local) opPlan(
 		// Write the backend if we have one
 		plan.Backend = op.PlanOutBackend
 
+		// This works around a bug (#12871) which is no longer possible to
+		// trigger but will exist for already corrupted upgrades.
+		if plan.Backend != nil && plan.State != nil {
+			plan.State.Remote = nil
+		}
+
 		log.Printf("[INFO] backend/local: writing plan output to: %s", path)
 		f, err := os.Create(path)
 		if err == nil {

backend/local/backend_refresh.go

@@ -9,7 +9,7 @@ import (
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/terraform/backend"
-	clistate "github.com/hashicorp/terraform/command/state"
+	"github.com/hashicorp/terraform/command/clistate"
 	"github.com/hashicorp/terraform/config/module"
 	"github.com/hashicorp/terraform/state"
 )
@@ -51,9 +51,12 @@ func (b *Local) opRefresh(
 	}
 
 	if op.LockState {
+		lockCtx, cancel := context.WithTimeout(ctx, op.StateLockTimeout)
+		defer cancel()
+
 		lockInfo := state.NewLockInfo()
 		lockInfo.Operation = op.Type.String()
-		lockID, err := clistate.Lock(opState, lockInfo, b.CLI, b.Colorize())
+		lockID, err := clistate.Lock(lockCtx, opState, lockInfo, b.CLI, b.Colorize())
 		if err != nil {
 			runningOp.Err = errwrap.Wrapf("Error locking state: {{err}}", err)
 			return

backend/local/backend_test.go

@@ -21,7 +21,8 @@ func TestLocal_impl(t *testing.T) {
 }
 
 func TestLocal_backend(t *testing.T) {
-	b := TestLocal(t)
+	defer testTmpDir(t)()
+	b := &Local{}
 	backend.TestBackend(t, b, b)
 }
 

backend/remote-state/consul/backend_state.go

@@ -56,7 +56,7 @@ func (b *Backend) States() ([]string, error) {
 }
 
 func (b *Backend) DeleteState(name string) error {
-	if name == backend.DefaultStateName {
+	if name == backend.DefaultStateName || name == "" {
 		return fmt.Errorf("can't delete default state")
 	}
 
@@ -102,22 +102,19 @@ func (b *Backend) State(name string) (state.State, error) {
 		stateMgr = &state.LockDisabled{Inner: stateMgr}
 	}
 
-	// Get the locker, which we know always exists
-	stateMgrLocker := stateMgr.(state.Locker)
-
 	// Grab a lock, we use this to write an empty state if one doesn't
 	// exist already. We have to write an empty state as a sentinel value
 	// so States() knows it exists.
 	lockInfo := state.NewLockInfo()
 	lockInfo.Operation = "init"
-	lockId, err := stateMgrLocker.Lock(lockInfo)
+	lockId, err := stateMgr.Lock(lockInfo)
 	if err != nil {
 		return nil, fmt.Errorf("failed to lock state in Consul: %s", err)
 	}
 
 	// Local helper function so we can call it multiple places
 	lockUnlock := func(parent error) error {
-		if err := stateMgrLocker.Unlock(lockId); err != nil {
+		if err := stateMgr.Unlock(lockId); err != nil {
 			return fmt.Errorf(strings.TrimSpace(errStateUnlock), lockId, err)
 		}
 

backend/remote-state/consul/client.go

@@ -121,16 +121,15 @@ func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
 	default:
 		if c.lockCh != nil {
 			// we have an active lock already
-			return "", nil
+			return "", fmt.Errorf("state %q already locked", c.Path)
 		}
 	}
 
 	if c.consulLock == nil {
 		opts := &consulapi.LockOptions{
 			Key: c.Path + lockSuffix,
-			// We currently don't procide any options to block terraform and
-			// retry lock acquisition, but we can wait briefly in case the
-			// lock is about to be freed.
+			// only wait briefly, so terraform has the choice to fail fast or
+			// retry as needed.
 			LockWaitTime: time.Second,
 			LockTryOnce:  true,
 		}
@@ -191,6 +190,10 @@ func (c *RemoteClient) Unlock(id string) error {
 	err := c.consulLock.Unlock()
 	c.lockCh = nil
 
+	// This is only cleanup, and will fail if the lock was immediately taken by
+	// another client, so we don't report an error to the user here.
+	c.consulLock.Destroy()
+
 	kv := c.Client.KV()
 	_, delErr := kv.Delete(c.Path+lockInfoSuffix, nil)
 	if delErr != nil {

backend/remote-state/consul/client_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/state/remote"
 )
 
@@ -98,3 +99,43 @@ func TestConsul_stateLock(t *testing.T) {
 
 	remote.TestRemoteLocks(t, sA.(*remote.State).Client, sB.(*remote.State).Client)
 }
+
+func TestConsul_destroyLock(t *testing.T) {
+	srv := newConsulTestServer(t)
+	defer srv.Stop()
+
+	// Get the backend
+	b := backend.TestBackendConfig(t, New(), map[string]interface{}{
+		"address": srv.HTTPAddr,
+		"path":    fmt.Sprintf("tf-unit/%s", time.Now().String()),
+	})
+
+	// Grab the client
+	s, err := b.State(backend.DefaultStateName)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	c := s.(*remote.State).Client.(*RemoteClient)
+
+	info := state.NewLockInfo()
+	id, err := c.Lock(info)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lockPath := c.Path + lockSuffix
+
+	if err := c.Unlock(id); err != nil {
+		t.Fatal(err)
+	}
+
+	// get the lock val
+	pair, _, err := c.Client.KV().Get(lockPath, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if pair != nil {
+		t.Fatalf("lock key not cleaned up at: %s", pair.Key)
+	}
+}