Skip to content

Commit

Permalink
Merge pull request #11944 from hashicorp/f-state-slow
Browse files Browse the repository at this point in the history
show message if state lock acquisition/release is slow
  • Loading branch information
mitchellh authored Feb 14, 2017
2 parents 7fa5731 + b7a143f commit 235b7eb
Show file tree
Hide file tree
Showing 12 changed files with 279 additions and 81 deletions.
24 changes: 8 additions & 16 deletions backend/local/backend_apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"github.com/hashicorp/errwrap"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/state"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/terraform"
)

Expand All @@ -35,22 +35,14 @@ func (b *Local) opApply(
return
}

// context acquired the state, and therefor the lock.
// Unlock it when the operation is complete
defer func() {
if s, ok := opState.(state.Locker); op.LockState && ok {
if err := s.Unlock(); err != nil {
runningOp.Err = multierror.Append(runningOp.Err,
errwrap.Wrapf("Error unlocking state:\n\n"+
"{{err}}\n\n"+
"The Terraform operation completed but there was an error unlocking the state.\n"+
"This may require unlocking the state manually with the `terraform unlock` command\n",
err,
),
)
// If we're locking state, unlock when we're done
if op.LockState {
defer func() {
if err := clistate.Unlock(opState, b.CLI, b.Colorize()); err != nil {
runningOp.Err = multierror.Append(runningOp.Err, err)
}
}
}()
}()
}

// Setup the state
runningOp.State = tfCtx.State()
Expand Down
6 changes: 4 additions & 2 deletions backend/local/backend_local.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/hashicorp/errwrap"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)
Expand All @@ -28,8 +29,9 @@ func (b *Local) context(op *backend.Operation) (*terraform.Context, state.State,
return nil, nil, errwrap.Wrapf("Error loading state: {{err}}", err)
}

if s, ok := s.(state.Locker); op.LockState && ok {
if err := s.Lock(op.Type.String()); err != nil {
if op.LockState {
err := clistate.Lock(s, op.Type.String(), b.CLI, b.Colorize())
if err != nil {
return nil, nil, errwrap.Wrapf("Error locking state: {{err}}", err)
}
}
Expand Down
18 changes: 9 additions & 9 deletions backend/local/backend_plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ import (
"strings"

"github.com/hashicorp/errwrap"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/command/format"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)

Expand Down Expand Up @@ -58,15 +59,14 @@ func (b *Local) opPlan(
return
}

// context acquired the state, and therefor the lock.
// Unlock it when the operation is complete
defer func() {
if s, ok := opState.(state.Locker); op.LockState && ok {
if err := s.Unlock(); err != nil {
log.Printf("[ERROR]: %s", err)
// If we're locking state, unlock when we're done
if op.LockState {
defer func() {
if err := clistate.Unlock(opState, b.CLI, b.Colorize()); err != nil {
runningOp.Err = multierror.Append(runningOp.Err, err)
}
}
}()
}()
}

// Setup the state
runningOp.State = tfCtx.State()
Expand Down
19 changes: 9 additions & 10 deletions backend/local/backend_refresh.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ package local
import (
"context"
"fmt"
"log"
"os"

"github.com/hashicorp/errwrap"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/state"
clistate "github.com/hashicorp/terraform/command/state"
)

func (b *Local) opRefresh(
Expand Down Expand Up @@ -48,15 +48,14 @@ func (b *Local) opRefresh(
return
}

// context acquired the state, and therefor the lock.
// Unlock it when the operation is complete
defer func() {
if s, ok := opState.(state.Locker); op.LockState && ok {
if err := s.Unlock(); err != nil {
log.Printf("[ERROR]: %s", err)
// If we're locking state, unlock when we're done
if op.LockState {
defer func() {
if err := clistate.Unlock(opState, b.CLI, b.Colorize()); err != nil {
runningOp.Err = multierror.Append(runningOp.Err, err)
}
}
}()
}()
}

// Set our state
runningOp.State = opState.State()
Expand Down
44 changes: 17 additions & 27 deletions command/meta_backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/hcl"
"github.com/hashicorp/terraform/backend"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
Expand Down Expand Up @@ -531,11 +532,12 @@ func (m *Meta) backendFromPlan(opts *BackendOpts) (backend.Backend, error) {
return nil, fmt.Errorf("Error reading state: %s", err)
}

unlock, err := lockState(realMgr, "backend from plan")
// Lock the state if we can
err = clistate.Lock(realMgr, "backend from plan", m.Ui, m.Colorize())
if err != nil {
return nil, err
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer unlock()
defer clistate.Unlock(realMgr, m.Ui, m.Colorize())

if err := realMgr.RefreshState(); err != nil {
return nil, fmt.Errorf("Error reading state: %s", err)
Expand Down Expand Up @@ -983,11 +985,12 @@ func (m *Meta) backend_C_r_s(
}
}

unlock, err := lockState(sMgr, "backend_C_r_s")
// Lock the state if we can
err = clistate.Lock(sMgr, "backend from config", m.Ui, m.Colorize())
if err != nil {
return nil, err
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer unlock()
defer clistate.Unlock(sMgr, m.Ui, m.Colorize())

// Store the metadata in our saved state location
s := sMgr.State()
Expand Down Expand Up @@ -1087,11 +1090,12 @@ func (m *Meta) backend_C_r_S_changed(
}
}

unlock, err := lockState(sMgr, "backend_C_r_S_changed")
// Lock the state if we can
err = clistate.Lock(sMgr, "backend from config", m.Ui, m.Colorize())
if err != nil {
return nil, err
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer unlock()
defer clistate.Unlock(sMgr, m.Ui, m.Colorize())

// Update the backend state
s = sMgr.State()
Expand Down Expand Up @@ -1244,11 +1248,12 @@ func (m *Meta) backend_C_R_S_unchanged(
}
}

unlock, err := lockState(sMgr, "backend_C_R_S_unchanged")
// Lock the state if we can
err = clistate.Lock(sMgr, "backend from config", m.Ui, m.Colorize())
if err != nil {
return nil, err
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer unlock()
defer clistate.Unlock(sMgr, m.Ui, m.Colorize())

// Unset the remote state
s = sMgr.State()
Expand Down Expand Up @@ -1399,21 +1404,6 @@ func init() {
backendlegacy.Init(Backends)
}

// simple wrapper to check for a state.Locker and always provide an unlock
// function to defer.
func lockState(s state.State, info string) (func() error, error) {
l, ok := s.(state.Locker)
if !ok {
return func() error { return nil }, nil
}

if err := l.Lock(info); err != nil {
return nil, err
}

return l.Unlock, nil
}

// errBackendInitRequired is the final error message shown when reinit
// is required for some reason. The error message includes the reason.
var errBackendInitRequired = errors.New(
Expand Down
13 changes: 7 additions & 6 deletions command/meta_backend_migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"path/filepath"
"strings"

clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)
Expand All @@ -23,17 +24,17 @@ import (
//
// This will attempt to lock both states for the migration.
func (m *Meta) backendMigrateState(opts *backendMigrateOpts) error {
unlockOne, err := lockState(opts.One, "migrate from")
err := clistate.Lock(opts.One, "migration source state", m.Ui, m.Colorize())
if err != nil {
return err
return fmt.Errorf("Error locking source state: %s", err)
}
defer unlockOne()
defer clistate.Unlock(opts.One, m.Ui, m.Colorize())

unlockTwo, err := lockState(opts.Two, "migrate to")
err = clistate.Lock(opts.Two, "migration destination state", m.Ui, m.Colorize())
if err != nil {
return err
return fmt.Errorf("Error locking destination state: %s", err)
}
defer unlockTwo()
defer clistate.Unlock(opts.Two, m.Ui, m.Colorize())

one := opts.One.State()
two := opts.Two.State()
Expand Down
96 changes: 96 additions & 0 deletions command/state/state.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
// Package state exposes common helpers for working with state from the CLI.
//
// This is a separate package so that backends can use this for consistent
// messaging without creating a circular reference to the command package.
package message

import (
"fmt"
"strings"
"time"

"github.com/hashicorp/errwrap"
"github.com/hashicorp/terraform/helper/slowmessage"
"github.com/hashicorp/terraform/state"
"github.com/mitchellh/cli"
"github.com/mitchellh/colorstring"
)

const (
LockThreshold = 400 * time.Millisecond
LockMessage = "Acquiring state lock. This may take a few moments..."
LockErrorMessage = `Error acquiring the state lock: {{err}}
Terraform acquires a state lock to protect the state from being written
by multiple users at the same time. Please resolve the issue above and try
again. For most commands, you can disable locking with the "-lock=false"
flag, but this is not recommended.`

UnlockMessage = "Releasing state lock. This may take a few moments..."
UnlockErrorMessage = `
[reset][bold][red]Error releasing the state lock![reset][red]
Error message: %s
Terraform acquires a lock when accessing your state to prevent others
running Terraform to potentially modify the state at the same time. An
error occurred while releasing this lock. This could mean that the lock
did or did not release properly. If the lock didn't release properly,
Terraform may not be able to run future commands since it'll appear as if
the lock is held.
In this scenario, please call the "force-unlock" command to unlock the
state manually. This is a very dangerous operation since if it is done
erroneously it could result in two people modifying state at the same time.
Only call this command if you're certain that the unlock above failed and
that no one else is holding a lock.
`
)

// Lock locks the given state and outputs to the user if locking
// is taking longer than the threshold.
func Lock(s state.State, info string, ui cli.Ui, color *colorstring.Colorize) error {
sl, ok := s.(state.Locker)
if !ok {
return nil
}

err := slowmessage.Do(LockThreshold, func() error {
return sl.Lock(info)
}, func() {
if ui != nil {
ui.Output(color.Color(LockMessage))
}
})

if err != nil {
err = errwrap.Wrapf(strings.TrimSpace(LockErrorMessage), err)
}

return err
}

// Unlock unlocks the given state and outputs to the user if the
// unlock fails what can be done.
func Unlock(s state.State, ui cli.Ui, color *colorstring.Colorize) error {
sl, ok := s.(state.Locker)
if !ok {
return nil
}

err := slowmessage.Do(LockThreshold, sl.Unlock, func() {
if ui != nil {
ui.Output(color.Color(UnlockMessage))
}
})

if err != nil {
ui.Output(color.Color(fmt.Sprintf(
"\n"+strings.TrimSpace(UnlockErrorMessage)+"\n", err)))

err = fmt.Errorf(
"Error releasing the state lock. Please see the longer error message above.")
}

return err
}
11 changes: 6 additions & 5 deletions command/taint.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"log"
"strings"

"github.com/hashicorp/terraform/state"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/terraform"
)

Expand Down Expand Up @@ -72,13 +72,14 @@ func (c *TaintCommand) Run(args []string) int {
return 1
}

if s, ok := st.(state.Locker); c.Meta.stateLock && ok {
if err := s.Lock("taint"); err != nil {
c.Ui.Error(fmt.Sprintf("Failed to lock state: %s", err))
if c.Meta.stateLock {
err := clistate.Lock(st, "taint", c.Ui, c.Colorize())
if err != nil {
c.Ui.Error(fmt.Sprintf("Error locking state: %s", err))
return 1
}

defer s.Unlock()
defer clistate.Unlock(st, c.Ui, c.Colorize())
}

// Get the actual state structure
Expand Down
2 changes: 1 addition & 1 deletion command/unlock.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func (c *UnlockCommand) Synopsis() string {
}

const outputUnlockSuccess = `
[reset][bold][red]Terraform state has been successfully unlocked![reset][red]
[reset][bold][green]Terraform state has been successfully unlocked![reset][green]
The state has been unlocked, and Terraform commands should now be able to
obtain a new lock on the remote state.
Expand Down
Loading

0 comments on commit 235b7eb

Please sign in to comment.