Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Retry validating token from vault till success #3957

Merged
merged 4 commits into from
Mar 9, 2018
Merged

Retry validating token from vault till success #3957

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2738a1c
Retry when vault is sealed
preetapan Mar 8, 2018
208cce8
Always retry on token validation instead of special casing vault sealing
preetapan Mar 9, 2018
646780c
update comment
preetapan Mar 9, 2018
d50338a
Update error message
preetapan Mar 9, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
44 changes: 23 additions & 21 deletions nomad/vault.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,35 +406,37 @@ func (v *vaultClient) establishConnection() {
// Create the retry timer and set initial duration to zero so it fires
// immediately
retryTimer := time.NewTimer(0)

initStatus := false
OUTER:
for {
select {
case <-v.tomb.Dying():
return
case <-retryTimer.C:
// Ensure the API is reachable
if _, err := v.client.Sys().InitStatus(); err != nil {
v.logger.Printf("[WARN] vault: failed to contact Vault API. Retrying in %v: %v",
v.config.ConnectionRetryIntv, err)
if !initStatus {
if _, err := v.client.Sys().InitStatus(); err != nil {
v.logger.Printf("[WARN] vault: failed to contact Vault API. Retrying in %v: %v",
v.config.ConnectionRetryIntv, err)
retryTimer.Reset(v.config.ConnectionRetryIntv)
continue OUTER
}
initStatus = true
}
// Retry validating the token till success
if err := v.parseSelfToken(); err != nil {
v.logger.Printf("[ERR] vault: failed to validate self token/role. Retrying in %v: %v", v.config.ConnectionRetryIntv, err)
retryTimer.Reset(v.config.ConnectionRetryIntv)
v.l.Lock()
v.connEstablished = true
v.connEstablishedErr = fmt.Errorf("Connection to Vault failed: %v", err)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

v.connEstablishedErr = fmt.Errorf("Nomad Server failed to establish connections to Vault: %v", err)

v.l.Unlock()
continue OUTER
}

break OUTER
}
}

// Retrieve our token, validate it and parse the lease duration
if err := v.parseSelfToken(); err != nil {
v.logger.Printf("[ERR] vault: failed to validate self token/role and not retrying: %v", err)
v.l.Lock()
v.connEstablished = false
v.connEstablishedErr = err
v.l.Unlock()
return
}

// Set the wrapping function such that token creation is wrapped now
// that we know our role
v.client.SetWrappingLookupFunc(v.getWrappingFn())
Expand Down Expand Up @@ -844,8 +846,8 @@ func (v *vaultClient) CreateToken(ctx context.Context, a *structs.Allocation, ta
// Check if we have established a connection with Vault
if established, err := v.ConnectionEstablished(); !established && err == nil {
return nil, structs.NewRecoverableError(fmt.Errorf("Connection to Vault has not been established"), true)
} else if !established {
return nil, fmt.Errorf("Connection to Vault failed: %v", err)
} else if err != nil {
return nil, err
}

// Track how long the request takes
Expand Down Expand Up @@ -922,8 +924,8 @@ func (v *vaultClient) LookupToken(ctx context.Context, token string) (*vapi.Secr
// Check if we have established a connection with Vault
if established, err := v.ConnectionEstablished(); !established && err == nil {
return nil, structs.NewRecoverableError(fmt.Errorf("Connection to Vault has not been established"), true)
} else if !established {
return nil, fmt.Errorf("Connection to Vault failed: %v", err)
} else if err != nil {
return nil, err
}

// Track how long the request takes
Expand Down Expand Up @@ -1041,8 +1043,8 @@ func (v *vaultClient) parallelRevoke(ctx context.Context, accessors []*structs.V
// Check if we have established a connection with Vault
if established, err := v.ConnectionEstablished(); !established && err == nil {
return structs.NewRecoverableError(fmt.Errorf("Connection to Vault has not been established"), true)
} else if !established {
return fmt.Errorf("Connection to Vault failed: %v", err)
} else if err != nil {
return err
}

g, pCtx := errgroup.WithContext(ctx)
Expand Down
17 changes: 8 additions & 9 deletions nomad/vault_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,8 +260,8 @@ func TestVaultClient_ValidateRole(t *testing.T) {
var connErr error
testutil.WaitForResult(func() (bool, error) {
conn, connErr = client.ConnectionEstablished()
if conn {
return false, fmt.Errorf("Should not connect")
if !conn {
return false, fmt.Errorf("Should connect")
}

if connErr == nil {
Expand Down Expand Up @@ -303,8 +303,8 @@ func TestVaultClient_ValidateRole_NonExistant(t *testing.T) {
var connErr error
testutil.WaitForResult(func() (bool, error) {
conn, connErr = client.ConnectionEstablished()
if conn {
return false, fmt.Errorf("Should not connect")
if !conn {
return false, fmt.Errorf("Should connect")
}

if connErr == nil {
Expand Down Expand Up @@ -351,8 +351,8 @@ func TestVaultClient_ValidateToken(t *testing.T) {
var connErr error
testutil.WaitForResult(func() (bool, error) {
conn, connErr = client.ConnectionEstablished()
if conn {
return false, fmt.Errorf("Should not connect")
if !conn {
return false, fmt.Errorf("Should connect")
}

if connErr == nil {
Expand Down Expand Up @@ -967,10 +967,9 @@ func TestVaultClient_CreateToken_Role_InvalidToken(t *testing.T) {

testutil.WaitForResult(func() (bool, error) {
established, err := client.ConnectionEstablished()
if established {
return false, fmt.Errorf("Shouldn't establish")
if !established {
return false, fmt.Errorf("Should establish")
}

return err != nil, nil
}, func(err error) {
t.Fatalf("Connection not established")
Expand Down