Improve handling of Vault errors

CHANGELOG.md

@@ -82,6 +82,7 @@ BUG FIXES:
  * client: Improve auto-detection of network interface when interface name has a
    space in it on Windows [[GH-3855](https://github.com/hashicorp/nomad/issues/3855)]
  * client/vault: Recognize renewing non-renewable Vault lease as fatal [[GH-3727](https://github.com/hashicorp/nomad/issues/3727)]
+ * client/vault: Improved error handling of network errors with Vault [[GH-4100](https://github.com/hashicorp/nomad/issues/4100)]
  * config: Revert minimum CPU limit back to 20 from 100 [[GH-3706](https://github.com/hashicorp/nomad/issues/3706)]
  * config: Always add core scheduler to enabled schedulers and add invalid
    EnabledScheduler detection [[GH-3978](https://github.com/hashicorp/nomad/issues/3978)]

client/client.go

@@ -1939,10 +1939,27 @@ func (c *Client) deriveToken(alloc *structs.Allocation, taskNames []string, vcli
 		// Unwrap the vault token
 		unwrapResp, err := vclient.Logical().Unwrap(wrappedToken)
 		if err != nil {
-			return nil, fmt.Errorf("failed to unwrap the token for task %q: %v", taskName, err)
+			if structs.VaultUnrecoverableError.MatchString(err.Error()) {
+				return nil, err
+			}
+
+			// The error is recoverable
+			return nil, structs.NewRecoverableError(
+				fmt.Errorf("failed to unwrap the token for task %q: %v", taskName, err), true)
+		}
+
+		// Validate the response
+		var validationErr error
+		if unwrapResp == nil {
+			validationErr = fmt.Errorf("Vault returned nil secret when unwrapping")
+		} else if unwrapResp.Auth == nil {
+			validationErr = fmt.Errorf("Vault returned unwrap secret with nil Auth. Secret warnings: %v", unwrapResp.Warnings)
+		} else if unwrapResp.Auth.ClientToken == "" {
+			validationErr = fmt.Errorf("Vault returned unwrap secret with empty Auth.ClientToken. Secret warnings: %v", unwrapResp.Warnings)
 		}
-		if unwrapResp == nil || unwrapResp.Auth == nil || unwrapResp.Auth.ClientToken == "" {
-			return nil, fmt.Errorf("failed to unwrap the token for task %q", taskName)
+		if validationErr != nil {
+			c.logger.Printf("[WARN] client.vault: failed to unwrap token: %v", err)
+			return nil, structs.NewRecoverableError(validationErr, true)
 		}
 
 		// Append the unwrapped token to the return value

nomad/node_endpoint.go

@@ -1398,10 +1398,6 @@ func (n *Node) DeriveVaultToken(args *structs.DeriveVaultTokenRequest,
 	tokens := make(map[string]string, len(results))
 	for task, secret := range results {
 		w := secret.WrapInfo
-		if w == nil {
-			return fmt.Errorf("Vault returned Secret without WrapInfo")
-		}
-
 		tokens[task] = w.Token
 		accessor := &structs.VaultAccessor{
 			Accessor:    w.WrappedAccessor,

nomad/structs/structs.go

@@ -5136,6 +5136,12 @@ func (d *EphemeralDisk) Copy() *EphemeralDisk {
 	return ld
 }
 
+var (
+	// VaultUnrecoverableError matches unrecoverable errors returned by a Vault
+	// server
+	VaultUnrecoverableError = regexp.MustCompile(`Code:\s+40(0|3|4)`)
+)
+
 const (
 	// VaultChangeModeNoop takes no action when a new token is retrieved.
 	VaultChangeModeNoop = "noop"

nomad/vault.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"log"
 	"math/rand"
-	"regexp"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -70,9 +69,6 @@ const (
 )
 
 var (
-	// vaultUnrecoverableError matches unrecoverable errors
-	vaultUnrecoverableError = regexp.MustCompile(`Code:\s+40(0|3|4)`)
-
 	// vaultCapabilitiesCapability is the expected capability of Nomad's Vault
 	// token on the the path. The token must have at least one of the
 	// capabilities.
@@ -695,7 +691,7 @@ func (v *vaultClient) validateCapabilities(role string, root bool) error {
 	_, _, err := v.hasCapability(vaultCapabilitiesLookupPath, vaultCapabilitiesCapability)
 	if err != nil {
 		// Check if there is a permission denied
-		if vaultUnrecoverableError.MatchString(err.Error()) {
+		if structs.VaultUnrecoverableError.MatchString(err.Error()) {
 			// Since we can't read permissions, we just log a warning that we
 			// can't tell if the Vault token will work
 			msg := fmt.Sprintf("Can not lookup token capabilities. "+
@@ -894,14 +890,29 @@ func (v *vaultClient) CreateToken(ctx context.Context, a *structs.Allocation, ta
 
 	// Determine whether it is unrecoverable
 	if err != nil {
-		if vaultUnrecoverableError.MatchString(err.Error()) {
+		if structs.VaultUnrecoverableError.MatchString(err.Error()) {
 			return secret, err
 		}
 
 		// The error is recoverable
 		return nil, structs.NewRecoverableError(err, true)
 	}
 
+	// Validate the response
+	var validationErr error
+	if secret == nil {
+		validationErr = fmt.Errorf("Vault returned nil Secret")
+	} else if secret.WrapInfo == nil {
+		validationErr = fmt.Errorf("Vault returned Secret with nil WrapInfo. Secret warnings: %v", secret.Warnings)
+	} else if secret.WrapInfo.WrappedAccessor == "" {
+		validationErr = fmt.Errorf("Vault returned WrapInfo without WrappedAccessor. Secret warnings: %v", secret.Warnings)
+	}
+	if validationErr != nil {
+		v.logger.Printf("[WARN] vault: failed to CreateToken: %v", err)
+		return nil, structs.NewRecoverableError(validationErr, true)
+	}
+
+	// Got a valid response
 	return secret, nil
 }