From c04da2a9bdc91174d88a627869abd41c3f7b013e Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Thu, 23 Jul 2020 11:33:08 -0400
Subject: [PATCH 1/4] vault: simply make the API call

Avoid checking if API is accessible, just make the API call and handle
when it fails.
---
 nomad/vault.go | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/nomad/vault.go b/nomad/vault.go
index 013f38be9214..c5efeefaf7e2 100644
--- a/nomad/vault.go
+++ b/nomad/vault.go
@@ -458,22 +458,12 @@ func (v *vaultClient) establishConnection() {
 	// Create the retry timer and set initial duration to zero so it fires
 	// immediately
 	retryTimer := time.NewTimer(0)
-	initStatus := false
 OUTER:
 	for {
 		select {
 		case <-v.tomb.Dying():
 			return
 		case <-retryTimer.C:
-			// Ensure the API is reachable
-			if !initStatus {
-				if _, err := v.clientSys.Sys().InitStatus(); err != nil {
-					v.logger.Warn("failed to contact Vault API", "retry", v.config.ConnectionRetryIntv, "error", err)
-					retryTimer.Reset(v.config.ConnectionRetryIntv)
-					continue OUTER
-				}
-				initStatus = true
-			}
 			// Retry validating the token till success
 			if err := v.parseSelfToken(); err != nil {
 				v.logger.Error("failed to validate self token/role", "retry", v.config.ConnectionRetryIntv, "error", err)

From 142c1815ae328563dc0a6dbc1a2d049cb3ca47f1 Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Thu, 23 Jul 2020 13:08:16 -0400
Subject: [PATCH 2/4] run revoke daemon if connection is successful

---
 nomad/vault.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nomad/vault.go b/nomad/vault.go
index c5efeefaf7e2..b06d79655133 100644
--- a/nomad/vault.go
+++ b/nomad/vault.go
@@ -1229,7 +1229,7 @@ func (v *vaultClient) revokeDaemon() {
 		case <-v.tomb.Dying():
 			return
 		case now := <-ticker.C:
-			if established, _ := v.ConnectionEstablished(); !established {
+			if established, err := v.ConnectionEstablished(); !established || err != nil {
 				continue
 			}
 

From f459aa6cfb89a8bb4d8e4198aa97885843108f2b Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Thu, 23 Jul 2020 13:25:25 -0400
Subject: [PATCH 3/4] test tweaks

---
 nomad/vault_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/nomad/vault_test.go b/nomad/vault_test.go
index 3f94d79446e6..07ee66a6d43a 100644
--- a/nomad/vault_test.go
+++ b/nomad/vault_test.go
@@ -1015,6 +1015,7 @@ func TestVaultClient_LookupToken_RateLimit(t *testing.T) {
 
 	// Spin up many requests. These should block
 	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	cancels := 0
 	numRequests := 20
@@ -1028,7 +1029,7 @@ func TestVaultClient_LookupToken_RateLimit(t *testing.T) {
 					cancels += 1
 					return
 				}
-				t.Fatalf("self lookup failed: %v", err)
+				t.Errorf("self lookup failed: %v", err)
 				return
 			}
 

From a3b4f060fb49ed2c6408ffce2fd01454d7fdd209 Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Mon, 10 Aug 2020 15:50:03 -0400
Subject: [PATCH 4/4] distinguish between transient and persistent errors

---
 nomad/vault.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/nomad/vault.go b/nomad/vault.go
index b06d79655133..aa58522bb906 100644
--- a/nomad/vault.go
+++ b/nomad/vault.go
@@ -458,6 +458,7 @@ func (v *vaultClient) establishConnection() {
 	// Create the retry timer and set initial duration to zero so it fires
 	// immediately
 	retryTimer := time.NewTimer(0)
+	initStatus := false
 OUTER:
 	for {
 		select {
@@ -466,6 +467,16 @@ OUTER:
 		case <-retryTimer.C:
 			// Retry validating the token till success
 			if err := v.parseSelfToken(); err != nil {
+				// if parsing token fails, try to distinguish legitimate token error from transient Vault initialization/connection issue
+				if !initStatus {
+					if _, err := v.clientSys.Sys().Health(); err != nil {
+						v.logger.Warn("failed to contact Vault API", "retry", v.config.ConnectionRetryIntv, "error", err)
+						retryTimer.Reset(v.config.ConnectionRetryIntv)
+						continue OUTER
+					}
+					initStatus = true
+				}
+
 				v.logger.Error("failed to validate self token/role", "retry", v.config.ConnectionRetryIntv, "error", err)
 				retryTimer.Reset(v.config.ConnectionRetryIntv)
 				v.l.Lock()
@@ -474,6 +485,7 @@ OUTER:
 				v.l.Unlock()
 				continue OUTER
 			}
+
 			break OUTER
 		}
 	}