From 2ed23de1712ac5ee36d177d467245a1bc7bb9429 Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Thu, 1 Nov 2018 09:27:39 -0400
Subject: [PATCH] More metrics for Server vault

Add a gauge to track remaining time-to-live, duration of renewal request
API call.
---
 nomad/vault.go      | 33 +++++++++++++++++++++++++------
 nomad/vault_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/nomad/vault.go b/nomad/vault.go
index 8220d4aa9dc8..775a442bc957 100644
--- a/nomad/vault.go
+++ b/nomad/vault.go
@@ -150,12 +150,13 @@ type PurgeVaultAccessorFn func(accessors []*structs.VaultAccessor) error
 // tokenData holds the relevant information about the Vault token passed to the
 // client.
 type tokenData struct {
-	CreationTTL int      `mapstructure:"creation_ttl"`
-	TTL         int      `mapstructure:"ttl"`
-	Renewable   bool     `mapstructure:"renewable"`
-	Policies    []string `mapstructure:"policies"`
-	Role        string   `mapstructure:"role"`
-	Root        bool
+	CreationTTL  int      `mapstructure:"creation_ttl"`
+	CreationTime int      `mapstructure:"creation_time"`
+	TTL          int      `mapstructure:"ttl"`
+	Renewable    bool     `mapstructure:"renewable"`
+	Policies     []string `mapstructure:"policies"`
+	Role         string   `mapstructure:"role"`
+	Root         bool
 }
 
 // vaultClient is the Servers implementation of the VaultClient interface. The
@@ -491,7 +492,9 @@ func (v *vaultClient) renewalLoop() {
 				break
 			}
 
+			metrics.IncrCounter([]string{"nomad", "vault", "renew_failed"}, 1)
 			v.logger.Warn("got error or bad auth, so backing off", "error", err)
+
 			backoff = nextBackoff(backoff, currentExpiration)
 			if backoff < 0 {
 				// We have failed to renew the token past its expiration. Stop
@@ -554,6 +557,9 @@ func nextBackoff(backoff float64, expiry time.Time) float64 {
 // renew attempts to renew our Vault token. If the renewal fails, an error is
 // returned. This method updates the lastRenewed time
 func (v *vaultClient) renew() error {
+	// Track how long the request takes
+	defer metrics.MeasureSince([]string{"nomad", "vault", "renew"}, time.Now())
+
 	// Attempt to renew the token
 	secret, err := v.auth.RenewSelf(v.tokenData.CreationTTL)
 	if err != nil {
@@ -650,6 +656,11 @@ func (v *vaultClient) parseSelfToken() error {
 			multierror.Append(&mErr, fmt.Errorf("Vault token is not renewable or root"))
 		}
 
+		// All non-root tokens must have creation time
+		if data.CreationTime == 0 {
+			multierror.Append(&mErr, fmt.Errorf("invalid lease creation time of zero"))
+		}
+
 		// All non-root tokens must have a lease duration
 		if data.CreationTTL == 0 {
 			multierror.Append(&mErr, fmt.Errorf("invalid lease duration of zero"))
@@ -1228,8 +1239,18 @@ func (v *vaultClient) EmitStats(period time.Duration, stopCh chan struct{}) {
 		case <-time.After(period):
 			stats := v.Stats()
 			metrics.SetGauge([]string{"nomad", "vault", "distributed_tokens_revoking"}, float32(stats.TrackedForRevoke))
+			metrics.SetGauge([]string{"nomad", "vault", "token_ttl"}, float32(tokenTTL(v.tokenData)/time.Millisecond))
 		case <-stopCh:
 			return
 		}
 	}
 }
+
+func tokenTTL(tokenData *tokenData) time.Duration {
+	if tokenData == nil {
+		return time.Duration(0)
+	}
+
+	ttl := int64(tokenData.CreationTime+tokenData.CreationTTL) - time.Now().Unix()
+	return time.Duration(ttl) * time.Second
+}
diff --git a/nomad/vault_test.go b/nomad/vault_test.go
index 3de2e079fbdd..fb63e64f2134 100644
--- a/nomad/vault_test.go
+++ b/nomad/vault_test.go
@@ -1297,3 +1297,50 @@ func TestVaultClient_nextBackoff(t *testing.T) {
 		}
 	})
 }
+
+func TestVaultClient_tokenTTL(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name         string
+		creationTime int64
+		creationTTL  int
+		ttl          time.Duration
+	}{
+		{
+			"in future",
+			time.Now().Unix() - 1000,
+			2500,
+			1500 * time.Second,
+		},
+		{
+			"in past",
+			time.Now().Unix() - 2000,
+			1000,
+			-1000 * time.Second,
+		},
+	}
+
+	tolerance := 2 * time.Second
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			tokenData := &tokenData{
+				CreationTTL:  c.creationTTL,
+				CreationTime: int(c.creationTime),
+			}
+
+			found := tokenTTL(tokenData)
+			if !(c.ttl-tolerance <= found && found <= c.ttl+tolerance) {
+				t.Fatalf("wrong token ttl, expected=%s found=%s", c.ttl, found)
+			}
+		})
+	}
+
+	t.Run("nil case", func(t *testing.T) {
+		found := tokenTTL(nil)
+		if found != 0 {
+			t.Fatalf("expected 0 ttl but found=%s", found)
+		}
+	})
+}