Plumb config

client/client.go

@@ -484,9 +484,7 @@ TRY:
 	}
 
 	// We can wait a bit and retry!
-	// TODO(alexdadgar): Plumb through the RPCHoldTimeout config
-	//if time.Since(firstCheck) < c.config.RPCHoldTimeout {
-	if time.Since(firstCheck) < 5*time.Second {
+	if time.Since(firstCheck) < c.config.RPCHoldTimeout {
 		jitter := lib.RandomStagger(5 * time.Second / nomad.JitterFraction)
 		select {
 		case <-time.After(jitter):

client/config/config.go

@@ -195,6 +195,13 @@ type Config struct {
 	// BackwardsCompatibleMetrics determines whether to show methods of
 	// displaying metrics for older verions, or to only show the new format
 	BackwardsCompatibleMetrics bool
+
+	// RPCHoldTimeout is how long an RPC can be "held" before it is errored.
+	// This is used to paper over a loss of leadership by instead holding RPCs,
+	// so that the caller experiences a slow response rather than an error.
+	// This period is meant to be long enough for a leader election to take
+	// place, and a small jitter is applied to avoid a thundering herd.
+	RPCHoldTimeout time.Duration
 }
 
 func (c *Config) Copy() *Config {
@@ -228,6 +235,7 @@ func DefaultConfig() *Config {
 		NoHostUUID:                 true,
 		DisableTaggedMetrics:       false,
 		BackwardsCompatibleMetrics: false,
+		RPCHoldTimeout:             5 * time.Second,
 	}
 }