Merge pull request hashicorp#100 from hashicorp/smooth-suspicion-timeout

Change suspicion timeout function to be smooth instead of stepwise

util.go

@@ -155,8 +155,9 @@ func randomOffset(n int) int {
 // suspicionTimeout computes the timeout that should be used when
 // a node is suspected
 func suspicionTimeout(suspicionMult, n int, interval time.Duration) time.Duration {
-	nodeScale := math.Ceil(math.Log10(float64(n + 1)))
-	timeout := time.Duration(suspicionMult) * time.Duration(nodeScale) * interval
+	nodeScale := math.Max(1.0, math.Log10(math.Max(1.0, float64(n))))
+	// multiply by 1000 to keep some precision because time.Duration is an int64 type
+	timeout := time.Duration(suspicionMult) * time.Duration(nodeScale*1000) * interval / 1000
 	return timeout
 }
 

util_test.go

@@ -158,9 +158,19 @@ func TestRandomOffset_Zero(t *testing.T) {
 }
 
 func TestSuspicionTimeout(t *testing.T) {
-	timeout := suspicionTimeout(3, 10, time.Second)
-	if timeout != 6*time.Second {
-		t.Fatalf("bad timeout")
+	timeouts := map[int]time.Duration{
+		5:    1000 * time.Millisecond,
+		10:   1000 * time.Millisecond,
+		50:   1698 * time.Millisecond,
+		100:  2000 * time.Millisecond,
+		500:  2698 * time.Millisecond,
+		1000: 3000 * time.Millisecond,
+	}
+	for n, expected := range timeouts {
+		timeout := suspicionTimeout(3, n, time.Second) / 3
+		if timeout != expected {
+			t.Fatalf("bad: %v, %v", expected, timeout)
+		}
 	}
 }