lease: randomize expiry on initial refresh call

Randomize the very first expiry on lease recovery
to prevent recovered leases from expiring all at
the same time.

Address #8096.

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>

lease/lessor.go

@@ -18,6 +18,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"math"
+	"math/rand"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -147,6 +148,9 @@ type lessor struct {
 	stopC chan struct{}
 	// doneC is a channel whose closure indicates that the lessor is stopped.
 	doneC chan struct{}
+
+	// 'true' when lease is just recovered
+	fresh bool
 }
 
 func NewLessor(b backend.Backend, minLeaseTTL int64) Lessor {
@@ -163,6 +167,7 @@ func newLessor(b backend.Backend, minLeaseTTL int64) *lessor {
 		expiredC: make(chan []*Lease, 16),
 		stopC:    make(chan struct{}),
 		doneC:    make(chan struct{}),
+		fresh:    true,
 	}
 	l.initAndRecover()
 
@@ -323,11 +328,25 @@ func (le *lessor) Promote(extend time.Duration) {
 	defer le.mu.Unlock()
 
 	le.demotec = make(chan struct{})
+	fresh := le.fresh
 
 	// refresh the expiries of all leases.
 	for _, l := range le.leaseMap {
-		l.refresh(extend)
+		ext := extend
+		if fresh {
+			// randomize expiry with 士10%, otherwise leases of same TTL
+			// will expire all at the same time,
+			var delta int64
+			if l.ttl > 10 {
+				delta = int64(float64(l.ttl) * 0.1 * rand.Float64())
+			} else {
+				delta = rand.Int63n(10)
+			}
+			ext += time.Duration(delta) * time.Second
+		}
+		l.refresh(ext)
 	}
+	le.fresh = false
 }
 
 func (le *lessor) Demote() {

lease/lessor_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/coreos/etcd/mvcc/backend"
+	"github.com/coreos/etcd/pkg/monotime"
 )
 
 const (
@@ -210,6 +211,52 @@ func TestLessorRenew(t *testing.T) {
 	}
 }
 
+// TestLessorRenewRandomize ensures Lessor renews with randomized expiry.
+func TestLessorRenewRandomize(t *testing.T) {
+	dir, be := NewTestBackend(t)
+	defer os.RemoveAll(dir)
+
+	le := newLessor(be, minLeaseTTL)
+	for i := LeaseID(1); i <= 10; i++ {
+		if _, err := le.Grant(i, 3600); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// simulate stop and recovery
+	le.Stop()
+	be.Close()
+	bcfg := backend.DefaultBackendConfig()
+	bcfg.Path = filepath.Join(dir, "be")
+	be = backend.New(bcfg)
+	defer be.Close()
+	le = newLessor(be, minLeaseTTL)
+
+	now := monotime.Now()
+
+	// first extend after recovery should randomize expiries
+	le.Promote(0)
+
+	for _, l := range le.leaseMap {
+		leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9))
+		pc := (float64(leftSeconds-3600) / float64(3600)) * 100
+		if pc > 10.0 || pc < -10.0 || pc == 0 { // should be within 士10%
+			t.Fatalf("expected randomized expiry, got %d seconds (ttl: 3600)", leftSeconds)
+		}
+	}
+
+	// second extend should not be randomized
+	le.Promote(0)
+
+	for _, l := range le.leaseMap {
+		leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9))
+		pc := (float64(leftSeconds-3600) / float64(3600)) * 100
+		if pc > .5 || pc < -.5 { // should be close to original ttl 3600
+			t.Fatalf("expected 3600-sec left, got %d", leftSeconds)
+		}
+	}
+}
+
 func TestLessorDetach(t *testing.T) {
 	dir, be := NewTestBackend(t)
 	defer os.RemoveAll(dir)