From 1a5f816584af06f8b4874f33e8ad5d430a0b3047 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Wed, 14 Jun 2017 11:43:02 -0700 Subject: [PATCH] etcdserver: rate limit lease revoke per second Instead of having maximum number of inflight revoke routine, rate limit revokes 'per second'. Fix https://github.com/coreos/etcd/issues/8097. Signed-off-by: Gyu-Ho Lee --- etcdserver/server.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/etcdserver/server.go b/etcdserver/server.go index 070063777573..ed1fbfcb9349 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -54,9 +54,11 @@ import ( "github.com/coreos/etcd/store" "github.com/coreos/etcd/version" "github.com/coreos/etcd/wal" + "github.com/coreos/go-semver/semver" "github.com/coreos/pkg/capnslog" "golang.org/x/net/context" + "golang.org/x/time/rate" ) const ( @@ -81,8 +83,9 @@ const ( releaseDelayAfterSnapshot = 30 * time.Second - // maxPendingRevokes is the maximum number of outstanding expired lease revocations. - maxPendingRevokes = 16 + // TODO: make this configurable? + // maximum number of leases to revoke per second + leaseRevokeRate = 1000 recommendedMaxRequestBytes = 10 * 1024 * 1024 ) @@ -730,6 +733,7 @@ func (s *EtcdServer) run() { expiredLeaseC = s.lessor.ExpiredLeasesC() } + revokerLimiter := rate.NewLimiter(rate.Limit(leaseRevokeRate), leaseRevokeRate) for { select { case ap := <-s.r.apply(): @@ -738,19 +742,18 @@ func (s *EtcdServer) run() { case leases := <-expiredLeaseC: s.goAttach(func() { // Increases throughput of expired leases deletion process through parallelization - c := make(chan struct{}, maxPendingRevokes) for _, lease := range leases { select { - case c <- struct{}{}: case <-s.stopping: return + default: } + revokerLimiter.Wait(context.Background()) lid := lease.ID s.goAttach(func() { ctx := s.authStore.WithRoot(s.ctx) s.LeaseRevoke(ctx, &pb.LeaseRevokeRequest{ID: int64(lid)}) leaseExpired.Inc() - <-c }) } })