From 537402a7485621c7135fc0e35339b0ff4652bb3c Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 11 Jan 2024 18:41:26 +0800 Subject: [PATCH 1/4] modify log Signed-off-by: Cabinfever_B --- pkg/election/lease.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/election/lease.go b/pkg/election/lease.go index eada4f8786d..d5aa6ce0cba 100644 --- a/pkg/election/lease.go +++ b/pkg/election/lease.go @@ -135,7 +135,7 @@ func (l *lease) KeepAlive(ctx context.Context) { // https://pkg.go.dev/time@master#Timer.Reset timer.Reset(l.leaseTimeout) case <-timer.C: - log.Info("lease timeout", zap.Time("expire", l.expireTime.Load().(time.Time)), zap.String("purpose", l.Purpose)) + log.Info("keep alive lease too slow", zap.Duration("timeout duration", l.leaseTimeout), zap.Time("actual-expire", l.expireTime.Load().(time.Time)), zap.String("purpose", l.Purpose)) return case <-ctx.Done(): return From 8848ba29f2a1c9ce426587fc8c2469cf1275204b Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 12 Jan 2024 09:51:06 +0800 Subject: [PATCH 2/4] modify log Signed-off-by: Cabinfever_B --- pkg/election/lease.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/election/lease.go b/pkg/election/lease.go index d5aa6ce0cba..84c7d48521c 100644 --- a/pkg/election/lease.go +++ b/pkg/election/lease.go @@ -135,7 +135,7 @@ func (l *lease) KeepAlive(ctx context.Context) { // https://pkg.go.dev/time@master#Timer.Reset timer.Reset(l.leaseTimeout) case <-timer.C: - log.Info("keep alive lease too slow", zap.Duration("timeout duration", l.leaseTimeout), zap.Time("actual-expire", l.expireTime.Load().(time.Time)), zap.String("purpose", l.Purpose)) + log.Info("keep alive lease too slow", zap.Duration("timeout-duration", l.leaseTimeout), zap.Time("actual-expire", l.expireTime.Load().(time.Time)), zap.String("purpose", l.Purpose)) return case <-ctx.Done(): return From 116c11073e484a7a3b0e0c9d22a14c3e9eee34f1 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 12 Jan 2024 10:19:35 +0800 Subject: [PATCH 3/4] modify log Signed-off-by: Cabinfever_B --- pkg/election/lease.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/election/lease.go b/pkg/election/lease.go index 84c7d48521c..31f08f55edd 100644 --- a/pkg/election/lease.go +++ b/pkg/election/lease.go @@ -29,9 +29,10 @@ import ( ) const ( - revokeLeaseTimeout = time.Second - requestTimeout = etcdutil.DefaultRequestTimeout - slowRequestTime = etcdutil.DefaultSlowRequestTime + revokeLeaseTimeout = time.Second + requestTimeout = etcdutil.DefaultRequestTimeout + slowRequestTime = etcdutil.DefaultSlowRequestTime + slowKeepLeaseInterval = 1500 * time.Millisecond ) // lease is used as the low-level mechanism for campaigning and renewing elected leadership. @@ -154,11 +155,14 @@ func (l *lease) keepAliveWorker(ctx context.Context, interval time.Duration) <-c log.Info("start lease keep alive worker", zap.Duration("interval", interval), zap.String("purpose", l.Purpose)) defer log.Info("stop lease keep alive worker", zap.String("purpose", l.Purpose)) - + lastTime := time.Now() for { - go func() { + start := time.Now() + if start.Sub(lastTime) > slowKeepLeaseInterval { + log.Warn("the interval between keeping alive lease is too long", zap.Time("last-time", lastTime)) + } + go func(start time.Time) { defer logutil.LogPanic() - start := time.Now() ctx1, cancel := context.WithTimeout(ctx, l.leaseTimeout) defer cancel() var leaseID clientv3.LeaseID @@ -180,12 +184,13 @@ func (l *lease) keepAliveWorker(ctx context.Context, interval time.Duration) <-c } else { log.Error("keep alive response ttl is zero", zap.String("purpose", l.Purpose)) } - }() + }(start) select { case <-ctx.Done(): return case <-ticker.C: + lastTime = start } } }() From 1441cc43c85c2d2d482d021275678013521f797e Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 12 Jan 2024 10:25:40 +0800 Subject: [PATCH 4/4] modify log Signed-off-by: Cabinfever_B --- pkg/election/lease.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/election/lease.go b/pkg/election/lease.go index 31f08f55edd..a6b49fb99f8 100644 --- a/pkg/election/lease.go +++ b/pkg/election/lease.go @@ -29,10 +29,9 @@ import ( ) const ( - revokeLeaseTimeout = time.Second - requestTimeout = etcdutil.DefaultRequestTimeout - slowRequestTime = etcdutil.DefaultSlowRequestTime - slowKeepLeaseInterval = 1500 * time.Millisecond + revokeLeaseTimeout = time.Second + requestTimeout = etcdutil.DefaultRequestTimeout + slowRequestTime = etcdutil.DefaultSlowRequestTime ) // lease is used as the low-level mechanism for campaigning and renewing elected leadership. @@ -158,7 +157,7 @@ func (l *lease) keepAliveWorker(ctx context.Context, interval time.Duration) <-c lastTime := time.Now() for { start := time.Now() - if start.Sub(lastTime) > slowKeepLeaseInterval { + if start.Sub(lastTime) > interval*2 { log.Warn("the interval between keeping alive lease is too long", zap.Time("last-time", lastTime)) } go func(start time.Time) {