Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

election: provide methods to read and write campaignTimes instead of export #7501

Merged
merged 1 commit into from
Dec 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions pkg/election/leadership.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
)

const (
defaultCampaignTimesSlot = 10
watchLoopUnhealthyTimeout = 60 * time.Second
campaignTimesRecordTimeout = 5 * time.Minute
)
Expand Down Expand Up @@ -65,9 +66,9 @@ type Leadership struct {
keepAliveCtx context.Context
keepAliveCancelFunc context.CancelFunc
keepAliveCancelFuncLock syncutil.Mutex
// CampaignTimes is used to record the campaign times of the leader within `campaignTimesRecordTimeout`.
// campaignTimes is used to record the campaign times of the leader within `campaignTimesRecordTimeout`.
// It is ordered by time to prevent the leader from campaigning too frequently.
CampaignTimes []time.Time
campaignTimes []time.Time
}

// NewLeadership creates a new Leadership.
Expand All @@ -76,7 +77,7 @@ func NewLeadership(client *clientv3.Client, leaderKey, purpose string) *Leadersh
purpose: purpose,
client: client,
leaderKey: leaderKey,
CampaignTimes: make([]time.Time, 0, 10),
campaignTimes: make([]time.Time, 0, defaultCampaignTimesSlot),
}
return leadership
}
Expand Down Expand Up @@ -111,18 +112,37 @@ func (ls *Leadership) GetLeaderKey() string {
return ls.leaderKey
}

// GetCampaignTimesNum is used to get the campaign times of the leader within `campaignTimesRecordTimeout`.
func (ls *Leadership) GetCampaignTimesNum() int {
if ls == nil {
return 0
}
return len(ls.campaignTimes)
}

// ResetCampaignTimes is used to reset the campaign times of the leader.
func (ls *Leadership) ResetCampaignTimes() {
if ls == nil {
return
}
ls.campaignTimes = make([]time.Time, 0, defaultCampaignTimesSlot)
}

// addCampaignTimes is used to add the campaign times of the leader.
func (ls *Leadership) addCampaignTimes() {
for i := len(ls.CampaignTimes) - 1; i >= 0; i-- {
if time.Since(ls.CampaignTimes[i]) > campaignTimesRecordTimeout {
if ls == nil {
return
}
for i := len(ls.campaignTimes) - 1; i >= 0; i-- {
if time.Since(ls.campaignTimes[i]) > campaignTimesRecordTimeout {
// remove the time which is more than `campaignTimesRecordTimeout`
// array is sorted by time
ls.CampaignTimes = ls.CampaignTimes[i:]
ls.campaignTimes = ls.campaignTimes[i:]
break
}
}

ls.CampaignTimes = append(ls.CampaignTimes, time.Now())
ls.campaignTimes = append(ls.campaignTimes, time.Now())
}

// Campaign is used to campaign the leader with given lease and returns a leadership
Expand Down
5 changes: 2 additions & 3 deletions pkg/member/member.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,11 +185,10 @@ func (m *EmbeddedEtcdMember) CampaignLeader(ctx context.Context, leaseTimeout in
failpoint.Inject("skipCampaignLeaderCheck", func() {
failpoint.Return(m.leadership.Campaign(leaseTimeout, m.MemberValue()))
})
if len(m.leadership.CampaignTimes) >= campaignLeaderFrequencyTimes {
if m.leadership.GetCampaignTimesNum() >= campaignLeaderFrequencyTimes {
log.Warn("campaign times is too frequent, resign and campaign again",
zap.String("leader-name", m.Name()), zap.String("leader-key", m.GetLeaderPath()))
// remove all campaign times
m.leadership.CampaignTimes = nil
m.leadership.ResetCampaignTimes()
return m.ResignEtcdLeader(ctx, m.Name(), "")
}
return m.leadership.Campaign(leaseTimeout, m.MemberValue())
Expand Down
3 changes: 1 addition & 2 deletions tests/server/member/member_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -341,8 +341,7 @@ func TestCampaignLeaderFrequently(t *testing.T) {
cluster.GetLeaderServer().ResetPDLeader()
cluster.WaitLeader()
}
// leader should be changed when campaign leader frequently
cluster.WaitLeader()
// PD leader should be different from before because etcd leader changed.
re.NotEmpty(cluster.GetLeader())
re.NotEqual(leader, cluster.GetLeader())
}
Expand Down