From 167e723f3cab7921ee76a6090343e5ddfd4abde7 Mon Sep 17 00:00:00 2001 From: fanmin shi Date: Tue, 2 May 2017 12:07:49 -0700 Subject: [PATCH] etcdserver: apply() sets consistIndex for any entry type previously, apply() doesn't set consistIndex for EntryConfChange type. this causes a misalignment between consistIndex and applied index where EntryConfChange entry results setting applied index but not consistIndex. suppose that addMember() is called and leader reflects that change. 1. applied index and consistIndex is now misaligned. 2. a new follower node joined. 3. leader sends the snapshot to follower where the applied index is the snapshot metadata index. 4. follower node saves the snapshot and database(includes consistIndex) from leader. 5. restarting follower loads snapshot and database. 6. follower checks snapshot metadata index(same as applied index) and database consistIndex, finds them don't match, and then panic. FIXES #7834 --- etcdserver/server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/etcdserver/server.go b/etcdserver/server.go index f5fc36952fa1..8f285d74c242 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -1272,6 +1272,10 @@ func (s *EtcdServer) apply(es []raftpb.Entry, confState *raftpb.ConfState) (appl case raftpb.EntryNormal: s.applyEntryNormal(&e) case raftpb.EntryConfChange: + // set the consistent index of current executing entry + if e.Index > s.consistIndex.ConsistentIndex() { + s.consistIndex.setConsistentIndex(e.Index) + } var cc raftpb.ConfChange pbutil.MustUnmarshal(&cc, e.Data) removedSelf, err := s.applyConfChange(cc, confState) @@ -1292,9 +1296,9 @@ func (s *EtcdServer) apply(es []raftpb.Entry, confState *raftpb.ConfState) (appl func (s *EtcdServer) applyEntryNormal(e *raftpb.Entry) { shouldApplyV3 := false if e.Index > s.consistIndex.ConsistentIndex() { + shouldApplyV3 = true // set the consistent index of current executing entry s.consistIndex.setConsistentIndex(e.Index) - shouldApplyV3 = true } defer s.setAppliedIndex(e.Index)