diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go
index f3e051691b..693a9e28b3 100644
--- a/consensus/parlia/parlia.go
+++ b/consensus/parlia/parlia.go
@@ -1326,7 +1326,14 @@ func (p *Parlia) backOffTime(snap *Snapshot, header *types.Header, val common.Ad
 		if p.chainConfig.IsPlanck(header.Number) {
 			// reverse the key/value of snap.Recents to get recentsMap
 			recentsMap := make(map[common.Address]uint64, len(snap.Recents))
+			bound := uint64(0)
+			if n, limit := header.Number.Uint64(), uint64(len(validators)/2+1); n > limit {
+				bound = n - limit
+			}
 			for seen, recent := range snap.Recents {
+				if seen <= bound {
+					continue
+				}
 				recentsMap[recent] = seen
 			}