EVM-689 The impact of database saving failures on Stake Manager funct…

…ionality (#1607)

consensus/polybft/blockchain_wrapper.go

@@ -60,6 +60,9 @@ type blockchainBackend interface {
 
 	// GetChainID returns chain id of the current blockchain
 	GetChainID() uint64
+
+	// GetReceiptsByHash retrieves receipts by hash
+	GetReceiptsByHash(hash types.Hash) ([]*types.Receipt, error)
 }
 
 var _ blockchainBackend = &blockchainWrapper{}
@@ -190,6 +193,10 @@ func (p *blockchainWrapper) GetChainID() uint64 {
 	return uint64(p.blockchain.Config().ChainID)
 }
 
+func (p *blockchainWrapper) GetReceiptsByHash(hash types.Hash) ([]*types.Receipt, error) {
+	return p.blockchain.GetReceiptsByHash(hash)
+}
+
 var _ contract.Provider = &stateProvider{}
 
 type stateProvider struct {

consensus/polybft/consensus_runtime.go

@@ -234,6 +234,7 @@ func (c *consensusRuntime) initStakeManager(logger hcf.Logger) error {
 		wallet.NewEcdsaSigner(c.config.Key),
 		contracts.ValidatorSetContract,
 		c.config.PolyBFTConfig.Bridge.CustomSupernetManagerAddr,
+		c.config.blockchain,
 		int(c.config.PolyBFTConfig.MaxValidatorSetSize),
 	)
 

consensus/polybft/mocks_test.go

@@ -125,6 +125,12 @@ func (m *blockchainMock) GetChainID() uint64 {
 	return 0
 }
 
+func (m *blockchainMock) GetReceiptsByHash(hash types.Hash) ([]*types.Receipt, error) {
+	args := m.Called(hash)
+
+	return args.Get(0).([]*types.Receipt), args.Error(1) //nolint:forcetypeassert
+}
+
 var _ polybftBackend = (*polybftBackendMock)(nil)
 
 type polybftBackendMock struct {

consensus/polybft/stake_manager.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/0xPolygon/polygon-edge/blockchain"
 	"github.com/0xPolygon/polygon-edge/consensus/polybft/bitmap"
 	"github.com/0xPolygon/polygon-edge/consensus/polybft/contractsapi"
 	bls "github.com/0xPolygon/polygon-edge/consensus/polybft/signer"
@@ -56,6 +57,7 @@ type stakeManager struct {
 	validatorSetContract    types.Address
 	supernetManagerContract types.Address
 	maxValidatorSetSize     int
+	blockchain              blockchainBackend
 }
 
 // newStakeManager returns a new instance of stake manager
@@ -65,6 +67,7 @@ func newStakeManager(
 	rootchainRelayer txrelayer.TxRelayer,
 	key ethgo.Key,
 	validatorSetAddr, supernetManagerAddr types.Address,
+	blockchain blockchainBackend,
 	maxValidatorSetSize int,
 ) *stakeManager {
 	return &stakeManager{
@@ -75,6 +78,7 @@ func newStakeManager(
 		validatorSetContract:    validatorSetAddr,
 		supernetManagerContract: supernetManagerAddr,
 		maxValidatorSetSize:     maxValidatorSetSize,
+		blockchain:              blockchain,
 	}
 }
 
@@ -86,71 +90,100 @@ func (s *stakeManager) PostEpoch(req *PostEpochRequest) error {
 
 	// save initial validator set as full validator set in db
 	return s.state.StakeStore.insertFullValidatorSet(validatorSetState{
-		BlockNumber: 0,
-		EpochID:     0,
-		Validators:  newValidatorStakeMap(req.ValidatorSet.Accounts()),
+		BlockNumber:          0,
+		EpochID:              0,
+		UpdatedAtBlockNumber: 0,
+		Validators:           newValidatorStakeMap(req.ValidatorSet.Accounts()),
 	})
 }
 
 // PostBlock is called on every insert of finalized block (either from consensus or syncer)
 // It will read any transfer event that happened in block and update full validator set in db
 func (s *stakeManager) PostBlock(req *PostBlockRequest) error {
-	events, err := s.getTransferEventsFromReceipts(req.FullBlock.Receipts)
+	fullValidatorSet, err := s.state.StakeStore.getFullValidatorSet()
 	if err != nil {
 		return err
 	}
 
-	if len(events) == 0 {
-		return nil
+	s.logger.Debug("Stake manager on post block", "block", req.FullBlock.Block.Number(),
+		"last saved", fullValidatorSet.BlockNumber, "last updated", fullValidatorSet.UpdatedAtBlockNumber)
+
+	// update with missing blocks
+	for i := fullValidatorSet.BlockNumber + 1; i < req.FullBlock.Block.Number(); i++ {
+		blockHeader, found := s.blockchain.GetHeaderByNumber(i)
+		if !found {
+			return blockchain.ErrNoBlock
+		}
+
+		receipts, err := s.blockchain.GetReceiptsByHash(blockHeader.Hash)
+		if err != nil {
+			return err
+		}
+
+		if err := s.updateWithReceipts(&fullValidatorSet, receipts, i); err != nil {
+			return err
+		}
+	}
+
+	// finally update with received block
+	err = s.updateWithReceipts(&fullValidatorSet, req.FullBlock.Receipts, req.FullBlock.Block.Number())
+	if err != nil {
+		return err
 	}
 
-	s.logger.Debug("Gotten transfer (stake changed) events from logs on block",
-		"eventsNum", len(events), "block", req.FullBlock.Block.Number())
+	fullValidatorSet.EpochID = req.Epoch
+	fullValidatorSet.BlockNumber = req.FullBlock.Block.Number()
 
-	fullValidatorSet, err := s.state.StakeStore.getFullValidatorSet()
+	return s.state.StakeStore.insertFullValidatorSet(fullValidatorSet)
+}
+
+func (s *stakeManager) updateWithReceipts(
+	fullValidatorSet *validatorSetState, receipts []*types.Receipt, block uint64) error {
+	events, err := s.getTransferEventsFromReceipts(receipts)
 	if err != nil {
 		return err
 	}
 
-	stakeMap := fullValidatorSet.Validators
+	s.logger.Debug("Full validator set before",
+		"block", block-1, "evnts", len(events), "data", fullValidatorSet.Validators)
 
-	s.logger.Debug("full validator set before", "block", fullValidatorSet.BlockNumber, "data", stakeMap)
+	if len(events) == 0 {
+		return nil
+	}
 
 	for _, event := range events {
 		if event.IsStake() {
-			s.logger.Debug("Stake transfer event", "To", event.To, "Value", event.Value)
+			s.logger.Debug("Stake transfer event", "to", event.To, "value", event.Value)
 
 			// then this amount was minted To validator address
-			stakeMap.addStake(event.To, event.Value)
+			fullValidatorSet.Validators.addStake(event.To, event.Value)
 		} else if event.IsUnstake() {
-			s.logger.Debug("Unstake transfer event", "From", event.From, "Value", event.Value)
+			s.logger.Debug("Unstake transfer event", "from", event.From, "value", event.Value)
 
 			// then this amount was burned From validator address
-			stakeMap.removeStake(event.From, event.Value)
+			fullValidatorSet.Validators.removeStake(event.From, event.Value)
 		} else {
 			// this should not happen, but lets log it if it does
 			s.logger.Warn("Found a transfer event that represents neither stake nor unstake")
 		}
 	}
 
-	for addr, data := range stakeMap {
+	for addr, data := range fullValidatorSet.Validators {
 		if data.BlsKey == nil {
 			data.BlsKey, err = s.getBlsKey(data.Address)
 			if err != nil {
-				s.logger.Warn("Could not get info for new validator", "epoch", req.Epoch, "address", addr)
+				s.logger.Warn("Could not get info for new validator", "block", block, "address", addr)
 			}
 		}
 
 		data.IsActive = data.VotingPower.Cmp(bigZero) > 0
 	}
 
-	s.logger.Debug("full validator set after", "block", req.FullBlock.Block.Number(), "data", stakeMap)
+	fullValidatorSet.UpdatedAtBlockNumber = block // mark on which block validator set has been updated
 
-	return s.state.StakeStore.insertFullValidatorSet(validatorSetState{
-		EpochID:     req.Epoch,
-		BlockNumber: req.FullBlock.Block.Number(),
-		Validators:  stakeMap,
-	})
+	s.logger.Debug("Full validator set after", "block", block, "data", fullValidatorSet.Validators)
+
+	return nil
 }
 
 // UpdateValidatorSet returns an updated validator set
@@ -308,9 +341,10 @@ func (s *stakeManager) getBlsKey(address types.Address) (*bls.PublicKey, error)
 }
 
 type validatorSetState struct {
-	BlockNumber uint64            `json:"block"`
-	EpochID     uint64            `json:"epoch"`
-	Validators  validatorStakeMap `json:"validators"`
+	BlockNumber          uint64            `json:"block"`
+	EpochID              uint64            `json:"epoch"`
+	UpdatedAtBlockNumber uint64            `json:"updated_at_block"`
+	Validators           validatorStakeMap `json:"validators"`
 }
 
 func (vs validatorSetState) Marshal() ([]byte, error) {

consensus/polybft/stake_manager_fuzz_test.go

@@ -154,6 +154,7 @@ func FuzzTestStakeManagerPostBlock(f *testing.F) {
 			wallet.NewEcdsaSigner(validators.GetValidator("A").Key()),
 			types.StringToAddress("0x0001"),
 			types.StringToAddress("0x0002"),
+			nil,
 			5,
 		)
 
@@ -200,6 +201,7 @@ func FuzzTestStakeManagerUpdateValidatorSet(f *testing.F) {
 		nil,
 		wallet.NewEcdsaSigner(validators.GetValidator("A").Key()),
 		types.StringToAddress("0x0001"), types.StringToAddress("0x0002"),
+		nil,
 		10,
 	)
 

consensus/polybft/stake_manager_test.go

@@ -76,12 +76,14 @@ func TestStakeManager_PostBlock(t *testing.T) {
 			nil,
 			wallet.NewEcdsaSigner(validators.GetValidator("A").Key()),
 			types.StringToAddress("0x0001"), types.StringToAddress("0x0002"),
+			nil,
 			5,
 		)
 
 		// insert initial full validator set
 		require.NoError(t, state.StakeStore.insertFullValidatorSet(validatorSetState{
-			Validators: newValidatorStakeMap(validators.GetPublicIdentities(initialSetAliases...)),
+			Validators:  newValidatorStakeMap(validators.GetPublicIdentities(initialSetAliases...)),
+			BlockNumber: block - 1,
 		}))
 
 		receipt := &types.Receipt{
@@ -130,12 +132,14 @@ func TestStakeManager_PostBlock(t *testing.T) {
 			nil,
 			wallet.NewEcdsaSigner(validators.GetValidator("A").Key()),
 			types.StringToAddress("0x0001"), types.StringToAddress("0x0002"),
+			nil,
 			5,
 		)
 
 		// insert initial full validator set
 		require.NoError(t, state.StakeStore.insertFullValidatorSet(validatorSetState{
-			Validators: newValidatorStakeMap(validators.GetPublicIdentities(initialSetAliases...)),
+			Validators:  newValidatorStakeMap(validators.GetPublicIdentities(initialSetAliases...)),
+			BlockNumber: block - 1,
 		}))
 
 		receipt := &types.Receipt{
@@ -194,12 +198,14 @@ func TestStakeManager_PostBlock(t *testing.T) {
 			txRelayerMock,
 			wallet.NewEcdsaSigner(validators.GetValidator("A").Key()),
 			types.StringToAddress("0x0001"), types.StringToAddress("0x0002"),
+			nil,
 			5,
 		)
 
 		// insert initial full validator set
 		require.NoError(t, state.StakeStore.insertFullValidatorSet(validatorSetState{
-			Validators: newValidatorStakeMap(validators.GetPublicIdentities(initialSetAliases...)),
+			Validators:  newValidatorStakeMap(validators.GetPublicIdentities(initialSetAliases...)),
+			BlockNumber: block - 1,
 		}))
 
 		receipts := make([]*types.Receipt, len(allAliases))
@@ -233,6 +239,71 @@ func TestStakeManager_PostBlock(t *testing.T) {
 			require.Equal(t, newStake+uint64(validatorsCount)-uint64(i)-1, v.VotingPower.Uint64())
 		}
 	})
+
+	t.Run("PostBlock - add stake to one validator + missing block", func(t *testing.T) {
+		t.Parallel()
+
+		receipt := &types.Receipt{}
+		header1, header2 := &types.Header{Hash: types.Hash{3, 2}}, &types.Header{Hash: types.Hash{6, 4}}
+
+		bcMock := new(blockchainMock)
+		bcMock.On("GetHeaderByNumber", block-2).Return(header1, true).Once()
+		bcMock.On("GetHeaderByNumber", block-1).Return(header2, true).Once()
+		bcMock.On("GetReceiptsByHash", header1.Hash).Return([]*types.Receipt{receipt}, error(nil)).Once()
+		bcMock.On("GetReceiptsByHash", header2.Hash).Return([]*types.Receipt{}, error(nil)).Once()
+
+		validators := validator.NewTestValidatorsWithAliases(t, allAliases)
+		stakeManager := newStakeManager(
+			hclog.NewNullLogger(),
+			state,
+			nil,
+			wallet.NewEcdsaSigner(validators.GetValidator("A").Key()),
+			types.StringToAddress("0x0001"), types.StringToAddress("0x0002"),
+			bcMock,
+			5,
+		)
+
+		// insert initial full validator set
+		require.NoError(t, state.StakeStore.insertFullValidatorSet(validatorSetState{
+			Validators:  newValidatorStakeMap(validators.GetPublicIdentities(initialSetAliases...)),
+			BlockNumber: block - 3,
+		}))
+
+		receipt.Logs = []*types.Log{
+			createTestLogForTransferEvent(
+				t,
+				stakeManager.validatorSetContract,
+				types.ZeroAddress,
+				validators.GetValidator(initialSetAliases[secondValidator]).Address(),
+				250,
+			),
+		}
+		receipt.SetStatus(types.ReceiptSuccess)
+
+		req := &PostBlockRequest{
+			FullBlock: &types.FullBlock{Block: &types.Block{Header: &types.Header{Number: block}},
+				Receipts: []*types.Receipt{receipt},
+			},
+			Epoch: epoch,
+		}
+
+		require.NoError(t, stakeManager.PostBlock(req))
+
+		fullValidatorSet, err := state.StakeStore.getFullValidatorSet()
+		require.NoError(t, err)
+		var firstValidaotor *validator.ValidatorMetadata
+		firstValidaotor = nil
+		for _, validator := range fullValidatorSet.Validators {
+			if validator.Address.String() == validators.GetValidator(initialSetAliases[secondValidator]).Address().String() {
+				firstValidaotor = validator
+			}
+		}
+		require.NotNil(t, firstValidaotor)
+		require.Equal(t, big.NewInt(501), firstValidaotor.VotingPower) // 250 + 250 + initial 1
+		require.True(t, firstValidaotor.IsActive)
+
+		bcMock.AssertExpectations(t)
+	})
 }
 
 func TestStakeManager_UpdateValidatorSet(t *testing.T) {
@@ -251,6 +322,7 @@ func TestStakeManager_UpdateValidatorSet(t *testing.T) {
 		nil,
 		wallet.NewEcdsaSigner(validators.GetValidator("A").Key()),
 		types.StringToAddress("0x0001"), types.StringToAddress("0x0002"),
+		nil,
 		10,
 	)