Update Web3Q validator set from PoS staking contract in remote chain (e…

…thereum#78)

* Update Web3Q validator set from PoS staking contract in remote chain

cmd/geth/main.go

@@ -151,6 +151,10 @@ var (
 		utils.Web3QMainnetFlag,
 		utils.ValPortFlag,
 		utils.ValNodeKeyFlag,
+		utils.ValRpcFlag,
+		utils.ValContractFlag,
+		utils.ValChainIdFlag,
+		utils.ValChangeEpochIdFlag,
 		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
 		utils.EthStatsURLFlag,

cmd/utils/flags.go

@@ -181,6 +181,22 @@ var (
 		Name:  "validator.nodekey",
 		Usage: "Validator P2P node key file",
 	}
+	ValContractFlag = cli.StringFlag{
+		Name:  "validator.contract",
+		Usage: "the contract used by Validator to get latest validator set from",
+	}
+	ValChainIdFlag = cli.StringFlag{
+		Name:  "validator.chainid",
+		Usage: "the chain used by Validator to get latest validator set from",
+	}
+	ValChangeEpochIdFlag = cli.StringFlag{
+		Name:  "validator.changeepochid",
+		Usage: "the epoch validator start to get validator set from contract, 0 means disable",
+	}
+	ValRpcFlag = cli.StringFlag{
+		Name:  "validator.rpc",
+		Usage: "rpc for Validator to update validator set",
+	}
 	DeveloperFlag = cli.BoolFlag{
 		Name:  "dev",
 		Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled",
@@ -1427,6 +1443,18 @@ func setTendermint(ctx *cli.Context, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(ValNodeKeyFlag.Name) {
 		cfg.ValNodeKey = ctx.GlobalString(ValNodeKeyFlag.Name)
 	}
+	if ctx.GlobalIsSet(ValRpcFlag.Name) {
+		cfg.ValRpc = ctx.GlobalString(ValRpcFlag.Name)
+	}
+	if ctx.GlobalIsSet(ValChangeEpochIdFlag.Name) {
+		cfg.ValidatorChangeEpochId = ctx.GlobalUint64(ValChangeEpochIdFlag.Name)
+	}
+	if ctx.GlobalIsSet(ValChainIdFlag.Name) {
+		cfg.ValChainId = ctx.GlobalUint64(ValChainIdFlag.Name)
+	}
+	if ctx.GlobalIsSet(ValContractFlag.Name) {
+		cfg.ValContract = ctx.GlobalString(ValContractFlag.Name)
+	}
 }
 
 func setMiner(ctx *cli.Context, cfg *miner.Config) {

consensus/tendermint/adapter/store.go

@@ -1,32 +1,41 @@
 package adapter
 
 import (
+	"bytes"
 	"context"
+	"encoding/binary"
 	"errors"
 	"fmt"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
 	pbft "github.com/ethereum/go-ethereum/consensus/tendermint/consensus"
+	"github.com/ethereum/go-ethereum/consensus/tendermint/gov"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
 )
 
+var prefix = []byte("HeaderNumber")
+
 type Store struct {
+	config           *params.TendermintConfig
 	chain            *core.BlockChain
 	verifyHeaderFunc func(chain consensus.ChainHeaderReader, header *types.Header, seal bool) error
 	makeBlock        func(parentHash common.Hash, coinbase common.Address, timestamp uint64) (block *types.Block, err error)
+	gov              *gov.Governance
 	mux              *event.TypeMux
 }
 
 func NewStore(
+	config *params.TendermintConfig,
 	chain *core.BlockChain,
 	verifyHeaderFunc func(chain consensus.ChainHeaderReader, header *types.Header, seal bool) error,
 	makeBlock func(parentHash common.Hash, coinbase common.Address, timestamp uint64) (block *types.Block, err error),
+	gov *gov.Governance,
 	mux *event.TypeMux) *Store {
-	return &Store{chain: chain, verifyHeaderFunc: verifyHeaderFunc, makeBlock: makeBlock, mux: mux}
+	return &Store{config: config, chain: chain, verifyHeaderFunc: verifyHeaderFunc, makeBlock: makeBlock, gov: gov, mux: mux}
 }
 
 func (s *Store) Base() uint64 {
@@ -82,11 +91,47 @@ func (s *Store) SaveBlock(block *types.FullBlock, commit *types.Commit) {
 
 // Validate a block without Commit and with LastCommit.
 func (s *Store) ValidateBlock(state pbft.ChainState, block *types.FullBlock) (err error) {
-	err = s.verifyHeaderFunc(s.chain, block.Header(), false)
+	header := block.Header()
+	err = s.verifyHeaderFunc(s.chain, header, false)
 	if err != nil {
 		return
 	}
 
+	validators, powers := []common.Address{}, []uint64{}
+	if header.Number.Uint64()%s.config.Epoch == 0 {
+		epochId := header.Number.Uint64() / s.config.Epoch
+		remoteChainNumber := uint64(0)
+		hash, emptyHash := common.Hash{}, common.Hash{}
+		if s.config.ValidatorChangeEpochId > 0 && s.config.ValidatorChangeEpochId <= epochId {
+			l := len(prefix)
+			if len(header.Extra) >= l+8+32 && bytes.Compare(header.Extra[:l], prefix) == 0 {
+				nb := header.Extra[l : l+8]
+				remoteChainNumber = binary.BigEndian.Uint64(nb)
+				if remoteChainNumber == 0 {
+					return errors.New("invalid remoteChainNumber in header.Extra")
+				}
+				hb := header.Extra[l+8 : l+8+32]
+				hash = common.BytesToHash(hb)
+				if hash == emptyHash {
+					return errors.New("invalid remote block hash in header.Extra")
+				}
+			} else {
+				return errors.New("header.Extra missing validator chain block height and hash")
+			}
+		}
+
+		validators, powers, err = s.gov.NextValidatorsAndPowersAt(epochId, remoteChainNumber, hash)
+		if err != nil {
+			return errors.New(fmt.Sprintf("verifyHeader failed with %s", err.Error()))
+		}
+	}
+	if !gov.CompareValidators(header.NextValidators, validators) {
+		return errors.New("NextValidators is incorrect")
+	}
+	if !gov.CompareValidatorPowers(header.NextValidatorPowers, powers) {
+		return errors.New("NextValidatorPowers is incorrect")
+	}
+
 	// Validate if the block matches current state.
 	if state.LastBlockHeight == 0 && block.NumberU64() != state.InitialHeight {
 		return fmt.Errorf("wrong Block.Header.Height. Expected %v for initial block, got %v",
@@ -235,6 +280,33 @@ func (s *Store) MakeBlock(
 	header.TimeMs = timestampMs
 	header.LastCommitHash = commit.Hash()
 
+	if height%s.config.Epoch == 0 {
+		epochId := height / s.config.Epoch
+		remoteChainNumber := uint64(0)
+		hash := common.Hash{}
+
+		header.NextValidators, header.NextValidatorPowers, remoteChainNumber, hash, err =
+			s.gov.NextValidatorsAndPowersForProposal(epochId)
+		if err != nil {
+			log.Error(err.Error())
+			return nil
+		}
+
+		// remoteChainNumber == 0 when NextValidatorsAndPowers return err or
+		// when update validator from contract is not enable. as err has been handled above.
+		// so only when remoteChainNumber != 0 need to add number and hash to header.Extra.
+		if remoteChainNumber != 0 {
+			data := prefix
+			b := make([]byte, 8)
+			binary.BigEndian.PutUint64(b, remoteChainNumber)
+			data = append(data, b...)
+			data = append(data, hash.Bytes()...)
+			header.Extra = append(data, header.Extra...)
+		}
+	} else {
+		header.NextValidators, header.NextValidatorPowers = []common.Address{}, []uint64{}
+	}
+
 	block = block.WithSeal(header)
 
 	return &types.FullBlock{Block: block, LastCommit: commit}

consensus/tendermint/gov/gov.go

@@ -1,22 +1,53 @@
 package gov
 
 import (
+	"context"
+	"fmt"
+	"math"
+	"math/big"
+	"strings"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 )
 
+const (
+	validatorsetABI           = `[{"inputs": [],"name": "GetEpochValidators","outputs": [{"internalType": "uint256","name": "EpochIdx","type": "uint256"},{"internalType": "address[]","name": "Validators","type": "address[]"},{"internalType": "uint256[]","name": "Powers","type": "uint256[]"}],"stateMutability": "view","type": "function"},{"inputs": [],"name": "proposedValidators","outputs": [{"internalType": "address[]","name": "Validators","type": "address[]"},{"internalType": "uint256[]","name": "Powers","type": "uint256[]"}],"stateMutability": "view","type": "function"}]`
+	confirmedNumber           = 96
+	contractFunc_GetValidator = "proposedValidators"
+	gas                       = uint64(math.MaxUint64 / 2)
+)
+
 type Governance struct {
-	config *params.TendermintConfig
-	chain  consensus.ChainHeaderReader
+	ctx             context.Context
+	config          *params.TendermintConfig
+	chain           consensus.ChainHeaderReader
+	validatorSetABI abi.ABI
+	client          *ethclient.Client
+	contract        *common.Address
 }
 
-func New(config *params.TendermintConfig, chain consensus.ChainHeaderReader) *Governance {
-	return &Governance{config: config, chain: chain}
+func New(config *params.TendermintConfig, chain consensus.ChainHeaderReader, client *ethclient.Client) *Governance {
+	vABI, _ := abi.JSON(strings.NewReader(validatorsetABI))
+	contract := common.HexToAddress(config.ValidatorContract)
+	return &Governance{
+		ctx:             context.Background(),
+		config:          config,
+		chain:           chain,
+		client:          client,
+		validatorSetABI: vABI,
+		contract:        &contract,
+	}
 }
 
-// Returns the validator sets for last, current blocks
+// GetValidatorSets Returns the validator sets for last, current blocks
 func (g *Governance) GetValidatorSets(height uint64) (*types.ValidatorSet, *types.ValidatorSet) {
 	if height == 0 {
 		panic("cannot get genesis validator set")
@@ -28,7 +59,6 @@ func (g *Governance) GetValidatorSets(height uint64) (*types.ValidatorSet, *type
 }
 
 // GetValidatorSet returns the validator set of a height
-
 func (g *Governance) GetValidatorSet(height uint64, lastVals *types.ValidatorSet) *types.ValidatorSet {
 	if height == 0 {
 		return &types.ValidatorSet{}
@@ -53,20 +83,116 @@ func (g *Governance) GetValidatorSet(height uint64, lastVals *types.ValidatorSet
 	return epochVals
 }
 
-func (g *Governance) NextValidators(height uint64) []common.Address {
-	if height%g.config.Epoch != 0 {
-		return []common.Address{}
+// NextValidatorsAndPowersForProposal get next validators according to block height and config
+func (g *Governance) NextValidatorsAndPowersForProposal(epochId uint64) ([]common.Address, []uint64, uint64, common.Hash, error) {
+	log.Debug("NextValidatorsAndPowersForProposal", "epoch", epochId)
+	if g.config.ValidatorChangeEpochId == 0 || g.config.ValidatorChangeEpochId > epochId {
+		header := g.chain.GetHeaderByNumber(0)
+		return header.NextValidators, header.NextValidatorPowers, 0, common.Hash{}, nil
 	}
 
-	switch {
-	case height == 0:
-		header := g.chain.GetHeaderByNumber(0)
-		return header.NextValidators
-	default:
-		// TODO: get real validators by calling contract, currently use genesis
+	number, err := g.client.BlockNumber(g.ctx)
+	if err != nil {
+		return nil, nil, 0, common.Hash{}, err
+	}
+
+	if number <= confirmedNumber {
+		return nil, nil, 0, common.Hash{}, fmt.Errorf(
+			"remote chain number %d smaller than confirmedNumber %d", number, confirmedNumber)
+	}
+	number = number - confirmedNumber
+
+	header, err := g.client.BlockByNumber(g.ctx, new(big.Int).SetUint64(number))
+	if err != nil {
+		return nil, nil, 0, common.Hash{}, err
+	}
+
+	validators, powers, err := g.getValidatorsAndPowersFromContract(header.Hash())
+	if err != nil {
+		return nil, nil, 0, common.Hash{}, err
+	}
+
+	log.Debug("get validators and powers", "validators", validators, "powers", powers)
+	return validators, powers, number, header.Hash(), err
+}
+
+// NextValidatorsAndPowersAt get next validators according to block height and config
+func (g *Governance) NextValidatorsAndPowersAt(epochId uint64, remoteChainNumber uint64, hash common.Hash) ([]common.Address, []uint64, error) {
+	log.Debug("NextValidatorsAndPowersAt", "epoch", epochId)
+	if g.config.ValidatorChangeEpochId == 0 || g.config.ValidatorChangeEpochId > epochId {
 		header := g.chain.GetHeaderByNumber(0)
-		return header.NextValidators
+		return header.NextValidators, header.NextValidatorPowers, nil
+	}
+
+	number, err := g.client.BlockNumber(g.ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if number-confirmedNumber/2*3 > remoteChainNumber || number-confirmedNumber/2 < remoteChainNumber {
+		return nil, nil, fmt.Errorf("remoteChainNumber %d is out of range [%d, %d]",
+			remoteChainNumber, number-confirmedNumber/2*3, number-confirmedNumber/2)
+	}
+
+	header, err := g.client.BlockByNumber(g.ctx, new(big.Int).SetUint64(remoteChainNumber))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if hash != header.Hash() {
+		fmt.Errorf("block hash mismatch", "remoteChainNumber hash", header.Hash(), "hash", hash)
+	}
+
+	validators, powers, err := g.getValidatorsAndPowersFromContract(hash)
+	if err != nil {
+		return nil, nil, err
 	}
+
+	log.Debug("get validators and powers", "validators", validators, "powers", powers)
+	return validators, powers, err
+}
+
+// getValidatorsAndPowersFromContract get next validators from contract
+func (g *Governance) getValidatorsAndPowersFromContract(blockHash common.Hash) ([]common.Address, []uint64, error) {
+	data, err := g.validatorSetABI.Pack(contractFunc_GetValidator)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// call
+	msgData := (hexutil.Bytes)(data)
+	msg := ethereum.CallMsg{
+		To:   g.contract,
+		Gas:  gas,
+		Data: msgData,
+	}
+	result, err := g.client.CallContractAtHash(g.ctx, msg, blockHash)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	type validators struct {
+		Validators []common.Address
+		Powers     []*big.Int
+	}
+
+	var v validators
+
+	if err := g.validatorSetABI.UnpackIntoInterface(&v, contractFunc_GetValidator, result); err != nil {
+		return nil, nil, err
+	}
+
+	if len(v.Validators) != len(v.Powers) {
+		return nil, nil, fmt.Errorf("invalid validator set: validator count %d is mismatch with power count %d",
+			len(v.Validators), len(v.Powers))
+	}
+
+	powers := make([]uint64, len(v.Powers))
+	for i, p := range v.Powers {
+		powers[i] = p.Uint64()
+	}
+
+	return v.Validators, powers, nil
 }
 
 func CompareValidators(lhs, rhs []common.Address) bool {
@@ -96,19 +222,3 @@ func CompareValidatorPowers(lhs, rhs []uint64) bool {
 
 	return true
 }
-
-func (g *Governance) NextValidatorPowers(height uint64) []uint64 {
-	if height%g.config.Epoch != 0 {
-		return []uint64{}
-	}
-
-	switch {
-	case height == 0:
-		header := g.chain.GetHeaderByNumber(0)
-		return header.NextValidatorPowers
-	default:
-		// TODO get real validators by calling contract
-		header := g.chain.GetHeaderByNumber(0)
-		return header.NextValidatorPowers
-	}
-}