added new option --use-in-memory-discovery-table

.circleci/config.yml

@@ -70,18 +70,13 @@ jobs:
             set -euo pipefail
             export CELO_MONOREPO_DIR="$PWD"
             git clone --depth 1 https://github.com/celo-org/celo-monorepo.git ${CELO_MONOREPO_DIR} -b master
-            cd ${CELO_MONOREPO_DIR}/packages
-            # TODO(ashishb): Delete unnecessary packages to speed up build time.
-            # It would be better whitelist certain packages and delete the rest.
-            # Deletion does not work right now and yarn fails with weird errors.
-            # This will be enabled and resolved later.
-            # rm -rf analytics blockchain-api cli docs faucet helm-charts mobile notification-service react-components transaction-metrics-exporter verification-pool-api verifier web
-            cd ${CELO_MONOREPO_DIR}/packages/celotool
-            yarn || yarn
-            yarn --cwd=${CELO_MONOREPO_DIR}/packages/utils build
-            yarn --cwd=${CELO_MONOREPO_DIR}/packages/walletkit build
-            yarn --cwd=${CELO_MONOREPO_DIR}/packages/celotool build
-
+            yarn install || yarn install
+            # separate build to avoid ENOMEM in CI :(
+            yarn build --scope @celo/utils
+            yarn build --scope @celo/protocol
+            yarn build --scope docs
+            yarn build --scope @celo/walletkit
+            yarn build --ignore @celo/protocol --ignore docs --ignore @celo/walletkit --ignore @celo/web --ignore @celo/mobile --ignore @celo/react-components
       - run:
           name: Setup Go language
           command: |

cmd/geth/main.go

@@ -139,6 +139,7 @@ var (
 		utils.IstanbulRequestTimeoutFlag,
 		utils.IstanbulBlockPeriodFlag,
 		utils.PingIPFromPacketFlag,
+		utils.UseInMemoryDiscoverTable,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/geth/usage.go

@@ -183,6 +183,8 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.NetrestrictFlag,
 			utils.NodeKeyFileFlag,
 			utils.NodeKeyHexFlag,
+			utils.PingIPFromPacketFlag,
+			utils.UseInMemoryDiscoverTable,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -563,6 +563,10 @@ var (
 		Name:  "ping-ip-from-packet",
 		Usage: "Has the discovery protocol use the IP address given by a ping packet",
 	}
+	UseInMemoryDiscoverTable = cli.BoolFlag{
+		Name:  "use-in-memory-discovery-table",
+		Usage: "Specifies whether to use an in memory discovery table",
+	}
 
 	// ATM the url is left to the user and deployment to
 	JSpathFlag = cli.StringFlag{
@@ -995,6 +999,9 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 	if ctx.GlobalIsSet(PingIPFromPacketFlag.Name) {
 		cfg.PingIPFromPacket = true
 	}
+	if ctx.GlobalIsSet(UseInMemoryDiscoverTable.Name) {
+		cfg.UseInMemoryNodeDatabase = true
+	}
 
 	// if we're running a light client or server, force enable the v5 peer discovery
 	// unless it is explicitly disabled with --nodiscover note that explicitly specifying

consensus/istanbul/backend.go

@@ -60,7 +60,7 @@ type Backend interface {
 
 	// Verify verifies the proposal. If a consensus.ErrFutureBlock error is returned,
 	// the time difference of the proposal and current time is also returned.
-	Verify(Proposal, Validator) (time.Duration, error)
+	Verify(Proposal) (time.Duration, error)
 
 	// Sign signs input data with the backend's private key
 	Sign([]byte) ([]byte, error)

consensus/istanbul/backend/backend.go

@@ -279,7 +279,7 @@ func (sb *Backend) EventMux() *event.TypeMux {
 }
 
 // Verify implements istanbul.Backend.Verify
-func (sb *Backend) Verify(proposal istanbul.Proposal, src istanbul.Validator) (time.Duration, error) {
+func (sb *Backend) Verify(proposal istanbul.Proposal) (time.Duration, error) {
 	// Check if the proposal is a valid block
 	block := &types.Block{}
 	block, ok := proposal.(*types.Block)
@@ -303,11 +303,17 @@ func (sb *Backend) Verify(proposal istanbul.Proposal, src istanbul.Validator) (t
 		return 0, errInvalidUncleHash
 	}
 
-	// verify the header of proposed block
-	if block.Header().Coinbase != src.Address() {
+	// The author should be the first person to propose the block to ensure that randomness matches up.
+	addr, err := sb.Author(block.Header())
+	if err != nil {
+		sb.logger.Error("Could not recover orignal author of the block to verify the randomness", "err", err, "func", "Verify")
+		return 0, errInvalidProposal
+	} else if addr != block.Header().Coinbase {
+		sb.logger.Error("Original author of the block does not match the coinbase", "addr", addr, "coinbase", block.Header().Coinbase, "func", "Verify")
 		return 0, errInvalidCoinbase
 	}
-	err := sb.VerifyHeader(sb.chain, block.Header(), false)
+
+	err = sb.VerifyHeader(sb.chain, block.Header(), false)
 
 	// ignore errEmptyCommittedSeals error because we don't have the committed seals yet
 	if err != nil && err != errEmptyCommittedSeals {

consensus/istanbul/core/backlog.go

@@ -40,10 +40,11 @@ func (c *core) checkMessage(msgCode uint64, view *istanbul.View) error {
 		return errInvalidMessage
 	}
 
+	// Round change messages should be in the same sequence but be >= the desired round
 	if msgCode == istanbul.MsgRoundChange {
 		if view.Sequence.Cmp(c.currentView().Sequence) > 0 {
 			return errFutureMessage
-		} else if view.Cmp(c.currentView()) < 0 {
+		} else if view.Round.Cmp(c.current.DesiredRound()) < 0 {
 			return errOldMessage
 		}
 		return nil
@@ -57,7 +58,8 @@ func (c *core) checkMessage(msgCode uint64, view *istanbul.View) error {
 		return errOldMessage
 	}
 
-	if c.waitingForRoundChange {
+	// Round change messages are already let through.
+	if c.state == StateWaitingForNewRound {
 		return errFutureMessage
 	}
 
@@ -76,9 +78,14 @@ func (c *core) checkMessage(msgCode uint64, view *istanbul.View) error {
 }
 
 func (c *core) storeBacklog(msg *istanbul.Message, src istanbul.Validator) {
-	logger := c.logger.New("from", src, "state", c.state)
+	logger := c.logger.New("from", msg.Address, "state", c.state, "func", "storeBacklog")
+	if c.current != nil {
+		logger = logger.New("cur_seq", c.current.Sequence(), "cur_round", c.current.Round())
+	} else {
+		logger = logger.New("cur_seq", 0, "cur_round", -1)
+	}
 
-	if src.Address() == c.Address() {
+	if msg.Address == c.Address() {
 		logger.Warn("Backlog from self")
 		return
 	}
@@ -99,13 +106,20 @@ func (c *core) storeBacklog(msg *istanbul.Message, src istanbul.Validator) {
 		if err == nil {
 			backlog.Push(msg, toPriority(msg.Code, p.View))
 		}
-		// for istanbul.MsgRoundChange, istanbul.MsgPrepare and istanbul.MsgCommit cases
-	default:
+	case istanbul.MsgPrepare:
+		fallthrough
+	case istanbul.MsgCommit:
 		var p *istanbul.Subject
 		err := msg.Decode(&p)
 		if err == nil {
 			backlog.Push(msg, toPriority(msg.Code, p.View))
 		}
+	case istanbul.MsgRoundChange:
+		var p *istanbul.RoundChange
+		err := msg.Decode(&p)
+		if err == nil {
+			backlog.Push(msg, toPriority(msg.Code, p.View))
+		}
 	}
 	c.backlogs[src] = backlog
 }
@@ -119,7 +133,7 @@ func (c *core) processBacklog() {
 			continue
 		}
 
-		logger := c.logger.New("from", src, "state", c.state)
+		logger := c.logger.New("from", src, "state", c.state, "cur_round", c.current.Round(), "cur_seq", c.current.Sequence(), "func", "processBacklog")
 		isFuture := false
 
 		// We stop processing if
@@ -136,13 +150,20 @@ func (c *core) processBacklog() {
 				if err == nil {
 					view = m.View
 				}
-				// for istanbul.MsgRoundChange, istanbul.MsgPrepare and istanbul.MsgCommit cases
-			default:
+			case istanbul.MsgPrepare:
+				fallthrough
+			case istanbul.MsgCommit:
 				var sub *istanbul.Subject
 				err := msg.Decode(&sub)
 				if err == nil {
 					view = sub.View
 				}
+			case istanbul.MsgRoundChange:
+				var rc *istanbul.RoundChange
+				err := msg.Decode(&rc)
+				if err == nil {
+					view = rc.View
+				}
 			}
 			if view == nil {
 				logger.Debug("Nil view", "msg", msg)