Decouple PartSetHeader from BlockID (#441)

* add partsetheader to block meta fields * add the partsetheader to the proposal * add partsetheader to Commit * add the partsetheader to vote * add the partsetheader to Header * add the partsetheader to the state * begin relying on decoupled partsetheader * fix TestHeaderHash by updating expected hash * fix all tests in the types package * fix all tests in state * fix prival tests * fix p2p tests * get evidence, light and consensus to compile * add partset header to and from proto for consensus messages * revert blockid.Key requiring the partsetheader * fix e2e by including missed lastpartsetheader to state * fix more hardcoded tests * linter * clean up todos * fix PartSetHeader field in EventDAtaCompleteProposal Co-authored-by: Hlib Kanunnikov <hlibwondertan@gmail.com> * remove unused tests Co-authored-by: Hlib Kanunnikov <hlibwondertan@gmail.com>
celestiaorg · Jul 25, 2021 · 9d4265d · 9d4265d
1 parent 3e32d42
commit 9d4265d
Show file tree

Hide file tree

Showing 90 changed files with 2,331 additions and 1,953 deletions.
diff --git a/abci/types/types.pb.go b/abci/types/types.pb.go
diff --git a/blockchain/msgs_test.go b/blockchain/msgs_test.go
@@ -97,7 +97,7 @@ func TestBlockchainMessageVectors(t *testing.T) {
 			BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}},
 			"0a0a08ffffffffffffffff7f"},
 		{"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{
-			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac2020abf020a5d0a02080b1803220b088092b8c398feffffff012a021200380142204c149a7cfadc92b669b0cbfa4951a1b18c2d9f3177a3b8756d39ebb96e9d63317220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001b81cb5596c28d044214b9f935e4af7dbe76e417f6182d86fbee68bfff7b2ff3a0a30ffffffffffffffffffffffffffffffffc4096ba8fccf882c309896e9168fa43fe62fccb752cb12d5160cc1d9c2ebffe7123000000000000000010000000000000001b81cb5596c28d044214b9f935e4af7dbe76e417f6182d86fbee68bfff7b2ff3a1230ffffffffffffffffffffffffffffffffc4096ba8fccf882c309896e9168fa43fe62fccb752cb12d5160cc1d9c2ebffe7"},
+			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ac2020abf020a5d0a02080b1803220b088092b8c398feffffff012a00320040014a204c149a7cfadc92b669b0cbfa4951a1b18c2d9f3177a3b8756d39ebb96e9d63317a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c6412001a0022001ac8010a3000000000000000010000000000000001b81cb5596c28d044214b9f935e4af7dbe76e417f6182d86fbee68bfff7b2ff3a0a30ffffffffffffffffffffffffffffffffc4096ba8fccf882c309896e9168fa43fe62fccb752cb12d5160cc1d9c2ebffe7123000000000000000010000000000000001b81cb5596c28d044214b9f935e4af7dbe76e417f6182d86fbee68bfff7b2ff3a1230ffffffffffffffffffffffffffffffffc4096ba8fccf882c309896e9168fa43fe62fccb752cb12d5160cc1d9c2ebffe7"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
 			NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{

diff --git a/blockchain/v0/reactor.go b/blockchain/v0/reactor.go
@@ -389,14 +389,14 @@ FOR_LOOP:
 			var (
 				firstParts         = first.MakePartSet(types.BlockPartSizeBytes)
 				firstPartSetHeader = firstParts.Header()
-				firstID            = types.BlockID{Hash: first.Hash(), PartSetHeader: firstPartSetHeader}
+				firstID            = types.BlockID{Hash: first.Hash()}
 			)
 
 			// Finally, verify the first block using the second's commit
 			// NOTE: we can probably make this more efficient, but note that calling
 			// first.Hash() doesn't verify the tx contents, so MakePartSet() is
 			// currently necessary.
-			err := state.Validators.VerifyCommitLight(chainID, firstID, first.Height, second.LastCommit)
+			err := state.Validators.VerifyCommitLight(chainID, firstID, firstPartSetHeader, first.Height, second.LastCommit)
 			if err != nil {
 				err = fmt.Errorf("invalid last commit: %w", err)
 				bcR.Logger.Error(err.Error(),
@@ -430,7 +430,7 @@ FOR_LOOP:
 
 				// TODO: same thing for app - but we would need a way to get the hash
 				// without persisting the state.
-				state, _, err = bcR.blockExec.ApplyBlock(state, firstID, first)
+				state, _, err = bcR.blockExec.ApplyBlock(state, firstID, firstPartSetHeader, first)
 				if err != nil {
 					// TODO This is bad, are we zombie?
 					panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))

diff --git a/blockchain/v0/reactor_test.go b/blockchain/v0/reactor_test.go
@@ -96,7 +96,7 @@ func newBlockchainReactor(
 
 	// let's add some blocks in
 	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
-		lastCommit := types.NewCommit(blockHeight-1, 0, types.BlockID{}, nil)
+		lastCommit := types.NewCommit(blockHeight-1, 0, types.BlockID{}, nil, types.PartSetHeader{})
 		if blockHeight > 1 {
 			lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
 			lastBlock, err := blockStore.LoadBlock(context.TODO(), blockHeight-1)
@@ -107,6 +107,7 @@ func newBlockchainReactor(
 			vote, err := types.MakeVote(
 				lastBlock.Header.Height,
 				lastBlockMeta.BlockID,
+				lastBlockMeta.PartSetHeader,
 				state.Validators,
 				privVals[0],
 				lastBlock.Header.ChainID,
@@ -116,15 +117,15 @@ func newBlockchainReactor(
 				panic(err)
 			}
 			lastCommit = types.NewCommit(vote.Height, vote.Round,
-				lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()})
+				lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()}, lastBlockMeta.PartSetHeader)
 		}
 
 		thisBlock := makeBlock(blockHeight, state, lastCommit)
 
 		thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes)
-		blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()}
+		blockID := types.BlockID{Hash: thisBlock.Hash()}
 
-		state, _, err = blockExec.ApplyBlock(state, blockID, thisBlock)
+		state, _, err = blockExec.ApplyBlock(state, blockID, thisParts.Header(), thisBlock)
 		if err != nil {
 			panic(fmt.Errorf("error apply block: %w", err))
 		}

diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go
@@ -385,8 +385,10 @@ func byzantineDecideProposalFunc(t *testing.T, height int64, round int32, cs *St
 
 	// Create a new proposal block from state/txs from the mempool.
 	block1, blockParts1 := cs.createProposalBlock()
-	polRound, propBlockID := cs.ValidRound, types.BlockID{Hash: block1.Hash(), PartSetHeader: blockParts1.Header()}
-	proposal1 := types.NewProposal(height, round, polRound, propBlockID, &block1.DataAvailabilityHeader)
+	polRound, propBlockID := cs.ValidRound, types.BlockID{Hash: block1.Hash()}
+	proposal1 := types.NewProposal(
+		height, round, polRound, propBlockID, &block1.DataAvailabilityHeader, blockParts1.Header(),
+	)
 	p1, err := proposal1.ToProto()
 	require.NoError(t, err)
 	if err := cs.privValidator.SignProposal(cs.state.ChainID, p1); err != nil {
@@ -400,8 +402,10 @@ func byzantineDecideProposalFunc(t *testing.T, height int64, round int32, cs *St
 
 	// Create a new proposal block from state/txs from the mempool.
 	block2, blockParts2 := cs.createProposalBlock()
-	polRound, propBlockID = cs.ValidRound, types.BlockID{Hash: block2.Hash(), PartSetHeader: blockParts2.Header()}
-	proposal2 := types.NewProposal(height, round, polRound, propBlockID, &block2.DataAvailabilityHeader)
+	polRound, propBlockID = cs.ValidRound, types.BlockID{Hash: block2.Hash()}
+	proposal2 := types.NewProposal(
+		height, round, polRound, propBlockID, &block2.DataAvailabilityHeader, blockParts2.Header(),
+	)
 	p2, err := proposal2.ToProto()
 	require.NoError(t, err)
 	if err := cs.privValidator.SignProposal(cs.state.ChainID, p2); err != nil {

diff --git a/consensus/common_test.go b/consensus/common_test.go
@@ -108,7 +108,8 @@ func (vs *validatorStub) signVote(
 		Round:            vs.Round,
 		Timestamp:        tmtime.Now(),
 		Type:             voteType,
-		BlockID:          types.BlockID{Hash: hash, PartSetHeader: header},
+		BlockID:          types.BlockID{Hash: hash},
+		PartSetHeader:    header,
 	}
 	v := vote.ToProto()
 	err = vs.PrivValidator.SignVote(config.ChainID(), v)
@@ -205,8 +206,8 @@ func decideProposal(
 	}
 
 	// Make proposal
-	polRound, propBlockID := validRound, types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()}
-	proposal = types.NewProposal(height, round, polRound, propBlockID, &block.DataAvailabilityHeader)
+	polRound, propBlockID := validRound, types.BlockID{Hash: block.Hash()}
+	proposal = types.NewProposal(height, round, polRound, propBlockID, &block.DataAvailabilityHeader, blockParts.Header())
 	p, err := proposal.ToProto()
 	if err != nil {
 		panic(err)
@@ -599,7 +600,13 @@ func ensureNewUnlock(unlockCh <-chan tmpubsub.Message, height int64, round int32
 		"Timeout expired while waiting for NewUnlock event")
 }
 
-func ensureProposal(proposalCh <-chan tmpubsub.Message, height int64, round int32, propID types.BlockID) {
+func ensureProposal(
+	proposalCh <-chan tmpubsub.Message,
+	height int64,
+	round int32,
+	propID types.BlockID,
+	propPartSetHeader types.PartSetHeader,
+) {
 	select {
 	case <-time.After(ensureTimeout):
 		panic("Timeout expired while waiting for NewProposal event")
@@ -615,7 +622,7 @@ func ensureProposal(proposalCh <-chan tmpubsub.Message, height int64, round int3
 		if proposalEvent.Round != round {
 			panic(fmt.Sprintf("expected round %v, got %v", round, proposalEvent.Round))
 		}
-		if !proposalEvent.BlockID.Equals(propID) {
+		if !proposalEvent.BlockID.Equals(propID) || !proposalEvent.PartSetHeader.Equals(propPartSetHeader) {
 			panic(fmt.Sprintf("Proposed block does not match expected block (%v != %v)", proposalEvent.BlockID, propID))
 		}
 	}

diff --git a/consensus/invalid_test.go b/consensus/invalid_test.go
@@ -83,8 +83,9 @@ func invalidDoPrevoteFunc(t *testing.T, height int64, round int32, cs *State, sw
 			Timestamp:        cs.voteTime(),
 			Type:             tmproto.PrecommitType,
 			BlockID: types.BlockID{
-				Hash:          blockHash,
-				PartSetHeader: types.PartSetHeader{Total: 1, Hash: tmrand.Bytes(32)}},
+				Hash: blockHash,
+			},
+			PartSetHeader: types.PartSetHeader{Total: 1, Hash: tmrand.Bytes(32)},
 		}
 		p := precommit.ToProto()
 		err = cs.privValidator.SignVote(cs.state.ChainID, p)

diff --git a/consensus/msgs.go b/consensus/msgs.go
@@ -109,26 +109,30 @@ func MsgToProto(msg Message) (*tmcons.Message, error) {
 		}
 	case *VoteSetMaj23Message:
 		bi := msg.BlockID.ToProto()
+		psh := msg.PartSetHeader.ToProto()
 		pb = tmcons.Message{
 			Sum: &tmcons.Message_VoteSetMaj23{
 				VoteSetMaj23: &tmcons.VoteSetMaj23{
-					Height:  msg.Height,
-					Round:   msg.Round,
-					Type:    msg.Type,
-					BlockID: bi,
+					Height:        msg.Height,
+					Round:         msg.Round,
+					Type:          msg.Type,
+					BlockID:       bi,
+					PartSetHeader: &psh,
 				},
 			},
 		}
 	case *VoteSetBitsMessage:
 		bi := msg.BlockID.ToProto()
+		psh := msg.PartSetHeader.ToProto()
 		bits := msg.Votes.ToProto()
 
 		vsb := &tmcons.Message_VoteSetBits{
 			VoteSetBits: &tmcons.VoteSetBits{
-				Height:  msg.Height,
-				Round:   msg.Round,
-				Type:    msg.Type,
-				BlockID: bi,
+				Height:        msg.Height,
+				Round:         msg.Round,
+				Type:          msg.Type,
+				BlockID:       bi,
+				PartSetHeader: &psh,
 			},
 		}
 
@@ -238,29 +242,39 @@ func MsgFromProto(msg *tmcons.Message) (Message, error) {
 		if err != nil {
 			return nil, fmt.Errorf("voteSetMaj23 msg to proto error: %w", err)
 		}
+		psh, err := types.PartSetHeaderFromProto(msg.VoteSetMaj23.PartSetHeader)
+		if err != nil {
+			return nil, fmt.Errorf("voteSetMaj23 msg to proto error: %w", err)
+		}
 		pb = &VoteSetMaj23Message{
-			Height:  msg.VoteSetMaj23.Height,
-			Round:   msg.VoteSetMaj23.Round,
-			Type:    msg.VoteSetMaj23.Type,
-			BlockID: *bi,
+			Height:        msg.VoteSetMaj23.Height,
+			Round:         msg.VoteSetMaj23.Round,
+			Type:          msg.VoteSetMaj23.Type,
+			BlockID:       *bi,
+			PartSetHeader: *psh,
 		}
 	case *tmcons.Message_VoteSetBits:
 		bi, err := types.BlockIDFromProto(&msg.VoteSetBits.BlockID)
 		if err != nil {
 			return nil, fmt.Errorf("block ID to proto error: %w", err)
 		}
+		psh, err := types.PartSetHeaderFromProto(msg.VoteSetBits.PartSetHeader)
+		if err != nil {
+			return nil, fmt.Errorf("part set header to proto error: %w", err)
+		}
 		bits := new(bits.BitArray)
 		err = bits.FromProto(&msg.VoteSetBits.Votes)
 		if err != nil {
 			return nil, fmt.Errorf("votes to proto error: %w", err)
 		}
 
 		pb = &VoteSetBitsMessage{
-			Height:  msg.VoteSetBits.Height,
-			Round:   msg.VoteSetBits.Round,
-			Type:    msg.VoteSetBits.Type,
-			BlockID: *bi,
-			Votes:   bits,
+			Height:        msg.VoteSetBits.Height,
+			Round:         msg.VoteSetBits.Round,
+			Type:          msg.VoteSetBits.Type,
+			BlockID:       *bi,
+			PartSetHeader: *psh,
+			Votes:         bits,
 		}
 	default:
 		return nil, fmt.Errorf("consensus: message not recognized: %T", msg)