From f7345258e6daf8ae229d05583648c08d5239e899 Mon Sep 17 00:00:00 2001 From: Bowen Wang Date: Mon, 18 May 2020 13:09:43 -0700 Subject: [PATCH] feat(chain): add block merkle root to BlockHeaderInnerLite (#2643) Add block merkle root to block header so that we light client can verify transaction outcomes in blocks that are not known by the light client. Resolves #2632. Test plan ---------- * Unit tests `test_merkle_tree` and `test_invalid_block_merkle_root`. * Manually trigger nightly to see if anything breaks. --- chain/chain/src/chain.rs | 13 ++- chain/chain/src/error.rs | 4 + chain/chain/src/store.rs | 104 +++++++++++++++---- chain/chain/src/types.rs | 1 + chain/chain/tests/gc.rs | 9 +- chain/chain/tests/simple_chain.rs | 2 + chain/chain/tests/sync_chain.rs | 8 +- chain/client/src/client.rs | 9 +- chain/client/src/lib.rs | 6 +- chain/client/src/sync.rs | 4 + chain/client/src/types.rs | 14 +++ chain/client/src/view_client.rs | 19 +++- chain/client/tests/challenges.rs | 13 ++- chain/client/tests/process_blocks.rs | 109 +++++++------------- chain/client/tests/query_client.rs | 15 ++- core/chain-configs/src/lib.rs | 2 +- core/primitives/benches/serialization.rs | 1 + core/primitives/src/block.rs | 9 ++ core/primitives/src/merkle.rs | 112 ++++++++++++++++++--- core/primitives/src/test_utils.rs | 36 ++++++- core/primitives/src/views.rs | 3 + core/store/src/db.rs | 5 +- genesis-tools/genesis-populate/src/lib.rs | 4 +- neard/res/genesis_config.json | 2 +- neard/tests/sync_nodes.rs | 7 ++ scripts/migrations/13-block-merkle-root.py | 22 ++++ 26 files changed, 400 insertions(+), 133 deletions(-) create mode 100644 scripts/migrations/13-block-merkle-root.py diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index 53971587b4f..49889857f87 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -322,7 +322,7 @@ impl Chain { vec![], chain_genesis.total_supply, )?; - store_update.save_block_header(genesis.header.clone()); + store_update.save_block_header(genesis.header.clone())?; store_update.save_block(genesis.clone()); store_update.save_block_extra( &genesis.hash(), @@ -784,7 +784,7 @@ impl Chain { } chain_update.validate_header(header, &Provenance::SYNC, on_challenge)?; - chain_update.chain_store_update.save_block_header(header.clone()); + chain_update.chain_store_update.save_block_header(header.clone())?; chain_update.commit()?; // Add validator proposals for given header. @@ -2826,7 +2826,7 @@ impl<'a> ChainUpdate<'a> { F: FnMut(ChallengeBody) -> (), { self.validate_header(header, provenance, on_challenge)?; - self.chain_store_update.save_block_header(header.clone()); + self.chain_store_update.save_block_header(header.clone())?; self.update_header_head_if_not_challenged(header)?; Ok(()) } @@ -2968,6 +2968,13 @@ impl<'a> ChainUpdate<'a> { { return Err(ErrorKind::InvalidFinalityInfo.into()); } + + let mut block_merkle_tree = + self.chain_store_update.get_block_merkle_tree(&header.prev_hash)?.clone(); + block_merkle_tree.insert(header.prev_hash); + if block_merkle_tree.root() != header.inner_lite.block_merkle_root { + return Err(ErrorKind::InvalidBlockMerkleRoot.into()); + } } Ok(()) diff --git a/chain/chain/src/error.rs b/chain/chain/src/error.rs index 73cf2818eda..d4cc887c063 100644 --- a/chain/chain/src/error.rs +++ b/chain/chain/src/error.rs @@ -140,6 +140,9 @@ pub enum ErrorKind { /// Invalid VRF proof, or incorrect random_output in the header #[fail(display = "Invalid Randomness Beacon Output")] InvalidRandomnessBeaconOutput, + /// Invalid block merkle root. + #[fail(display = "Invalid Block Merkle Root")] + InvalidBlockMerkleRoot, /// Someone is not a validator. Usually happens in signature verification #[fail(display = "Not A Validator")] NotAValidator, @@ -249,6 +252,7 @@ impl Error { | ErrorKind::InvalidShardId(_) | ErrorKind::InvalidStateRequest(_) | ErrorKind::InvalidRandomnessBeaconOutput + | ErrorKind::InvalidBlockMerkleRoot | ErrorKind::NotAValidator => true, } } diff --git a/chain/chain/src/store.rs b/chain/chain/src/store.rs index a15e057d589..3c678e8e405 100644 --- a/chain/chain/src/store.rs +++ b/chain/chain/src/store.rs @@ -12,7 +12,7 @@ use tracing::debug; use near_primitives::block::Approval; use near_primitives::errors::InvalidTxError; use near_primitives::hash::CryptoHash; -use near_primitives::merkle::MerklePath; +use near_primitives::merkle::{MerklePath, PartialMerkleTree}; use near_primitives::receipt::Receipt; use near_primitives::sharding::{ ChunkHash, EncodedShardChunk, PartialEncodedChunk, ReceiptProof, ShardChunk, ShardChunkHeader, @@ -28,13 +28,14 @@ use near_primitives::types::{ use near_primitives::utils::{index_to_bytes, to_timestamp}; use near_primitives::views::LightClientBlockView; use near_store::{ - read_with_cache, ColBlock, ColBlockExtra, ColBlockHeader, ColBlockHeight, ColBlockMisc, - ColBlockPerHeight, ColBlockRefCount, ColBlocksToCatchup, ColChallengedBlocks, ColChunkExtra, - ColChunkPerHeightShard, ColChunks, ColEpochLightClientBlocks, ColIncomingReceipts, - ColInvalidChunks, ColLastBlockWithNewChunk, ColNextBlockHashes, ColNextBlockWithNewChunk, - ColOutgoingReceipts, ColPartialChunks, ColReceiptIdToShardId, ColState, ColStateChanges, - ColStateDlInfos, ColStateHeaders, ColTransactionResult, ColTransactions, ColTrieChanges, - KeyForStateChanges, ShardTries, Store, StoreUpdate, TrieChanges, WrappedTrieChanges, + read_with_cache, ColBlock, ColBlockExtra, ColBlockHeader, ColBlockHeight, ColBlockMerkleTree, + ColBlockMisc, ColBlockPerHeight, ColBlockRefCount, ColBlocksToCatchup, ColChallengedBlocks, + ColChunkExtra, ColChunkPerHeightShard, ColChunks, ColEpochLightClientBlocks, + ColIncomingReceipts, ColInvalidChunks, ColLastBlockWithNewChunk, ColNextBlockHashes, + ColNextBlockWithNewChunk, ColOutgoingReceipts, ColPartialChunks, ColReceiptIdToShardId, + ColState, ColStateChanges, ColStateDlInfos, ColStateHeaders, ColTransactionResult, + ColTransactions, ColTrieChanges, KeyForStateChanges, ShardTries, Store, StoreUpdate, + TrieChanges, WrappedTrieChanges, }; use crate::byzantine_assert; @@ -253,6 +254,11 @@ pub trait ChainStoreAccess { ) -> Result; fn get_genesis_height(&self) -> BlockHeight; + + fn get_block_merkle_tree( + &mut self, + block_hash: &CryptoHash, + ) -> Result<&PartialMerkleTree, Error>; } /// All chain-related database operations. @@ -307,6 +313,8 @@ pub struct ChainStore { transactions: SizedCache, SignedTransaction>, /// Cache with height to hash on any chain. block_refcounts: SizedCache, u64>, + /// Cache of block hash -> block merkle tree at the current block + block_merkle_tree: SizedCache, PartialMerkleTree>, } pub fn option_to_not_found(res: io::Result>, field_name: &str) -> Result { @@ -345,6 +353,7 @@ impl ChainStore { next_block_with_new_chunk: SizedCache::with_size(CHUNK_CACHE_SIZE), last_block_with_new_chunk: SizedCache::with_size(CHUNK_CACHE_SIZE), transactions: SizedCache::with_size(CHUNK_CACHE_SIZE), + block_merkle_tree: SizedCache::with_size(CACHE_SIZE), } } @@ -961,6 +970,21 @@ impl ChainStoreAccess for ChainStore { fn get_genesis_height(&self) -> BlockHeight { self.genesis_height } + + fn get_block_merkle_tree( + &mut self, + block_hash: &CryptoHash, + ) -> Result<&PartialMerkleTree, Error> { + option_to_not_found( + read_with_cache( + &*self.store, + ColBlockMerkleTree, + &mut self.block_merkle_tree, + block_hash.as_ref(), + ), + &format!("BLOCK MERKLE TREE: {}", block_hash), + ) + } } /// Cache update for ChainStore @@ -987,6 +1011,7 @@ struct ChainStoreCacheUpdate { last_block_with_new_chunk: HashMap, transactions: HashSet, block_refcounts: HashMap, + block_merkle_tree: HashMap, } impl ChainStoreCacheUpdate { @@ -1014,6 +1039,7 @@ impl ChainStoreCacheUpdate { last_block_with_new_chunk: Default::default(), transactions: Default::default(), block_refcounts: HashMap::default(), + block_merkle_tree: HashMap::default(), } } } @@ -1435,6 +1461,17 @@ impl<'a> ChainStoreAccess for ChainStoreUpdate<'a> { fn get_genesis_height(&self) -> BlockHeight { self.chain_store.genesis_height } + + fn get_block_merkle_tree( + &mut self, + block_hash: &CryptoHash, + ) -> Result<&PartialMerkleTree, Error> { + if let Some(merkle_tree) = self.chain_store_cache_update.block_merkle_tree.get(block_hash) { + Ok(merkle_tree) + } else { + self.chain_store.get_block_merkle_tree(block_hash) + } + } } impl<'a> ChainStoreUpdate<'a> { @@ -1576,8 +1613,30 @@ impl<'a> ChainStoreUpdate<'a> { self.chain_store_cache_update.partial_chunks.insert(chunk_hash.clone(), partial_chunk); } - pub fn save_block_header(&mut self, header: BlockHeader) { + pub fn save_block_merkle_tree( + &mut self, + block_hash: CryptoHash, + block_merkle_tree: PartialMerkleTree, + ) { + self.chain_store_cache_update.block_merkle_tree.insert(block_hash, block_merkle_tree); + } + + fn update_and_save_block_merkle_tree(&mut self, header: &BlockHeader) -> Result<(), Error> { + let prev_hash = header.prev_hash; + if prev_hash == CryptoHash::default() { + self.save_block_merkle_tree(header.hash(), PartialMerkleTree::default()); + } else { + let mut block_merkle_tree = self.get_block_merkle_tree(&prev_hash)?.clone(); + block_merkle_tree.insert(prev_hash); + self.save_block_merkle_tree(header.hash(), block_merkle_tree); + } + Ok(()) + } + + pub fn save_block_header(&mut self, header: BlockHeader) -> Result<(), Error> { + self.update_and_save_block_merkle_tree(&header)?; self.chain_store_cache_update.headers.insert(header.hash(), header); + Ok(()) } pub fn save_next_block_hash(&mut self, hash: &CryptoHash, next_hash: CryptoHash) { @@ -2072,6 +2131,11 @@ impl<'a> ChainStoreUpdate<'a> { for (block_hash, refcount) in self.chain_store_cache_update.block_refcounts.iter() { store_update.set_ser(ColBlockRefCount, block_hash.as_ref(), refcount)?; } + for (block_hash, block_merkle_tree) in + self.chain_store_cache_update.block_merkle_tree.iter() + { + store_update.set_ser(ColBlockMerkleTree, block_hash.as_ref(), block_merkle_tree)?; + } for mut wrapped_trie_changes in self.trie_changes.drain(..) { wrapped_trie_changes .wrapped_into(&mut store_update) @@ -2183,6 +2247,7 @@ impl<'a> ChainStoreUpdate<'a> { last_block_with_new_chunk, transactions, block_refcounts, + block_merkle_tree, } = self.chain_store_cache_update; for (hash, block) in blocks { self.chain_store.blocks.cache_set(hash.into(), block); @@ -2267,6 +2332,9 @@ impl<'a> ChainStoreUpdate<'a> { for (block_hash, refcount) in block_refcounts { self.chain_store.block_refcounts.cache_set(block_hash.into(), refcount); } + for (block_hash, merkle_tree) in block_merkle_tree { + self.chain_store.block_merkle_tree.cache_set(block_hash.into(), merkle_tree); + } Ok(()) } @@ -2322,7 +2390,7 @@ mod tests { Arc::new(InMemoryValidatorSigner::from_seed("test1", KeyType::ED25519, "test1")); let short_fork = vec![Block::empty_with_height(&genesis, 1, &*signer.clone())]; let mut store_update = chain.mut_store().store_update(); - store_update.save_block_header(short_fork[0].header.clone()); + store_update.save_block_header(short_fork[0].header.clone()).unwrap(); store_update.commit().unwrap(); let short_fork_head = short_fork[0].clone().header; @@ -2340,7 +2408,7 @@ mod tests { for i in 1..(transaction_validity_period + 3) { let block = Block::empty_with_height(&prev_block, i, &*signer.clone()); prev_block = block.clone(); - store_update.save_block_header(block.header.clone()); + store_update.save_block_header(block.header.clone()).unwrap(); store_update .update_height_if_not_challenged(block.header.inner_lite.height, block.hash()) .unwrap(); @@ -2381,7 +2449,7 @@ mod tests { for i in 1..(transaction_validity_period + 2) { let block = Block::empty_with_height(&prev_block, i, &*signer.clone()); prev_block = block.clone(); - store_update.save_block_header(block.header.clone()); + store_update.save_block_header(block.header.clone()).unwrap(); store_update .update_height_if_not_challenged(block.header.inner_lite.height, block.hash()) .unwrap(); @@ -2404,7 +2472,7 @@ mod tests { &*signer.clone(), ); let mut store_update = chain.mut_store().store_update(); - store_update.save_block_header(new_block.header.clone()); + store_update.save_block_header(new_block.header.clone()).unwrap(); store_update .update_height_if_not_challenged(new_block.header.inner_lite.height, new_block.hash()) .unwrap(); @@ -2432,7 +2500,7 @@ mod tests { for i in 1..(transaction_validity_period + 2) { let block = Block::empty_with_height(&prev_block, i, &*signer.clone()); prev_block = block.clone(); - store_update.save_block_header(block.header.clone()); + store_update.save_block_header(block.header.clone()).unwrap(); short_fork.push(block); } store_update.commit().unwrap(); @@ -2452,7 +2520,7 @@ mod tests { for i in 1..(transaction_validity_period * 5) { let block = Block::empty_with_height(&prev_block, i, &*signer.clone()); prev_block = block.clone(); - store_update.save_block_header(block.header.clone()); + store_update.save_block_header(block.header.clone()).unwrap(); long_fork.push(block); } store_update.commit().unwrap(); @@ -2523,7 +2591,7 @@ mod tests { store_update.save_block(block.clone()); store_update.inc_block_refcount(&block.header.prev_hash).unwrap(); store_update.save_head(&Tip::from_header(&block.header)).unwrap(); - store_update.save_block_header(block.header.clone()); + store_update.save_block_header(block.header.clone()).unwrap(); store_update .chain_store_cache_update .height_to_hashes @@ -2574,7 +2642,7 @@ mod tests { store_update.save_block(block.clone()); store_update.inc_block_refcount(&block.header.prev_hash).unwrap(); store_update.save_head(&Tip::from_header(&block.header)).unwrap(); - store_update.save_block_header(block.header.clone()); + store_update.save_block_header(block.header.clone()).unwrap(); store_update .chain_store_cache_update .height_to_hashes @@ -2639,7 +2707,7 @@ mod tests { store_update.save_block(block.clone()); store_update.inc_block_refcount(&block.header.prev_hash).unwrap(); store_update.save_head(&Tip::from_header(&block.header)).unwrap(); - store_update.save_block_header(block.header.clone()); + store_update.save_block_header(block.header.clone()).unwrap(); store_update .chain_store_cache_update .height_to_hashes diff --git a/chain/chain/src/types.rs b/chain/chain/src/types.rs index d16ae00ae33..059d9a22831 100644 --- a/chain/chain/src/types.rs +++ b/chain/chain/src/types.rs @@ -572,6 +572,7 @@ mod tests { approvals, &signer, genesis.header.inner_lite.next_bp_hash, + CryptoHash::default(), ); b2.header.verify_block_producer(&signer.public_key()); } diff --git a/chain/chain/tests/gc.rs b/chain/chain/tests/gc.rs index d300b92eff6..b166d9fd76c 100644 --- a/chain/chain/tests/gc.rs +++ b/chain/chain/tests/gc.rs @@ -8,6 +8,7 @@ mod tests { use near_chain::DoomslugThresholdMode; use near_crypto::KeyType; use near_primitives::block::Block; + use near_primitives::merkle::PartialMerkleTree; use near_primitives::types::{NumBlocks, StateRoot}; use near_primitives::validator_signer::InMemoryValidatorSigner; use near_store::test_utils::{create_test_store, gen_changes}; @@ -53,14 +54,18 @@ mod tests { let signer = Arc::new(InMemoryValidatorSigner::from_seed("test1", KeyType::ED25519, "test1")); let num_shards = prev_state_roots.len() as u64; - for _ in 0..num_blocks { + for i in 0..num_blocks { let block = Block::empty(&prev_block, &*signer); let head = chain.head().unwrap(); let mut store_update = chain.mut_store().store_update(); + if i == 0 { + store_update + .save_block_merkle_tree(prev_block.hash(), PartialMerkleTree::default()); + } store_update.save_block(block.clone()); store_update.inc_block_refcount(&block.header.prev_hash).unwrap(); - store_update.save_block_header(block.header.clone()); + store_update.save_block_header(block.header.clone()).unwrap(); let tip = Tip::from_header(&block.header); if head.height < tip.height { store_update.save_head(&tip).unwrap(); diff --git a/chain/chain/tests/simple_chain.rs b/chain/chain/tests/simple_chain.rs index a04203ca08e..2d2f07f04b7 100644 --- a/chain/chain/tests/simple_chain.rs +++ b/chain/chain/tests/simple_chain.rs @@ -1,6 +1,7 @@ use near_chain::test_utils::setup; use near_chain::{Block, ChainStoreAccess, ErrorKind, Provenance}; use near_logger_utils::init_test_logger; +use near_primitives::hash::CryptoHash; use num_rational::Rational; #[test] @@ -50,6 +51,7 @@ fn build_chain_with_orhpans() { vec![], &*signer, last_block.header.inner_lite.next_bp_hash.clone(), + CryptoHash::default(), ); assert_eq!( chain diff --git a/chain/chain/tests/sync_chain.rs b/chain/chain/tests/sync_chain.rs index 413a303419b..fa3af8f59f2 100644 --- a/chain/chain/tests/sync_chain.rs +++ b/chain/chain/tests/sync_chain.rs @@ -1,6 +1,7 @@ use near_chain::test_utils::setup; use near_chain::Block; use near_logger_utils::init_test_logger; +use near_primitives::merkle::PartialMerkleTree; #[test] fn chain_sync_headers() { @@ -8,8 +9,13 @@ fn chain_sync_headers() { let (mut chain, _, bls_signer) = setup(); assert_eq!(chain.sync_head().unwrap().height, 0); let mut blocks = vec![chain.get_block(&chain.genesis().hash()).unwrap().clone()]; + let mut block_merkle_tree = PartialMerkleTree::default(); for i in 0..4 { - blocks.push(Block::empty(&blocks[i], &*bls_signer)); + blocks.push(Block::empty_with_block_merkle_tree( + &blocks[i], + &*bls_signer, + &mut block_merkle_tree, + )); } chain .sync_block_headers(blocks.drain(1..).map(|block| block.header).collect(), |_| { diff --git a/chain/client/src/client.rs b/chain/client/src/client.rs index c1a03c6d9c8..70889dd90d3 100644 --- a/chain/client/src/client.rs +++ b/chain/client/src/client.rs @@ -353,8 +353,12 @@ impl Client { }; // Get block extra from previous block. - let prev_block_extra = self.chain.get_block_extra(&head.last_block_hash)?.clone(); - let prev_block = self.chain.get_block(&head.last_block_hash)?; + let mut block_merkle_tree = + self.chain.mut_store().get_block_merkle_tree(&prev_hash)?.clone(); + block_merkle_tree.insert(prev_hash); + let block_merkle_root = block_merkle_tree.root(); + let prev_block_extra = self.chain.get_block_extra(&prev_hash)?.clone(); + let prev_block = self.chain.get_block(&prev_hash)?; let mut chunks = prev_block.chunks.clone(); // Collect new chunks. @@ -393,6 +397,7 @@ impl Client { vec![], &*validator_signer, next_bp_hash, + block_merkle_root, ); // Update latest known even before returning block out, to prevent race conditions. diff --git a/chain/client/src/lib.rs b/chain/client/src/lib.rs index 305cdca05cb..78427ca1cc1 100644 --- a/chain/client/src/lib.rs +++ b/chain/client/src/lib.rs @@ -4,9 +4,9 @@ extern crate lazy_static; pub use crate::client::Client; pub use crate::client_actor::ClientActor; pub use crate::types::{ - Error, GetBlock, GetChunk, GetGasPrice, GetNetworkInfo, GetNextLightClientBlock, - GetStateChanges, GetStateChangesInBlock, GetValidatorInfo, Query, Status, StatusResponse, - SyncStatus, TxStatus, TxStatusError, + Error, GetBlock, GetBlockWithMerkleTree, GetChunk, GetGasPrice, GetNetworkInfo, + GetNextLightClientBlock, GetStateChanges, GetStateChangesInBlock, GetValidatorInfo, Query, + Status, StatusResponse, SyncStatus, TxStatus, TxStatusError, }; pub use crate::view_client::ViewClientActor; diff --git a/chain/client/src/sync.rs b/chain/client/src/sync.rs index d464b653f86..6e622bef7e0 100644 --- a/chain/client/src/sync.rs +++ b/chain/client/src/sync.rs @@ -876,6 +876,7 @@ mod test { use near_primitives::network::PeerId; use super::*; + use near_primitives::merkle::PartialMerkleTree; use near_primitives::types::EpochId; use near_primitives::validator_signer::InMemoryValidatorSigner; use num_rational::Ratio; @@ -1005,6 +1006,7 @@ mod test { let mut last_block = &genesis; let mut all_blocks = vec![]; + let mut block_merkle_tree = PartialMerkleTree::default(); for i in 0..61 { let current_height = 3 + i * 5; @@ -1050,7 +1052,9 @@ mod test { vec![], &*signers[3], last_block.header.inner_lite.next_bp_hash.clone(), + block_merkle_tree.root(), ); + block_merkle_tree.insert(block.hash()); all_blocks.push(block); diff --git a/chain/client/src/types.rs b/chain/client/src/types.rs index 9af57466f77..76a81167446 100644 --- a/chain/client/src/types.rs +++ b/chain/client/src/types.rs @@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize}; use near_network::types::{AccountOrPeerIdOrHash, KnownProducer}; use near_network::PeerInfo; use near_primitives::hash::CryptoHash; +use near_primitives::merkle::PartialMerkleTree; use near_primitives::sharding::ChunkHash; use near_primitives::types::{AccountId, BlockHeight, BlockIdOrFinality, MaybeBlockId, ShardId}; use near_primitives::utils::generate_random_string; @@ -155,6 +156,19 @@ impl Message for GetBlock { type Result = Result; } +/// Get block with the block merkle tree. Used for testing +pub struct GetBlockWithMerkleTree(pub BlockIdOrFinality); + +impl GetBlockWithMerkleTree { + pub fn latest() -> Self { + Self(BlockIdOrFinality::latest()) + } +} + +impl Message for GetBlockWithMerkleTree { + type Result = Result<(BlockView, PartialMerkleTree), String>; +} + /// Actor message requesting a chunk by chunk hash and block hash + shard id. pub enum GetChunk { Height(BlockHeight, ShardId), diff --git a/chain/client/src/view_client.rs b/chain/client/src/view_client.rs index 11fd53ed0bb..cf6c141785c 100644 --- a/chain/client/src/view_client.rs +++ b/chain/client/src/view_client.rs @@ -23,7 +23,7 @@ use near_network::types::{ use near_network::{NetworkAdapter, NetworkRequests}; use near_primitives::block::{BlockHeader, GenesisId}; use near_primitives::hash::CryptoHash; -use near_primitives::merkle::verify_path; +use near_primitives::merkle::{verify_path, PartialMerkleTree}; use near_primitives::network::AnnounceAccount; use near_primitives::types::{ AccountId, BlockHeight, BlockId, BlockIdOrFinality, Finality, MaybeBlockId, @@ -34,7 +34,9 @@ use near_primitives::views::{ StateChangesView, }; -use crate::types::{Error, GetBlock, GetGasPrice, Query, TxStatus, TxStatusError}; +use crate::types::{ + Error, GetBlock, GetBlockWithMerkleTree, GetGasPrice, Query, TxStatus, TxStatusError, +}; use crate::{ sync, GetChunk, GetNextLightClientBlock, GetStateChanges, GetStateChangesInBlock, GetValidatorInfo, @@ -396,6 +398,19 @@ impl Handler for ViewClientActor { } } +impl Handler for ViewClientActor { + type Result = Result<(BlockView, PartialMerkleTree), String>; + + fn handle(&mut self, msg: GetBlockWithMerkleTree, ctx: &mut Context) -> Self::Result { + let block_view = self.handle(GetBlock(msg.0), ctx)?; + self.chain + .mut_store() + .get_block_merkle_tree(&block_view.header.hash) + .map(|merkle_tree| (block_view, merkle_tree.clone())) + .map_err(|e| e.to_string()) + } +} + impl Handler for ViewClientActor { type Result = Result; diff --git a/chain/client/tests/challenges.rs b/chain/client/tests/challenges.rs index 93bdd17e388..353f844f982 100644 --- a/chain/client/tests/challenges.rs +++ b/chain/client/tests/challenges.rs @@ -22,7 +22,7 @@ use near_primitives::challenge::{ BlockDoubleSign, Challenge, ChallengeBody, ChunkProofs, MaybeEncodedShardChunk, }; use near_primitives::hash::CryptoHash; -use near_primitives::merkle::{merklize, MerklePath}; +use near_primitives::merkle::{merklize, MerklePath, PartialMerkleTree}; use near_primitives::receipt::Receipt; use near_primitives::serialize::BaseDecode; use near_primitives::sharding::{EncodedShardChunk, ReedSolomonWrapper}; @@ -44,6 +44,8 @@ fn test_verify_block_double_sign_challenge() { env.process_block(0, b1.clone(), Provenance::NONE); let signer = InMemoryValidatorSigner::from_seed("test0", KeyType::ED25519, "test0"); + let mut block_merkle_tree = PartialMerkleTree::default(); + block_merkle_tree.insert(genesis.hash()); let b2 = Block::produce( &genesis.header, 2, @@ -58,6 +60,7 @@ fn test_verify_block_double_sign_challenge() { vec![], &signer, b1.header.inner_lite.next_bp_hash.clone(), + block_merkle_tree.root(), ); let epoch_id = b1.header.inner_lite.epoch_id.clone(); let valid_challenge = Challenge::produce( @@ -182,6 +185,9 @@ fn create_chunk( chunk.header.hash = hash; chunk.header.signature = signature; } + let mut block_merkle_tree = + client.chain.mut_store().get_block_merkle_tree(&last_block.hash()).unwrap().clone(); + block_merkle_tree.insert(last_block.hash()); let block = Block::produce( &last_block.header, 2, @@ -196,6 +202,7 @@ fn create_chunk( vec![], &*client.validator_signer.as_ref().unwrap().clone(), last_block.header.inner_lite.next_bp_hash, + block_merkle_tree.root(), ); (chunk, merkle_paths, receipts, block) } @@ -422,6 +429,9 @@ fn test_verify_chunk_invalid_state_challenge() { .unwrap(); invalid_chunk.header.height_included = last_block.header.inner_lite.height + 1; + let mut block_merkle_tree = + client.chain.mut_store().get_block_merkle_tree(&last_block.hash()).unwrap().clone(); + block_merkle_tree.insert(last_block.hash()); let block = Block::produce( &last_block.header, last_block.header.inner_lite.height + 1, @@ -436,6 +446,7 @@ fn test_verify_chunk_invalid_state_challenge() { vec![], &validator_signer, last_block.header.inner_lite.next_bp_hash, + block_merkle_tree.root(), ); let challenge_body = { diff --git a/chain/client/tests/process_blocks.rs b/chain/client/tests/process_blocks.rs index dda23aa00f7..d9bbfa81222 100644 --- a/chain/client/tests/process_blocks.rs +++ b/chain/client/tests/process_blocks.rs @@ -14,7 +14,7 @@ use near_chain_configs::Genesis; use near_chunks::{ChunkStatus, ShardsManager}; use near_client::test_utils::setup_mock_all_validators; use near_client::test_utils::{setup_client, setup_mock, TestEnv}; -use near_client::{Client, GetBlock}; +use near_client::{Client, GetBlock, GetBlockWithMerkleTree}; use near_crypto::{InMemorySigner, KeyType, Signature, Signer}; use near_logger_utils::init_test_logger; #[cfg(feature = "metric_recorder")] @@ -157,9 +157,10 @@ fn receive_network_block() { NetworkResponses::NoResponse }), ); - actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { - let last_block = res.unwrap().unwrap(); + actix::spawn(view_client.send(GetBlockWithMerkleTree::latest()).then(move |res| { + let (last_block, mut block_merkle_tree) = res.unwrap().unwrap(); let signer = InMemoryValidatorSigner::from_seed("test1", KeyType::ED25519, "test1"); + block_merkle_tree.insert(last_block.header.hash); let block = Block::produce( &last_block.header.clone().into(), last_block.header.height + 1, @@ -178,6 +179,7 @@ fn receive_network_block() { vec![], &signer, last_block.header.next_bp_hash, + block_merkle_tree.root(), ); client.do_send(NetworkClientMessages::Block(block, PeerInfo::random().id, false)); future::ready(()) @@ -187,71 +189,6 @@ fn receive_network_block() { .unwrap(); } -/// Runs client that receives a block from network and announces header to the network. -#[test] -fn receive_network_block_header() { - let block_holder: Arc>> = Arc::new(RwLock::new(None)); - init_test_logger(); - System::run(|| { - let block_holder1 = block_holder.clone(); - let (client, view_client) = setup_mock( - vec!["test"], - "other", - true, - false, - Box::new(move |msg, _ctx, client_addr| match msg { - NetworkRequests::BlockRequest { hash, peer_id } => { - let block = block_holder1.read().unwrap().clone().unwrap(); - assert_eq!(hash, &block.hash()); - actix::spawn( - client_addr - .send(NetworkClientMessages::Block(block, peer_id.clone(), false)) - .map(drop), - ); - NetworkResponses::NoResponse - } - NetworkRequests::Approval { .. } => { - System::current().stop(); - NetworkResponses::NoResponse - } - _ => NetworkResponses::NoResponse, - }), - ); - actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { - let last_block = res.unwrap().unwrap(); - let signer = InMemoryValidatorSigner::from_seed("test", KeyType::ED25519, "test"); - let block = Block::produce( - &last_block.header.clone().into(), - last_block.header.height + 1, - last_block.chunks.into_iter().map(Into::into).collect(), - EpochId::default(), - if last_block.header.prev_hash == CryptoHash::default() { - EpochId(last_block.header.hash) - } else { - EpochId(last_block.header.next_epoch_id.clone()) - }, - vec![], - Rational::from_integer(0), - 0, - None, - vec![], - vec![], - &signer, - last_block.header.next_bp_hash, - ); - client.do_send(NetworkClientMessages::Block( - block.clone(), - PeerInfo::random().id, - false, - )); - *block_holder.write().unwrap() = Some(block); - future::ready(()) - })); - near_network::test_utils::wait_or_panic(5000); - }) - .unwrap(); -} - /// Include approvals to the next block in newly produced block. #[test] fn produce_block_with_approvals() { @@ -288,9 +225,10 @@ fn produce_block_with_approvals() { NetworkResponses::NoResponse }), ); - actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { - let last_block = res.unwrap().unwrap(); + actix::spawn(view_client.send(GetBlockWithMerkleTree::latest()).then(move |res| { + let (last_block, mut block_merkle_tree) = res.unwrap().unwrap(); let signer1 = InMemoryValidatorSigner::from_seed("test2", KeyType::ED25519, "test2"); + block_merkle_tree.insert(last_block.header.hash); let block = Block::produce( &last_block.header.clone().into(), last_block.header.height + 1, @@ -309,6 +247,7 @@ fn produce_block_with_approvals() { vec![], &signer1, last_block.header.next_bp_hash, + block_merkle_tree.root(), ); client.do_send(NetworkClientMessages::Block( block.clone(), @@ -435,8 +374,8 @@ fn invalid_blocks() { NetworkResponses::NoResponse }), ); - actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { - let last_block = res.unwrap().unwrap(); + actix::spawn(view_client.send(GetBlockWithMerkleTree::latest()).then(move |res| { + let (last_block, mut block_merkle_tree) = res.unwrap().unwrap(); let signer = InMemoryValidatorSigner::from_seed("test", KeyType::ED25519, "test"); // Send block with invalid chunk mask let mut block = Block::produce( @@ -457,6 +396,7 @@ fn invalid_blocks() { vec![], &signer, last_block.header.next_bp_hash, + CryptoHash::default(), ); block.header.inner_rest.chunk_mask = vec![]; client.do_send(NetworkClientMessages::Block( @@ -466,6 +406,7 @@ fn invalid_blocks() { )); // Send proper block. + block_merkle_tree.insert(last_block.header.hash); let block2 = Block::produce( &last_block.header.clone().into(), last_block.header.height + 1, @@ -484,6 +425,7 @@ fn invalid_blocks() { vec![], &signer, last_block.header.next_bp_hash, + block_merkle_tree.root(), ); client.do_send(NetworkClientMessages::Block(block2, PeerInfo::random().id, false)); future::ready(()) @@ -1188,3 +1130,26 @@ fn test_gas_price_change() { env.produce_block(0, i); } } + +#[test] +fn test_invalid_block_root() { + let mut env = TestEnv::new(ChainGenesis::test(), 1, 1); + let mut b1 = env.clients[0].produce_block(1).unwrap().unwrap(); + let signer = InMemoryValidatorSigner::from_seed("test0", KeyType::ED25519, "test0"); + b1.header.inner_lite.block_merkle_root = CryptoHash::default(); + let (hash, signature) = signer.sign_block_header_parts( + b1.header.prev_hash, + &b1.header.inner_lite, + &b1.header.inner_rest, + ); + b1.header.hash = hash; + b1.header.signature = signature; + let (_, tip) = env.clients[0].process_block(b1, Provenance::NONE); + match tip { + Err(e) => match e.kind() { + ErrorKind::InvalidBlockMerkleRoot => {} + _ => assert!(false, "wrong error: {}", e), + }, + _ => assert!(false, "succeeded, tip: {:?}", tip), + } +} diff --git a/chain/client/tests/query_client.rs b/chain/client/tests/query_client.rs index 4deb9ace8c5..6906d3f1c48 100644 --- a/chain/client/tests/query_client.rs +++ b/chain/client/tests/query_client.rs @@ -2,11 +2,11 @@ use actix::System; use futures::{future, FutureExt}; use near_client::test_utils::setup_no_network; -use near_client::{GetBlock, Query, Status}; +use near_client::{GetBlockWithMerkleTree, Query, Status}; use near_crypto::KeyType; use near_logger_utils::init_test_logger; use near_network::{NetworkClientMessages, PeerInfo}; -use near_primitives::block::Block; +use near_primitives::block::{Block, BlockHeader}; use near_primitives::types::{BlockIdOrFinality, EpochId}; use near_primitives::utils::to_timestamp; use near_primitives::validator_signer::{InMemoryValidatorSigner, ValidatorSigner}; @@ -46,11 +46,10 @@ fn query_status_not_crash() { System::run(|| { let (client, view_client) = setup_no_network(vec!["test"], "other", true, false); let signer = InMemoryValidatorSigner::from_seed("test", KeyType::ED25519, "test"); - //let new_block = Arc::new(RwLock::new(None)); - //let new_block_sent = Arc::new(AtomicBool::new(false)); - actix::spawn(view_client.send(GetBlock::latest()).then(move |res| { - let block = res.unwrap().unwrap(); - let header = block.header.clone().into(); + actix::spawn(view_client.send(GetBlockWithMerkleTree::latest()).then(move |res| { + let (block, mut block_merkle_tree) = res.unwrap().unwrap(); + let header: BlockHeader = block.header.clone().into(); + block_merkle_tree.insert(header.hash); let mut next_block = Block::produce( &header, block.header.height + 1, @@ -65,6 +64,7 @@ fn query_status_not_crash() { vec![], &signer, block.header.next_bp_hash, + block_merkle_tree.root(), ); next_block.header.inner_lite.timestamp = to_timestamp(next_block.header.timestamp() + chrono::Duration::seconds(60)); @@ -79,7 +79,6 @@ fn query_status_not_crash() { client .send(NetworkClientMessages::Block(next_block, PeerInfo::random().id, false)) .then(move |_| { - //new_block_sent1.store(true, SeqCst); actix::spawn(client.send(Status { is_health_check: true }).then( move |_| { System::current().stop(); diff --git a/core/chain-configs/src/lib.rs b/core/chain-configs/src/lib.rs index c32e90011f8..07222d9ae3f 100644 --- a/core/chain-configs/src/lib.rs +++ b/core/chain-configs/src/lib.rs @@ -7,4 +7,4 @@ pub use genesis_config::{ }; /// Current latest version of the protocol -pub const PROTOCOL_VERSION: u32 = 12; +pub const PROTOCOL_VERSION: u32 = 13; diff --git a/core/primitives/benches/serialization.rs b/core/primitives/benches/serialization.rs index 5b3c97231fe..d5f9487ab92 100644 --- a/core/primitives/benches/serialization.rs +++ b/core/primitives/benches/serialization.rs @@ -58,6 +58,7 @@ fn create_block() -> Block { vec![], &signer, CryptoHash::default(), + CryptoHash::default(), ) } diff --git a/core/primitives/src/block.rs b/core/primitives/src/block.rs index 9f27ca3dfe3..97da62f6853 100644 --- a/core/primitives/src/block.rs +++ b/core/primitives/src/block.rs @@ -35,6 +35,8 @@ pub struct BlockHeaderInnerLite { pub timestamp: u64, /// Hash of the next epoch block producers set pub next_bp_hash: CryptoHash, + /// Merkle root of block hashes up to the current block. + pub block_merkle_root: CryptoHash, } #[derive(BorshSerialize, BorshDeserialize, Serialize, Debug, Clone, Eq, PartialEq)] @@ -80,6 +82,7 @@ impl BlockHeaderInnerLite { outcome_root: MerkleHash, timestamp: u64, next_bp_hash: CryptoHash, + block_merkle_root: CryptoHash, ) -> Self { Self { height, @@ -89,6 +92,7 @@ impl BlockHeaderInnerLite { outcome_root, timestamp, next_bp_hash, + block_merkle_root, } } @@ -263,6 +267,7 @@ impl BlockHeader { last_ds_final_block: CryptoHash, approvals: Vec>, next_bp_hash: CryptoHash, + block_merkle_root: CryptoHash, ) -> Self { let inner_lite = BlockHeaderInnerLite::new( height, @@ -272,6 +277,7 @@ impl BlockHeader { outcome_root, timestamp, next_bp_hash, + block_merkle_root, ); let inner_rest = BlockHeaderInnerRest::new( chunk_receipts_root, @@ -314,6 +320,7 @@ impl BlockHeader { CryptoHash::default(), to_timestamp(timestamp), next_bp_hash, + CryptoHash::default(), ); let inner_rest = BlockHeaderInnerRest::new( chunk_receipts_root, @@ -454,6 +461,7 @@ impl Block { challenges: Challenges, signer: &dyn ValidatorSigner, next_bp_hash: CryptoHash, + block_merkle_root: CryptoHash, ) -> Self { // Collect aggregate of validators and gas usage/limits from chunks. let mut validator_proposals = vec![]; @@ -531,6 +539,7 @@ impl Block { last_ds_final_block, approvals, next_bp_hash, + block_merkle_root, ), chunks, challenges, diff --git a/core/primitives/src/merkle.rs b/core/primitives/src/merkle.rs index 03c4aaba357..6c5410c48e3 100644 --- a/core/primitives/src/merkle.rs +++ b/core/primitives/src/merkle.rs @@ -1,7 +1,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; -use crate::hash::hash; +use crate::hash::{hash, CryptoHash}; use crate::types::MerkleHash; #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, Serialize, Deserialize)] @@ -29,24 +29,28 @@ pub fn merklize(arr: &[T]) -> (MerkleHash, Vec) { if arr.is_empty() { return (MerkleHash::default(), vec![]); } - let mut len = (arr.len() as u32).next_power_of_two(); - let mut hashes: Vec<_> = (0..len) - .map(|i| { - if i < arr.len() as u32 { - hash(&arr[i as usize].try_to_vec().expect("Failed to serialize")) - } else { - hash(&[0]) - } - }) - .collect(); + let mut len = arr.len().next_power_of_two(); + let mut hashes = arr + .iter() + .map(|elem| hash(&elem.try_to_vec().expect("Failed to serialize"))) + .collect::>(); + // degenerate case if len == 1 { return (hashes[0], vec![vec![]]); } - let mut paths: Vec = (0..arr.len()) + let mut arr_len = arr.len(); + let mut paths: Vec = (0..arr_len) .map(|i| { if i % 2 == 0 { - vec![MerklePathItem { hash: hashes[(i + 1) as usize], direction: Direction::Right }] + if i + 1 < arr_len { + vec![MerklePathItem { + hash: hashes[(i + 1) as usize], + direction: Direction::Right, + }] + } else { + vec![] + } } else { vec![MerklePathItem { hash: hashes[(i - 1) as usize], direction: Direction::Left }] } @@ -58,8 +62,14 @@ pub fn merklize(arr: &[T]) -> (MerkleHash, Vec) { len /= 2; counter *= 2; for i in 0..len { - let hash = combine_hash(hashes[2 * i as usize], hashes[(2 * i + 1) as usize]); - hashes[i as usize] = hash; + let hash = if 2 * i >= arr_len { + continue; + } else if 2 * i + 1 >= arr_len { + hashes[2 * i] + } else { + combine_hash(hashes[2 * i], hashes[2 * i + 1]) + }; + hashes[i] = hash; if len > 1 { if i % 2 == 0 { for j in 0..counter { @@ -78,7 +88,9 @@ pub fn merklize(arr: &[T]) -> (MerkleHash, Vec) { } } } + arr_len = (arr_len + 1) / 2; } + println!("paths: {:?}", paths); (hashes[0], paths) } @@ -98,6 +110,46 @@ pub fn verify_path(root: MerkleHash, path: &MerklePath, item: hash == root } +/// Merkle tree that only maintains the path for the next leaf, i.e, +/// when a new leaf is inserted, the existing `path` is its proof. +/// The root can be computed by folding `path` from right but is not explicitly +/// maintained to save space. +/// The size of the object is O(log(n)) where n is the number of leaves in the tree, i.e, `size`. +#[derive(Default, Clone, BorshSerialize, BorshDeserialize)] +pub struct PartialMerkleTree { + /// Path for the next leaf. + path: Vec, + /// Number of leaves in the tree. + size: u64, +} + +impl PartialMerkleTree { + pub fn root(&self) -> MerkleHash { + if self.path.is_empty() { + CryptoHash::default() + } else { + let mut res = *self.path.last().unwrap(); + let len = self.path.len(); + for i in (0..len - 1).rev() { + res = combine_hash(self.path[i], res); + } + res + } + } + + pub fn insert(&mut self, elem: MerkleHash) { + let mut s = self.size; + let mut node = elem; + while s % 2 == 1 { + let last_path_elem = self.path.pop().unwrap(); + node = combine_hash(last_path_elem, node); + s /= 2; + } + self.path.push(node); + self.size += 1; + } +} + #[cfg(test)] mod tests { use rand::rngs::StdRng; @@ -113,6 +165,7 @@ mod tests { let (root, paths) = merklize(&arr); assert_eq!(paths.len() as u32, n); for (i, item) in arr.iter().enumerate() { + println!("i: {} item: {:?} path: {:?}", i, item, paths[i]); assert!(verify_path(root, &paths[i], item)); } } @@ -121,7 +174,7 @@ mod tests { fn test_merkle_path() { let mut rng: StdRng = SeedableRng::seed_from_u64(1); for _ in 0..10 { - let len: u32 = rng.gen_range(1, 50); + let len: u32 = rng.gen_range(1, 100); test_with_len(len, &mut rng); } } @@ -143,4 +196,31 @@ mod tests { let (root2, _) = merklize(&items2); assert_ne!(root, root2); } + + /// Compute the merkle root of a given array. + fn compute_root(hashes: &[CryptoHash]) -> CryptoHash { + if hashes.is_empty() { + CryptoHash::default() + } else if hashes.len() == 1 { + hashes[0] + } else { + let len = hashes.len(); + let subtree_len = len.next_power_of_two() / 2; + let left_root = compute_root(&hashes[0..subtree_len]); + let right_root = compute_root(&hashes[subtree_len..len]); + combine_hash(left_root, right_root) + } + } + + #[test] + fn test_merkle_tree() { + let mut tree = PartialMerkleTree::default(); + let mut hashes = vec![]; + for i in 0..50 { + assert_eq!(compute_root(&hashes), tree.root()); + let cur_hash = hash(&[i]); + hashes.push(cur_hash); + tree.insert(cur_hash); + } + } } diff --git a/core/primitives/src/test_utils.rs b/core/primitives/src/test_utils.rs index 4101c38c03a..ebd006d902a 100644 --- a/core/primitives/src/test_utils.rs +++ b/core/primitives/src/test_utils.rs @@ -3,6 +3,7 @@ use near_crypto::{EmptySigner, PublicKey, Signature, Signer}; use crate::account::{AccessKey, AccessKeyPermission, Account}; use crate::block::Block; use crate::hash::CryptoHash; +use crate::merkle::PartialMerkleTree; use crate::transaction::{ Action, AddKeyAction, CreateAccountAction, DeleteAccountAction, DeleteKeyAction, DeployContractAction, FunctionCallAction, SignedTransaction, StakeAction, Transaction, @@ -245,7 +246,9 @@ impl Block { next_epoch_id: EpochId, next_bp_hash: CryptoHash, signer: &dyn ValidatorSigner, + block_merkle_tree: &mut PartialMerkleTree, ) -> Self { + block_merkle_tree.insert(prev.hash()); Self::empty_with_approvals( prev, height, @@ -254,6 +257,7 @@ impl Block { vec![], signer, next_bp_hash, + block_merkle_tree.root(), ) } @@ -261,6 +265,20 @@ impl Block { prev: &Block, height: BlockHeight, signer: &dyn ValidatorSigner, + ) -> Self { + Self::empty_with_height_and_block_merkle_tree( + prev, + height, + signer, + &mut PartialMerkleTree::default(), + ) + } + + pub fn empty_with_height_and_block_merkle_tree( + prev: &Block, + height: BlockHeight, + signer: &dyn ValidatorSigner, + block_merkle_tree: &mut PartialMerkleTree, ) -> Self { Self::empty_with_epoch( prev, @@ -273,11 +291,25 @@ impl Block { }, prev.header.inner_lite.next_bp_hash, signer, + block_merkle_tree, + ) + } + + pub fn empty_with_block_merkle_tree( + prev: &Block, + signer: &dyn ValidatorSigner, + block_merkle_tree: &mut PartialMerkleTree, + ) -> Self { + Self::empty_with_height_and_block_merkle_tree( + prev, + prev.header.inner_lite.height + 1, + signer, + block_merkle_tree, ) } pub fn empty(prev: &Block, signer: &dyn ValidatorSigner) -> Self { - Self::empty_with_height(prev, prev.header.inner_lite.height + 1, signer) + Self::empty_with_block_merkle_tree(prev, signer, &mut PartialMerkleTree::default()) } /// This is not suppose to be used outside of chain tests, because this doesn't refer to correct chunks. @@ -290,6 +322,7 @@ impl Block { approvals: Vec>, signer: &dyn ValidatorSigner, next_bp_hash: CryptoHash, + block_merkle_root: CryptoHash, ) -> Self { Block::produce( &prev.header, @@ -305,6 +338,7 @@ impl Block { vec![], signer, next_bp_hash, + block_merkle_root, ) } } diff --git a/core/primitives/src/views.rs b/core/primitives/src/views.rs index 0870ccaf177..920a5c7d243 100644 --- a/core/primitives/src/views.rs +++ b/core/primitives/src/views.rs @@ -338,6 +338,7 @@ pub struct BlockHeaderView { pub last_final_block: CryptoHash, pub last_ds_final_block: CryptoHash, pub next_bp_hash: CryptoHash, + pub block_merkle_root: CryptoHash, pub approvals: Vec>, pub signature: Signature, } @@ -374,6 +375,7 @@ impl From for BlockHeaderView { last_final_block: header.inner_rest.last_final_block, last_ds_final_block: header.inner_rest.last_ds_final_block, next_bp_hash: header.inner_lite.next_bp_hash, + block_merkle_root: header.inner_lite.block_merkle_root, approvals: header.inner_rest.approvals.clone(), signature: header.signature, } @@ -392,6 +394,7 @@ impl From for BlockHeader { outcome_root: view.outcome_root, timestamp: view.timestamp, next_bp_hash: view.next_bp_hash, + block_merkle_root: view.block_merkle_root, }, inner_rest: BlockHeaderInnerRest { chunk_receipts_root: view.chunk_receipts_root, diff --git a/core/store/src/db.rs b/core/store/src/db.rs index a0cfc4323a3..5a9294d116f 100644 --- a/core/store/src/db.rs +++ b/core/store/src/db.rs @@ -80,10 +80,12 @@ pub enum DBCol { ColStateChanges = 34, ColBlockRefCount = 35, ColTrieChanges = 36, + /// Merkle tree of block hashes + ColBlockMerkleTree = 37, } // Do not move this line from enum DBCol -const NUM_COLS: usize = 37; +const NUM_COLS: usize = 38; impl std::fmt::Display for DBCol { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { @@ -125,6 +127,7 @@ impl std::fmt::Display for DBCol { Self::ColStateChanges => "key value changes", Self::ColBlockRefCount => "refcount per block", Self::ColTrieChanges => "trie changes", + Self::ColBlockMerkleTree => "block merkle tree", }; write!(formatter, "{}", desc) } diff --git a/genesis-tools/genesis-populate/src/lib.rs b/genesis-tools/genesis-populate/src/lib.rs index 385b301841e..fb6921a75c3 100644 --- a/genesis-tools/genesis-populate/src/lib.rs +++ b/genesis-tools/genesis-populate/src/lib.rs @@ -212,7 +212,9 @@ impl GenesisBuilder { self.genesis.config.total_supply.clone(), ) .unwrap(); - store_update.save_block_header(genesis.header.clone()); + store_update + .save_block_header(genesis.header.clone()) + .expect("save genesis block header shouldn't fail"); store_update.save_block(genesis.clone()); for (chunk_header, state_root) in genesis.chunks.iter().zip(self.roots.values()) { diff --git a/neard/res/genesis_config.json b/neard/res/genesis_config.json index 0f015fd3aae..a2ee4c6f85a 100644 --- a/neard/res/genesis_config.json +++ b/neard/res/genesis_config.json @@ -1,6 +1,6 @@ { "config_version": 1, - "protocol_version": 12, + "protocol_version": 13, "genesis_time": "1970-01-01T00:00:00.000000000Z", "chain_id": "sample", "genesis_height": 0, diff --git a/neard/tests/sync_nodes.rs b/neard/tests/sync_nodes.rs index 8aae15f2782..f445912d5a8 100644 --- a/neard/tests/sync_nodes.rs +++ b/neard/tests/sync_nodes.rs @@ -14,6 +14,7 @@ use near_logger_utils::init_integration_logger; use near_network::test_utils::{convert_boot_nodes, open_port, WaitOrTimeout}; use near_network::{NetworkClientMessages, PeerInfo}; use near_primitives::block::Approval; +use near_primitives::merkle::PartialMerkleTree; use near_primitives::transaction::SignedTransaction; use near_primitives::types::{BlockHeightDelta, EpochId, ValidatorStake}; use near_primitives::validator_signer::{InMemoryValidatorSigner, ValidatorSigner}; @@ -30,6 +31,10 @@ fn add_blocks( signer: &dyn ValidatorSigner, ) -> Vec { let mut prev = &blocks[blocks.len() - 1]; + let mut block_merkle_tree = PartialMerkleTree::default(); + for block in blocks.iter() { + block_merkle_tree.insert(block.hash()); + } for _ in 0..num { let epoch_id = match prev.header.inner_lite.height + 1 { height if height <= epoch_length => EpochId::default(), @@ -68,7 +73,9 @@ fn add_blocks( stake: TESTING_INIT_STAKE, }]) .unwrap(), + block_merkle_tree.root(), ); + block_merkle_tree.insert(block.hash()); let _ = client.do_send(NetworkClientMessages::Block( block.clone(), PeerInfo::random().id, diff --git a/scripts/migrations/13-block-merkle-root.py b/scripts/migrations/13-block-merkle-root.py new file mode 100644 index 00000000000..0e68ddc3154 --- /dev/null +++ b/scripts/migrations/13-block-merkle-root.py @@ -0,0 +1,22 @@ +""" +Add block merkle root to block header lite. + +No state migration needed for this change. + +""" + +import sys +import os +import json +from collections import OrderedDict + +home = sys.argv[1] +output_home = sys.argv[2] + +config = json.load(open(os.path.join(home, 'output.json')), object_pairs_hook=OrderedDict) + +assert config['protocol_version'] == 12 + +config['protocol_version'] = 13 + +json.dump(config, open(os.path.join(output_home, 'output.json'), 'w'), indent=2)