diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index acc25c7a280..1e8c1c5f36e 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -7,6 +7,7 @@ use chrono::prelude::{DateTime, Utc}; use chrono::Duration; use log::{debug, info}; +use near_primitives::block::genesis_chunks; use near_primitives::hash::{hash, CryptoHash}; use near_primitives::merkle::{merklize, verify_path}; use near_primitives::receipt::Receipt; @@ -185,11 +186,15 @@ impl Chain { // Get runtime initial state and create genesis block out of it. let (state_store_update, state_roots) = runtime_adapter.genesis_state(); - let genesis = Block::genesis( + let genesis_chunks = genesis_chunks( state_roots.clone(), - chain_genesis.time, runtime_adapter.num_shards(), chain_genesis.gas_limit, + ); + let genesis = Block::genesis( + genesis_chunks.iter().map(|chunk| chunk.header.clone()).collect(), + chain_genesis.time, + chain_genesis.gas_limit, chain_genesis.gas_price, chain_genesis.total_supply, ); @@ -227,6 +232,9 @@ impl Chain { } Err(err) => match err.kind() { ErrorKind::DBNotFoundErr(_) => { + for chunk in genesis_chunks { + store_update.save_chunk(&chunk.chunk_hash, chunk.clone()); + } runtime_adapter.add_validator_proposals( CryptoHash::default(), genesis.hash(), @@ -247,7 +255,15 @@ impl Chain { store_update.save_chunk_extra( &genesis.hash(), chunk_header.inner.shard_id, - ChunkExtra::new(state_root, vec![], 0, chain_genesis.gas_limit, 0, 0, 0), + ChunkExtra::new( + state_root, + vec![], + 0, + chain_genesis.gas_limit, + 0, + 0, + 0, + ), ); } @@ -1652,11 +1668,14 @@ impl<'a> ChainUpdate<'a> { self.chain_store_update.get_chunk_clone_from_header(&chunk_header)?; let any_transaction_is_invalid = chunk.transactions.iter().any(|t| { - self.chain_store_update.get_chain_store().check_blocks_on_same_chain( - &block.header, - &t.transaction.block_hash, - self.transaction_validity_period, - ).is_err() + self.chain_store_update + .get_chain_store() + .check_blocks_on_same_chain( + &block.header, + &t.transaction.block_hash, + self.transaction_validity_period, + ) + .is_err() }); if any_transaction_is_invalid { debug!(target: "chain", "Invalid transactions in the chunk: {:?}", chunk.transactions); @@ -1693,7 +1712,7 @@ impl<'a> ChainUpdate<'a> { gas_limit, apply_result.total_rent_paid, apply_result.total_validator_reward, - apply_result.total_balance_burnt + apply_result.total_balance_burnt, ), ); // Save resulting receipts. @@ -2200,7 +2219,7 @@ impl<'a> ChainUpdate<'a> { gas_limit, apply_result.total_rent_paid, apply_result.total_validator_reward, - apply_result.total_balance_burnt + apply_result.total_balance_burnt, ); self.chain_store_update.save_chunk_extra(&block_header.hash, shard_id, chunk_extra); diff --git a/chain/chain/src/types.rs b/chain/chain/src/types.rs index 1bf5d7d134d..681b332beee 100644 --- a/chain/chain/src/types.rs +++ b/chain/chain/src/types.rs @@ -483,17 +483,22 @@ mod tests { use chrono::Utc; use near_crypto::{InMemorySigner, KeyType, Signature}; + use near_primitives::block::genesis_chunks; use super::*; #[test] fn test_block_produce() { let num_shards = 32; - let genesis = Block::genesis( + let genesis_chunks = genesis_chunks( vec![StateRoot { hash: CryptoHash::default(), num_parts: 9 /* TODO MOO */ }], - Utc::now(), num_shards, 1_000_000, + ); + let genesis = Block::genesis( + genesis_chunks.into_iter().map(|chunk| chunk.header).collect(), + Utc::now(), + 1_000_000, 100, 1_000_000_000, ); diff --git a/chain/client/src/types.rs b/chain/client/src/types.rs index 98c3678e062..db603825473 100644 --- a/chain/client/src/types.rs +++ b/chain/client/src/types.rs @@ -255,6 +255,7 @@ impl Message for GetBlock { /// Actor message requesting a chunk by chunk hash and block hash + shard id. pub enum GetChunk { + BlockHeight(BlockIndex, ShardId), BlockHash(CryptoHash, ShardId), ChunkHash(ChunkHash), } diff --git a/chain/client/src/view_client.rs b/chain/client/src/view_client.rs index 3cdc93b79b4..2163b928dbf 100644 --- a/chain/client/src/view_client.rs +++ b/chain/client/src/view_client.rs @@ -174,6 +174,13 @@ impl Handler for ViewClientActor { .get_chunk(&block.chunks[shard_id as usize].chunk_hash()) .map(Clone::clone) }) + }, + GetChunk::BlockHeight(block_height, shard_id) => { + self.chain.get_block_by_height(block_height).map(Clone::clone).and_then(|block| { + self.chain + .get_chunk(&block.chunks[shard_id as usize].chunk_hash()) + .map(Clone::clone) + }) } } .map(|chunk| chunk.into()) diff --git a/chain/jsonrpc/client/src/lib.rs b/chain/jsonrpc/client/src/lib.rs index 11c07d3fabe..1debac4c3c0 100644 --- a/chain/jsonrpc/client/src/lib.rs +++ b/chain/jsonrpc/client/src/lib.rs @@ -6,9 +6,9 @@ use serde::Deserialize; use serde::Serialize; use near_primitives::hash::CryptoHash; -use near_primitives::types::BlockIndex; +use near_primitives::types::{BlockIndex, ShardId}; use near_primitives::views::{ - BlockView, ExecutionOutcomeView, FinalExecutionOutcomeView, QueryResponse, StatusResponse, + BlockView, ChunkView, ExecutionOutcomeView, FinalExecutionOutcomeView, QueryResponse, StatusResponse, }; use crate::message::{from_slice, Message}; @@ -22,6 +22,13 @@ pub enum BlockId { Hash(CryptoHash), } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ChunkId { + BlockShardId(BlockId, ShardId), + Hash(CryptoHash), +} + /// Timeout for establishing connection. const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); @@ -180,6 +187,7 @@ jsonrpc_client!(pub struct JsonRpcClient { pub fn tx(&mut self, hash: String) -> RpcRequest; pub fn tx_details(&mut self, hash: String) -> RpcRequest; pub fn block(&mut self, id: BlockId) -> RpcRequest; + pub fn chunk(&mut self, id: ChunkId) -> RpcRequest; }); /// Create new JSON RPC client that connects to the given address. diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index d2a5f712106..5c11911ff31 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -19,10 +19,10 @@ use async_utils::{delay, timeout}; use message::Message; use message::{Request, RpcError}; use near_client::{ - ClientActor, GetBlock, GetNetworkInfo, Query, Status, TxDetails, TxStatus, ViewClientActor, + ClientActor, GetBlock, GetChunk, GetNetworkInfo, Query, Status, TxDetails, TxStatus, ViewClientActor, }; pub use near_jsonrpc_client as client; -use near_jsonrpc_client::{message, BlockId}; +use near_jsonrpc_client::{message, BlockId, ChunkId}; use near_metrics::{Encoder, TextEncoder}; use near_network::{NetworkClientMessages, NetworkClientResponses}; use near_primitives::hash::CryptoHash; @@ -143,6 +143,7 @@ impl JsonRpcHandler { "tx" => self.tx_status(request.params).await, "tx_details" => self.tx_details(request.params).await, "block" => self.block(request.params).await, + "chunk" => self.chunk(request.params).await, "network_info" => self.network_info().await, _ => Err(RpcError::method_not_found(request.method)), } @@ -239,6 +240,22 @@ impl JsonRpcHandler { ) } + async fn chunk(&self, params: Option) -> Result { + let (chunk_id,) = parse_params::<(ChunkId,)>(params)?; + jsonify( + self.view_client_addr + .send(match chunk_id { + ChunkId::BlockShardId(block_id, shard_id) => match block_id { + BlockId::Height(block_height) => GetChunk::BlockHeight(block_height, shard_id), + BlockId::Hash(block_hash) => GetChunk::BlockHash(block_hash.into(), shard_id), + }, + ChunkId::Hash(chunk_hash) => GetChunk::ChunkHash(chunk_hash.into()), + }) + .compat() + .await, + ) + } + async fn network_info(&self) -> Result { jsonify(self.client_addr.send(GetNetworkInfo {}).compat().await) } diff --git a/chain/jsonrpc/tests/rpc_query.rs b/chain/jsonrpc/tests/rpc_query.rs index 4c78346ef3d..61b063e486e 100644 --- a/chain/jsonrpc/tests/rpc_query.rs +++ b/chain/jsonrpc/tests/rpc_query.rs @@ -4,11 +4,13 @@ use actix::System; use futures::future; use futures::future::Future; +use near_crypto::Signature; use near_jsonrpc::client::new_client; use near_jsonrpc::test_utils::start_all; -use near_jsonrpc_client::BlockId; +use near_jsonrpc_client::{BlockId, ChunkId}; use near_primitives::hash::CryptoHash; use near_primitives::test_utils::init_test_logger; +use near_primitives::types::ShardId; /// Retrieve blocks via json rpc #[test] @@ -65,6 +67,55 @@ fn test_block_by_hash() { .unwrap(); } +/// Retrieve blocks via json rpc +#[test] +fn test_chunk_by_hash() { + init_test_logger(); + + System::run(|| { + let (_view_client_addr, addr) = start_all(true); + + let mut client = new_client(&format!("http://{}", addr.clone())); + actix::spawn( + client.chunk(ChunkId::BlockShardId(BlockId::Height(0), ShardId::from(0u64))).then( + move |chunk| { + let chunk = chunk.unwrap(); + assert_eq!(chunk.header.balance_burnt, 0); + assert_eq!(chunk.header.chunk_hash.as_ref().len(), 32); + assert_eq!(chunk.header.encoded_length, 8); + assert_eq!(chunk.header.encoded_merkle_root.as_ref().len(), 32); + assert_eq!(chunk.header.gas_limit, 1000000); + assert_eq!(chunk.header.gas_used, 0); + assert_eq!(chunk.header.height_created, 0); + assert_eq!(chunk.header.height_included, 0); + assert_eq!(chunk.header.outgoing_receipts_root.as_ref().len(), 32); + assert_eq!(chunk.header.prev_block_hash.as_ref().len(), 32); + assert_eq!(chunk.header.prev_state_num_parts, 17); + assert_eq!(chunk.header.prev_state_root_hash.as_ref().len(), 32); + assert_eq!(chunk.header.rent_paid, 0); + assert_eq!(chunk.header.shard_id, 0); + assert!(if let Signature::ED25519(_) = chunk.header.signature { + true + } else { + false + }); + assert_eq!(chunk.header.tx_root.as_ref(), &[0; 32]); + assert_eq!(chunk.header.validator_proposals, vec![]); + assert_eq!(chunk.header.validator_reward, 0); + let mut client = new_client(&format!("http://{}", addr)); + client.chunk(ChunkId::Hash(chunk.header.chunk_hash)).then(move |same_chunk| { + let same_chunk = same_chunk.unwrap(); + assert_eq!(chunk.header.chunk_hash, same_chunk.header.chunk_hash); + System::current().stop(); + future::ok(()) + }) + }, + ), + ); + }) + .unwrap(); +} + /// Connect to json rpc and query the client. #[test] fn test_query() { diff --git a/core/primitives/benches/serialization.rs b/core/primitives/benches/serialization.rs index cb68f3057b1..5f70bc7642e 100644 --- a/core/primitives/benches/serialization.rs +++ b/core/primitives/benches/serialization.rs @@ -9,7 +9,7 @@ use chrono::Utc; use near_crypto::{InMemorySigner, KeyType, PublicKey, Signature}; use near_primitives::account::Account; -use near_primitives::block::Block; +use near_primitives::block::{genesis_chunks, Block}; use near_primitives::hash::CryptoHash; use near_primitives::transaction::{Action, SignedTransaction, Transaction, TransferAction}; use near_primitives::types::{EpochId, StateRoot}; @@ -33,11 +33,15 @@ fn create_transaction() -> SignedTransaction { } fn create_block() -> Block { - let genesis = Block::genesis( + let genesis_chunks = genesis_chunks( vec![StateRoot { hash: CryptoHash::default(), num_parts: 1 /* TODO MOO */ }], - Utc::now(), 1, 1_000, + ); + let genesis = Block::genesis( + genesis_chunks.into_iter().map(|chunk| chunk.header).collect(), + Utc::now(), + 1_000, 1_000, 1_000, ); diff --git a/core/primitives/src/block.rs b/core/primitives/src/block.rs index 84b332a7a30..584eff3ebf0 100644 --- a/core/primitives/src/block.rs +++ b/core/primitives/src/block.rs @@ -7,7 +7,7 @@ use near_crypto::{EmptySigner, KeyType, PublicKey, Signature, Signer}; use crate::hash::{hash, CryptoHash}; use crate::merkle::merklize; -use crate::sharding::{ChunkHashHeight, ShardChunkHeader}; +use crate::sharding::{ChunkHashHeight, EncodedShardChunk, ShardChunk, ShardChunkHeader}; use crate::types::{ Balance, BlockIndex, EpochId, Gas, MerkleHash, ShardId, StateRoot, ValidatorStake, }; @@ -242,38 +242,48 @@ pub struct Block { pub chunks: Vec, } +pub fn genesis_chunks( + state_roots: Vec, + num_shards: ShardId, + initial_gas_limit: Gas, +) -> Vec { + assert!(state_roots.len() == 1 || state_roots.len() == (num_shards as usize)); + (0..num_shards) + .map(|i| { + let (encoded_chunk, _) = EncodedShardChunk::new( + CryptoHash::default(), + state_roots[i as usize % state_roots.len()].clone(), + 0, + i, + 3, + 1, + 0, + initial_gas_limit, + 0, + 0, + 0, + CryptoHash::default(), + vec![], + &vec![], + &vec![], + CryptoHash::default(), + &EmptySigner {}, + ) + .expect("Failed to decode genesis chunk"); + encoded_chunk.decode_chunk(1).expect("Failed to decode genesis chunk") + }) + .collect() +} + impl Block { /// Returns genesis block for given genesis date and state root. pub fn genesis( - state_roots: Vec, + chunks: Vec, timestamp: DateTime, - num_shards: ShardId, initial_gas_limit: Gas, initial_gas_price: Balance, initial_total_supply: Balance, ) -> Self { - assert!(state_roots.len() == 1 || state_roots.len() == (num_shards as usize)); - let chunks = (0..num_shards) - .map(|i| { - ShardChunkHeader::new( - CryptoHash::default(), - state_roots[i as usize % state_roots.len()].clone(), - CryptoHash::default(), - 0, - 0, - i, - 0, - initial_gas_limit, - 0, - 0, - 0, - CryptoHash::default(), - CryptoHash::default(), - vec![], - &EmptySigner {}, - ) - }) - .collect(); Block { header: BlockHeader::genesis( Block::compute_state_root(&chunks), diff --git a/core/primitives/src/sharding.rs b/core/primitives/src/sharding.rs index 2108aa6db2f..c2a3d5f2b25 100644 --- a/core/primitives/src/sharding.rs +++ b/core/primitives/src/sharding.rs @@ -18,6 +18,12 @@ impl AsRef<[u8]> for ChunkHash { } } +impl From for ChunkHash { + fn from(crypto_hash: CryptoHash) -> Self { + Self(crypto_hash) + } +} + #[derive(BorshSerialize, BorshDeserialize, Clone, PartialEq, Eq, Debug)] pub struct ShardChunkHeaderInner { /// Previous block hash. diff --git a/core/primitives/src/views.rs b/core/primitives/src/views.rs index fc3a44ef473..7ecceaca16d 100644 --- a/core/primitives/src/views.rs +++ b/core/primitives/src/views.rs @@ -326,6 +326,7 @@ impl From for BlockHeader { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct ChunkHeaderView { + pub chunk_hash: CryptoHash, pub prev_block_hash: CryptoHash, pub prev_state_root_hash: CryptoHash, pub prev_state_num_parts: u64, @@ -351,10 +352,11 @@ pub struct ChunkHeaderView { impl From for ChunkHeaderView { fn from(chunk: ShardChunkHeader) -> Self { ChunkHeaderView { - prev_block_hash: chunk.inner.prev_block_hash.into(), - prev_state_root_hash: chunk.inner.prev_state_root.hash.into(), + chunk_hash: chunk.hash.0, + prev_block_hash: chunk.inner.prev_block_hash, + prev_state_root_hash: chunk.inner.prev_state_root.hash, prev_state_num_parts: chunk.inner.prev_state_root.num_parts, - encoded_merkle_root: chunk.inner.encoded_merkle_root.into(), + encoded_merkle_root: chunk.inner.encoded_merkle_root, encoded_length: chunk.inner.encoded_length, height_created: chunk.inner.height_created, height_included: chunk.height_included, @@ -425,7 +427,6 @@ impl From for BlockView { #[derive(Serialize, Deserialize, Debug)] pub struct ChunkView { - pub chunk_hash: CryptoHash, pub header: ChunkHeaderView, pub transactions: Vec, pub receipts: Vec, @@ -434,7 +435,6 @@ pub struct ChunkView { impl From for ChunkView { fn from(chunk: ShardChunk) -> Self { Self { - chunk_hash: chunk.chunk_hash.0.into(), header: chunk.header.into(), transactions: chunk.transactions.into_iter().map(Into::into).collect(), receipts: chunk.receipts.into_iter().map(Into::into).collect(), diff --git a/genesis-tools/genesis-populate/src/lib.rs b/genesis-tools/genesis-populate/src/lib.rs index 89a76dc3f64..546246b72c7 100644 --- a/genesis-tools/genesis-populate/src/lib.rs +++ b/genesis-tools/genesis-populate/src/lib.rs @@ -1,11 +1,20 @@ //! Tools for creating a genesis block. +use std::collections::BTreeMap; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + use borsh::BorshSerialize; use indicatif::{ProgressBar, ProgressStyle}; +use tempdir::TempDir; + use near::{get_store_path, GenesisConfig, NightshadeRuntime}; use near_chain::{Block, ChainStore, RuntimeAdapter, Tip}; use near_crypto::{InMemorySigner, KeyType}; use near_primitives::account::AccessKey; +use near_primitives::block::genesis_chunks; use near_primitives::contract::ContractCode; use near_primitives::hash::{hash, CryptoHash}; use near_primitives::serialize::to_base64; @@ -15,12 +24,6 @@ use near_store::{ create_store, get_account, set_access_key, set_account, set_code, Store, TrieUpdate, COL_STATE, }; use node_runtime::StateRecord; -use std::collections::BTreeMap; -use std::fs::File; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use tempdir::TempDir; fn get_account_id(account_index: u64) -> String { format!("near_{}_{}", account_index, account_index) @@ -179,13 +182,17 @@ impl GenesisBuilder { } fn write_genesis_block(&mut self) -> Result<()> { - let genesis = Block::genesis( + let genesis_chunks = genesis_chunks( self.roots.values().cloned().collect(), - self.config.genesis_time.clone(), self.runtime.num_shards(), - self.config.gas_limit.clone(), - self.config.gas_price.clone(), - self.config.total_supply.clone(), + self.config.gas_limit, + ); + let genesis = Block::genesis( + genesis_chunks.into_iter().map(|chunk| chunk.header).collect(), + self.config.genesis_time, + self.config.gas_limit, + self.config.gas_price, + self.config.total_supply, ); let mut store = ChainStore::new(self.store.clone());