From e1dc79e3175329e50aaebb2aa8d09bb0f66b3f00 Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 6 Jul 2020 12:14:00 +0800 Subject: [PATCH] Problem (Fix #1881): wallet sync logic adapt to new tendermint-rs light client design Solution: - running light client superivsor, and sync wallet to recent trusted block - rpc call_batch ignore the first failed request and return the successed ones. - run ra-sp-server before prepare env, and gen fresh keypackage when prepare test env - fix tendermint-rs deserialization issues --- .gitignore | 2 +- Cargo.lock | 109 ++++- .../enclave-ra/ra-enclave/Cargo.toml | 6 +- chain-tx-enclave-next/mls/Cargo.toml | 2 +- .../tx-query-next/enclave-app/Cargo.toml | 3 - client-cli/src/command.rs | 22 +- client-common/Cargo.toml | 5 +- client-common/src/cipher/mock.rs | 16 +- client-common/src/tendermint/client.rs | 8 - .../tendermint/rpc_client/async_rpc_client.rs | 29 +- .../tendermint/rpc_client/sync_rpc_client.rs | 103 +--- client-common/src/tendermint/types.rs | 22 +- .../src/tendermint/types/block_results.rs | 9 +- .../src/tendermint/unauthorized_client.rs | 10 +- client-core/Cargo.toml | 13 +- client-core/src/service/sync_state_service.rs | 41 +- client-core/src/wallet/syncer.rs | 454 +++++++++++++++--- client-core/src/wallet/syncer_logic.rs | 6 + client-network/Cargo.toml | 2 +- .../network_ops/default_network_ops_client.rs | 21 +- client-rpc/Cargo.toml | 3 +- client-rpc/src/handler.rs | 17 +- client-rpc/src/rpc/multisig_rpc.rs | 9 - client-rpc/src/rpc/sync_rpc.rs | 40 +- client-rpc/src/rpc/wallet_rpc.rs | 27 +- docker/unittest.sh | 6 +- integration-tests/bot/chainbot.py | 48 +- integration-tests/bot/client_cli.py | 2 +- integration-tests/cleanup.sh | 4 + integration-tests/run.sh | 39 +- integration-tests/run_multinode.sh | 32 +- test-common/Cargo.toml | 6 +- test-common/src/block_generator.rs | 54 ++- 33 files changed, 758 insertions(+), 412 deletions(-) diff --git a/.gitignore b/.gitignore index 27f2f4317..f0e486d36 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,7 @@ id_ecdsa # integration tests temparary artifacts /integration-tests/*.so -/integration-tests/bot/.venv +/integration-tests/.venv **/chain-abci/Enclave_u.* diff --git a/Cargo.lock b/Cargo.lock index 20db5dffb..8bb753310 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,6 +120,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "550632e31568ae1a5f47998c3aa48563030fc49b9ec91913ca337cf64fbc5ccb" dependencies = [ "backtrace", + "serde", ] [[package]] @@ -181,6 +182,20 @@ dependencies = [ "syn", ] +[[package]] +name = "async-tungstenite" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "182617e5bbfe7001b6f2317883506f239c77313171620a04cc11292704d3e171" +dependencies = [ + "futures-io", + "futures-util", + "log", + "pin-project", + "tokio 0.2.20", + "tungstenite", +] + [[package]] name = "async_zmq" version = "0.3.2" @@ -228,6 +243,7 @@ dependencies = [ "cfg-if", "libc", "rustc-demangle", + "serde", ] [[package]] @@ -758,6 +774,7 @@ dependencies = [ "serde_json", "sled", "tendermint", + "tendermint-rpc", "tokio 0.2.20", "tokio-tungstenite", "uuid", @@ -797,7 +814,9 @@ dependencies = [ "secstr", "serde", "serde_json", + "sled", "tendermint", + "tendermint-light-client", "test-common", "thiserror", "tiny-bip39", @@ -861,6 +880,7 @@ dependencies = [ "secstr", "serde", "serde_json", + "tendermint-light-client", "zeroize", ] @@ -899,6 +919,17 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "contracts" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9424f2ca1e42776615720e5746eed6efa19866fdbaac2923ab51c294ac4d1f2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "core-foundation" version = "0.7.0" @@ -1220,6 +1251,17 @@ dependencies = [ "syn", ] +[[package]] +name = "derive_more" +version = "0.99.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298998b1cf6b5b2c8a7b023dfd45821825ce3ba8a8af55c921a0e734e4653f76" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dev-utils" version = "0.6.0" @@ -3928,6 +3970,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc6b7951b17b051f3210b063f12cc17320e2fe30ae05b0fe2a3abb068551c76" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.6.1" @@ -4425,23 +4478,22 @@ dependencies = [ [[package]] name = "tendermint" -version = "0.12.0-rc0" -source = "git+https://github.com/crypto-com/tendermint-rs.git?rev=ceca4219d6ae4e4ce906a6579e865af15458fdd6#ceca4219d6ae4e4ce906a6579e865af15458fdd6" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3920b9773738bcd15ed4c9088faa0e73671f9b93ba538f04909e893e571725" dependencies = [ "anomaly", "async-trait", "bytes 0.5.4", "chrono", "futures 0.3.5", - "getrandom", - "http 0.2.1", - "hyper 0.13.5", "once_cell", "prost-amino", "prost-amino-derive", "serde", "serde_bytes", "serde_json", + "serde_repr", "sha2 0.9.1", "signatory", "signatory-dalek", @@ -4450,10 +4502,53 @@ dependencies = [ "tai64", "thiserror", "toml 0.5.6", - "uuid", "zeroize", ] +[[package]] +name = "tendermint-light-client" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4607ed34754d39ac3261010ba275ff76b0d622b5a726617f0734e5adbfdcbe" +dependencies = [ + "anomaly", + "contracts", + "crossbeam-channel", + "derive_more", + "futures 0.3.5", + "prost-amino", + "serde", + "serde_cbor", + "serde_derive", + "sled", + "static_assertions 1.1.0", + "tendermint", + "tendermint-rpc", + "thiserror", + "tokio 0.2.20", +] + +[[package]] +name = "tendermint-rpc" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee992790a74bf6c83150ca8ff4a060a5db5e0166bf3bc4b6361fb88ae174961" +dependencies = [ + "async-tungstenite", + "bytes 0.5.4", + "futures 0.3.5", + "getrandom", + "http 0.2.1", + "hyper 0.13.5", + "serde", + "serde_bytes", + "serde_json", + "tendermint", + "thiserror", + "tokio 0.2.20", + "uuid", +] + [[package]] name = "termcolor" version = "1.1.0" @@ -4498,6 +4593,8 @@ dependencies = [ "subtle 2.2.3", "subtle-encoding", "tendermint", + "tendermint-light-client", + "tendermint-rpc", ] [[package]] diff --git a/chain-tx-enclave-next/enclave-ra/ra-enclave/Cargo.toml b/chain-tx-enclave-next/enclave-ra/ra-enclave/Cargo.toml index 24faf669c..a3ab475d4 100644 --- a/chain-tx-enclave-next/enclave-ra/ra-enclave/Cargo.toml +++ b/chain-tx-enclave-next/enclave-ra/ra-enclave/Cargo.toml @@ -14,7 +14,7 @@ cmac = "0.3" crypto-mac = { version = "0.8", features = ["std"] } log = "0.4" rcgen = "0.8" -ring = "0.16" +ring = "0.16.15" rustls = "0.18" serde_json = "1.0" sgx-isa = { version = "0.3", features = ["sgxstd"] } @@ -23,7 +23,3 @@ thiserror = "1.0" ra-common = { path = "../ra-common" } ra-sp-client = { path = "../ra-sp-client" } - -# Add these lines in workpace Cargo.toml -# [patch.crates-io] -# ring = { git = "https://github.com/crypto-com/ring.git", rev = "4e1862fb0df9efaf2d2c1ec8cacb1e53104f3daa" } diff --git a/chain-tx-enclave-next/mls/Cargo.toml b/chain-tx-enclave-next/mls/Cargo.toml index 78a305fad..7fd831165 100644 --- a/chain-tx-enclave-next/mls/Cargo.toml +++ b/chain-tx-enclave-next/mls/Cargo.toml @@ -7,7 +7,7 @@ readme = "../../README.md" edition = "2018" [dependencies] -ring = "0.16" +ring = "0.16.15" thiserror = "1.0" rustls = "0.18" x509-parser = "0.8.0-beta4" diff --git a/chain-tx-enclave-next/tx-query-next/enclave-app/Cargo.toml b/chain-tx-enclave-next/tx-query-next/enclave-app/Cargo.toml index 15f916e30..73f28f29b 100644 --- a/chain-tx-enclave-next/tx-query-next/enclave-app/Cargo.toml +++ b/chain-tx-enclave-next/tx-query-next/enclave-app/Cargo.toml @@ -21,6 +21,3 @@ chain-core = { path = "../../../chain-core", default-features = false, features enclave-protocol = { path = "../../../enclave-protocol", features = ["edp"] } enclave-utils = { path = "../../../chain-tx-enclave-next/enclave-utils", features = ["sgxstd"] } ra-enclave = { path = "../../../chain-tx-enclave-next/enclave-ra/ra-enclave" } - -# [patch.crates-io] -# ring = { git = "https://github.com/crypto-com/ring.git", rev = "4e1862fb0df9efaf2d2c1ec8cacb1e53104f3daa" } diff --git a/client-cli/src/command.rs b/client-cli/src/command.rs index 877ec8b29..73f904748 100644 --- a/client-cli/src/command.rs +++ b/client-cli/src/command.rs @@ -28,7 +28,8 @@ use client_core::signer::WalletSignerManager; use client_core::transaction_builder::DefaultWalletTransactionBuilder; use client_core::types::BalanceChange; use client_core::wallet::syncer::{ - ObfuscationSyncerConfig, ProgressReport, SyncerOptions, WalletSyncer, + spawn_light_client_supervisor, Handle, ObfuscationSyncerConfig, ProgressReport, SyncerOptions, + WalletSyncer, }; use client_core::wallet::{DefaultWalletClient, WalletClient}; use client_network::network_ops::{DefaultNetworkOpsClient, NetworkOpsClient}; @@ -385,10 +386,17 @@ impl Command { block_height_ensure, } => { let enckey = ask_seckey(None)?; - let tendermint_client = WebsocketRpcClient::new(&tendermint_url())?; + let rpc_url = tendermint_url(); + let tendermint_client = WebsocketRpcClient::new(&rpc_url)?; let tx_obfuscation = get_tx_query(tendermint_client.clone())?; - let storage = SledStorage::new(storage_path())?; + let db_path = storage_path(); + let storage = SledStorage::new(&db_path)?; + let handle = spawn_light_client_supervisor( + db_path.as_ref(), + &rpc_url, + tendermint_client.genesis()?.trusting_period(), + )?; let config = ObfuscationSyncerConfig::new( storage.clone(), tendermint_client, @@ -399,8 +407,12 @@ impl Command { batch_size: *batch_size, block_height_ensure: *block_height_ensure, }, + handle.clone(), ); Self::resync(config, name.clone(), enckey, *force, storage)?; + handle + .terminate() + .expect("terminate light client supervisor in client-cli"); Ok(()) } Command::MultiSig { multisig_command } => { @@ -637,8 +649,8 @@ impl Command { Ok(()) } - fn resync( - config: ObfuscationSyncerConfig, + fn resync( + config: ObfuscationSyncerConfig, name: String, enckey: SecKey, force: bool, diff --git a/client-common/Cargo.toml b/client-common/Cargo.toml index 5d90eb110..73bd2216e 100644 --- a/client-common/Cargo.toml +++ b/client-common/Cargo.toml @@ -37,7 +37,8 @@ secstr = { version = "0.4.0", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sled = { version = "0.33.0", optional = true } -tendermint = { git = "https://github.com/crypto-com/tendermint-rs.git", default-features = false, rev = "ceca4219d6ae4e4ce906a6579e865af15458fdd6" } +tendermint = "0.15" +tendermint-rpc = "0.15" tokio = { version = "0.2", features = ["rt-threaded", "sync", "time", "tcp"], optional = true } tokio-tungstenite = { version = "0.10", features = ["tls"], optional = true } uuid = { version = "0.8.1", features = ["v4"] } @@ -51,4 +52,4 @@ quickcheck = "0.9" default = ["sled", "websocket-rpc"] websocket-rpc = ["futures-util", "tokio", "tokio-tungstenite"] mock-enclave = [] -experimental = [] \ No newline at end of file +experimental = [] diff --git a/client-common/src/cipher/mock.rs b/client-common/src/cipher/mock.rs index 9190ab0ba..cb7a3e7a4 100644 --- a/client-common/src/cipher/mock.rs +++ b/client-common/src/cipher/mock.rs @@ -48,10 +48,7 @@ where .collect::>>() .expect("abci_query failed"); - let sealed_logs = rsps - .into_iter() - .map(|rsp| rsp.value.expect("sealed log query failed")) - .collect::>(); + let sealed_logs = rsps.into_iter().map(|rsp| rsp.value).collect::>(); let txs = sealed_logs .into_iter() @@ -95,7 +92,6 @@ fn checked_unseal(payload: &[u8], _private_key: &PrivateKey) -> Option>( - &self, - _state: lite::TrustedState, - _heights: T, - ) -> Result<(Vec, lite::TrustedState)> { - unreachable!() - } - fn broadcast_transaction(&self, _transaction: &[u8]) -> Result { unreachable!() } @@ -155,7 +143,7 @@ mod tests { _prove: bool, ) -> Result { Ok(AbciQuery { - value: Some(seal(&TxWithOutputs::Transfer(Tx::default()))), + value: seal(&TxWithOutputs::Transfer(Tx::default())), ..Default::default() }) } diff --git a/client-common/src/tendermint/client.rs b/client-common/src/tendermint/client.rs index 5018c7417..800456efb 100644 --- a/client-common/src/tendermint/client.rs +++ b/client-common/src/tendermint/client.rs @@ -1,4 +1,3 @@ -use crate::tendermint::lite; use crate::tendermint::types::*; use crate::Result; use chain_core::state::ChainState; @@ -26,13 +25,6 @@ pub trait Client: Send + Sync + Clone { heights: T, ) -> Result>; - /// Fetch continuous blocks and verify them. - fn block_batch_verified<'a, T: Clone + Iterator>( - &self, - state: lite::TrustedState, - heights: T, - ) -> Result<(Vec, lite::TrustedState)>; - /// Makes `broadcast_tx_sync` call to tendermint fn broadcast_transaction(&self, transaction: &[u8]) -> Result; diff --git a/client-common/src/tendermint/rpc_client/async_rpc_client.rs b/client-common/src/tendermint/rpc_client/async_rpc_client.rs index 6f1da8881..673454192 100644 --- a/client-common/src/tendermint/rpc_client/async_rpc_client.rs +++ b/client-common/src/tendermint/rpc_client/async_rpc_client.rs @@ -123,8 +123,11 @@ impl AsyncRpcClient { let method = batch_params[i].0; let params = &batch_params[i].1; - let response = self.receive_response(method, params, &id, receiver).await?; - responses.push(response); + if let Ok(response) = self.receive_response(method, params, &id, receiver).await { + responses.push(response); + } else { + break; + } } Ok(responses) @@ -154,18 +157,16 @@ impl AsyncRpcClient { let response_values = self.request_batch(batch_params).await?; let mut responses = Vec::with_capacity(response_values.len()); - for (i, response_value) in response_values.into_iter().enumerate() { - let method = batch_params[i].0; - let params = &batch_params[i].1; - - let response = serde_json::from_value(response_value).with_context(|| { - format!( - "Unable to deserialize `{}` from JSON-RPC response for params: {:?}", - method, params - ) - })?; - - responses.push(response); + for response_value in response_values.into_iter() { + match serde_json::from_value(response_value) { + Ok(response) => { + responses.push(response); + } + Err(err) => { + log::error!("rpc call fail: {:?}", err); + break; + } + } } Ok(responses) diff --git a/client-common/src/tendermint/rpc_client/sync_rpc_client.rs b/client-common/src/tendermint/rpc_client/sync_rpc_client.rs index 736fa3baf..87509f240 100644 --- a/client-common/src/tendermint/rpc_client/sync_rpc_client.rs +++ b/client-common/src/tendermint/rpc_client/sync_rpc_client.rs @@ -1,14 +1,12 @@ use std::{ convert::TryFrom, sync::{mpsc::sync_channel, Arc}, - time::{Duration, SystemTime}, + time::Duration, }; -use itertools::izip; use once_cell::sync::OnceCell; use serde::Deserialize; use serde_json::{json, Value}; -use tendermint::{lite, validator}; use tokio::runtime::Runtime; use chain_core::state::ChainState; @@ -16,7 +14,7 @@ use std::sync::Mutex; use super::async_rpc_client::AsyncRpcClient; use crate::{ - tendermint::{lite::TrustedState, types::*, Client}, + tendermint::{types::*, Client}, Error, ErrorKind, PrivateKey, Result, ResultExt, SignedTransaction, Transaction, TransactionObfuscation, }; @@ -200,31 +198,6 @@ impl SyncRpcClient { ) }) } - - fn validators_batch>( - &self, - heights: T, - ) -> Result> { - let params = heights - .map(|height| { - ( - "validators", - vec![json!(height.to_string()), json!("0"), json!("100")], - ) - }) - .collect::)>>(); - self.call_batch(params) - } - - fn commit_batch<'a, T: Iterator>( - &self, - heights: T, - ) -> Result> { - let params = heights - .map(|height| ("commit", vec![json!(height.to_string())])) - .collect::)>>(); - self.call_batch(params) - } } impl Client for SyncRpcClient { @@ -272,48 +245,6 @@ impl Client for SyncRpcClient { self.call_batch(params) } - /// Fetch continuous blocks and verify them. - fn block_batch_verified<'a, T: Clone + Iterator>( - &self, - mut state: TrustedState, - heights: T, - ) -> Result<(Vec, TrustedState)> { - let commits = self.commit_batch(heights.clone())?; - let validators: Vec = self - .validators_batch(heights.clone().map(|h| h.saturating_add(1)))? - .into_iter() - .map(|rsp| validator::Set::new(rsp.validators)) - .collect(); - let blocks = self.block_batch(heights)?; - for (commit, next_vals, block) in izip!(&commits, &validators, &blocks) { - let signed_header = - lite::SignedHeader::new(commit.signed_header.clone(), block.header.clone()); - state = if let Some(state) = &state.0 { - lite::verifier::verify_single( - state.clone(), - &signed_header, - state.validators(), - next_vals, - // FIXME make parameters configurable - lite::TrustThresholdFraction::new(1, 3).unwrap(), - Duration::from_secs(std::u32::MAX as u64), - SystemTime::now(), - ) - .map_err(|err| { - Error::new( - ErrorKind::VerifyError, - format!("block verify failed: {:?}", err), - ) - })? - .into() - } else { - // TODO verify block1 against genesis block - lite::TrustedState::new(signed_header, next_vals.clone()).into() - }; - } - Ok((blocks, state)) - } - /// Makes `broadcast_tx_sync` call to tendermint fn broadcast_transaction(&self, transaction: &[u8]) -> Result { let params = vec![json!(transaction)]; @@ -376,27 +307,17 @@ impl Client for SyncRpcClient { .collect(); let rsps = self.call_batch::(params)?; - rsps.into_iter() - .map(|rsp| { - if let Some(value) = rsp.response.value { - let state = serde_json::from_str( - &String::from_utf8(value) - .chain(|| (ErrorKind::InvalidInput, "chain state decode failed"))?, - ) - .chain(|| (ErrorKind::InvalidInput, "chain state decode failed"))?; - Ok(state) - } else { - Err(Error::new( - ErrorKind::InvalidInput, - format!( - "abci query fail: {}, {}", - rsp.response.code.value(), - rsp.response.log, - ), - )) + let mut states = Vec::new(); + for rsp in rsps.into_iter() { + if let Ok(s) = String::from_utf8(rsp.response.value) { + if let Ok(state) = serde_json::from_str(&s) { + states.push(state); + continue; } - }) - .collect() + } + break; + } + Ok(states) } } diff --git a/client-common/src/tendermint/types.rs b/client-common/src/tendermint/types.rs index 71af51a31..058a1d3c7 100644 --- a/client-common/src/tendermint/types.rs +++ b/client-common/src/tendermint/types.rs @@ -3,6 +3,7 @@ mod block_results; use parity_scale_codec::Decode; use serde::{Deserialize, Serialize}; +use std::time::Duration; use crate::{ErrorKind, Result, ResultExt, Transaction}; use chain_core::init::config::InitConfig; @@ -11,17 +12,16 @@ use chain_core::tx::fee::LinearFee; use chain_core::tx::{TxAux, TxEnclaveAux, TxPublicAux}; pub use self::block_results::BlockResults; -pub use tendermint::rpc::endpoint::{ - abci_query::AbciQuery, abci_query::Response as AbciQueryResponse, - block::Response as BlockResponse, block_results::Response as BlockResultsResponse, - broadcast::tx_sync::Response as BroadcastTxResponse, commit::Response as CommitResponse, - status::Response as StatusResponse, validators::Response as ValidatorsResponse, -}; -pub use tendermint::rpc::endpoint::{broadcast, status}; pub use tendermint::{ abci, abci::transaction::Data, abci::Code, block::Header, block::Height, Block, Genesis as GenericGenesis, Hash, Time, }; +pub use tendermint_rpc::endpoint::{ + abci_query::AbciQuery, abci_query::Response as AbciQueryResponse, + block::Response as BlockResponse, block_results::Response as BlockResultsResponse, broadcast, + broadcast::tx_sync::Response as BroadcastTxResponse, commit::Response as CommitResponse, + status, status::Response as StatusResponse, validators::Response as ValidatorsResponse, +}; /// crypto-com instantiated genesis type pub type Genesis = GenericGenesis>; @@ -98,6 +98,8 @@ impl BlockExt for Block { pub trait GenesisExt { /// get fee policy fn fee_policy(&self) -> LinearFee; + /// get light client trusting period + fn trusting_period(&self) -> Duration; } impl GenesisExt for Genesis { @@ -108,6 +110,10 @@ impl GenesisExt for Genesis { .network_params .initial_fee_policy } + + fn trusting_period(&self) -> Duration { + self.consensus_params.evidence.max_age_duration.into() + } } /// crypto-chain specific methods. @@ -118,6 +124,6 @@ pub trait AbciQueryExt { impl AbciQueryExt for AbciQuery { fn bytes(&self) -> Vec { - self.value.clone().unwrap_or_default() + self.value.clone() } } diff --git a/client-common/src/tendermint/types/block_results.rs b/client-common/src/tendermint/types/block_results.rs index 1e41e8615..20fd34fbe 100644 --- a/client-common/src/tendermint/types/block_results.rs +++ b/client-common/src/tendermint/types/block_results.rs @@ -318,7 +318,14 @@ mod tests { #[test] fn check_null_deliver_tx() { - let block_results = BlockResultsResponse::default(); + let block_results = BlockResultsResponse { + height: Default::default(), + txs_results: None, + begin_block_events: None, + end_block_events: None, + validator_updates: vec![], + consensus_param_updates: None, + }; assert_eq!(0, block_results.fees().unwrap().len()); } diff --git a/client-common/src/tendermint/unauthorized_client.rs b/client-common/src/tendermint/unauthorized_client.rs index cfdf63e77..8a0ed9125 100644 --- a/client-common/src/tendermint/unauthorized_client.rs +++ b/client-common/src/tendermint/unauthorized_client.rs @@ -1,5 +1,5 @@ use crate::{ - tendermint::{lite, types::*, Client}, + tendermint::{types::*, Client}, ErrorKind, Result, }; use chain_core::state::ChainState; @@ -50,14 +50,6 @@ impl Client for UnauthorizedClient { Err(ErrorKind::PermissionDenied.into()) } - fn block_batch_verified<'a, T: Clone + Iterator>( - &self, - _state: lite::TrustedState, - _heights: T, - ) -> Result<(Vec, lite::TrustedState)> { - Err(ErrorKind::PermissionDenied.into()) - } - fn query_state_batch>(&self, _heights: T) -> Result> { Err(ErrorKind::PermissionDenied.into()) } diff --git a/client-core/Cargo.toml b/client-core/Cargo.toml index f16af3d34..b28d18259 100644 --- a/client-core/Cargo.toml +++ b/client-core/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" [dependencies] blake3 = { version = "0.3.5", default-features = false } chain-core = { path = "../chain-core" } -client-common = { path = "../client-common" } +client-common = { path = "../client-common", features = ["sled"] } chain-tx-filter = { path = "../chain-tx-filter" } chain-tx-validation = { path = "../chain-tx-validation" } chain-storage = { path = "../chain-storage", default-features = false } @@ -34,12 +34,14 @@ tokio = { version = "0.1.22", default-features = false, features = ["rt-full"] } tiny-bip39 = { version = "0.7", default-features = false } unicase = "2.6.0" lazy_static = "1.4.0" -ring = "0.16.12" -tendermint = { git = "https://github.com/crypto-com/tendermint-rs.git", default-features = false, rev = "ceca4219d6ae4e4ce906a6579e865af15458fdd6" } +ring = "0.16.15" +tendermint = "0.15" +tendermint-light-client = "0.15" thiserror = { version = "1.0", default-features = false } non-empty-vec = "0.1" zxcvbn = "2.0" indexmap = "1.5" +sled = "0.33.0" [dev-dependencies] base58 = "0.1.0" @@ -48,8 +50,7 @@ ripemd160 = "0.9" test-common = { path = "../test-common" } [features] -default = ["sled"] -sled = ["client-common/sled"] websocket-rpc = ["client-common/websocket-rpc"] mock-hardware-wallet = [] -experimental = ["client-common/experimental"] \ No newline at end of file +experimental = ["client-common/experimental"] +mock-enclave = ["client-common/mock-enclave"] diff --git a/client-core/src/service/sync_state_service.rs b/client-core/src/service/sync_state_service.rs index 680425a49..fecf80f13 100644 --- a/client-core/src/service/sync_state_service.rs +++ b/client-core/src/service/sync_state_service.rs @@ -1,9 +1,6 @@ use chain_core::common::H256; -use client_common::tendermint::lite; use client_common::{ErrorKind, Result, ResultExt, Storage}; use parity_scale_codec::{Decode, Encode}; -use tendermint::validator; - /// key space of wallet sync state const KEYSPACE: &str = "core_wallet_sync"; @@ -14,20 +11,23 @@ pub struct SyncState { pub last_block_height: u64, /// last app hash pub last_app_hash: String, - /// current trusted state for lite client verification - pub trusted_state: lite::TrustedState, + /// last block hash + pub last_block_hash: String, /// current trusted staking_root pub staking_root: H256, + /// Is current synced wallet state trusted + pub trusted: bool, } impl SyncState { /// construct genesis global state - pub fn genesis(genesis_validators: Vec, staking_root: H256) -> SyncState { + pub fn genesis(staking_root: H256) -> SyncState { SyncState { last_block_height: 0, last_app_hash: "".to_owned(), - trusted_state: lite::TrustedState::genesis(genesis_validators), + last_block_hash: "".to_owned(), staking_root, + trusted: true, } } } @@ -109,11 +109,9 @@ where #[cfg(test)] mod tests { use parity_scale_codec::{Decode, Encode}; - use tendermint::{block::Height, lite}; - use super::{lite::TrustedState, SyncState, SyncStateService}; + use super::{SyncState, SyncStateService}; use client_common::storage::MemoryStorage; - use test_common::block_generator::{BlockGenerator, GeneratorClient}; #[test] fn check_flow() { @@ -134,8 +132,11 @@ mod tests { last_app_hash: "3891040F29C6A56A5E36B17DCA6992D8F91D1EAAB4439D008D19A9D703271D3C" .to_string(), - trusted_state: TrustedState::genesis(vec![]), + last_block_hash: + "3891040F29C6A56A5E36B17DCA6992D8F91D1EAAB4439D008D19A9D703271D3C" + .to_string(), staking_root: [0u8; 32], + trusted: true, } ) .is_ok()); @@ -164,26 +165,10 @@ mod tests { #[test] fn check_sync_state_serialization() { - let c = GeneratorClient::new(BlockGenerator::one_node()); - { - let mut gen = c.gen.write().unwrap(); - gen.gen_block(&[]); - gen.gen_block(&[]); - } - - let gen = c.gen.read().unwrap(); - let header = gen.signed_header(Height::default()); - - let trusted_state = lite::TrustedState::new( - lite::SignedHeader::new(header.clone(), header.header.clone()), - gen.validators.clone(), - ) - .into(); - let mut state = SyncState::genesis(vec![], [0u8; 32]); + let mut state = SyncState::genesis([0u8; 32]); state.last_block_height = 1; state.last_app_hash = "0F46E113C21F9EACB26D752F9523746CF8D47ECBEA492736D176005911F973A5".to_owned(); - state.trusted_state = trusted_state; let bytes = state.encode(); let state2 = SyncState::decode(&mut bytes.as_slice()).unwrap(); diff --git a/client-core/src/wallet/syncer.rs b/client-core/src/wallet/syncer.rs index ff9476754..313a47416 100644 --- a/client-core/src/wallet/syncer.rs +++ b/client-core/src/wallet/syncer.rs @@ -2,8 +2,33 @@ use indexmap::IndexMap; use itertools::{izip, Itertools}; use non_empty_vec::NonEmpty; +use std::cmp::{max, Ordering}; use std::collections::HashMap; use std::iter; +use std::path::Path; +use std::result; +use std::str::FromStr; +use std::sync::Arc; +use std::thread; +use std::time::Duration; +pub use tendermint_light_client::supervisor::Handle; +use tendermint_light_client::{ + components::{ + clock::SystemClock, + io::{AtHeight, Io, ProdIo}, + scheduler, + verifier::ProdVerifier, + }, + evidence::ProdEvidenceReporter, + fork_detector::ProdForkDetector, + light_client::{self, LightClient}, + operations::hasher::{Hasher, ProdHasher}, + peer_list::PeerList, + state::State, + store::{sled::SledStore, LightStore}, + supervisor::{Instance, Supervisor}, + types::{LightBlock, PeerId, Status, TrustThreshold}, +}; use chain_core::common::H256; use chain_core::state::account::StakedStateAddress; @@ -15,7 +40,7 @@ use chain_core::tx::TransactionId; use chain_storage::jellyfish::compute_staking_root; use chain_tx_filter::BlockFilter; use client_common::tendermint::types::{ - Block, BlockExt, BlockResults, BlockResultsResponse, Genesis, StatusResponse, Time, + Block, BlockExt, BlockResults, BlockResultsResponse, Genesis, Time, }; use client_common::tendermint::Client; use client_common::{ @@ -27,6 +52,9 @@ use super::syncer_logic::handle_blocks; use crate::service; use crate::service::{KeyService, SyncState, Wallet, WalletState, WalletStateMemento}; +pub trait LightClientHandle: Handle + Send + Sync + Clone {} +impl LightClientHandle for T {} + pub trait AddressRecovery: Clone + Send + Sync { // new_address: transfer address in TxOut // return: true, new addresses are generated @@ -88,39 +116,50 @@ pub struct SyncerOptions { /// Common configs for wallet syncer with `TransactionObfuscation` #[derive(Clone)] -pub struct ObfuscationSyncerConfig { +pub struct ObfuscationSyncerConfig< + S: SecureStorage, + C: Client, + O: TransactionObfuscation, + L: LightClientHandle, +> { // services pub storage: S, pub client: C, pub obfuscation: O, + pub light_client: L, // configs pub options: SyncerOptions, } -impl ObfuscationSyncerConfig { +impl + ObfuscationSyncerConfig +{ /// Construct ObfuscationSyncerConfig pub fn new( storage: S, client: C, obfuscation: O, options: SyncerOptions, - ) -> ObfuscationSyncerConfig { + light_client: L, + ) -> ObfuscationSyncerConfig { ObfuscationSyncerConfig { storage, client, obfuscation, options, + light_client, } } } /// Common configs for wallet syncer #[derive(Clone)] -pub struct SyncerConfig { +pub struct SyncerConfig { // services storage: S, client: C, + light_client: L, // configs options: SyncerOptions, @@ -128,7 +167,13 @@ pub struct SyncerConfig { /// Wallet Syncer #[derive(Clone)] -pub struct WalletSyncer { +pub struct WalletSyncer< + S: SecureStorage, + C: Client, + D: TxDecryptor, + T: AddressRecovery, + L: LightClientHandle, +> { // common storage: S, client: C, @@ -139,23 +184,25 @@ pub struct WalletSyncer WalletSyncer +impl WalletSyncer where S: SecureStorage, C: Client, D: TxDecryptor, T: AddressRecovery, + L: LightClientHandle, { /// Construct with common config pub fn with_config( - config: SyncerConfig, + config: SyncerConfig, decryptor: D, name: String, enckey: SecKey, recover_address: T, - ) -> WalletSyncer { + ) -> WalletSyncer { Self { storage: config.storage, client: config.client, @@ -164,6 +211,7 @@ where enckey, options: config.options, recover_address, + light_client: config.light_client, } } @@ -188,20 +236,21 @@ fn load_view_key(storage: &S, name: &str, enckey: &SecKey) -> }) } -impl WalletSyncer, T> +impl WalletSyncer, T, L> where S: SecureStorage, C: Client, O: TransactionObfuscation, T: AddressRecovery, + L: LightClientHandle, { /// Construct with obfuscation config pub fn with_obfuscation_config( - config: ObfuscationSyncerConfig, + config: ObfuscationSyncerConfig, name: String, enckey: SecKey, wallet_client: T, - ) -> Result, T>> + ) -> Result, T, L>> where O: TransactionObfuscation, { @@ -212,6 +261,7 @@ where storage: config.storage, client: config.client, options: config.options, + light_client: config.light_client, }, decryptor, name, @@ -229,8 +279,9 @@ struct WalletSyncerImpl< D: TxDecryptor, F: FnMut(ProgressReport) -> bool, T: AddressRecovery, + L: LightClientHandle, > { - env: &'a mut WalletSyncer, + env: &'a mut WalletSyncer, progress_callback: F, // cached state @@ -246,9 +297,10 @@ impl< D: TxDecryptor, F: FnMut(ProgressReport) -> bool, T: AddressRecovery, - > WalletSyncerImpl<'a, S, C, D, F, T> + L: LightClientHandle, + > WalletSyncerImpl<'a, S, C, D, F, T, L> { - fn new(env: &'a mut WalletSyncer, progress_callback: F) -> Result { + fn new(env: &'a mut WalletSyncer, progress_callback: F) -> Result { let wallet = service::load_wallet(&env.storage, &env.name, &env.enckey)? .err_kind(ErrorKind::InvalidInput, || { format!("wallet not found: {}", env.name) @@ -376,6 +428,7 @@ impl< let block = blocks.last(); self.sync_state.last_block_height = block.block_height; self.sync_state.last_app_hash = block.app_hash.clone(); + self.sync_state.last_block_hash = block.block_hash.clone(); self.sync_state.staking_root = block.staking_root; self.save(&memento)?; @@ -394,19 +447,58 @@ impl< "Tendermint node is catching up with full node (retry after some time)", )); } - let current_block_height = status.sync_info.latest_block_height.value(); - if !self.init_progress(current_block_height) { + let light_block = self + .env + .light_client + .verify_to_highest() + .err_kind(ErrorKind::VerifyError, || "")?; + + let target_height = light_block.signed_header.header.height.value(); + let target_app_hash = hex::encode_upper(&light_block.signed_header.header.app_hash); + let target_block_hash = ProdHasher {} + .hash_header(&light_block.signed_header.header) + .to_string(); + if !self.init_progress(target_height) { return Err(Error::new(ErrorKind::InvalidInput, "Cancelled by user")); } + { + // wait for the target block results to become available + let mut success = false; + for _ in 0..10 { + if self.env.client.block_results(target_height).is_ok() { + success = true; + break; + } + thread::sleep(Duration::from_millis(100)); + } + if !success { + return Err(Error::new( + ErrorKind::TendermintRpcError, + "block result for highest light block is not available", + )); + } + } + self.sync_to(target_height, &target_app_hash, &target_block_hash)?; + + Ok(()) + } + + fn sync_to( + &mut self, + target_height: u64, + target_app_hash: &str, + target_block_hash: &str, + ) -> Result<()> { + self.sync_state.trusted = false; // Send batch RPC requests to tendermint in chunks of `batch_size` requests per batch call - for chunk in ((self.sync_state.last_block_height + 1)..=current_block_height) + for chunk in ((self.sync_state.last_block_height + 1)..=target_height) .chunks(self.env.options.batch_size) .into_iter() { let mut batch = Vec::with_capacity(self.env.options.batch_size); if self.env.options.enable_fast_forward { - if let Some(block) = self.fast_forward_status(&status)? { + if let Some(block) = self.fast_forward_status(&target_app_hash, target_height)? { // Fast forward to latest state if possible self.handle_batch((batch, block).into())?; return Ok(()); @@ -426,39 +518,15 @@ impl< } // Fetch batch details if it cannot be fast forwarded - let (blocks, trusted_state) = self - .env - .client - .block_batch_verified(self.sync_state.trusted_state.clone(), range.iter())?; - self.sync_state.trusted_state = trusted_state; + let blocks = self.env.client.block_batch(range.iter())?; let block_results = self.env.client.block_results_batch(range.iter())?; let states = self.env.client.query_state_batch(range.iter().cloned())?; - let mut app_hash: Option = None; for (block, block_result, state) in izip!( blocks.into_iter(), block_results.into_iter(), states.into_iter() ) { - if let Some(app_hash) = app_hash { - if app_hash != block.header.app_hash.as_slice() { - return Err(Error::new( - ErrorKind::VerifyError, - "state app hash don't match block header", - )); - } - } - app_hash = Some( - state.compute_app_hash( - block_result - .fees() - .chain(|| (ErrorKind::VerifyError, "verify block results"))? - .keys() - .cloned() - .collect(), - ), - ); - let block = FilteredBlock::from_block( &self.wallet, &self.wallet_state, @@ -466,6 +534,29 @@ impl< &block_result, &state, )?; + + // verify app hash chain + if !self.sync_state.last_app_hash.is_empty() + && self.sync_state.last_app_hash != block.last_app_hash + { + return Err(Error::new( + ErrorKind::VerifyError, + "last app hash don't match", + )); + } + self.sync_state.last_app_hash = block.app_hash.clone(); + + // verify block hash chain + if !self.sync_state.last_block_hash.is_empty() + && self.sync_state.last_block_hash != block.last_block_hash + { + return Err(Error::new( + ErrorKind::VerifyError, + "last block hash don't match", + )); + } + self.sync_state.last_block_hash = block.block_hash.clone(); + self.update_progress(block.block_height); batch.push(block); } @@ -473,8 +564,34 @@ impl< self.handle_batch(non_empty_batch)?; } } - // rollback the pending transaction - self.rollback_pending_tx(current_block_height) + + match self.sync_state.last_block_height.cmp(&target_height) { + Ordering::Equal => { + // rollback the pending transaction + self.rollback_pending_tx(target_height)?; + + if self.sync_state.last_block_hash != target_block_hash { + return Err(Error::new( + ErrorKind::VerifyError, + "target block hash dont match", + )); + }; + self.sync_state.trusted = true; + service::save_sync_state(&self.env.storage, &self.env.name, &self.sync_state) + } + Ordering::Greater => { + // impossible + Err(Error::new( + ErrorKind::VerifyError, + "sync block higher than target", + )) + } + Ordering::Less => { + // not up-to-date, try again + log::warn!("not up to date, sync again"); + self.sync_to(target_height, target_app_hash, target_block_hash) + } + } } fn rollback_pending_tx(&mut self, current_block_height: u64) -> Result<()> { @@ -491,16 +608,12 @@ impl< } /// Fast forwards state to given status if app hashes match - fn fast_forward_status(&self, status: &StatusResponse) -> Result> { - let current_app_hash = status - .sync_info - .latest_app_hash - .ok_or_else(|| Error::new(ErrorKind::TendermintRpcError, "latest_app_hash not found"))? - .to_string(); - + fn fast_forward_status( + &self, + current_app_hash: &str, + current_block_height: u64, + ) -> Result> { if current_app_hash == self.sync_state.last_app_hash { - let current_block_height = status.sync_info.latest_block_height.value(); - let block = self.env.client.block(current_block_height)?; let block_result = self.env.client.block_results(current_block_height)?; let states = self @@ -604,10 +717,7 @@ pub fn get_genesis_sync_state( .expect("invalid genesis time") .as_secs(), ); - Ok(SyncState::genesis( - genesis.validators, - compute_staking_root(&accounts), - )) + Ok(SyncState::genesis(compute_staking_root(&accounts))) } /// A struct for providing progress report for synchronization @@ -635,10 +745,16 @@ pub enum ProgressReport { /// already filtered for current wallet. #[derive(Debug)] pub(crate) struct FilteredBlock { - /// App hash of block + /// The result app hash of last block + pub last_app_hash: String, + /// The result app hash of this block pub app_hash: String, /// Block height pub block_height: u64, + /// Hash of last block + pub last_block_hash: String, + /// Block hash + pub block_hash: String, /// Block time pub block_time: Time, /// List of successfully committed transaction ids in this block and their fees @@ -662,9 +778,26 @@ impl FilteredBlock { block_result: &BlockResultsResponse, state: &ChainState, ) -> Result { - let app_hash = hex::encode(&block.header.app_hash); + let last_app_hash = hex::encode_upper(&block.header.app_hash); + let app_hash = hex::encode_upper( + &state.compute_app_hash( + block_result + .fees() + .chain(|| (ErrorKind::VerifyError, "verify block results"))? + .keys() + .cloned() + .collect(), + ), + ); let block_height = block.header.height.value(); let block_time = block.header.time; + let last_block_hash = block + .header + .last_block_id + .as_ref() + .map(|block_id| block_id.hash.to_string()) + .unwrap_or_default(); + let block_hash = ProdHasher {}.hash_header(&block.header).to_string(); let block_filter = block_result.block_filter()?; @@ -690,9 +823,12 @@ impl FilteredBlock { }; Ok(FilteredBlock { + last_app_hash, app_hash, block_height, block_time, + block_hash, + last_block_hash, valid_transaction_fees, enclave_transaction_ids, block_filter, @@ -750,6 +886,64 @@ fn filter_incomming_staking_transactions<'a>( Ok(Default::default()) } +fn make_light_client_instance( + peer_id: PeerId, + addr: tendermint::net::Address, + db_path: impl AsRef, + trusting_period: Duration, +) -> Result { + let mut peer_map = HashMap::new(); + peer_map.insert(peer_id, addr); + + let timeout = Duration::from_secs(10); + let io = ProdIo::new(peer_map, Some(timeout)); + + let db = sled::open(&db_path).err_kind(ErrorKind::InitializationError, || { + format!( + "Unable to initialize sled storage for light client peer at path: {}", + db_path.as_ref().display() + ) + })?; + + let mut light_store = SledStore::new(db); + + if light_store.latest(Status::Verified).is_none() { + // FIXME trust height 1 automatically + let trusted_state = io + .fetch_light_block(peer_id, AtHeight::At(1)) + .err_kind(ErrorKind::InitializationError, || { + "could not retrieve trusted header of block 1" + })?; + light_store.insert(trusted_state, Status::Verified); + } + let state = State { + light_store: Box::new(light_store), + verification_trace: HashMap::new(), + }; + + let options = light_client::Options { + trust_threshold: TrustThreshold { + numerator: 1, + denominator: 3, + }, + // https://docs.tendermint.com/master/spec/consensus/light-client/verification.html#high-level-solution + // set a minimal duration because integration test's unbonding period is too short for + // trusting period. + trusting_period: max(trusting_period, Duration::from_secs(600)), + // allowed clock drift between local clocks and BFT time + // https://docs.tendermint.com/master/spec/consensus/light-client/verification.html#failure-model + clock_drift: Duration::from_secs(1), + }; + + let verifier = ProdVerifier::default(); + let clock = SystemClock; + let scheduler = scheduler::basic_bisecting_schedule; + + let light_client = LightClient::new(peer_id, options, clock, scheduler, verifier, io); + + Ok(Instance::new(light_client, state)) +} + #[cfg(test)] mod tests { use super::*; @@ -764,7 +958,7 @@ mod tests { use chain_core::state::ChainState; use client_common::storage::MemoryStorage; use client_common::tendermint::types::*; - use client_common::tendermint::{lite, Client}; + use client_common::tendermint::Client; use test_common::block_generator::{BlockGenerator, GeneratorClient}; use crate::service::save_sync_state; @@ -795,11 +989,13 @@ mod tests { gen.gen_block(&[]); } } + let light_client = client.clone(); let mut syncer = WalletSyncer::with_config( SyncerConfig { storage, client, + light_client, options: SyncerOptions { enable_fast_forward, enable_address_recovery: false, @@ -863,21 +1059,6 @@ mod tests { )) .expect("tendermint block results batch")); } - fn block_batch_verified<'a, T: Clone + Iterator>( - &self, - _state: lite::TrustedState, - _heights: T, - ) -> Result<(Vec, lite::TrustedState)> { - let blocks: Vec = serde_json::from_str(&read_asset_file( - "tendermint_block_batch_verified_blocks.json", - )) - .expect("tendermint block batch verified blocks"); - let trusted_state: lite::TrustedState = serde_json::from_str(&read_asset_file( - "tendermint_block_batch_verified_trusted_state.json", - )) - .expect("tendermint block batch verified trusted state"); - Ok((blocks, trusted_state)) - } fn broadcast_transaction(&self, _transaction: &[u8]) -> Result { unreachable!() } @@ -902,11 +1083,10 @@ mod tests { ) } } + impl Handle for MockTendermintClient {} let storage = MemoryStorage::default(); let name = "name"; - let trusted_state: lite::TrustedState = - serde_json::from_str(&read_asset_file("sync_state_trusted_state.json")).unwrap(); save_sync_state( &storage, name, @@ -914,8 +1094,10 @@ mod tests { last_block_height: 1745, last_app_hash: "3fe291fd64f1140acfe38988a9f8c5b0cb5da43a0214bbd4000035509ce34205" .to_string(), + last_block_hash: "3fe291fd64f1140acfe38988a9f8c5b0cb5da43a0214bbd4000035509ce34205" + .to_string(), staking_root: [0u8; 32], - trusted_state, + trusted: true, }, ) .expect("should save sync state"); @@ -927,6 +1109,7 @@ mod tests { .new_wallet(name, &wallet_passphrase, WalletKind::Basic, None) .expect("create wallet failed"); let client = MockTendermintClient {}; + let light_client = client.clone(); let enable_fast_forward = false; @@ -934,6 +1117,7 @@ mod tests { SyncerConfig { storage, client, + light_client, options: SyncerOptions { enable_fast_forward, enable_address_recovery: false, @@ -979,11 +1163,13 @@ mod tests { gen.gen_block(&[]); } } + let light_client = client.clone(); let mut syncer = WalletSyncer::with_config( SyncerConfig { storage, client, + light_client, options: SyncerOptions { enable_fast_forward: false, enable_address_recovery: true, @@ -1036,11 +1222,13 @@ mod tests { gen.gen_block(&[]); } } + let light_client = client.clone(); let mut syncer = WalletSyncer::with_config( SyncerConfig { storage, client, + light_client, options: SyncerOptions { enable_fast_forward: false, enable_address_recovery: true, @@ -1101,3 +1289,109 @@ mod tests { ); } } + +/// [new light client design](https://github.com/informalsystems/tendermint-rs/blob/master/docs/architecture/adr-006-light-client-refactor.md) +pub fn spawn_light_client_supervisor( + db_path: &Path, + addr: &str, + trusting_period: Duration, +) -> Result> { + // convert "ws://host:port/websocket" to "tcp://host:port" + let addr = format!( + "tcp://{}", + strip_prefix(addr, "ws://") + .and_then(|addr| strip_suffix(addr, "/websocket")) + .err_kind(ErrorKind::InvalidInput, || "invalid tendermint rpc address")? + ); + let addr = tendermint::net::Address::from_str(&addr).unwrap(); + + // FIXME use actual tendermint node id as peer id + let primary: PeerId = "BADFADAD0BEFEEDC0C0ADEADBEEFC0FFEEFACADE".parse().unwrap(); + let witness: PeerId = "CEFEEDBADFADAD0C0CEEFACADE0ADEADBEEFC0FF".parse().unwrap(); + + let primary_path = db_path.join(primary.to_string()); + let witness_path = db_path.join(witness.to_string()); + + let primary_instance = + make_light_client_instance(primary, addr.clone(), primary_path, trusting_period)?; + let witness_instance = + make_light_client_instance(witness, addr.clone(), witness_path, trusting_period)?; + + let mut peer_addr = HashMap::new(); + peer_addr.insert(primary, addr.clone()); + peer_addr.insert(witness, addr); + + let peer_list = PeerList::builder() + .primary(primary, primary_instance) + .witness(witness, witness_instance) + .build(); + let mut supervisor = Supervisor::new( + peer_list, + ProdForkDetector::default(), + ProdEvidenceReporter::new(peer_addr), + ); + let handle = supervisor.handle(); + std::thread::spawn(|| supervisor.run()); + Ok(LightClientWrapper { + inner: Arc::new(handle), + }) +} + +/// A wrapper over light client `Handle` which supports `Clone` +pub struct LightClientWrapper { + inner: Arc, +} + +impl Clone for LightClientWrapper { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl Handle for LightClientWrapper { + fn latest_trusted( + &self, + ) -> result::Result, tendermint_light_client::errors::Error> { + self.inner.latest_trusted() + } + + /// Verify to the highest block. + fn verify_to_highest( + &self, + ) -> result::Result { + self.inner.verify_to_highest() + } + + /// Verify to the block at the given height. + fn verify_to_target( + &self, + height: u64, + ) -> result::Result { + self.inner.verify_to_target(height) + } + + /// Terminate the underlying [`Supervisor`]. + fn terminate(&self) -> result::Result<(), tendermint_light_client::errors::Error> { + self.inner.terminate() + } +} + +/// FIXME change to str::strip_prefix after toolchain upgraded. +fn strip_prefix<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { + if s.len() >= prefix.len() && &s[0..prefix.len()] == prefix { + Some(&s[prefix.len()..]) + } else { + None + } +} + +/// FIXME change to str::strip_suffix after toolchain upgraded. +fn strip_suffix<'a>(s: &'a str, suffix: &str) -> Option<&'a str> { + if s.len() >= suffix.len() && &s[s.len() - suffix.len()..s.len()] == suffix { + Some(&s[0..s.len() - suffix.len()]) + } else { + None + } +} diff --git a/client-core/src/wallet/syncer_logic.rs b/client-core/src/wallet/syncer_logic.rs index 3a534d724..c110bb299 100644 --- a/client-core/src/wallet/syncer_logic.rs +++ b/client-core/src/wallet/syncer_logic.rs @@ -343,6 +343,12 @@ mod tests { } FilteredBlock { app_hash: "3891040F29C6A56A5E36B17DCA6992D8F91D1EAAB4439D008D19A9D703271D3C".to_owned(), + last_app_hash: "3891040F29C6A56A5E36B17DCA6992D8F91D1EAAB4439D008D19A9D703271D3C" + .to_owned(), + block_hash: "3891040F29C6A56A5E36B17DCA6992D8F91D1EAAB4439D008D19A9D703271D3C" + .to_owned(), + last_block_hash: "3891040F29C6A56A5E36B17DCA6992D8F91D1EAAB4439D008D19A9D703271D3C" + .to_owned(), block_height: 1, block_time: Time::from_str("2019-04-09T09:38:41.735577Z").unwrap(), valid_transaction_fees, diff --git a/client-network/Cargo.toml b/client-network/Cargo.toml index 864f2fcfc..faa3f9c29 100644 --- a/client-network/Cargo.toml +++ b/client-network/Cargo.toml @@ -18,7 +18,7 @@ chrono = { version = "0.4", features = ["serde"] } parity-scale-codec = { features = ["derive"], version = "1.3" } hex = "0.4.2" secp256k1 = { git = "https://github.com/crypto-com/rust-secp256k1-zkp.git", rev = "535790e91fac1b3b00c770cb339a06feadc5f48d", features = ["recovery"] } -tendermint = { git = "https://github.com/crypto-com/tendermint-rs.git", default-features = false, rev = "ceca4219d6ae4e4ce906a6579e865af15458fdd6" } +tendermint = "0.15" [dev-dependencies] secp256k1 = { git = "https://github.com/crypto-com/rust-secp256k1-zkp.git", rev = "535790e91fac1b3b00c770cb339a06feadc5f48d", features = ["serde", "rand", "recovery", "endomorphism"] } diff --git a/client-network/src/network_ops/default_network_ops_client.rs b/client-network/src/network_ops/default_network_ops_client.rs index d417fa450..5b5610950 100644 --- a/client-network/src/network_ops/default_network_ops_client.rs +++ b/client-network/src/network_ops/default_network_ops_client.rs @@ -554,7 +554,6 @@ mod tests { use chain_core::tx::{PlainTxAux, TxEnclaveAux, TxObfuscated}; use chain_tx_validation::witness::verify_tx_recover_address; use client_common::storage::MemoryStorage; - use client_common::tendermint::lite; use client_common::tendermint::mock; use client_common::tendermint::types::*; use client_common::{seckey::derive_enckey, PrivateKey, PublicKey, Transaction}; @@ -645,14 +644,6 @@ mod tests { unreachable!() } - fn block_batch_verified<'a, T: Clone + Iterator>( - &self, - _state: lite::TrustedState, - _heights: T, - ) -> Result<(Vec, lite::TrustedState)> { - unreachable!() - } - fn block_results_batch<'a, T: Iterator>( &self, _heights: T, @@ -690,7 +681,7 @@ mod tests { ); Ok(AbciQuery { - value: Some(Some(staked_state).encode()), + value: Some(staked_state).encode(), ..Default::default() }) } @@ -741,14 +732,6 @@ mod tests { unreachable!() } - fn block_batch_verified<'a, T: Clone + Iterator>( - &self, - _state: lite::TrustedState, - _heights: T, - ) -> Result<(Vec, lite::TrustedState)> { - unreachable!() - } - fn broadcast_transaction(&self, _: &[u8]) -> Result { unreachable!() } @@ -770,7 +753,7 @@ mod tests { ); Ok(AbciQuery { - value: Some(Some(staked_state).encode()), + value: Some(staked_state).encode(), ..Default::default() }) } diff --git a/client-rpc/Cargo.toml b/client-rpc/Cargo.toml index 89ebdf4c7..6427fe512 100644 --- a/client-rpc/Cargo.toml +++ b/client-rpc/Cargo.toml @@ -21,7 +21,8 @@ env_logger="0.7.1" log ="0.4.11" zeroize = "1.1" parity-scale-codec = "1.3" +tendermint-light-client = "0.15" [features] mock-enclave = ["client-common/mock-enclave"] -experimental = ["client-common/experimental", "client-core/experimental"] \ No newline at end of file +experimental = ["client-common/experimental", "client-core/experimental"] diff --git a/client-rpc/src/handler.rs b/client-rpc/src/handler.rs index 2b2a2e545..f49d31d44 100644 --- a/client-rpc/src/handler.rs +++ b/client-rpc/src/handler.rs @@ -3,12 +3,14 @@ use jsonrpc_core::IoHandler; use chain_core::tx::fee::FeeAlgorithm; use client_common::cipher::TransactionObfuscation; use client_common::storage::SledStorage; -use client_common::tendermint::WebsocketRpcClient; +use client_common::tendermint::{types::GenesisExt, Client, WebsocketRpcClient}; use client_common::Result; use client_core::service::HwKeyService; use client_core::signer::WalletSignerManager; use client_core::transaction_builder::DefaultWalletTransactionBuilder; -use client_core::wallet::syncer::{ObfuscationSyncerConfig, SyncerOptions}; +use client_core::wallet::syncer::{ + spawn_light_client_supervisor, ObfuscationSyncerConfig, SyncerOptions, +}; use client_core::wallet::DefaultWalletClient; use client_network::network_ops::DefaultNetworkOpsClient; @@ -30,7 +32,7 @@ type AppWalletClient = DefaultWalletClient< >; type AppOpsClient = DefaultNetworkOpsClient, SledStorage, WebsocketRpcClient, F, O>; -type AppSyncerConfig = ObfuscationSyncerConfig; +type AppSyncerConfig = ObfuscationSyncerConfig; #[derive(Clone)] pub struct RpcHandler { @@ -63,11 +65,17 @@ impl RpcHandler { fee_policy.clone(), tendermint_client.clone(), )?; + let handle = spawn_light_client_supervisor( + storage_dir.as_ref(), + websocket_url, + tendermint_client.genesis()?.trusting_period(), + )?; let syncer_config = AppSyncerConfig::new( storage.clone(), tendermint_client.clone(), obfuscation.clone(), sync_options, + handle.clone(), ); #[cfg(feature = "experimental")] @@ -80,7 +88,8 @@ impl RpcHandler { let sync_wallet_client = make_wallet_client(storage, tendermint_client, fee_policy, obfuscation)?; - let sync_rpc = SyncRpcImpl::new(syncer_config, progress_callback, sync_wallet_client); + let sync_rpc = + SyncRpcImpl::new(syncer_config, progress_callback, sync_wallet_client, handle); let wallet_rpc = WalletRpcImpl::new(wallet_client, network_id); #[cfg(feature = "experimental")] diff --git a/client-rpc/src/rpc/multisig_rpc.rs b/client-rpc/src/rpc/multisig_rpc.rs index 9fa5b13c2..2d9a6681b 100644 --- a/client-rpc/src/rpc/multisig_rpc.rs +++ b/client-rpc/src/rpc/multisig_rpc.rs @@ -342,7 +342,6 @@ mod test { use chain_core::tx::fee::{Fee, FeeAlgorithm}; use chain_core::tx::TxAux; use client_common::storage::MemoryStorage; - use client_common::tendermint::lite; use client_common::tendermint::types::*; use client_common::tendermint::Client; use client_common::TransactionObfuscation; @@ -489,14 +488,6 @@ mod test { unreachable!("block_results_batch") } - fn block_batch_verified<'a, T: Clone + Iterator>( - &self, - _state: lite::TrustedState, - _heights: T, - ) -> CommonResult<(Vec, lite::TrustedState)> { - unreachable!() - } - fn broadcast_transaction(&self, _transaction: &[u8]) -> CommonResult { unreachable!("broadcast_transaction") } diff --git a/client-rpc/src/rpc/sync_rpc.rs b/client-rpc/src/rpc/sync_rpc.rs index ef015ba93..eb9377cf3 100644 --- a/client-rpc/src/rpc/sync_rpc.rs +++ b/client-rpc/src/rpc/sync_rpc.rs @@ -4,9 +4,9 @@ use crate::to_rpc_error; use client_common::tendermint::Client; use client_common::Storage; use client_common::TransactionObfuscation; -use client_core::wallet::syncer::AddressRecovery; -use client_core::wallet::syncer::ProgressReport; -use client_core::wallet::syncer::{ObfuscationSyncerConfig, WalletSyncer}; +use client_core::wallet::syncer::{ + AddressRecovery, Handle, ObfuscationSyncerConfig, ProgressReport, WalletSyncer, +}; use client_core::wallet::WalletRequest; use jsonrpc_core::Result; use jsonrpc_derive::rpc; @@ -14,6 +14,7 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use std::sync::Mutex; use std::thread; + // seconds const NOTIFICATION_TIME: u64 = 2; const ERROR_NOTIFICATION_TIME: u64 = 30; @@ -74,32 +75,36 @@ pub trait SyncRpc: Send + Sync { fn sync_stop(&self, request: WalletRequest) -> Result<()>; } -pub struct SyncRpcImpl +pub struct SyncRpcImpl where S: Storage, C: Client, O: TransactionObfuscation, T: AddressRecovery, + L: Handle + Send + Sync + Clone, { - config: ObfuscationSyncerConfig, + config: ObfuscationSyncerConfig, progress_callback: Option, worker: WorkerShared, recover_address: T, + light_client_handle: L, } -impl SyncRpcImpl +impl SyncRpcImpl where S: Storage + 'static, C: Client + 'static, O: TransactionObfuscation + 'static, T: AddressRecovery + 'static, + L: Handle + Send + Sync + Clone + 'static, { pub fn new( - config: ObfuscationSyncerConfig, + config: ObfuscationSyncerConfig, progress_callback: Option, recover_address: T, + light_client_handle: L, ) -> Self { SyncRpcImpl { config, @@ -108,12 +113,13 @@ where worker: Arc::new(Mutex::new(SyncWorker::new())), recover_address, + light_client_handle, } } } -fn process_sync( - config: ObfuscationSyncerConfig, +fn process_sync( + config: ObfuscationSyncerConfig, request: WalletRequest, reset: bool, progress_callback: Option, @@ -124,6 +130,7 @@ where C: Client, O: TransactionObfuscation, T: AddressRecovery, + L: Handle + Send + Sync + Clone, { let mut syncer = WalletSyncer::with_obfuscation_config( config, @@ -185,12 +192,13 @@ where .map_err(to_rpc_error) } -impl SyncRpcImpl +impl SyncRpcImpl where S: Storage + 'static, C: Client + 'static, O: TransactionObfuscation + 'static, T: AddressRecovery + 'static, + L: Handle + Send + Sync + Clone + 'static, { fn do_run_sync( &self, @@ -279,12 +287,13 @@ where } } -impl SyncRpc for SyncRpcImpl +impl SyncRpc for SyncRpcImpl where S: Storage + 'static, C: Client + 'static, O: TransactionObfuscation + 'static, T: AddressRecovery + 'static, + L: Handle + Send + Sync + Clone + 'static, { #[inline] fn sync(&self, request: WalletRequest, sync_request: SyncRequest) -> Result { @@ -320,12 +329,17 @@ where } } -impl Drop for SyncRpcImpl +impl Drop for SyncRpcImpl where S: Storage, C: Client, O: TransactionObfuscation, T: AddressRecovery, + L: Handle + Send + Sync + Clone, { - fn drop(&mut self) {} + fn drop(&mut self) { + self.light_client_handle + .terminate() + .expect("terminate light client supervisor in drop"); + } } diff --git a/client-rpc/src/rpc/wallet_rpc.rs b/client-rpc/src/rpc/wallet_rpc.rs index 74e814a23..bd9ef56b8 100644 --- a/client-rpc/src/rpc/wallet_rpc.rs +++ b/client-rpc/src/rpc/wallet_rpc.rs @@ -477,7 +477,6 @@ pub mod tests { use chain_core::tx::fee::{Fee, FeeAlgorithm}; use chain_core::tx::{PlainTxAux, TransactionId, TxAux, TxEnclaveAux, TxObfuscated}; use client_common::storage::MemoryStorage; - use client_common::tendermint::lite; use client_common::tendermint::mock; use client_common::tendermint::types::*; use client_common::tendermint::Client; @@ -626,22 +625,28 @@ pub mod tests { } fn block_results(&self, _height: u64) -> CommonResult { - Ok(BlockResultsResponse::default()) + Ok(BlockResultsResponse { + height: Default::default(), + txs_results: None, + begin_block_events: None, + end_block_events: None, + validator_updates: vec![], + consensus_param_updates: None, + }) } fn block_results_batch<'a, T: Iterator>( &self, _heights: T, ) -> CommonResult> { - Ok(vec![BlockResultsResponse::default()]) - } - - fn block_batch_verified<'a, T: Clone + Iterator>( - &self, - _state: lite::TrustedState, - _heights: T, - ) -> CommonResult<(Vec, lite::TrustedState)> { - unreachable!() + Ok(vec![BlockResultsResponse { + height: Default::default(), + txs_results: None, + begin_block_events: None, + end_block_events: None, + validator_updates: vec![], + consensus_param_updates: None, + }]) } fn broadcast_transaction(&self, _transaction: &[u8]) -> CommonResult { diff --git a/docker/unittest.sh b/docker/unittest.sh index d57b9918c..3387de941 100755 --- a/docker/unittest.sh +++ b/docker/unittest.sh @@ -14,15 +14,15 @@ echo "Test $BUILD_MODE $BUILD_PROFILE" if [ $BUILD_MODE == "sgx" ]; then cargo test $CARGO_ARGS else + cargo test $CARGO_ARGS --features mock-enclave --manifest-path client-common/Cargo.toml + cargo test $CARGO_ARGS --features mock-enclave --manifest-path client-core/Cargo.toml + cargo test $CARGO_ARGS --features mock-enclave --manifest-path client-rpc/Cargo.toml cargo test $CARGO_ARGS --features mock-enclave --manifest-path client-rpc/server/Cargo.toml cargo test $CARGO_ARGS --features mock-enclave --manifest-path client-cli/Cargo.toml cargo test $CARGO_ARGS --features mock-enclave --manifest-path dev-utils/Cargo.toml cargo test $CARGO_ARGS --features mock-enclave --manifest-path chain-abci/Cargo.toml for pkg in \ - client-common \ client-network \ - client-core \ - client-rpc \ test-common \ enclave-protocol \ chain-core \ diff --git a/integration-tests/bot/chainbot.py b/integration-tests/bot/chainbot.py index f1e8529a1..18a19b22b 100755 --- a/integration-tests/bot/chainbot.py +++ b/integration-tests/bot/chainbot.py @@ -12,14 +12,22 @@ import binascii # import time import shutil +import datetime import jsonpatch import fire import toml import nacl.signing from nacl.encoding import HexEncoder +from decouple import config PASSPHRASE = '123456' +CARGO_TARGET_DIR = config('CARGO_TARGET_DIR', '../target') +MLS_ENCLAVE_PATH = config( + 'MLS_ENCLAVE_PATH', + os.path.join(CARGO_TARGET_DIR, + 'x86_64-fortanix-unknown-sgx/debug/mls.sgxs') +) class SigningKey: @@ -172,11 +180,20 @@ def extract_enckey(s): return re.search(rb'Authentication token: ([0-9a-fA-F]+)', s).group(1).decode() -def app_state_cfg(cfg): - mock_keypackage = open(os.path.join( - os.path.dirname(__file__), - '../../chain-tx-enclave-next/mls/tests/test_vectors/keypackage.bin' - ), 'rb').read() +async def gen_keypackage(mock_mode): + if mock_mode: + return open(os.path.join( + os.path.dirname(__file__), + '../../chain-tx-enclave-next/mls/tests/test_vectors/keypackage.bin' + ), 'rb').read() + else: + temp = tempfile.NamedTemporaryFile() + await run(f'dev-utils keypackage generate --path {MLS_ENCLAVE_PATH} --output {temp.name}') + return base64.b64decode(temp.read()) + + +async def app_state_cfg(cfg): + keypackage = await gen_keypackage(cfg.get('mock_mode', False)) return { "distribution": gen_distribution(cfg), "required_council_node_stake": "100000000", # 10 coins @@ -211,7 +228,7 @@ def app_state_cfg(cfg): 'type': 'tendermint/PubKeyEd25519', 'value': SigningKey(node['validator_seed']).pub_key_base64(), }, - {'keypackage': base64.b64encode(mock_keypackage).decode()} # FIXME: to be designed and implemented + {'keypackage': base64.b64encode(keypackage).decode()} # FIXME: to be designed and implemented ] for node in cfg['nodes'] if node['bonded_coin'] > 0 }, @@ -280,17 +297,6 @@ def tasks_ini(node_cfgs, app_hash, root_path, cfg): 'serverurl': 'unix://%(here)s/supervisor.sock', }, } - if not cfg.get('mock_mode'): - ini['program:ra-sp-server'] = { - 'command': f'ra-sp-server --quote-type Unlinkable --ias-key {os.environ["IAS_API_KEY"]} --spid {os.environ["SPID"]}', - 'stdout_logfile': '%(here)s/logs/ra-sp-server.log', - 'autostart': 'true', - 'autorestart': 'true', - 'redirect_stderr': 'true', - 'priority': '10', - 'startsecs': '3', - 'startretries': '10', - } ini['program:mock_hardware_key_storage'] = { 'command': f'mock_hardware_wallet', @@ -395,7 +401,7 @@ async def gen_genesis(cfg): } patch = jsonpatch.JsonPatch(cfg['chain_config_patch']) - cfg['genesis_fingerprint'], genesis = await fix_genesis(genesis, patch.apply(app_state_cfg(cfg))) + cfg['genesis_fingerprint'], genesis = await fix_genesis(genesis, patch.apply(await app_state_cfg(cfg))) return genesis @@ -617,7 +623,7 @@ def gen(self, count=1, expansion_cap=1000000000000000000, def _prepare(self, cfg): asyncio.run(init_cluster(cfg)) - def prepare(self, spec=None, base_port=None, mock_mode=None, start_client_rpc=None): + def prepare(self, spec=None, base_port=None, mock_mode=None, start_client_rpc=None, genesis_time=None): '''Prepare tendermint testnet based on specification :param spec: Path of specification file, [default: stdin] ''' @@ -629,6 +635,10 @@ def prepare(self, spec=None, base_port=None, mock_mode=None, start_client_rpc=No cfg['mock_mode'] = mock_mode if start_client_rpc is not None: cfg['start_client_rpc'] = start_client_rpc + if genesis_time is not None: + if genesis_time == 'now': + genesis_time = datetime.datetime.utcnow().isoformat('T') + 'Z' + cfg['genesis_time'] = genesis_time self._prepare(cfg) print( 'Prepared succesfully', diff --git a/integration-tests/bot/client_cli.py b/integration-tests/bot/client_cli.py index 151c86835..4a9c7e195 100755 --- a/integration-tests/bot/client_cli.py +++ b/integration-tests/bot/client_cli.py @@ -390,4 +390,4 @@ def can_view_tx(self, tx_id): "Transaction outputs" in text: return True else: - return False \ No newline at end of file + return False diff --git a/integration-tests/cleanup.sh b/integration-tests/cleanup.sh index b2d454ebd..b9c16db36 100755 --- a/integration-tests/cleanup.sh +++ b/integration-tests/cleanup.sh @@ -23,4 +23,8 @@ fi if [ -L tx_validation_enclave.signed.so ]; then rm -f tx_validation_enclave.signed.so fi +if [ -f data/ra-sp-server.pid ]; then + kill `cat data/ra-sp-server.pid` + rm data/ra-sp-server.pid +fi exit 0 diff --git a/integration-tests/run.sh b/integration-tests/run.sh index c9f89d9e8..bc67c9978 100755 --- a/integration-tests/run.sh +++ b/integration-tests/run.sh @@ -33,13 +33,18 @@ export CLIENT_RPC_PORT=$(($BASE_PORT + 9)) export TENDERMINT_RPC_PORT=$(($BASE_PORT + 7)) export CLIENT_RPC_ZEROFEE_PORT=$CLIENT_RPC_PORT export TENDERMINT_ZEROFEE_RPC_PORT=$TENDERMINT_RPC_PORT +# client-cli output detailed error message +export CRYPTO_CLIENT_DEBUG=true function wait_port() { echo "Wait for tcp port $1" for i in $(seq 0 20); do + set +e python -c "import socket; sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM); sock.connect(('127.0.0.1', $1))" 2> /dev/null - if [ $? -eq 0 ]; then + RETCODE=$? + set -e + if [ $RETCODE -eq 0 ]; then echo "Tcp port $1 is available now" return 0 fi @@ -49,30 +54,20 @@ function wait_port() { return 1 } -function wait_service() { - if [ $BUILD_MODE == "sgx" ]; then - # ra-sp-server - wait_port 8989 - fi - # tendermint rpc of first node - wait_port $TENDERMINT_RPC_PORT -} - function runtest() { echo "Preparing... $1" LOWERED_TYPE=`echo $1 | tr "[:upper:]" "[:lower:]"` - chainbot.py prepare ${LOWERED_TYPE}_cluster.json --base_port $BASE_PORT --start_client_rpc $CHAINBOT_ARGS + chainbot.py prepare ${LOWERED_TYPE}_cluster.json --base_port $BASE_PORT --start_client_rpc $CHAINBOT_ARGS --genesis_time now export CRYPTO_GENESIS_FINGERPRINT=`python -c "import json; print(json.load(open('data/info.json'))['genesis_fingerprint'])"` export CRYPTO_CHAIN_ID=`python -c "import json; print(json.load(open('data/info.json'))['chain_id'])"` export CRYPTO_CLIENT_STORAGE=`pwd`/data/wallet echo "genesis fingerprint: $CRYPTO_GENESIS_FINGERPRINT" echo "crypto_chain_id: $CRYPTO_CHAIN_ID" - echo "Startup..." supervisord -n -c data/tasks.ini & - if ! wait_service; then - echo 'tendermint rpc not ready, giveup.' + if ! wait_port $CLIENT_RPC_PORT; then + echo 'client-rpc still not ready, giveup.' RETCODE=1 else set +e @@ -102,7 +97,7 @@ function runtest() { echo "Quit supervisord..." kill -QUIT `cat data/supervisord.pid` - wait + wait `cat data/supervisord.pid` rm -r data rm supervisord.log @@ -113,8 +108,22 @@ if [ -d data ]; then echo "Last run doesn't quit cleanly, please quit supervisord daemon and remove integration-tests/data manually." exit 1; fi +mkdir data + +if [ $BUILD_MODE == "sgx" ]; then + echo "Starting ra-sp-server..." + (set +e; while true; do ra-sp-server --ias-key $IAS_API_KEY --quote-type Unlinkable --spid $SPID; done) & + echo $! > data/ra-sp-server.pid + wait_port 8989 +fi runtest "WITH_FEE" runtest "ZERO_FEE" +if [ -f data/ra-sp-server.pid ]; then + kill `cat data/ra-sp-server.pid` + wait + rm data/ra-sp-server.pid +fi + ./cleanup.sh diff --git a/integration-tests/run_multinode.sh b/integration-tests/run_multinode.sh index ce52ad241..8443b705a 100755 --- a/integration-tests/run_multinode.sh +++ b/integration-tests/run_multinode.sh @@ -35,8 +35,11 @@ function wait_port() { echo "Wait for tcp port $1" for i in $(seq 0 20); do + set +e python -c "import socket; sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM); sock.connect(('127.0.0.1', $1))" 2> /dev/null - if [ $? -eq 0 ]; then + RETCODE=$? + set -e + if [ $RETCODE -eq 0 ]; then echo "Tcp port $1 is available now" return 0 fi @@ -46,22 +49,15 @@ function wait_port() { return 1 } -function wait_service() { - # ra-sp-server - wait_port 8989 && - # tendermint rpc of first node - wait_port $TENDERMINT_RPC_PORT -} - function runtest() { echo "Preparing... $1" - chainbot.py prepare multinode/$1_cluster.json --base_port $BASE_PORT $CHAINBOT_ARGS + chainbot.py prepare multinode/$1_cluster.json --base_port $BASE_PORT $CHAINBOT_ARGS --genesis_time now export CRYPTO_GENESIS_FINGERPRINT=`python -c "import json; print(json.load(open('data/info.json'))['genesis_fingerprint'])"` echo "genesis fingerprint: $CRYPTO_GENESIS_FINGERPRINT" echo "Startup..." supervisord -n -c data/tasks.ini & - if ! wait_service; then + if ! wait_port $TENDERMINT_RPC_PORT; then echo 'tendermint of first node still not ready, giveup.' RETCODE=1 else @@ -77,7 +73,7 @@ function runtest() { echo "Quit supervisord..." kill -QUIT `cat data/supervisord.pid` - wait + wait `cat data/supervisord.pid` rm -r data rm supervisord.log @@ -88,10 +84,24 @@ if [ -d data ]; then echo "Last run doesn't quit cleanly, please quit supervisord daemon and remove integration-tests/data manually." exit 1; fi +mkdir data + +if [ $BUILD_MODE == "sgx" ]; then + echo "Starting ra-sp-server..." + (set +e; while true; do ra-sp-server --ias-key $IAS_API_KEY --quote-type Unlinkable --spid $SPID; done) & + echo $! > data/ra-sp-server.pid + wait_port 8989 +fi runtest "join" # non-live fault slash, re-join, unbond, re-join runtest "byzantine" # make byzantine fault and check jailed, then unjail and re-join again runtest "multitx" # make multiple transactions in one block runtest "reward" # check reward amount, no reward for jailed node +if [ -f data/ra-sp-server.pid ]; then + kill `cat data/ra-sp-server.pid` + wait + rm data/ra-sp-server.pid +fi + ./cleanup.sh diff --git a/test-common/Cargo.toml b/test-common/Cargo.toml index 0b7163322..0496a387a 100644 --- a/test-common/Cargo.toml +++ b/test-common/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "test-common" version = "0.6.0" -authors = ["yihuang "] +authors = ["Crypto.com "] edition = "2018" [dependencies] @@ -22,7 +22,9 @@ parity-scale-codec = { features = ["derive"], version = "1.3" } base64 = "0.12" hex = "0.4" -tendermint = { git = "https://github.com/crypto-com/tendermint-rs.git", default-features = false, rev = "ceca4219d6ae4e4ce906a6579e865af15458fdd6" } +tendermint = "0.15" +tendermint-rpc = "0.15" +tendermint-light-client = "0.15" chain-core = { path = "../chain-core" } chain-abci = { path = "../chain-abci" } chain-storage = { path = "../chain-storage" } diff --git a/test-common/src/block_generator.rs b/test-common/src/block_generator.rs index 9db8233df..f694f5887 100644 --- a/test-common/src/block_generator.rs +++ b/test-common/src/block_generator.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::result; use std::str::FromStr; use std::sync::{Arc, RwLock}; use std::time::{Duration, UNIX_EPOCH}; @@ -12,12 +13,13 @@ use signature::Signer; use subtle_encoding::{base64, hex}; use tendermint::amino_types::message::AminoMessage; use tendermint::lite::{Header, ValidatorSet}; -use tendermint::rpc::endpoint::status; use tendermint::{ account, amino_types, block, block::signed_header::SignedHeader, block::Height, chain, consensus, evidence, hash, node, public_key, validator, vote, Block, Hash, PublicKey, Signature, Time, }; +use tendermint_light_client::{errors::Error, types::LightBlock}; +use tendermint_rpc::endpoint::status; use chain_abci::app::ChainNodeState; use chain_abci::staking::StakingTable; @@ -37,12 +39,11 @@ use chain_core::tx::TxAux; use chain_storage::buffer::MemStore; use chain_storage::jellyfish::{put_stakings, StakingGetter}; use client_common::tendermint::types::{AbciQuery, BroadcastTxResponse, Genesis}; -use client_common::tendermint::{lite, Client}; +use client_common::tendermint::Client; use client_common::Result; +use client_core::wallet::syncer::Handle; use client_core::{service::HDAccountType, HDSeed, Mnemonic}; -use tendermint::block::BlockIDFlag::BlockIDFlagCommit; use tendermint::block::{CommitSig, CommitSigs}; -use tendermint::hash::Algorithm; use crate::chain_env::mock_confidential_init; @@ -169,11 +170,10 @@ impl Node { }; let signature = self.sign_msg(&canonical_vote.bytes_vec_length_delimited()); - CommitSig { - block_id_flag: BlockIDFlagCommit, - validator_address: Some(self.validator_address()), + CommitSig::BlockIDFlagCommit { + validator_address: self.validator_address(), timestamp: now, - signature: Some(signature), + signature, } } @@ -318,7 +318,7 @@ impl TestnetSpec { }, evidence: evidence::Params { max_age_num_blocks: 100_000, - max_age_duration: Duration::from_nanos(172_800_000_000_000).into(), + max_age_duration: serde_json::from_str("\"172800000000000\"").unwrap(), }, validator: consensus::params::ValidatorParams { pub_key_types: vec![public_key::Algorithm::Ed25519], @@ -410,12 +410,12 @@ impl BlockGenerator { } pub fn sync_info(&self) -> status::SyncInfo { + let genesis_app_hash = Hash::new(hash::Algorithm::Sha256, &self.genesis.app_hash).unwrap(); if let Some(height) = self.current_height { let index = (height.value() - 1) as usize; status::SyncInfo { latest_block_hash: Some(self.blocks[index].block.header.hash()), - latest_app_hash: Hash::from_utf8(Algorithm::Sha256, &self.genesis.app_hash) - .unwrap(), + latest_app_hash: Some(genesis_app_hash), latest_block_height: height, latest_block_time: self.blocks[index].block.header.time, catching_up: false, @@ -423,8 +423,7 @@ impl BlockGenerator { } else { status::SyncInfo { latest_block_hash: None, - latest_app_hash: Hash::from_utf8(Algorithm::Sha256, &self.genesis.app_hash) - .unwrap(), + latest_app_hash: Some(genesis_app_hash), latest_block_height: Height::default(), latest_block_time: Time::unix_epoch(), catching_up: false, @@ -570,14 +569,6 @@ impl Client for GeneratorClient { heights.map(|height| self.block_results(*height)).collect() } - fn block_batch_verified<'a, T: Clone + Iterator>( - &self, - state: lite::TrustedState, - heights: T, - ) -> Result<(Vec, lite::TrustedState)> { - Ok((self.block_batch(heights)?, state)) - } - fn broadcast_transaction(&self, _transaction: &[u8]) -> Result { unreachable!(); } @@ -608,6 +599,27 @@ impl Client for GeneratorClient { } } +impl Handle for GeneratorClient { + fn verify_to_highest(&self) -> result::Result { + Ok(LightBlock { + signed_header: self + .gen + .read() + .unwrap() + .blocks + .last() + .unwrap() + .signed_header(), + validators: validator::Set::new(vec![]), + next_validators: validator::Set::new(vec![]), + provider: "BADFADAD0BEFEEDC0C0ADEADBEEFC0FFEEFACADE".parse().unwrap(), + }) + } + fn terminate(&self) -> result::Result<(), Error> { + Ok(()) + } +} + fn gen_network_params( base_fee: Milli, per_byte_fee: Milli,