From 6b0313c919314c4cebb32665a0bd0c52ab022d65 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Fri, 12 Jun 2020 15:38:37 -0700 Subject: [PATCH 01/18] core,graph,runtime: Create FullBlock struct and pass to mappings --- graph/src/components/ethereum/adapter.rs | 12 +- graph/src/components/ethereum/mod.rs | 6 +- graph/src/components/ethereum/types.rs | 224 ++++++++++++++++++++++- graph/src/lib.rs | 4 +- runtime/wasm/src/asc_abi/class.rs | 39 ++++ runtime/wasm/src/host.rs | 31 +++- runtime/wasm/src/host_exports.rs | 2 +- runtime/wasm/src/mapping.rs | 2 +- runtime/wasm/src/module/mod.rs | 20 +- runtime/wasm/src/to_from/external.rs | 65 +++++++ 10 files changed, 381 insertions(+), 24 deletions(-) diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index 97cbc6860c0..65e367c1315 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -411,6 +411,7 @@ impl From for EthereumCallFilter { pub struct EthereumBlockFilter { pub contract_addresses: HashSet<(u64, Address)>, pub trigger_every_block: bool, + pub full_block: bool, } impl EthereumBlockFilter { @@ -435,8 +436,10 @@ impl EthereumBlockFilter { .into_iter() .any(|block_handler| block_handler.filter.is_none()); + // TODO: process data source to get full_block boolean filter_opt.extend(Self { trigger_every_block: has_block_handler_without_filter, + full_block: true, contract_addresses: if has_block_handler_with_call_filter { vec![( data_source.source.start_block, @@ -816,7 +819,7 @@ fn parse_block_triggers( if trigger_every_block { triggers.push(EthereumTrigger::Block( block_ptr, - EthereumBlockTriggerType::Every, + EthereumBlockTriggerType::Every(BlockType::Full), )); } triggers @@ -921,7 +924,12 @@ pub fn blocks_with_triggers( .block_range_to_ptrs(logger.clone(), from, to) .map(move |ptrs| { ptrs.into_iter() - .map(|ptr| EthereumTrigger::Block(ptr, EthereumBlockTriggerType::Every)) + .map(|ptr| { + EthereumTrigger::Block( + ptr, + EthereumBlockTriggerType::Every(BlockType::Full), + ) + }) .collect() }), )) diff --git a/graph/src/components/ethereum/mod.rs b/graph/src/components/ethereum/mod.rs index e348586a690..e5a56d3ad40 100644 --- a/graph/src/components/ethereum/mod.rs +++ b/graph/src/components/ethereum/mod.rs @@ -13,8 +13,8 @@ pub use self::adapter::{ pub use self::listener::{ChainHeadUpdate, ChainHeadUpdateListener, ChainHeadUpdateStream}; pub use self::stream::{BlockStream, BlockStreamBuilder, BlockStreamEvent}; pub use self::types::{ - BlockFinality, EthereumBlock, EthereumBlockData, EthereumBlockPointer, + EthereumBlockType, BlockFinality, BlockType, EthereumBlock, EthereumBlockData, EthereumBlockPointer, EthereumBlockTriggerType, EthereumBlockWithCalls, EthereumBlockWithTriggers, EthereumCall, - EthereumCallData, EthereumEventData, EthereumTransactionData, EthereumTrigger, - LightEthereumBlock, LightEthereumBlockExt, + EthereumCallData, EthereumEventData, EthereumTransactionData, EthereumTransactionReceiptData, + EthereumTrigger, FullEthereumBlockData, LightEthereumBlock, LightEthereumBlockExt, }; diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index 214e075a36f..f328bb31f8a 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize}; use stable_hash::prelude::*; use stable_hash::utils::AsBytes; use std::cmp::Ordering; +use std::convert::TryFrom; use std::fmt; use web3::types::*; @@ -56,6 +57,15 @@ impl LightEthereumBlockExt for LightEthereumBlock { } } +impl<'a> From<&'a EthereumBlockType> for LightEthereumBlock { + fn from(block: &'a EthereumBlockType) -> LightEthereumBlock { + match block { + EthereumBlockType::Full(block) => block.block.clone(), + EthereumBlockType::Light(block) => block.clone(), + } + } +} + /// This is used in `EthereumAdapter::triggers_in_block`, called when re-processing a block for /// newly created data sources. This allows the re-processing to be reorg safe without having to /// always fetch the full block data. @@ -193,9 +203,37 @@ impl PartialEq for EthereumTrigger { impl Eq for EthereumTrigger {} +/// This is used in `EthereumAdapter::triggers_in_block`, called when re-processing a block for +/// newly created data sources. This allows the re-processing to be reorg safe without having to +/// always fetch the full block data. +#[derive(Clone, Debug)] +pub enum EthereumBlockType { + Light(LightEthereumBlock), + + Full(EthereumBlock), +} + +impl Default for EthereumBlockType { + fn default() -> Self { + EthereumBlockType::Light(LightEthereumBlock::default()) + } +} + +impl<'a> From<&'a LightEthereumBlock> for EthereumBlockType { + fn from(block: &'a LightEthereumBlock) -> EthereumBlockType { + EthereumBlockType::Light(block.clone()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum BlockType { + Light, + Full, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum EthereumBlockTriggerType { - Every, + Every(BlockType), WithCallTo(Address), } @@ -263,6 +301,158 @@ impl PartialOrd for EthereumTrigger { } } +pub struct EthereumTransactionReceiptData { + // from receipts + pub transaction_hash: H256, + pub transaction_index: Index, + pub cumulative_gas_used: U256, + pub gas_used: Option, + pub contract_address: Option, + pub status: Option, + pub root: Option, + + // from txs + pub from: H160, + pub to: Option, + pub value: U256, + pub gas_price: U256, + pub gas: U256, + pub input: Bytes, +} + +pub struct FullEthereumBlockData { + pub hash: H256, + pub parent_hash: H256, + pub uncles_hash: H256, + pub author: H160, + pub state_root: H256, + pub transactions_root: H256, + pub receipts_root: H256, + pub number: U64, + pub gas_used: U256, + pub gas_limit: U256, + pub timestamp: U256, + pub difficulty: U256, + pub total_difficulty: U256, + pub size: Option, + pub transaction_receipts: Vec, +} + +impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockData { + type Error = String; + + fn try_from(block: &'a EthereumBlockType) -> Result { + let fullblock = match block { + EthereumBlockType::Full(full_block) => full_block, + EthereumBlockType::Light(_) => return Err(format!( + "Failed to convert EthereumBlockType to FUllEthereumBlockData, requires a EthereumBlockType::Full()" + )) + }; + let block = &fullblock.block; + let transaction_receipts_data = block + .transactions + .iter() + .cloned() + .zip(fullblock.transaction_receipts.iter().cloned()) + .map(|transaction_and_receipt| { + assert_eq!( + transaction_and_receipt.0.hash, + transaction_and_receipt.1.transaction_hash + ); + EthereumTransactionReceiptData { + transaction_hash: transaction_and_receipt.0.hash, + transaction_index: transaction_and_receipt.1.transaction_index, + cumulative_gas_used: transaction_and_receipt.1.cumulative_gas_used, + gas_used: transaction_and_receipt.1.gas_used, + contract_address: transaction_and_receipt.1.contract_address, + status: transaction_and_receipt.1.status, + root: transaction_and_receipt.1.root, + + // from txs + from: transaction_and_receipt.0.from, + to: transaction_and_receipt.0.to, + value: transaction_and_receipt.0.value, + gas_price: transaction_and_receipt.0.gas_price, + gas: transaction_and_receipt.0.gas, + input: transaction_and_receipt.0.input, + } + }) + .collect::>(); + + Ok(FullEthereumBlockData { + hash: block.hash.unwrap(), + parent_hash: block.parent_hash, + uncles_hash: block.uncles_hash, + author: block.author, + state_root: block.state_root, + transactions_root: block.transactions_root, + receipts_root: block.receipts_root, + number: block.number.unwrap(), + gas_used: block.gas_used, + gas_limit: block.gas_limit, + timestamp: block.timestamp, + difficulty: block.difficulty, + total_difficulty: block.total_difficulty.unwrap_or_default(), + size: block.size, + transaction_receipts: transaction_receipts_data, + }) + } +} + +impl<'a> From<&'a EthereumBlock> for FullEthereumBlockData { + fn from(block: &'a EthereumBlock) -> FullEthereumBlockData { + let transaction_receipts_data = block + .block + .transactions + .iter() + .cloned() + .zip(block.transaction_receipts.iter().cloned()) + .map(|transaction_and_receipt| { + assert_eq!( + transaction_and_receipt.0.hash, + transaction_and_receipt.1.transaction_hash + ); + EthereumTransactionReceiptData { + transaction_hash: transaction_and_receipt.0.hash, + transaction_index: transaction_and_receipt.1.transaction_index, + cumulative_gas_used: transaction_and_receipt.1.cumulative_gas_used, + gas_used: transaction_and_receipt.1.gas_used, + contract_address: transaction_and_receipt.1.contract_address, + status: transaction_and_receipt.1.status, + root: transaction_and_receipt.1.root, + + // from txs + from: transaction_and_receipt.0.from, + to: transaction_and_receipt.0.to, + value: transaction_and_receipt.0.value, + gas_price: transaction_and_receipt.0.gas_price, + gas: transaction_and_receipt.0.gas, + input: transaction_and_receipt.0.input, + } + }) + .collect::>(); + let block = &block.block; + + FullEthereumBlockData { + hash: block.hash.unwrap(), + parent_hash: block.parent_hash, + uncles_hash: block.uncles_hash, + author: block.author, + state_root: block.state_root, + transactions_root: block.transactions_root, + receipts_root: block.receipts_root, + number: block.number.unwrap(), + gas_used: block.gas_used, + gas_limit: block.gas_limit, + timestamp: block.timestamp, + difficulty: block.difficulty, + total_difficulty: block.total_difficulty.unwrap_or_default(), + size: block.size, + transaction_receipts: transaction_receipts_data, + } + } +} + /// Ethereum block data. #[derive(Clone, Debug, Default)] pub struct EthereumBlockData { @@ -303,6 +493,32 @@ impl<'a, T> From<&'a Block> for EthereumBlockData { } } +impl<'a> From<&'a EthereumBlockType> for EthereumBlockData { + fn from(block: &'a EthereumBlockType) -> EthereumBlockData { + let block = match block { + EthereumBlockType::Full(full_block) => &full_block.block, + EthereumBlockType::Light(light_block) => light_block, + }; + + EthereumBlockData { + hash: block.hash.unwrap(), + parent_hash: block.parent_hash, + uncles_hash: block.uncles_hash, + author: block.author, + state_root: block.state_root, + transactions_root: block.transactions_root, + receipts_root: block.receipts_root, + number: block.number.unwrap(), + gas_used: block.gas_used, + gas_limit: block.gas_limit, + timestamp: block.timestamp, + difficulty: block.difficulty, + total_difficulty: block.total_difficulty.unwrap_or_default(), + size: block.size, + } + } +} + /// Ethereum transaction data. #[derive(Clone, Debug)] pub struct EthereumTransactionData { @@ -536,7 +752,9 @@ impl ToEntityKey for EthereumBlockPointer { #[cfg(test)] mod test { - use super::{EthereumBlockPointer, EthereumBlockTriggerType, EthereumCall, EthereumTrigger}; + use super::{ + BlockType, EthereumBlockPointer, EthereumBlockTriggerType, EthereumCall, EthereumTrigger, + }; use web3::types::*; #[test] @@ -546,7 +764,7 @@ mod test { number: 1, hash: H256::random(), }, - EthereumBlockTriggerType::Every, + EthereumBlockTriggerType::Every(BlockType::Light), ); let block2 = EthereumTrigger::Block( diff --git a/graph/src/lib.rs b/graph/src/lib.rs index 1dda564ee46..32495d42d08 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -80,8 +80,8 @@ pub mod prelude { EthereumBlockPointer, EthereumBlockTriggerType, EthereumBlockWithCalls, EthereumBlockWithTriggers, EthereumCall, EthereumCallData, EthereumCallFilter, EthereumContractCall, EthereumContractCallError, EthereumEventData, EthereumLogFilter, - EthereumNetworkIdentifier, EthereumTransactionData, EthereumTrigger, LightEthereumBlock, - LightEthereumBlockExt, ProviderEthRpcMetrics, SubgraphEthRpcMetrics, + EthereumNetworkIdentifier, EthereumTransactionData, EthereumTrigger, FullEthereumBlockData, + LightEthereumBlock, LightEthereumBlockExt, ProviderEthRpcMetrics, SubgraphEthRpcMetrics, }; pub use crate::components::graphql::{GraphQlRunner, SubscriptionResultFuture}; pub use crate::components::link_resolver::{JsonStreamValue, JsonValueStream, LinkResolver}; diff --git a/runtime/wasm/src/asc_abi/class.rs b/runtime/wasm/src/asc_abi/class.rs index f548d3d8fdc..03ec59b0371 100644 --- a/runtime/wasm/src/asc_abi/class.rs +++ b/runtime/wasm/src/asc_abi/class.rs @@ -418,6 +418,26 @@ pub(crate) struct AscEthereumBlock { pub size: AscPtr, } +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscFullEthereumBlock { + pub hash: AscPtr, + pub parent_hash: AscPtr, + pub uncles_hash: AscPtr, + pub author: AscPtr, + pub state_root: AscPtr, + pub transactions_root: AscPtr, + pub receipts_root: AscPtr, + pub number: AscPtr, + pub gas_used: AscPtr, + pub gas_limit: AscPtr, + pub timestamp: AscPtr, + pub difficulty: AscPtr, + pub total_difficulty: AscPtr, + pub size: AscPtr, + pub transaction_receipts: AscPtr>>, +} + #[repr(C)] #[derive(AscType)] pub(crate) struct AscEthereumTransaction { @@ -430,6 +450,25 @@ pub(crate) struct AscEthereumTransaction { pub gas_price: AscPtr, } +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumTransactionReceipt { + pub transaction_hash: AscPtr, + pub transaction_index: AscPtr, + pub cumulative_gas_used: AscPtr, + pub gas_used: AscPtr, + pub contract_address: AscPtr, + pub status: AscPtr, + pub root: AscPtr, + + pub from: AscPtr, + pub to: AscPtr, + pub value: AscPtr, + pub gas_price: AscPtr, + pub gas: AscPtr, + pub input: AscPtr, +} + #[repr(C)] #[derive(AscType)] pub(crate) struct AscEthereumTransaction_0_0_2 { diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index 19683496df3..c3e6509aaa5 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -291,7 +291,7 @@ impl RuntimeHost { // Do not match if this datasource has no address .map_or(false, |addr| addr == *address) } - EthereumBlockTriggerType::Every => true, + EthereumBlockTriggerType::Every(_) => true, }; source_address_matches && self.handler_for_block(block_trigger_type).is_ok() } @@ -347,7 +347,7 @@ impl RuntimeHost { trigger_type: &EthereumBlockTriggerType, ) -> Result { match trigger_type { - EthereumBlockTriggerType::Every => self + EthereumBlockTriggerType::Every(_type) => self .data_source_block_handlers .iter() .find(move |handler| handler.filter == None) @@ -386,7 +386,7 @@ impl RuntimeHost { state: BlockState, handler: &str, trigger: MappingTrigger, - block: &Arc, + block: &Arc, proof_of_indexing: SharedProofOfIndexing, ) -> Result { let trigger_type = trigger.as_static(); @@ -561,7 +561,7 @@ impl RuntimeHostTrait for RuntimeHost { outputs, handler: call_handler.clone(), }, - block, + &Arc::new(EthereumBlockType::from(block.as_ref())), proof_of_indexing, ) .await @@ -576,6 +576,25 @@ impl RuntimeHostTrait for RuntimeHost { proof_of_indexing: SharedProofOfIndexing, ) -> Result { let block_handler = self.handler_for_block(trigger_type)?; + let theblock: EthereumBlockType = match trigger_type { + EthereumBlockTriggerType::Every(BlockType::Full) => match graph::block_on_allow_panic( + future::lazy(move || { + self.host_exports + .ethereum_adapter + .load_full_block(logger, block.as_ref().clone()) + }) + .compat(), + ) { + Ok(block) => Ok(EthereumBlockType::Full(block)), + Err(e) => Err(anyhow::anyhow!( + "Failed to load full block: {}, error: {}", + &block.number.unwrap().to_string(), + e + )), + }?, + _ => EthereumBlockType::from(block.as_ref()), + }; + self.send_mapping_request( logger, o! { @@ -587,7 +606,7 @@ impl RuntimeHostTrait for RuntimeHost { MappingTrigger::Block { handler: block_handler.clone(), }, - block, + &Arc::new(theblock), proof_of_indexing, ) .await @@ -701,7 +720,7 @@ impl RuntimeHostTrait for RuntimeHost { params, handler: event_handler.clone(), }, - block, + &Arc::new(EthereumBlockType::from(block.as_ref())), proof_of_indexing, ) .await diff --git a/runtime/wasm/src/host_exports.rs b/runtime/wasm/src/host_exports.rs index cea48e20320..6be9b25518c 100644 --- a/runtime/wasm/src/host_exports.rs +++ b/runtime/wasm/src/host_exports.rs @@ -34,7 +34,7 @@ pub(crate) struct HostExports { causality_region: String, templates: Arc>, abis: Vec, - ethereum_adapter: Arc, + pub(crate) ethereum_adapter: Arc, pub(crate) link_resolver: Arc, call_cache: Arc, store: Arc, diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs index 50937843f84..b3985f0d313 100644 --- a/runtime/wasm/src/mapping.rs +++ b/runtime/wasm/src/mapping.rs @@ -143,7 +143,7 @@ pub struct MappingRequest { pub(crate) struct MappingContext { pub(crate) logger: Logger, pub(crate) host_exports: Arc, - pub(crate) block: Arc, + pub(crate) block: Arc, pub(crate) state: BlockState, pub(crate) proof_of_indexing: SharedProofOfIndexing, } diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index aba6f37668c..f57dd9e913f 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -161,10 +161,17 @@ impl WasmInstance { mut self, handler_name: &str, ) -> Result { - let block = EthereumBlockData::from(self.instance_ctx().ctx.block.as_ref()); + let block = self.instance_ctx().ctx.block.clone(); // Prepare an EthereumBlock for the WASM runtime - let arg = self.asc_new(&block); + let arg = match block.as_ref() { + EthereumBlockType::Full(block) => self + .asc_new::(&FullEthereumBlockData::from(block)) + .erase(), + EthereumBlockType::Light(light_block) => self + .asc_new::(&EthereumBlockData::from(light_block)) + .erase(), + }; self.invoke_handler(handler_name, arg)?; @@ -704,10 +711,11 @@ impl WasmInstanceContext { &mut self, call: UnresolvedContractCall, ) -> Result, Trap> { - let result = - self.ctx - .host_exports - .ethereum_call(&self.ctx.logger, &self.ctx.block, call)?; + let result = self.ctx.host_exports.ethereum_call( + &self.ctx.logger, + &LightEthereumBlock::from(self.ctx.block.as_ref()), + call, + )?; Ok(match result { Some(tokens) => self.asc_new(tokens.as_slice()), None => AscPtr::null(), diff --git a/runtime/wasm/src/to_from/external.rs b/runtime/wasm/src/to_from/external.rs index 4d0eab0d2eb..384568e9ace 100644 --- a/runtime/wasm/src/to_from/external.rs +++ b/runtime/wasm/src/to_from/external.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use graph::components::ethereum::{ EthereumBlockData, EthereumCallData, EthereumEventData, EthereumTransactionData, + EthereumTransactionReceiptData, FullEthereumBlockData, }; use graph::data::store; use graph::prelude::anyhow::{ensure, Error}; @@ -324,6 +325,70 @@ impl ToAscObj for EthereumBlockData { } } +impl ToAscObj for FullEthereumBlockData { + fn to_asc_obj(&self, heap: &mut H) -> AscFullEthereumBlock { + AscFullEthereumBlock { + hash: heap.asc_new(&self.hash), + parent_hash: heap.asc_new(&self.parent_hash), + uncles_hash: heap.asc_new(&self.uncles_hash), + author: heap.asc_new(&self.author), + state_root: heap.asc_new(&self.state_root), + transactions_root: heap.asc_new(&self.transactions_root), + receipts_root: heap.asc_new(&self.receipts_root), + number: heap.asc_new(&BigInt::from(self.number)), + gas_used: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_used)), + gas_limit: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_limit)), + timestamp: heap.asc_new(&BigInt::from_unsigned_u256(&self.timestamp)), + difficulty: heap.asc_new(&BigInt::from_unsigned_u256(&self.difficulty)), + total_difficulty: heap.asc_new(&BigInt::from_unsigned_u256(&self.total_difficulty)), + size: self + .size + .map(|size| heap.asc_new(&BigInt::from_unsigned_u256(&size))) + .unwrap_or_else(|| AscPtr::null()), + transaction_receipts: heap.asc_new(self.transaction_receipts.as_slice()), + } + } +} + +impl ToAscObj for EthereumTransactionReceiptData { + fn to_asc_obj(&self, heap: &mut H) -> AscEthereumTransactionReceipt { + AscEthereumTransactionReceipt { + //from receipts + transaction_hash: heap.asc_new(&self.transaction_hash), + transaction_index: heap.asc_new(&BigInt::from(self.transaction_index)), + cumulative_gas_used: heap + .asc_new(&BigInt::from_unsigned_u256(&self.cumulative_gas_used)), + gas_used: self + .gas_used + .map(|gas_used| heap.asc_new(&BigInt::from_unsigned_u256(&gas_used))) + .unwrap_or_else(|| AscPtr::null()), + contract_address: self + .contract_address + .map(|contract_address| heap.asc_new(&contract_address)) + .unwrap_or_else(|| AscPtr::null()), + status: self + .status + .map(|status| heap.asc_new(&BigInt::from(status))) + .unwrap_or_else(|| AscPtr::null()), + root: self + .root + .map(|root| heap.asc_new(&root)) + .unwrap_or_else(|| AscPtr::null()), + + // // from txs + from: heap.asc_new(&self.from), + to: self + .to + .map(|to| heap.asc_new(&to)) + .unwrap_or_else(|| AscPtr::null()), + value: heap.asc_new(&BigInt::from_unsigned_u256(&self.value)), + gas_price: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_price)), + gas: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas)), + input: heap.asc_new(&*self.input.0), + } + } +} + impl ToAscObj for EthereumTransactionData { fn to_asc_obj(&self, heap: &mut H) -> AscEthereumTransaction { AscEthereumTransaction { From 80d17f44f70ec781405d3f44d4a63a4a10e578c0 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Fri, 19 Jun 2020 13:48:43 -0700 Subject: [PATCH 02/18] graph,runtime: Parse Block handler data type from the manifest --- core/src/subgraph/instance_manager.rs | 2 +- graph/src/components/ethereum/adapter.rs | 19 ++++--- graph/src/data/subgraph/mod.rs | 25 +++++++-- graph/src/data/subgraph/schema.rs | 67 ++++++++++++++++++++++++ graph/src/lib.rs | 2 +- runtime/wasm/src/host.rs | 6 +-- 6 files changed, 106 insertions(+), 15 deletions(-) diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index e6aec8e8328..2c1c7d5e02d 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -879,7 +879,7 @@ async fn process_triggers, - block: &Arc, + block: &Arc, triggers: Vec, ) -> Result<(IndexingContext, BlockState), CancelableError> { for trigger in triggers.into_iter() { diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index 65e367c1315..179c7db87e7 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -14,6 +14,7 @@ use web3::types::*; use super::types::*; use crate::components::metrics::{CounterVec, GaugeVec, HistogramVec}; +use crate::data::subgraph::BlockHandlerData; use crate::prelude::*; pub type EventSignature = H256; @@ -422,8 +423,7 @@ impl EthereumBlockFilter { let has_block_handler_with_call_filter = data_source .mapping .block_handlers - .clone() - .into_iter() + .iter() .any(|block_handler| match block_handler.filter { Some(ref filter) if *filter == BlockHandlerFilter::Call => return true, _ => return false, @@ -432,14 +432,21 @@ impl EthereumBlockFilter { let has_block_handler_without_filter = data_source .mapping .block_handlers - .clone() - .into_iter() + .iter() .any(|block_handler| block_handler.filter.is_none()); - // TODO: process data source to get full_block boolean + let has_block_handler_with_fullblock = data_source + .mapping + .block_handlers + .iter() + .any(|block_handler| match block_handler.input { + BlockHandlerData::FullBlock => return true, + _ => return false, + }); + filter_opt.extend(Self { trigger_every_block: has_block_handler_without_filter, - full_block: true, + full_block: has_block_handler_with_fullblock, contract_addresses: if has_block_handler_with_call_filter { vec![( data_source.source.start_block, diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index 5516ca90ce9..c1f80007c49 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -22,10 +22,10 @@ use crate::data::query::QueryExecutionError; use crate::data::schema::{Schema, SchemaImportError, SchemaValidationError}; use crate::data::store::Entity; use crate::data::subgraph::schema::{ - EthereumBlockHandlerEntity, EthereumCallHandlerEntity, EthereumContractAbiEntity, - EthereumContractDataSourceTemplateEntity, EthereumContractDataSourceTemplateSourceEntity, - EthereumContractEventHandlerEntity, EthereumContractMappingEntity, - EthereumContractSourceEntity, SUBGRAPHS_ID, + EthereumBlockHandlerData, EthereumBlockHandlerEntity, EthereumCallHandlerEntity, + EthereumContractAbiEntity, EthereumContractDataSourceTemplateEntity, + EthereumContractDataSourceTemplateSourceEntity, EthereumContractEventHandlerEntity, + EthereumContractMappingEntity, EthereumContractSourceEntity, SUBGRAPHS_ID, }; use crate::prelude::{ anyhow::{self, Context}, @@ -515,6 +515,22 @@ impl UnresolvedMappingABI { pub struct MappingBlockHandler { pub handler: String, pub filter: Option, + pub input: BlockHandlerData, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub enum BlockHandlerData { + Block, + FullBlock, +} + +impl From for BlockHandlerData { + fn from(data: EthereumBlockHandlerData) -> Self { + match data { + EthereumBlockHandlerData::FullBlock => BlockHandlerData::FullBlock, + _ => BlockHandlerData::Block, + } + } } #[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] @@ -530,6 +546,7 @@ impl From for MappingBlockHandler { Self { handler: entity.handler, filter: None, + input: BlockHandlerData::from(entity.input), } } } diff --git a/graph/src/data/subgraph/schema.rs b/graph/src/data/subgraph/schema.rs index 695420f511b..6cec5796a65 100644 --- a/graph/src/data/subgraph/schema.rs +++ b/graph/src/data/subgraph/schema.rs @@ -1009,6 +1009,7 @@ impl TryFromValue for EthereumContractAbiEntity { pub struct EthereumBlockHandlerEntity { pub handler: String, pub filter: Option, + pub input: EthereumBlockHandlerData, } impl WriteOperations for EthereumBlockHandlerEntity { @@ -1025,6 +1026,7 @@ impl WriteOperations for EthereumBlockHandlerEntity { if let Some(filter_id) = filter_id { entity.set("filter", filter_id); } + entity.set("data", String::from(self.input)); ops.add(Self::TYPENAME, id.to_owned(), entity); } } @@ -1048,6 +1050,7 @@ impl From for EthereumBlockHandlerEntity { EthereumBlockHandlerEntity { handler: block_handler.handler, filter, + input: EthereumBlockHandlerData::from(block_handler.input), } } } @@ -1065,6 +1068,7 @@ impl TryFromValue for EthereumBlockHandlerEntity { Ok(EthereumBlockHandlerEntity { handler: map.get_required("handler")?, filter: map.get_optional("filter")?, + input: map.get_optional("input")?.unwrap_or_default(), }) } } @@ -1106,6 +1110,69 @@ impl TryFromValue for EthereumBlockHandlerFilterEntity { } } +#[derive(Debug, PartialEq)] +pub enum EthereumBlockHandlerData { + Block, + FullBlock, +} + +impl Default for EthereumBlockHandlerData { + fn default() -> EthereumBlockHandlerData { + EthereumBlockHandlerData::Block + } +} + +impl FromStr for EthereumBlockHandlerData { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "FullBlock" => Ok(EthereumBlockHandlerData::FullBlock), + "Block" => Ok(EthereumBlockHandlerData::Block), + _ => Err(format_err!( + "failed to parse `{}` as EthereumBlockHandlerData", + s + )), + } + } +} + +impl From for String { + fn from(data: EthereumBlockHandlerData) -> String { + match data { + EthereumBlockHandlerData::FullBlock => "FullBlock".into(), + EthereumBlockHandlerData::Block => "Block".into(), + } + } +} + +impl From for EthereumBlockHandlerData { + fn from(data: BlockHandlerData) -> Self { + match data { + BlockHandlerData::FullBlock => EthereumBlockHandlerData::FullBlock, + _ => EthereumBlockHandlerData::Block, + } + } +} + +impl From for q::Value { + fn from(data: EthereumBlockHandlerData) -> q::Value { + q::Value::Enum(data.into()) + } +} + +impl TryFromValue for EthereumBlockHandlerData { + fn try_from_value(value: &q::Value) -> Result { + match value { + q::Value::Enum(data) => EthereumBlockHandlerData::from_str(data), + _ => Err(format_err!( + "cannot parse value as EthereumBlockHandlerData: `{:?}`", + value + )), + } + } +} + #[derive(Debug)] pub struct EthereumCallHandlerEntity { pub function: String, diff --git a/graph/src/lib.rs b/graph/src/lib.rs index 32495d42d08..a970deecebd 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -126,7 +126,7 @@ pub mod prelude { }; pub use crate::data::subgraph::schema::{SubgraphDeploymentEntity, TypedEntity}; pub use crate::data::subgraph::{ - BlockHandlerFilter, CreateSubgraphResult, DataSource, DataSourceContext, + BlockHandlerData, BlockHandlerFilter, CreateSubgraphResult, DataSource, DataSourceContext, DataSourceTemplate, Link, MappingABI, MappingBlockHandler, MappingCallHandler, MappingEventHandler, SubgraphAssignmentProviderError, SubgraphAssignmentProviderEvent, SubgraphDeploymentId, SubgraphManifest, SubgraphManifestResolveError, diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index c3e6509aaa5..71368bf3112 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -576,7 +576,7 @@ impl RuntimeHostTrait for RuntimeHost { proof_of_indexing: SharedProofOfIndexing, ) -> Result { let block_handler = self.handler_for_block(trigger_type)?; - let theblock: EthereumBlockType = match trigger_type { + let theblock: EthereumBlockType = match dbg!(trigger_type) { EthereumBlockTriggerType::Every(BlockType::Full) => match graph::block_on_allow_panic( future::lazy(move || { self.host_exports @@ -585,7 +585,7 @@ impl RuntimeHostTrait for RuntimeHost { }) .compat(), ) { - Ok(block) => Ok(EthereumBlockType::Full(block)), + Ok(block) => Ok(EthereumBlockType::Full(dbg!(block))), Err(e) => Err(anyhow::anyhow!( "Failed to load full block: {}, error: {}", &block.number.unwrap().to_string(), @@ -606,7 +606,7 @@ impl RuntimeHostTrait for RuntimeHost { MappingTrigger::Block { handler: block_handler.clone(), }, - &Arc::new(theblock), + &Arc::new(dbg!(theblock)), proof_of_indexing, ) .await From f8e7ebfed5069303e308b342106aca4819f8bfe1 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Tue, 23 Jun 2020 11:53:31 -0700 Subject: [PATCH 03/18] core,graph,runtime: Update AscEthereumBlock struct --- core/src/subgraph/instance_manager.rs | 2 +- graph/src/components/ethereum/adapter.rs | 10 ++++++-- graph/src/components/ethereum/mod.rs | 9 +++---- graph/src/components/ethereum/types.rs | 30 +++++++++++++++--------- runtime/wasm/src/asc_abi/class.rs | 24 +++++++++++++++++-- runtime/wasm/src/host.rs | 6 ++--- runtime/wasm/src/to_from/external.rs | 29 +++++++++++++++++++++-- 7 files changed, 85 insertions(+), 25 deletions(-) diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index 2c1c7d5e02d..e6aec8e8328 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -879,7 +879,7 @@ async fn process_triggers, - block: &Arc, + block: &Arc, triggers: Vec, ) -> Result<(IndexingContext, BlockState), CancelableError> { for trigger in triggers.into_iter() { diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index 179c7db87e7..1e08a402530 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -813,6 +813,10 @@ fn parse_block_triggers( ) -> Vec { let block_ptr = EthereumBlockPointer::from(&block.ethereum_block); let trigger_every_block = block_filter.trigger_every_block; + let block_type = match block_filter.full_block { + true => BlockType::Full, + false => BlockType::Light, + }; let call_filter = EthereumCallFilter::from(block_filter); let mut triggers = block.calls.as_ref().map_or(vec![], |calls| { calls @@ -826,7 +830,7 @@ fn parse_block_triggers( if trigger_every_block { triggers.push(EthereumTrigger::Block( block_ptr, - EthereumBlockTriggerType::Every(BlockType::Full), + EthereumBlockTriggerType::Every(block_type), )); } triggers @@ -934,7 +938,9 @@ pub fn blocks_with_triggers( .map(|ptr| { EthereumTrigger::Block( ptr, - EthereumBlockTriggerType::Every(BlockType::Full), + EthereumBlockTriggerType::Every(BlockType::from( + block_filter.full_block, + )), ) }) .collect() diff --git a/graph/src/components/ethereum/mod.rs b/graph/src/components/ethereum/mod.rs index e5a56d3ad40..3a8d82c38a8 100644 --- a/graph/src/components/ethereum/mod.rs +++ b/graph/src/components/ethereum/mod.rs @@ -13,8 +13,9 @@ pub use self::adapter::{ pub use self::listener::{ChainHeadUpdate, ChainHeadUpdateListener, ChainHeadUpdateStream}; pub use self::stream::{BlockStream, BlockStreamBuilder, BlockStreamEvent}; pub use self::types::{ - EthereumBlockType, BlockFinality, BlockType, EthereumBlock, EthereumBlockData, EthereumBlockPointer, - EthereumBlockTriggerType, EthereumBlockWithCalls, EthereumBlockWithTriggers, EthereumCall, - EthereumCallData, EthereumEventData, EthereumTransactionData, EthereumTransactionReceiptData, - EthereumTrigger, FullEthereumBlockData, LightEthereumBlock, LightEthereumBlockExt, + BlockFinality, BlockType, EthereumBlock, EthereumBlockData, EthereumBlockPointer, + EthereumBlockTriggerType, EthereumBlockType, EthereumBlockWithCalls, EthereumBlockWithTriggers, + EthereumCall, EthereumCallData, EthereumEventData, EthereumTransactionData, + EthereumTransactionReceiptData, EthereumTrigger, FullEthereumBlockData, LightEthereumBlock, + LightEthereumBlockExt, }; diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index f328bb31f8a..688596c4a6b 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -203,9 +203,6 @@ impl PartialEq for EthereumTrigger { impl Eq for EthereumTrigger {} -/// This is used in `EthereumAdapter::triggers_in_block`, called when re-processing a block for -/// newly created data sources. This allows the re-processing to be reorg safe without having to -/// always fetch the full block data. #[derive(Clone, Debug)] pub enum EthereumBlockType { Light(LightEthereumBlock), @@ -303,8 +300,8 @@ impl PartialOrd for EthereumTrigger { pub struct EthereumTransactionReceiptData { // from receipts - pub transaction_hash: H256, - pub transaction_index: Index, + pub hash: H256, + pub index: Index, pub cumulative_gas_used: U256, pub gas_used: Option, pub contract_address: Option, @@ -360,8 +357,8 @@ impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockData { transaction_and_receipt.1.transaction_hash ); EthereumTransactionReceiptData { - transaction_hash: transaction_and_receipt.0.hash, - transaction_index: transaction_and_receipt.1.transaction_index, + hash: transaction_and_receipt.0.hash, + index: transaction_and_receipt.1.transaction_index, cumulative_gas_used: transaction_and_receipt.1.cumulative_gas_used, gas_used: transaction_and_receipt.1.gas_used, contract_address: transaction_and_receipt.1.contract_address, @@ -413,8 +410,8 @@ impl<'a> From<&'a EthereumBlock> for FullEthereumBlockData { transaction_and_receipt.1.transaction_hash ); EthereumTransactionReceiptData { - transaction_hash: transaction_and_receipt.0.hash, - transaction_index: transaction_and_receipt.1.transaction_index, + hash: transaction_and_receipt.0.hash, + index: transaction_and_receipt.1.transaction_index, cumulative_gas_used: transaction_and_receipt.1.cumulative_gas_used, gas_used: transaction_and_receipt.1.gas_used, contract_address: transaction_and_receipt.1.contract_address, @@ -470,10 +467,11 @@ pub struct EthereumBlockData { pub difficulty: U256, pub total_difficulty: U256, pub size: Option, + pub transactions: Vec, } -impl<'a, T> From<&'a Block> for EthereumBlockData { - fn from(block: &'a Block) -> EthereumBlockData { +impl<'a> From<&'a LightEthereumBlock> for EthereumBlockData { + fn from(block: &'a LightEthereumBlock) -> EthereumBlockData { EthereumBlockData { hash: block.hash.unwrap(), parent_hash: block.parent_hash, @@ -489,6 +487,11 @@ impl<'a, T> From<&'a Block> for EthereumBlockData { difficulty: block.difficulty, total_difficulty: block.total_difficulty.unwrap_or_default(), size: block.size, + transactions: block + .transactions + .iter() + .map(|tx| EthereumTransactionData::from(tx)) + .collect(), } } } @@ -515,6 +518,11 @@ impl<'a> From<&'a EthereumBlockType> for EthereumBlockData { difficulty: block.difficulty, total_difficulty: block.total_difficulty.unwrap_or_default(), size: block.size, + transactions: block + .transactions + .iter() + .map(|tx| EthereumTransactionData::from(tx)) + .collect(), } } } diff --git a/runtime/wasm/src/asc_abi/class.rs b/runtime/wasm/src/asc_abi/class.rs index 03ec59b0371..0b06068d55b 100644 --- a/runtime/wasm/src/asc_abi/class.rs +++ b/runtime/wasm/src/asc_abi/class.rs @@ -418,6 +418,26 @@ pub(crate) struct AscEthereumBlock { pub size: AscPtr, } +#[repr(C)] +#[derive(AscType)] +pub(crate) struct AscEthereumBlock_0_0_5 { + pub hash: AscPtr, + pub parent_hash: AscPtr, + pub uncles_hash: AscPtr, + pub author: AscPtr, + pub state_root: AscPtr, + pub transactions_root: AscPtr, + pub receipts_root: AscPtr, + pub number: AscPtr, + pub gas_used: AscPtr, + pub gas_limit: AscPtr, + pub timestamp: AscPtr, + pub difficulty: AscPtr, + pub total_difficulty: AscPtr, + pub size: AscPtr, + pub transactions: AscPtr>>, +} + #[repr(C)] #[derive(AscType)] pub(crate) struct AscFullEthereumBlock { @@ -453,8 +473,8 @@ pub(crate) struct AscEthereumTransaction { #[repr(C)] #[derive(AscType)] pub(crate) struct AscEthereumTransactionReceipt { - pub transaction_hash: AscPtr, - pub transaction_index: AscPtr, + pub hash: AscPtr, + pub index: AscPtr, pub cumulative_gas_used: AscPtr, pub gas_used: AscPtr, pub contract_address: AscPtr, diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index 71368bf3112..a3d26c3280d 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -576,7 +576,7 @@ impl RuntimeHostTrait for RuntimeHost { proof_of_indexing: SharedProofOfIndexing, ) -> Result { let block_handler = self.handler_for_block(trigger_type)?; - let theblock: EthereumBlockType = match dbg!(trigger_type) { + let mapping_block: EthereumBlockType = match trigger_type { EthereumBlockTriggerType::Every(BlockType::Full) => match graph::block_on_allow_panic( future::lazy(move || { self.host_exports @@ -585,7 +585,7 @@ impl RuntimeHostTrait for RuntimeHost { }) .compat(), ) { - Ok(block) => Ok(EthereumBlockType::Full(dbg!(block))), + Ok(block) => Ok(EthereumBlockType::Full(block)), Err(e) => Err(anyhow::anyhow!( "Failed to load full block: {}, error: {}", &block.number.unwrap().to_string(), @@ -606,7 +606,7 @@ impl RuntimeHostTrait for RuntimeHost { MappingTrigger::Block { handler: block_handler.clone(), }, - &Arc::new(dbg!(theblock)), + &Arc::new(mapping_block), proof_of_indexing, ) .await diff --git a/runtime/wasm/src/to_from/external.rs b/runtime/wasm/src/to_from/external.rs index 384568e9ace..2d85a2ea8bb 100644 --- a/runtime/wasm/src/to_from/external.rs +++ b/runtime/wasm/src/to_from/external.rs @@ -325,6 +325,31 @@ impl ToAscObj for EthereumBlockData { } } +impl ToAscObj for EthereumBlockData { + fn to_asc_obj(&self, heap: &mut H) -> AscEthereumBlock_0_0_5 { + AscEthereumBlock_0_0_5 { + hash: heap.asc_new(&self.hash), + parent_hash: heap.asc_new(&self.parent_hash), + uncles_hash: heap.asc_new(&self.uncles_hash), + author: heap.asc_new(&self.author), + state_root: heap.asc_new(&self.state_root), + transactions_root: heap.asc_new(&self.transactions_root), + receipts_root: heap.asc_new(&self.receipts_root), + number: heap.asc_new(&BigInt::from(self.number)), + gas_used: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_used)), + gas_limit: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_limit)), + timestamp: heap.asc_new(&BigInt::from_unsigned_u256(&self.timestamp)), + difficulty: heap.asc_new(&BigInt::from_unsigned_u256(&self.difficulty)), + total_difficulty: heap.asc_new(&BigInt::from_unsigned_u256(&self.total_difficulty)), + size: self + .size + .map(|size| heap.asc_new(&BigInt::from_unsigned_u256(&size))) + .unwrap_or_else(|| AscPtr::null()), + transactions: heap.asc_new(self.transactions.as_slice()), + } + } +} + impl ToAscObj for FullEthereumBlockData { fn to_asc_obj(&self, heap: &mut H) -> AscFullEthereumBlock { AscFullEthereumBlock { @@ -354,8 +379,8 @@ impl ToAscObj for EthereumTransactionReceiptData fn to_asc_obj(&self, heap: &mut H) -> AscEthereumTransactionReceipt { AscEthereumTransactionReceipt { //from receipts - transaction_hash: heap.asc_new(&self.transaction_hash), - transaction_index: heap.asc_new(&BigInt::from(self.transaction_index)), + hash: heap.asc_new(&self.hash), + index: heap.asc_new(&BigInt::from(self.index)), cumulative_gas_used: heap .asc_new(&BigInt::from_unsigned_u256(&self.cumulative_gas_used)), gas_used: self From 5a4446edb49bf5769da78a1eb834ff5d96044602 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Tue, 23 Jun 2020 21:51:37 -0700 Subject: [PATCH 04/18] graph,runtime: Support three data input structs for blockhandlers --- graph/src/components/ethereum/adapter.rs | 31 ++++-- graph/src/components/ethereum/mod.rs | 2 +- graph/src/components/ethereum/types.rs | 127 ++++++++++++++++++----- graph/src/data/subgraph/mod.rs | 4 +- graph/src/data/subgraph/schema.rs | 2 + runtime/wasm/src/asc_abi/class.rs | 4 +- runtime/wasm/src/host.rs | 11 +- runtime/wasm/src/module/mod.rs | 5 +- runtime/wasm/src/to_from/external.rs | 14 +-- 9 files changed, 146 insertions(+), 54 deletions(-) diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index 1e08a402530..d92a685c220 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -412,7 +412,7 @@ impl From for EthereumCallFilter { pub struct EthereumBlockFilter { pub contract_addresses: HashSet<(u64, Address)>, pub trigger_every_block: bool, - pub full_block: bool, + pub block_type: BlockType, } impl EthereumBlockFilter { @@ -435,18 +435,32 @@ impl EthereumBlockFilter { .iter() .any(|block_handler| block_handler.filter.is_none()); - let has_block_handler_with_fullblock = data_source + let block_type = if data_source + .mapping + .block_handlers + .iter() + .any(|block_handler| match block_handler.input { + BlockHandlerData::FullBlockWithReceipts => return true, + _ => return false, + }) { + BlockType::FullWithReceipts + } else if data_source .mapping .block_handlers .iter() .any(|block_handler| match block_handler.input { BlockHandlerData::FullBlock => return true, _ => return false, - }); + }) + { + BlockType::Full + } else { + BlockType::Light + }; filter_opt.extend(Self { trigger_every_block: has_block_handler_without_filter, - full_block: has_block_handler_with_fullblock, + block_type, contract_addresses: if has_block_handler_with_call_filter { vec![( data_source.source.start_block, @@ -813,10 +827,7 @@ fn parse_block_triggers( ) -> Vec { let block_ptr = EthereumBlockPointer::from(&block.ethereum_block); let trigger_every_block = block_filter.trigger_every_block; - let block_type = match block_filter.full_block { - true => BlockType::Full, - false => BlockType::Light, - }; + let block_type = block_filter.block_type; let call_filter = EthereumCallFilter::from(block_filter); let mut triggers = block.calls.as_ref().map_or(vec![], |calls| { calls @@ -938,9 +949,7 @@ pub fn blocks_with_triggers( .map(|ptr| { EthereumTrigger::Block( ptr, - EthereumBlockTriggerType::Every(BlockType::from( - block_filter.full_block, - )), + EthereumBlockTriggerType::Every(block_filter.block_type), ) }) .collect() diff --git a/graph/src/components/ethereum/mod.rs b/graph/src/components/ethereum/mod.rs index 3a8d82c38a8..3c875870dab 100644 --- a/graph/src/components/ethereum/mod.rs +++ b/graph/src/components/ethereum/mod.rs @@ -13,7 +13,7 @@ pub use self::adapter::{ pub use self::listener::{ChainHeadUpdate, ChainHeadUpdateListener, ChainHeadUpdateStream}; pub use self::stream::{BlockStream, BlockStreamBuilder, BlockStreamEvent}; pub use self::types::{ - BlockFinality, BlockType, EthereumBlock, EthereumBlockData, EthereumBlockPointer, + BlockFinality, BlockType, EthereumBlock, EthereumBlockData, FullEthereumBlockDataWithReceipts, EthereumBlockPointer, EthereumBlockTriggerType, EthereumBlockType, EthereumBlockWithCalls, EthereumBlockWithTriggers, EthereumCall, EthereumCallData, EthereumEventData, EthereumTransactionData, EthereumTransactionReceiptData, EthereumTrigger, FullEthereumBlockData, LightEthereumBlock, diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index 688596c4a6b..4871324131c 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -60,7 +60,8 @@ impl LightEthereumBlockExt for LightEthereumBlock { impl<'a> From<&'a EthereumBlockType> for LightEthereumBlock { fn from(block: &'a EthereumBlockType) -> LightEthereumBlock { match block { - EthereumBlockType::Full(block) => block.block.clone(), + EthereumBlockType::FullWithReceipts(block) => block.block.clone(), + EthereumBlockType::Full(block) => block.clone(), EthereumBlockType::Light(block) => block.clone(), } } @@ -207,7 +208,9 @@ impl Eq for EthereumTrigger {} pub enum EthereumBlockType { Light(LightEthereumBlock), - Full(EthereumBlock), + Full(LightEthereumBlock), + + FullWithReceipts(EthereumBlock), } impl Default for EthereumBlockType { @@ -216,16 +219,17 @@ impl Default for EthereumBlockType { } } -impl<'a> From<&'a LightEthereumBlock> for EthereumBlockType { - fn from(block: &'a LightEthereumBlock) -> EthereumBlockType { - EthereumBlockType::Light(block.clone()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum BlockType { Light, Full, + FullWithReceipts, +} + +impl Default for BlockType { + fn default() -> BlockType { + BlockType::Light + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -317,7 +321,7 @@ pub struct EthereumTransactionReceiptData { pub input: Bytes, } -pub struct FullEthereumBlockData { +pub struct FullEthereumBlockDataWithReceipts { pub hash: H256, pub parent_hash: H256, pub uncles_hash: H256, @@ -335,14 +339,17 @@ pub struct FullEthereumBlockData { pub transaction_receipts: Vec, } -impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockData { +impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockDataWithReceipts { type Error = String; - fn try_from(block: &'a EthereumBlockType) -> Result { + fn try_from(block: &'a EthereumBlockType) -> Result { let fullblock = match block { - EthereumBlockType::Full(full_block) => full_block, + EthereumBlockType::FullWithReceipts(full_block) => full_block, + EthereumBlockType::Full(_) => return Err(format!( + "Failed to convert EthereumBlockType to FullEthereumBlockDataWithReceipts, requires a EthereumBlockType::FullWithReceipts()" + )), EthereumBlockType::Light(_) => return Err(format!( - "Failed to convert EthereumBlockType to FUllEthereumBlockData, requires a EthereumBlockType::Full()" + "Failed to convert EthereumBlockType to FullEthereumBlockDataWithReceipts, requires a EthereumBlockType::FullWithReceipts()" )) }; let block = &fullblock.block; @@ -376,7 +383,7 @@ impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockData { }) .collect::>(); - Ok(FullEthereumBlockData { + Ok(FullEthereumBlockDataWithReceipts { hash: block.hash.unwrap(), parent_hash: block.parent_hash, uncles_hash: block.uncles_hash, @@ -396,8 +403,8 @@ impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockData { } } -impl<'a> From<&'a EthereumBlock> for FullEthereumBlockData { - fn from(block: &'a EthereumBlock) -> FullEthereumBlockData { +impl<'a> From<&'a EthereumBlock> for FullEthereumBlockDataWithReceipts { + fn from(block: &'a EthereumBlock) -> FullEthereumBlockDataWithReceipts { let transaction_receipts_data = block .block .transactions @@ -430,7 +437,7 @@ impl<'a> From<&'a EthereumBlock> for FullEthereumBlockData { .collect::>(); let block = &block.block; - FullEthereumBlockData { + FullEthereumBlockDataWithReceipts { hash: block.hash.unwrap(), parent_hash: block.parent_hash, uncles_hash: block.uncles_hash, @@ -452,7 +459,7 @@ impl<'a> From<&'a EthereumBlock> for FullEthereumBlockData { /// Ethereum block data. #[derive(Clone, Debug, Default)] -pub struct EthereumBlockData { +pub struct FullEthereumBlockData { pub hash: H256, pub parent_hash: H256, pub uncles_hash: H256, @@ -470,9 +477,9 @@ pub struct EthereumBlockData { pub transactions: Vec, } -impl<'a> From<&'a LightEthereumBlock> for EthereumBlockData { - fn from(block: &'a LightEthereumBlock) -> EthereumBlockData { - EthereumBlockData { +impl<'a> From<&'a LightEthereumBlock> for FullEthereumBlockData { + fn from(block: &'a LightEthereumBlock) -> FullEthereumBlockData { + FullEthereumBlockData { hash: block.hash.unwrap(), parent_hash: block.parent_hash, uncles_hash: block.uncles_hash, @@ -496,14 +503,15 @@ impl<'a> From<&'a LightEthereumBlock> for EthereumBlockData { } } -impl<'a> From<&'a EthereumBlockType> for EthereumBlockData { - fn from(block: &'a EthereumBlockType) -> EthereumBlockData { +impl<'a> From<&'a EthereumBlockType> for FullEthereumBlockData { + fn from(block: &'a EthereumBlockType) -> FullEthereumBlockData { let block = match block { - EthereumBlockType::Full(full_block) => &full_block.block, + EthereumBlockType::FullWithReceipts(full_block) => &full_block.block, + EthereumBlockType::Full(block) => &block, EthereumBlockType::Light(light_block) => light_block, }; - EthereumBlockData { + FullEthereumBlockData { hash: block.hash.unwrap(), parent_hash: block.parent_hash, uncles_hash: block.uncles_hash, @@ -527,6 +535,73 @@ impl<'a> From<&'a EthereumBlockType> for EthereumBlockData { } } +/// Ethereum block data. +#[derive(Clone, Debug, Default)] +pub struct EthereumBlockData { + pub hash: H256, + pub parent_hash: H256, + pub uncles_hash: H256, + pub author: H160, + pub state_root: H256, + pub transactions_root: H256, + pub receipts_root: H256, + pub number: U64, + pub gas_used: U256, + pub gas_limit: U256, + pub timestamp: U256, + pub difficulty: U256, + pub total_difficulty: U256, + pub size: Option, +} + +impl<'a> From<&'a LightEthereumBlock> for EthereumBlockData { + fn from(block: &'a LightEthereumBlock) -> EthereumBlockData { + EthereumBlockData { + hash: block.hash.unwrap(), + parent_hash: block.parent_hash, + uncles_hash: block.uncles_hash, + author: block.author, + state_root: block.state_root, + transactions_root: block.transactions_root, + receipts_root: block.receipts_root, + number: block.number.unwrap(), + gas_used: block.gas_used, + gas_limit: block.gas_limit, + timestamp: block.timestamp, + difficulty: block.difficulty, + total_difficulty: block.total_difficulty.unwrap_or_default(), + size: block.size, + } + } +} + +impl<'a> From<&'a EthereumBlockType> for EthereumBlockData { + fn from(block: &'a EthereumBlockType) -> EthereumBlockData { + let block = match block { + EthereumBlockType::FullWithReceipts(full_block) => &full_block.block, + EthereumBlockType::Full(full_block) => &full_block, + EthereumBlockType::Light(light_block) => light_block, + }; + + EthereumBlockData { + hash: block.hash.unwrap(), + parent_hash: block.parent_hash, + uncles_hash: block.uncles_hash, + author: block.author, + state_root: block.state_root, + transactions_root: block.transactions_root, + receipts_root: block.receipts_root, + number: block.number.unwrap(), + gas_used: block.gas_used, + gas_limit: block.gas_limit, + timestamp: block.timestamp, + difficulty: block.difficulty, + total_difficulty: block.total_difficulty.unwrap_or_default(), + size: block.size, + } + } +} + /// Ethereum transaction data. #[derive(Clone, Debug)] pub struct EthereumTransactionData { diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index c1f80007c49..6dd4ffad59a 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -522,13 +522,15 @@ pub struct MappingBlockHandler { pub enum BlockHandlerData { Block, FullBlock, + FullBlockWithReceipts, } impl From for BlockHandlerData { fn from(data: EthereumBlockHandlerData) -> Self { match data { + EthereumBlockHandlerData::FullBlockWithReceipts => BlockHandlerData::FullBlockWithReceipts, EthereumBlockHandlerData::FullBlock => BlockHandlerData::FullBlock, - _ => BlockHandlerData::Block, + EthereumBlockHandlerData::Block => BlockHandlerData::Block, } } } diff --git a/graph/src/data/subgraph/schema.rs b/graph/src/data/subgraph/schema.rs index 6cec5796a65..2353c651f01 100644 --- a/graph/src/data/subgraph/schema.rs +++ b/graph/src/data/subgraph/schema.rs @@ -1114,6 +1114,7 @@ impl TryFromValue for EthereumBlockHandlerFilterEntity { pub enum EthereumBlockHandlerData { Block, FullBlock, + FullBlockWithReceipts } impl Default for EthereumBlockHandlerData { @@ -1140,6 +1141,7 @@ impl FromStr for EthereumBlockHandlerData { impl From for String { fn from(data: EthereumBlockHandlerData) -> String { match data { + EthereumBlockHandlerData::FullBlockWithReceipts => "FullBlockWithReceipts".into(), EthereumBlockHandlerData::FullBlock => "FullBlock".into(), EthereumBlockHandlerData::Block => "Block".into(), } diff --git a/runtime/wasm/src/asc_abi/class.rs b/runtime/wasm/src/asc_abi/class.rs index 0b06068d55b..10feaaec548 100644 --- a/runtime/wasm/src/asc_abi/class.rs +++ b/runtime/wasm/src/asc_abi/class.rs @@ -420,7 +420,7 @@ pub(crate) struct AscEthereumBlock { #[repr(C)] #[derive(AscType)] -pub(crate) struct AscEthereumBlock_0_0_5 { +pub(crate) struct AscFullEthereumBlock { pub hash: AscPtr, pub parent_hash: AscPtr, pub uncles_hash: AscPtr, @@ -440,7 +440,7 @@ pub(crate) struct AscEthereumBlock_0_0_5 { #[repr(C)] #[derive(AscType)] -pub(crate) struct AscFullEthereumBlock { +pub(crate) struct AscFullEthereumBlockWithReceipts { pub hash: AscPtr, pub parent_hash: AscPtr, pub uncles_hash: AscPtr, diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index a3d26c3280d..a649afb9aa6 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -561,7 +561,7 @@ impl RuntimeHostTrait for RuntimeHost { outputs, handler: call_handler.clone(), }, - &Arc::new(EthereumBlockType::from(block.as_ref())), + &Arc::new(EthereumBlockType::Light(LightEthereumBlock::from(block.as_ref().clone()))), proof_of_indexing, ) .await @@ -577,7 +577,7 @@ impl RuntimeHostTrait for RuntimeHost { ) -> Result { let block_handler = self.handler_for_block(trigger_type)?; let mapping_block: EthereumBlockType = match trigger_type { - EthereumBlockTriggerType::Every(BlockType::Full) => match graph::block_on_allow_panic( + EthereumBlockTriggerType::Every(BlockType::FullWithReceipts) => match graph::block_on_allow_panic( future::lazy(move || { self.host_exports .ethereum_adapter @@ -585,14 +585,15 @@ impl RuntimeHostTrait for RuntimeHost { }) .compat(), ) { - Ok(block) => Ok(EthereumBlockType::Full(block)), + Ok(block) => Ok(EthereumBlockType::FullWithReceipts(block)), Err(e) => Err(anyhow::anyhow!( "Failed to load full block: {}, error: {}", &block.number.unwrap().to_string(), e )), }?, - _ => EthereumBlockType::from(block.as_ref()), + EthereumBlockTriggerType::Every(BlockType::Full) => EthereumBlockType::Full(LightEthereumBlock::from(block.as_ref().clone())), + _ => EthereumBlockType::Light(LightEthereumBlock::from(block.as_ref().clone())) }; self.send_mapping_request( @@ -720,7 +721,7 @@ impl RuntimeHostTrait for RuntimeHost { params, handler: event_handler.clone(), }, - &Arc::new(EthereumBlockType::from(block.as_ref())), + &Arc::new(EthereumBlockType::Light(block.as_ref()))), proof_of_indexing, ) .await diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index f57dd9e913f..36bf2159a9b 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -163,8 +163,11 @@ impl WasmInstance { ) -> Result { let block = self.instance_ctx().ctx.block.clone(); - // Prepare an EthereumBlock for the WASM runtime + // Prepare an Ethereum Block for the WASM runtime let arg = match block.as_ref() { + EthereumBlockType::FullWithReceipts(block) => self + .asc_new::(&FullEthereumBlockDataWithReceipts::from(block)) + .erase(), EthereumBlockType::Full(block) => self .asc_new::(&FullEthereumBlockData::from(block)) .erase(), diff --git a/runtime/wasm/src/to_from/external.rs b/runtime/wasm/src/to_from/external.rs index 2d85a2ea8bb..15ce2807468 100644 --- a/runtime/wasm/src/to_from/external.rs +++ b/runtime/wasm/src/to_from/external.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use graph::components::ethereum::{ EthereumBlockData, EthereumCallData, EthereumEventData, EthereumTransactionData, - EthereumTransactionReceiptData, FullEthereumBlockData, + EthereumTransactionReceiptData, FullEthereumBlockData, FullEthereumBlockDataWithReceipts }; use graph::data::store; use graph::prelude::anyhow::{ensure, Error}; @@ -325,9 +325,9 @@ impl ToAscObj for EthereumBlockData { } } -impl ToAscObj for EthereumBlockData { - fn to_asc_obj(&self, heap: &mut H) -> AscEthereumBlock_0_0_5 { - AscEthereumBlock_0_0_5 { +impl ToAscObj for FullEthereumBlockData { + fn to_asc_obj(&self, heap: &mut H) -> AscFullEthereumBlock { + AscFullEthereumBlock { hash: heap.asc_new(&self.hash), parent_hash: heap.asc_new(&self.parent_hash), uncles_hash: heap.asc_new(&self.uncles_hash), @@ -350,9 +350,9 @@ impl ToAscObj for EthereumBlockData { } } -impl ToAscObj for FullEthereumBlockData { - fn to_asc_obj(&self, heap: &mut H) -> AscFullEthereumBlock { - AscFullEthereumBlock { +impl ToAscObj for FullEthereumBlockDataWithReceipts { + fn to_asc_obj(&self, heap: &mut H) -> AscFullEthereumBlockWithReceipts { + AscFullEthereumBlockWithReceipts { hash: heap.asc_new(&self.hash), parent_hash: heap.asc_new(&self.parent_hash), uncles_hash: heap.asc_new(&self.uncles_hash), From 0105c585aa645828c9c959bc3c1dfa58b7ef0db5 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Wed, 24 Jun 2020 00:40:09 -0700 Subject: [PATCH 05/18] core,graph,runtime: Pass EthereumBlockType into process_triggers() - Rather converting to EthereumBlockType before sending the mapping request, pass EthereumBlockType down from process_block(). - Add utility functions to EthereumBlockType. --- core/src/subgraph/instance.rs | 4 +- core/src/subgraph/instance_manager.rs | 4 +- graph/src/components/ethereum/mod.rs | 6 +- graph/src/components/ethereum/types.rs | 67 ++++++++++++++++++----- graph/src/components/subgraph/host.rs | 6 +- graph/src/components/subgraph/instance.rs | 4 +- graph/src/data/subgraph/mod.rs | 4 +- graph/src/data/subgraph/schema.rs | 2 +- graph/src/lib.rs | 2 +- runtime/wasm/src/host.rs | 52 ++++++++++-------- runtime/wasm/src/module/mod.rs | 4 +- runtime/wasm/src/to_from/external.rs | 2 +- 12 files changed, 101 insertions(+), 56 deletions(-) diff --git a/core/src/subgraph/instance.rs b/core/src/subgraph/instance.rs index e27c8580586..0a81082aa6c 100644 --- a/core/src/subgraph/instance.rs +++ b/core/src/subgraph/instance.rs @@ -135,7 +135,7 @@ where async fn process_trigger( &self, logger: &Logger, - block: &Arc, + block: &Arc, trigger: EthereumTrigger, state: BlockState, proof_of_indexing: SharedProofOfIndexing, @@ -154,7 +154,7 @@ where async fn process_trigger_in_runtime_hosts( logger: &Logger, hosts: &[Arc], - block: &Arc, + block: &Arc, trigger: EthereumTrigger, mut state: BlockState, proof_of_indexing: SharedProofOfIndexing, diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index e6aec8e8328..b86b2664e54 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -638,7 +638,7 @@ where } // Obtain current and new block pointer (after this block is processed) - let light_block = Arc::new(block.light_block()); + let light_block = Arc::new(EthereumBlockType::Light(block.light_block())); let block_ptr_after = EthereumBlockPointer::from(&block); let block_ptr_for_new_data_sources = block_ptr_after.clone(); @@ -879,7 +879,7 @@ async fn process_triggers, - block: &Arc, + block: &Arc, triggers: Vec, ) -> Result<(IndexingContext, BlockState), CancelableError> { for trigger in triggers.into_iter() { diff --git a/graph/src/components/ethereum/mod.rs b/graph/src/components/ethereum/mod.rs index 3c875870dab..fb148b10270 100644 --- a/graph/src/components/ethereum/mod.rs +++ b/graph/src/components/ethereum/mod.rs @@ -13,9 +13,9 @@ pub use self::adapter::{ pub use self::listener::{ChainHeadUpdate, ChainHeadUpdateListener, ChainHeadUpdateStream}; pub use self::stream::{BlockStream, BlockStreamBuilder, BlockStreamEvent}; pub use self::types::{ - BlockFinality, BlockType, EthereumBlock, EthereumBlockData, FullEthereumBlockDataWithReceipts, EthereumBlockPointer, + BlockFinality, BlockType, EthereumBlock, EthereumBlockData, EthereumBlockPointer, EthereumBlockTriggerType, EthereumBlockType, EthereumBlockWithCalls, EthereumBlockWithTriggers, EthereumCall, EthereumCallData, EthereumEventData, EthereumTransactionData, - EthereumTransactionReceiptData, EthereumTrigger, FullEthereumBlockData, LightEthereumBlock, - LightEthereumBlockExt, + EthereumTransactionReceiptData, EthereumTrigger, FullEthereumBlockData, + FullEthereumBlockDataWithReceipts, LightEthereumBlock, LightEthereumBlockExt, }; diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index 4871324131c..1ace326c1c7 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -13,8 +13,6 @@ pub type LightEthereumBlock = Block; pub trait LightEthereumBlockExt { fn number(&self) -> u64; - fn transaction_for_log(&self, log: &Log) -> Option; - fn transaction_for_call(&self, call: &EthereumCall) -> Option; fn parent_ptr(&self) -> Option; fn format(&self) -> String; } @@ -24,18 +22,6 @@ impl LightEthereumBlockExt for LightEthereumBlock { self.number.unwrap().as_u64() } - fn transaction_for_log(&self, log: &Log) -> Option { - log.transaction_hash - .and_then(|hash| self.transactions.iter().find(|tx| tx.hash == hash)) - .cloned() - } - - fn transaction_for_call(&self, call: &EthereumCall) -> Option { - call.transaction_hash - .and_then(|hash| self.transactions.iter().find(|tx| tx.hash == hash)) - .cloned() - } - fn parent_ptr(&self) -> Option { match self.number() { 0 => None, @@ -212,6 +198,45 @@ pub enum EthereumBlockType { FullWithReceipts(EthereumBlock), } +impl EthereumBlockType { + pub fn light_block(&self) -> &LightEthereumBlock { + match self { + EthereumBlockType::Light(block) => block, + EthereumBlockType::Full(block) => block, + EthereumBlockType::FullWithReceipts(block) => &block.block, + } + } + + pub fn hash(&self) -> H256 { + self.light_block().hash.unwrap() + } + + pub fn number(&self) -> u64 { + self.light_block().number.unwrap().as_u64() + } + + pub fn transaction_for_log(&self, log: &Log) -> Option { + log.transaction_hash + .and_then(|hash| { + self.light_block() + .transactions + .iter() + .find(|tx| tx.hash == hash) + }) + .cloned() + } + + pub fn transaction_for_call(&self, call: &EthereumCall) -> Option { + call.transaction_hash + .and_then(|hash| { + self.light_block() + .transactions + .iter() + .find(|tx| tx.hash == hash) + }) + .cloned() + } +} impl Default for EthereumBlockType { fn default() -> Self { @@ -342,7 +367,9 @@ pub struct FullEthereumBlockDataWithReceipts { impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockDataWithReceipts { type Error = String; - fn try_from(block: &'a EthereumBlockType) -> Result { + fn try_from( + block: &'a EthereumBlockType, + ) -> Result { let fullblock = match block { EthereumBlockType::FullWithReceipts(full_block) => full_block, EthereumBlockType::Full(_) => return Err(format!( @@ -770,6 +797,16 @@ impl<'a> From<&'a EthereumBlock> for EthereumBlockPointer { } } +impl<'a> From<&'a EthereumBlockType> for EthereumBlockPointer { + fn from(b: &'a EthereumBlockType) -> EthereumBlockPointer { + match b { + EthereumBlockType::Light(block) => EthereumBlockPointer::from(block), + EthereumBlockType::Full(block) => EthereumBlockPointer::from(block), + EthereumBlockType::FullWithReceipts(block) => EthereumBlockPointer::from(block), + } + } +} + impl From<(H256, u64)> for EthereumBlockPointer { fn from((hash, number): (H256, u64)) -> EthereumBlockPointer { if number >= (1 << 63) { diff --git a/graph/src/components/subgraph/host.rs b/graph/src/components/subgraph/host.rs index 11b61093923..ceb56b515fa 100644 --- a/graph/src/components/subgraph/host.rs +++ b/graph/src/components/subgraph/host.rs @@ -27,7 +27,7 @@ pub trait RuntimeHost: Send + Sync + Debug + 'static { async fn process_log( &self, logger: &Logger, - block: &Arc, + block: &Arc, transaction: &Arc, log: &Arc, state: BlockState, @@ -38,7 +38,7 @@ pub trait RuntimeHost: Send + Sync + Debug + 'static { async fn process_call( &self, logger: &Logger, - block: &Arc, + block: &Arc, transaction: &Arc, call: &Arc, state: BlockState, @@ -49,7 +49,7 @@ pub trait RuntimeHost: Send + Sync + Debug + 'static { async fn process_block( &self, logger: &Logger, - block: &Arc, + block: &Arc, trigger_type: &EthereumBlockTriggerType, state: BlockState, proof_of_indexing: SharedProofOfIndexing, diff --git a/graph/src/components/subgraph/instance.rs b/graph/src/components/subgraph/instance.rs index 1b48a638f16..ce7d3560d27 100644 --- a/graph/src/components/subgraph/instance.rs +++ b/graph/src/components/subgraph/instance.rs @@ -38,7 +38,7 @@ pub trait SubgraphInstance { async fn process_trigger( &self, logger: &Logger, - block: &Arc, + block: &Arc, trigger: EthereumTrigger, state: BlockState, proof_of_indexing: SharedProofOfIndexing, @@ -48,7 +48,7 @@ pub trait SubgraphInstance { async fn process_trigger_in_runtime_hosts( logger: &Logger, hosts: &[Arc], - block: &Arc, + block: &Arc, trigger: EthereumTrigger, state: BlockState, proof_of_indexing: SharedProofOfIndexing, diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index 6dd4ffad59a..866eb956b03 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -528,7 +528,9 @@ pub enum BlockHandlerData { impl From for BlockHandlerData { fn from(data: EthereumBlockHandlerData) -> Self { match data { - EthereumBlockHandlerData::FullBlockWithReceipts => BlockHandlerData::FullBlockWithReceipts, + EthereumBlockHandlerData::FullBlockWithReceipts => { + BlockHandlerData::FullBlockWithReceipts + } EthereumBlockHandlerData::FullBlock => BlockHandlerData::FullBlock, EthereumBlockHandlerData::Block => BlockHandlerData::Block, } diff --git a/graph/src/data/subgraph/schema.rs b/graph/src/data/subgraph/schema.rs index 2353c651f01..65130d0f646 100644 --- a/graph/src/data/subgraph/schema.rs +++ b/graph/src/data/subgraph/schema.rs @@ -1114,7 +1114,7 @@ impl TryFromValue for EthereumBlockHandlerFilterEntity { pub enum EthereumBlockHandlerData { Block, FullBlock, - FullBlockWithReceipts + FullBlockWithReceipts, } impl Default for EthereumBlockHandlerData { diff --git a/graph/src/lib.rs b/graph/src/lib.rs index a970deecebd..642d0037671 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -77,7 +77,7 @@ pub mod prelude { BlockFinality, BlockStream, BlockStreamBuilder, BlockStreamEvent, BlockStreamMetrics, ChainHeadUpdate, ChainHeadUpdateListener, ChainHeadUpdateStream, EthereumAdapter, EthereumAdapterError, EthereumBlock, EthereumBlockData, EthereumBlockFilter, - EthereumBlockPointer, EthereumBlockTriggerType, EthereumBlockWithCalls, + EthereumBlockPointer, EthereumBlockTriggerType, EthereumBlockType, EthereumBlockWithCalls, EthereumBlockWithTriggers, EthereumCall, EthereumCallData, EthereumCallFilter, EthereumContractCall, EthereumContractCallError, EthereumEventData, EthereumLogFilter, EthereumNetworkIdentifier, EthereumTransactionData, EthereumTrigger, FullEthereumBlockData, diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index a649afb9aa6..fbc00830037 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -474,7 +474,7 @@ impl RuntimeHostTrait for RuntimeHost { async fn process_call( &self, logger: &Logger, - block: &Arc, + block: &Arc, transaction: &Arc, call: &Arc, state: BlockState, @@ -561,7 +561,7 @@ impl RuntimeHostTrait for RuntimeHost { outputs, handler: call_handler.clone(), }, - &Arc::new(EthereumBlockType::Light(LightEthereumBlock::from(block.as_ref().clone()))), + block, proof_of_indexing, ) .await @@ -570,37 +570,41 @@ impl RuntimeHostTrait for RuntimeHost { async fn process_block( &self, logger: &Logger, - block: &Arc, + block: &Arc, trigger_type: &EthereumBlockTriggerType, state: BlockState, proof_of_indexing: SharedProofOfIndexing, ) -> Result { let block_handler = self.handler_for_block(trigger_type)?; let mapping_block: EthereumBlockType = match trigger_type { - EthereumBlockTriggerType::Every(BlockType::FullWithReceipts) => match graph::block_on_allow_panic( - future::lazy(move || { - self.host_exports - .ethereum_adapter - .load_full_block(logger, block.as_ref().clone()) - }) - .compat(), - ) { - Ok(block) => Ok(EthereumBlockType::FullWithReceipts(block)), - Err(e) => Err(anyhow::anyhow!( - "Failed to load full block: {}, error: {}", - &block.number.unwrap().to_string(), - e - )), - }?, - EthereumBlockTriggerType::Every(BlockType::Full) => EthereumBlockType::Full(LightEthereumBlock::from(block.as_ref().clone())), - _ => EthereumBlockType::Light(LightEthereumBlock::from(block.as_ref().clone())) + EthereumBlockTriggerType::Every(BlockType::FullWithReceipts) => { + match graph::block_on_allow_panic( + future::lazy(move || { + self.host_exports + .ethereum_adapter + .load_full_block(logger, block.light_block().clone()) + }) + .compat(), + ) { + Ok(block) => Ok(EthereumBlockType::FullWithReceipts(block)), + Err(e) => Err(anyhow::anyhow!( + "Failed to load full block: {}, error: {}", + &block.number().to_string(), + e + )), + }? + } + EthereumBlockTriggerType::Every(BlockType::Full) => { + EthereumBlockType::Full(block.light_block().clone()) + } + _ => block.as_ref().clone(), }; self.send_mapping_request( logger, o! { - "hash" => block.hash.unwrap().to_string(), - "number" => &block.number.unwrap().to_string(), + "hash" => block.hash().to_string(), + "number" => &block.number().to_string(), }, state, &block_handler.handler, @@ -616,7 +620,7 @@ impl RuntimeHostTrait for RuntimeHost { async fn process_log( &self, logger: &Logger, - block: &Arc, + block: &Arc, transaction: &Arc, log: &Arc, state: BlockState, @@ -721,7 +725,7 @@ impl RuntimeHostTrait for RuntimeHost { params, handler: event_handler.clone(), }, - &Arc::new(EthereumBlockType::Light(block.as_ref()))), + block, proof_of_indexing, ) .await diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index 36bf2159a9b..3a665cf8486 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -166,7 +166,9 @@ impl WasmInstance { // Prepare an Ethereum Block for the WASM runtime let arg = match block.as_ref() { EthereumBlockType::FullWithReceipts(block) => self - .asc_new::(&FullEthereumBlockDataWithReceipts::from(block)) + .asc_new::( + &FullEthereumBlockDataWithReceipts::from(block), + ) .erase(), EthereumBlockType::Full(block) => self .asc_new::(&FullEthereumBlockData::from(block)) diff --git a/runtime/wasm/src/to_from/external.rs b/runtime/wasm/src/to_from/external.rs index 15ce2807468..b14d1446282 100644 --- a/runtime/wasm/src/to_from/external.rs +++ b/runtime/wasm/src/to_from/external.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use graph::components::ethereum::{ EthereumBlockData, EthereumCallData, EthereumEventData, EthereumTransactionData, - EthereumTransactionReceiptData, FullEthereumBlockData, FullEthereumBlockDataWithReceipts + EthereumTransactionReceiptData, FullEthereumBlockData, FullEthereumBlockDataWithReceipts, }; use graph::data::store; use graph::prelude::anyhow::{ensure, Error}; From 77b0495d2358142c9c190f6567d3e20d3b1e941e Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Wed, 24 Jun 2020 02:43:16 -0700 Subject: [PATCH 06/18] graph: Update EthereumBlockFilter.extend() with block_type field --- graph/src/components/ethereum/adapter.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index d92a685c220..3b827add4e9 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -478,6 +478,15 @@ impl EthereumBlockFilter { pub fn extend(&mut self, other: EthereumBlockFilter) { self.trigger_every_block = self.trigger_every_block || other.trigger_every_block; + self.block_type = match self.block_type { + BlockType::FullWithReceipts => BlockType::FullWithReceipts, + BlockType::Full => match other.block_type { + BlockType::FullWithReceipts => BlockType::FullWithReceipts, + _ => BlockType::Full, + }, + BlockType::Light => other.block_type, + }; + self.contract_addresses = self.contract_addresses.iter().cloned().fold( HashSet::new(), |mut addresses, (start_block, address)| { @@ -839,10 +848,10 @@ fn parse_block_triggers( .collect::>() }); if trigger_every_block { - triggers.push(EthereumTrigger::Block( + triggers.push(dbg!(EthereumTrigger::Block( block_ptr, EthereumBlockTriggerType::Every(block_type), - )); + ))); } triggers } From a112e474692c14f83177507d53fbf0d188d1498d Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Wed, 24 Jun 2020 02:45:25 -0700 Subject: [PATCH 07/18] graph,runtime: Support different block type inputs to call filters --- graph/src/components/ethereum/adapter.rs | 10 +++++-- graph/src/components/ethereum/types.rs | 4 +-- runtime/wasm/src/host.rs | 37 +++++++++++------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index 3b827add4e9..2e8da244a41 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -843,7 +843,10 @@ fn parse_block_triggers( .iter() .filter(move |call| call_filter.matches(call)) .map(move |call| { - EthereumTrigger::Block(block_ptr, EthereumBlockTriggerType::WithCallTo(call.to)) + EthereumTrigger::Block( + block_ptr, + EthereumBlockTriggerType::WithCallTo(call.to, block_type), + ) }) .collect::>() }); @@ -968,13 +971,14 @@ pub fn blocks_with_triggers( // To determine which blocks include a call to addresses // in the block filter, transform the `block_filter` into // a `call_filter` and run `blocks_with_calls` + let block_type = block_filter.block_type.clone(); let call_filter = EthereumCallFilter::from(block_filter); trigger_futs.push(Box::new( eth.calls_in_block_range(&logger, subgraph_metrics.clone(), from, to, call_filter) - .map(|call| { + .map(move |call| { EthereumTrigger::Block( EthereumBlockPointer::from(&call), - EthereumBlockTriggerType::WithCallTo(call.to), + EthereumBlockTriggerType::WithCallTo(call.to, block_type.clone()), ) }) .collect(), diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index 1ace326c1c7..bd7a85bd8f9 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -260,7 +260,7 @@ impl Default for BlockType { #[derive(Clone, Debug, PartialEq, Eq)] pub enum EthereumBlockTriggerType { Every(BlockType), - WithCallTo(Address), + WithCallTo(Address, BlockType), } impl EthereumTrigger { @@ -892,7 +892,7 @@ mod test { number: 0, hash: H256::random(), }, - EthereumBlockTriggerType::WithCallTo(Address::random()), + EthereumBlockTriggerType::WithCallTo(Address::random(), BlockType::Light), ); let mut call1 = EthereumCall::default(); diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index fbc00830037..eae43107d6e 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -285,7 +285,7 @@ impl RuntimeHost { fn matches_block_trigger(&self, block_trigger_type: &EthereumBlockTriggerType) -> bool { let source_address_matches = match block_trigger_type { - EthereumBlockTriggerType::WithCallTo(address) => { + EthereumBlockTriggerType::WithCallTo(address, _block_type) => { self.data_source_contract .address // Do not match if this datasource has no address @@ -347,7 +347,7 @@ impl RuntimeHost { trigger_type: &EthereumBlockTriggerType, ) -> Result { match trigger_type { - EthereumBlockTriggerType::Every(_type) => self + EthereumBlockTriggerType::Every(_block_type) => self .data_source_block_handlers .iter() .find(move |handler| handler.filter == None) @@ -359,7 +359,7 @@ impl RuntimeHost { self.data_source_name, ) }), - EthereumBlockTriggerType::WithCallTo(_address) => self + EthereumBlockTriggerType::WithCallTo(_address, _block_type) => self .data_source_block_handlers .iter() .find(move |handler| { @@ -577,23 +577,20 @@ impl RuntimeHostTrait for RuntimeHost { ) -> Result { let block_handler = self.handler_for_block(trigger_type)?; let mapping_block: EthereumBlockType = match trigger_type { - EthereumBlockTriggerType::Every(BlockType::FullWithReceipts) => { - match graph::block_on_allow_panic( - future::lazy(move || { - self.host_exports - .ethereum_adapter - .load_full_block(logger, block.light_block().clone()) - }) - .compat(), - ) { - Ok(block) => Ok(EthereumBlockType::FullWithReceipts(block)), - Err(e) => Err(anyhow::anyhow!( - "Failed to load full block: {}, error: {}", - &block.number().to_string(), - e - )), - }? - } + EthereumBlockTriggerType::Every(BlockType::FullWithReceipts) => match self + .host_exports + .ethereum_adapter + .load_full_block(logger, block.light_block().clone()) + .compat() + .await + { + Ok(block) => Ok(EthereumBlockType::FullWithReceipts(block)), + Err(e) => Err(anyhow::anyhow!( + "Failed to load full block: {}, error: {}", + &block.number().to_string(), + e + )), + }?, EthereumBlockTriggerType::Every(BlockType::Full) => { EthereumBlockType::Full(block.light_block().clone()) } From 2b98eafaedeac349e3804b50e7d85df53096921d Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Sun, 5 Jul 2020 19:43:27 -0700 Subject: [PATCH 08/18] core,graph,runtime: Move BlockType out of EthereumBlockTriggerType --- core/src/subgraph/instance.rs | 6 +++--- graph/src/components/ethereum/adapter.rs | 26 +++++++++++++++++------- graph/src/components/ethereum/mod.rs | 9 ++++---- graph/src/components/ethereum/types.rs | 19 +++++++++++------ graph/src/components/subgraph/host.rs | 2 +- graph/src/lib.rs | 11 +++++----- runtime/wasm/src/host.rs | 22 +++++++++----------- 7 files changed, 57 insertions(+), 38 deletions(-) diff --git a/core/src/subgraph/instance.rs b/core/src/subgraph/instance.rs index 0a81082aa6c..3ce29774182 100644 --- a/core/src/subgraph/instance.rs +++ b/core/src/subgraph/instance.rs @@ -206,16 +206,16 @@ where .await?; } } - EthereumTrigger::Block(ptr, trigger_type) => { + EthereumTrigger::Block(ptr, block_trigger_) => { let matching_hosts = hosts .iter() - .filter(|host| host.matches_block(&trigger_type, ptr.number)); + .filter(|host| host.matches_block(&block_trigger_.trigger_type, ptr.number)); for host in matching_hosts { state = host .process_block( logger, block, - &trigger_type, + &block_trigger_, state, proof_of_indexing.cheap_clone(), ) diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index 2e8da244a41..270e07ee1dd 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -845,16 +845,22 @@ fn parse_block_triggers( .map(move |call| { EthereumTrigger::Block( block_ptr, - EthereumBlockTriggerType::WithCallTo(call.to, block_type), + EthereumBlockTrigger { + block_type, + trigger_type: EthereumBlockTriggerType::WithCallTo(call.to), + }, ) }) .collect::>() }); if trigger_every_block { - triggers.push(dbg!(EthereumTrigger::Block( + triggers.push(EthereumTrigger::Block( block_ptr, - EthereumBlockTriggerType::Every(block_type), - ))); + EthereumBlockTrigger { + block_type, + trigger_type: EthereumBlockTriggerType::Every, + }, + )); } triggers } @@ -961,7 +967,10 @@ pub fn blocks_with_triggers( .map(|ptr| { EthereumTrigger::Block( ptr, - EthereumBlockTriggerType::Every(block_filter.block_type), + EthereumBlockTrigger { + block_type: block_filter.block_type, + trigger_type: EthereumBlockTriggerType::Every, + }, ) }) .collect() @@ -971,14 +980,17 @@ pub fn blocks_with_triggers( // To determine which blocks include a call to addresses // in the block filter, transform the `block_filter` into // a `call_filter` and run `blocks_with_calls` - let block_type = block_filter.block_type.clone(); + let block_type = block_filter.block_type; let call_filter = EthereumCallFilter::from(block_filter); trigger_futs.push(Box::new( eth.calls_in_block_range(&logger, subgraph_metrics.clone(), from, to, call_filter) .map(move |call| { EthereumTrigger::Block( EthereumBlockPointer::from(&call), - EthereumBlockTriggerType::WithCallTo(call.to, block_type.clone()), + EthereumBlockTrigger { + block_type, + trigger_type: EthereumBlockTriggerType::WithCallTo(call.to), + }, ) }) .collect(), diff --git a/graph/src/components/ethereum/mod.rs b/graph/src/components/ethereum/mod.rs index fb148b10270..13ba61f1f24 100644 --- a/graph/src/components/ethereum/mod.rs +++ b/graph/src/components/ethereum/mod.rs @@ -14,8 +14,9 @@ pub use self::listener::{ChainHeadUpdate, ChainHeadUpdateListener, ChainHeadUpda pub use self::stream::{BlockStream, BlockStreamBuilder, BlockStreamEvent}; pub use self::types::{ BlockFinality, BlockType, EthereumBlock, EthereumBlockData, EthereumBlockPointer, - EthereumBlockTriggerType, EthereumBlockType, EthereumBlockWithCalls, EthereumBlockWithTriggers, - EthereumCall, EthereumCallData, EthereumEventData, EthereumTransactionData, - EthereumTransactionReceiptData, EthereumTrigger, FullEthereumBlockData, - FullEthereumBlockDataWithReceipts, LightEthereumBlock, LightEthereumBlockExt, + EthereumBlockTrigger, EthereumBlockTriggerType, EthereumBlockType, EthereumBlockWithCalls, + EthereumBlockWithTriggers, EthereumCall, EthereumCallData, EthereumEventData, + EthereumTransactionData, EthereumTransactionReceiptData, EthereumTrigger, + FullEthereumBlockData, FullEthereumBlockDataWithReceipts, LightEthereumBlock, + LightEthereumBlockExt, }; diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index bd7a85bd8f9..6c13bb3fca4 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -165,7 +165,7 @@ impl EthereumCall { #[derive(Clone, Debug)] pub enum EthereumTrigger { - Block(EthereumBlockPointer, EthereumBlockTriggerType), + Block(EthereumBlockPointer, EthereumBlockTrigger), Call(EthereumCall), Log(Log), } @@ -259,8 +259,8 @@ impl Default for BlockType { #[derive(Clone, Debug, PartialEq, Eq)] pub enum EthereumBlockTriggerType { - Every(BlockType), - WithCallTo(Address, BlockType), + Every, + WithCallTo(Address), } impl EthereumTrigger { @@ -873,7 +873,8 @@ impl ToEntityKey for EthereumBlockPointer { #[cfg(test)] mod test { use super::{ - BlockType, EthereumBlockPointer, EthereumBlockTriggerType, EthereumCall, EthereumTrigger, + BlockType, EthereumBlockPointer, EthereumBlockTrigger, EthereumBlockTriggerType, + EthereumCall, EthereumTrigger, }; use web3::types::*; @@ -884,7 +885,10 @@ mod test { number: 1, hash: H256::random(), }, - EthereumBlockTriggerType::Every(BlockType::Light), + EthereumBlockTrigger { + block_type: BlockType::Light, + trigger_type: EthereumBlockTriggerType::Every, + }, ); let block2 = EthereumTrigger::Block( @@ -892,7 +896,10 @@ mod test { number: 0, hash: H256::random(), }, - EthereumBlockTriggerType::WithCallTo(Address::random(), BlockType::Light), + EthereumBlockTrigger { + block_type: BlockType::Light, + trigger_type: EthereumBlockTriggerType::WithCallTo(Address::random()), + }, ); let mut call1 = EthereumCall::default(); diff --git a/graph/src/components/subgraph/host.rs b/graph/src/components/subgraph/host.rs index ceb56b515fa..31a485db17c 100644 --- a/graph/src/components/subgraph/host.rs +++ b/graph/src/components/subgraph/host.rs @@ -50,7 +50,7 @@ pub trait RuntimeHost: Send + Sync + Debug + 'static { &self, logger: &Logger, block: &Arc, - trigger_type: &EthereumBlockTriggerType, + trigger: &EthereumBlockTrigger, state: BlockState, proof_of_indexing: SharedProofOfIndexing, ) -> Result; diff --git a/graph/src/lib.rs b/graph/src/lib.rs index 642d0037671..57c8164746f 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -77,11 +77,12 @@ pub mod prelude { BlockFinality, BlockStream, BlockStreamBuilder, BlockStreamEvent, BlockStreamMetrics, ChainHeadUpdate, ChainHeadUpdateListener, ChainHeadUpdateStream, EthereumAdapter, EthereumAdapterError, EthereumBlock, EthereumBlockData, EthereumBlockFilter, - EthereumBlockPointer, EthereumBlockTriggerType, EthereumBlockType, EthereumBlockWithCalls, - EthereumBlockWithTriggers, EthereumCall, EthereumCallData, EthereumCallFilter, - EthereumContractCall, EthereumContractCallError, EthereumEventData, EthereumLogFilter, - EthereumNetworkIdentifier, EthereumTransactionData, EthereumTrigger, FullEthereumBlockData, - LightEthereumBlock, LightEthereumBlockExt, ProviderEthRpcMetrics, SubgraphEthRpcMetrics, + EthereumBlockPointer, EthereumBlockTrigger, EthereumBlockTriggerType, EthereumBlockType, + EthereumBlockWithCalls, EthereumBlockWithTriggers, EthereumCall, EthereumCallData, + EthereumCallFilter, EthereumContractCall, EthereumContractCallError, EthereumEventData, + EthereumLogFilter, EthereumNetworkIdentifier, EthereumTransactionData, EthereumTrigger, + FullEthereumBlockData, LightEthereumBlock, LightEthereumBlockExt, ProviderEthRpcMetrics, + SubgraphEthRpcMetrics, }; pub use crate::components::graphql::{GraphQlRunner, SubscriptionResultFuture}; pub use crate::components::link_resolver::{JsonStreamValue, JsonValueStream, LinkResolver}; diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index eae43107d6e..b98e573c263 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -285,13 +285,13 @@ impl RuntimeHost { fn matches_block_trigger(&self, block_trigger_type: &EthereumBlockTriggerType) -> bool { let source_address_matches = match block_trigger_type { - EthereumBlockTriggerType::WithCallTo(address, _block_type) => { + EthereumBlockTriggerType::WithCallTo(address) => { self.data_source_contract .address // Do not match if this datasource has no address .map_or(false, |addr| addr == *address) } - EthereumBlockTriggerType::Every(_) => true, + EthereumBlockTriggerType::Every => true, }; source_address_matches && self.handler_for_block(block_trigger_type).is_ok() } @@ -347,7 +347,7 @@ impl RuntimeHost { trigger_type: &EthereumBlockTriggerType, ) -> Result { match trigger_type { - EthereumBlockTriggerType::Every(_block_type) => self + EthereumBlockTriggerType::Every => self .data_source_block_handlers .iter() .find(move |handler| handler.filter == None) @@ -359,7 +359,7 @@ impl RuntimeHost { self.data_source_name, ) }), - EthereumBlockTriggerType::WithCallTo(_address, _block_type) => self + EthereumBlockTriggerType::WithCallTo(_address) => self .data_source_block_handlers .iter() .find(move |handler| { @@ -571,13 +571,13 @@ impl RuntimeHostTrait for RuntimeHost { &self, logger: &Logger, block: &Arc, - trigger_type: &EthereumBlockTriggerType, + trigger: &EthereumBlockTrigger, state: BlockState, proof_of_indexing: SharedProofOfIndexing, ) -> Result { - let block_handler = self.handler_for_block(trigger_type)?; - let mapping_block: EthereumBlockType = match trigger_type { - EthereumBlockTriggerType::Every(BlockType::FullWithReceipts) => match self + let block_handler = self.handler_for_block(&trigger.trigger_type)?; + let mapping_block: EthereumBlockType = match trigger.block_type { + BlockType::FullWithReceipts => match self .host_exports .ethereum_adapter .load_full_block(logger, block.light_block().clone()) @@ -591,10 +591,8 @@ impl RuntimeHostTrait for RuntimeHost { e )), }?, - EthereumBlockTriggerType::Every(BlockType::Full) => { - EthereumBlockType::Full(block.light_block().clone()) - } - _ => block.as_ref().clone(), + BlockType::Full => EthereumBlockType::Full(block.light_block().clone()), + BlockType::Light => block.as_ref().clone(), }; self.send_mapping_request( From 8ec65f00683d694e49f6c33f94e28febaa420774 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Sun, 5 Jul 2020 19:45:22 -0700 Subject: [PATCH 09/18] graph: Fix Ethereum Block handler from string and write to store --- graph/src/data/subgraph/schema.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graph/src/data/subgraph/schema.rs b/graph/src/data/subgraph/schema.rs index 65130d0f646..4b6227d0c5a 100644 --- a/graph/src/data/subgraph/schema.rs +++ b/graph/src/data/subgraph/schema.rs @@ -1026,7 +1026,7 @@ impl WriteOperations for EthereumBlockHandlerEntity { if let Some(filter_id) = filter_id { entity.set("filter", filter_id); } - entity.set("data", String::from(self.input)); + entity.set("input", String::from(self.input)); ops.add(Self::TYPENAME, id.to_owned(), entity); } } @@ -1128,6 +1128,7 @@ impl FromStr for EthereumBlockHandlerData { fn from_str(s: &str) -> Result { match s { + "FullBlockWithReceipts" => Ok(EthereumBlockHandlerData::FullBlockWithReceipts), "FullBlock" => Ok(EthereumBlockHandlerData::FullBlock), "Block" => Ok(EthereumBlockHandlerData::Block), _ => Err(format_err!( From 2a8ec863646b9412b0a739c1c7a58b239286fe69 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Sun, 5 Jul 2020 20:19:11 -0700 Subject: [PATCH 10/18] graph: Use Ord impl for EthereumBlockHandlerData and BlockType --- graph/src/components/ethereum/adapter.rs | 33 +++++------------------- graph/src/components/ethereum/types.rs | 2 +- graph/src/data/subgraph/schema.rs | 2 +- 3 files changed, 8 insertions(+), 29 deletions(-) diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index 270e07ee1dd..f95d2fe9ea0 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -435,28 +435,14 @@ impl EthereumBlockFilter { .iter() .any(|block_handler| block_handler.filter.is_none()); - let block_type = if data_source + let block_type = data_source .mapping .block_handlers .iter() - .any(|block_handler| match block_handler.input { - BlockHandlerData::FullBlockWithReceipts => return true, - _ => return false, - }) { - BlockType::FullWithReceipts - } else if data_source - .mapping - .block_handlers - .iter() - .any(|block_handler| match block_handler.input { - BlockHandlerData::FullBlock => return true, - _ => return false, - }) - { - BlockType::Full - } else { - BlockType::Light - }; + .map(|handler| &handler.input) + .max() + .unwrap_or(&EthereumBlockHandlerData::Block) + .into(); filter_opt.extend(Self { trigger_every_block: has_block_handler_without_filter, @@ -478,14 +464,7 @@ impl EthereumBlockFilter { pub fn extend(&mut self, other: EthereumBlockFilter) { self.trigger_every_block = self.trigger_every_block || other.trigger_every_block; - self.block_type = match self.block_type { - BlockType::FullWithReceipts => BlockType::FullWithReceipts, - BlockType::Full => match other.block_type { - BlockType::FullWithReceipts => BlockType::FullWithReceipts, - _ => BlockType::Full, - }, - BlockType::Light => other.block_type, - }; + self.block_type = self.block_type.max(other.block_type); self.contract_addresses = self.contract_addresses.iter().cloned().fold( HashSet::new(), diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index 6c13bb3fca4..a6e655c474d 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -244,7 +244,7 @@ impl Default for EthereumBlockType { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq)] pub enum BlockType { Light, Full, diff --git a/graph/src/data/subgraph/schema.rs b/graph/src/data/subgraph/schema.rs index 4b6227d0c5a..bc5b063bf1e 100644 --- a/graph/src/data/subgraph/schema.rs +++ b/graph/src/data/subgraph/schema.rs @@ -1110,7 +1110,7 @@ impl TryFromValue for EthereumBlockHandlerFilterEntity { } } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Deserialize)] pub enum EthereumBlockHandlerData { Block, FullBlock, From ad83da2646d524e79db1d408050927c7df755359 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Sun, 5 Jul 2020 20:20:08 -0700 Subject: [PATCH 11/18] graph: Remove duplicate struct BlockHandlerData EthereumBlockHandlerData has the same variants, no need for both. --- graph/src/components/ethereum/adapter.rs | 1 - graph/src/components/ethereum/types.rs | 20 +++++++++++++++++++- graph/src/data/subgraph/mod.rs | 23 ++--------------------- graph/src/data/subgraph/schema.rs | 9 --------- graph/src/lib.rs | 6 ++++-- 5 files changed, 25 insertions(+), 34 deletions(-) diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index f95d2fe9ea0..a4d447ff86c 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -14,7 +14,6 @@ use web3::types::*; use super::types::*; use crate::components::metrics::{CounterVec, GaugeVec, HistogramVec}; -use crate::data::subgraph::BlockHandlerData; use crate::prelude::*; pub type EventSignature = H256; diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index a6e655c474d..9abf3752f25 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -7,7 +7,9 @@ use std::convert::TryFrom; use std::fmt; use web3::types::*; -use crate::prelude::{EntityKey, SubgraphDeploymentId, ToEntityKey}; +use crate::prelude::{ + anyhow, EntityKey, EthereumBlockHandlerData, SubgraphDeploymentId, ToEntityKey, +}; pub type LightEthereumBlock = Block; @@ -257,6 +259,22 @@ impl Default for BlockType { } } +impl<'a> From<&'a EthereumBlockHandlerData> for BlockType { + fn from(block: &'a EthereumBlockHandlerData) -> BlockType { + match block { + EthereumBlockHandlerData::Block => BlockType::Light, + EthereumBlockHandlerData::FullBlock => BlockType::Full, + EthereumBlockHandlerData::FullBlockWithReceipts => BlockType::FullWithReceipts, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EthereumBlockTrigger { + pub block_type: BlockType, + pub trigger_type: EthereumBlockTriggerType, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum EthereumBlockTriggerType { Every, diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index 866eb956b03..59c1032622b 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -515,26 +515,7 @@ impl UnresolvedMappingABI { pub struct MappingBlockHandler { pub handler: String, pub filter: Option, - pub input: BlockHandlerData, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub enum BlockHandlerData { - Block, - FullBlock, - FullBlockWithReceipts, -} - -impl From for BlockHandlerData { - fn from(data: EthereumBlockHandlerData) -> Self { - match data { - EthereumBlockHandlerData::FullBlockWithReceipts => { - BlockHandlerData::FullBlockWithReceipts - } - EthereumBlockHandlerData::FullBlock => BlockHandlerData::FullBlock, - EthereumBlockHandlerData::Block => BlockHandlerData::Block, - } - } + pub input: EthereumBlockHandlerData, } #[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] @@ -550,7 +531,7 @@ impl From for MappingBlockHandler { Self { handler: entity.handler, filter: None, - input: BlockHandlerData::from(entity.input), + input: entity.input, } } } diff --git a/graph/src/data/subgraph/schema.rs b/graph/src/data/subgraph/schema.rs index bc5b063bf1e..b5fd3ba188f 100644 --- a/graph/src/data/subgraph/schema.rs +++ b/graph/src/data/subgraph/schema.rs @@ -1149,15 +1149,6 @@ impl From for String { } } -impl From for EthereumBlockHandlerData { - fn from(data: BlockHandlerData) -> Self { - match data { - BlockHandlerData::FullBlock => EthereumBlockHandlerData::FullBlock, - _ => EthereumBlockHandlerData::Block, - } - } -} - impl From for q::Value { fn from(data: EthereumBlockHandlerData) -> q::Value { q::Value::Enum(data.into()) diff --git a/graph/src/lib.rs b/graph/src/lib.rs index 57c8164746f..f9cd563dc85 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -125,9 +125,11 @@ pub mod prelude { AssignmentEvent, Attribute, Entity, NodeId, SubgraphEntityPair, SubgraphVersionSummary, ToEntityId, ToEntityKey, TryIntoEntity, Value, ValueType, }; - pub use crate::data::subgraph::schema::{SubgraphDeploymentEntity, TypedEntity}; + pub use crate::data::subgraph::schema::{ + EthereumBlockHandlerData, SubgraphDeploymentEntity, TypedEntity, + }; pub use crate::data::subgraph::{ - BlockHandlerData, BlockHandlerFilter, CreateSubgraphResult, DataSource, DataSourceContext, + BlockHandlerFilter, CreateSubgraphResult, DataSource, DataSourceContext, DataSourceTemplate, Link, MappingABI, MappingBlockHandler, MappingCallHandler, MappingEventHandler, SubgraphAssignmentProviderError, SubgraphAssignmentProviderEvent, SubgraphDeploymentId, SubgraphManifest, SubgraphManifestResolveError, From 1bb1fac3d1cfd0e583fb3e3bc931291e9b3f3347 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Sun, 5 Jul 2020 20:21:11 -0700 Subject: [PATCH 12/18] graph/ethereum: Use the From<> impls to reduce code redundancy --- graph/src/components/ethereum/types.rs | 128 +++++-------------------- 1 file changed, 24 insertions(+), 104 deletions(-) diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index 9abf3752f25..2b11a47ef09 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -382,72 +382,6 @@ pub struct FullEthereumBlockDataWithReceipts { pub transaction_receipts: Vec, } -impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockDataWithReceipts { - type Error = String; - - fn try_from( - block: &'a EthereumBlockType, - ) -> Result { - let fullblock = match block { - EthereumBlockType::FullWithReceipts(full_block) => full_block, - EthereumBlockType::Full(_) => return Err(format!( - "Failed to convert EthereumBlockType to FullEthereumBlockDataWithReceipts, requires a EthereumBlockType::FullWithReceipts()" - )), - EthereumBlockType::Light(_) => return Err(format!( - "Failed to convert EthereumBlockType to FullEthereumBlockDataWithReceipts, requires a EthereumBlockType::FullWithReceipts()" - )) - }; - let block = &fullblock.block; - let transaction_receipts_data = block - .transactions - .iter() - .cloned() - .zip(fullblock.transaction_receipts.iter().cloned()) - .map(|transaction_and_receipt| { - assert_eq!( - transaction_and_receipt.0.hash, - transaction_and_receipt.1.transaction_hash - ); - EthereumTransactionReceiptData { - hash: transaction_and_receipt.0.hash, - index: transaction_and_receipt.1.transaction_index, - cumulative_gas_used: transaction_and_receipt.1.cumulative_gas_used, - gas_used: transaction_and_receipt.1.gas_used, - contract_address: transaction_and_receipt.1.contract_address, - status: transaction_and_receipt.1.status, - root: transaction_and_receipt.1.root, - - // from txs - from: transaction_and_receipt.0.from, - to: transaction_and_receipt.0.to, - value: transaction_and_receipt.0.value, - gas_price: transaction_and_receipt.0.gas_price, - gas: transaction_and_receipt.0.gas, - input: transaction_and_receipt.0.input, - } - }) - .collect::>(); - - Ok(FullEthereumBlockDataWithReceipts { - hash: block.hash.unwrap(), - parent_hash: block.parent_hash, - uncles_hash: block.uncles_hash, - author: block.author, - state_root: block.state_root, - transactions_root: block.transactions_root, - receipts_root: block.receipts_root, - number: block.number.unwrap(), - gas_used: block.gas_used, - gas_limit: block.gas_limit, - timestamp: block.timestamp, - difficulty: block.difficulty, - total_difficulty: block.total_difficulty.unwrap_or_default(), - size: block.size, - transaction_receipts: transaction_receipts_data, - }) - } -} - impl<'a> From<&'a EthereumBlock> for FullEthereumBlockDataWithReceipts { fn from(block: &'a EthereumBlock) -> FullEthereumBlockDataWithReceipts { let transaction_receipts_data = block @@ -502,6 +436,25 @@ impl<'a> From<&'a EthereumBlock> for FullEthereumBlockDataWithReceipts { } } +impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockDataWithReceipts { + type Error = anyhow::Error; + + fn try_from( + block: &'a EthereumBlockType, + ) -> Result { + let fullblock = match block { + EthereumBlockType::FullWithReceipts(full_block) => full_block, + EthereumBlockType::Full(_) => return Err(anyhow::anyhow!( + "Failed to convert EthereumBlockType to FullEthereumBlockDataWithReceipts, requires an EthereumBlockType::FullWithReceipts()" + )), + EthereumBlockType::Light(_) => return Err(anyhow::anyhow!( + "Failed to convert EthereumBlockType to FullEthereumBlockDataWithReceipts, requires an EthereumBlockType::FullWithReceipts()" + )) + }; + Ok(fullblock.into()) + } +} + /// Ethereum block data. #[derive(Clone, Debug, Default)] pub struct FullEthereumBlockData { @@ -550,33 +503,8 @@ impl<'a> From<&'a LightEthereumBlock> for FullEthereumBlockData { impl<'a> From<&'a EthereumBlockType> for FullEthereumBlockData { fn from(block: &'a EthereumBlockType) -> FullEthereumBlockData { - let block = match block { - EthereumBlockType::FullWithReceipts(full_block) => &full_block.block, - EthereumBlockType::Full(block) => &block, - EthereumBlockType::Light(light_block) => light_block, - }; - - FullEthereumBlockData { - hash: block.hash.unwrap(), - parent_hash: block.parent_hash, - uncles_hash: block.uncles_hash, - author: block.author, - state_root: block.state_root, - transactions_root: block.transactions_root, - receipts_root: block.receipts_root, - number: block.number.unwrap(), - gas_used: block.gas_used, - gas_limit: block.gas_limit, - timestamp: block.timestamp, - difficulty: block.difficulty, - total_difficulty: block.total_difficulty.unwrap_or_default(), - size: block.size, - transactions: block - .transactions - .iter() - .map(|tx| EthereumTransactionData::from(tx)) - .collect(), - } + let block = &LightEthereumBlock::from(block); + block.into() } } @@ -622,11 +550,7 @@ impl<'a> From<&'a LightEthereumBlock> for EthereumBlockData { impl<'a> From<&'a EthereumBlockType> for EthereumBlockData { fn from(block: &'a EthereumBlockType) -> EthereumBlockData { - let block = match block { - EthereumBlockType::FullWithReceipts(full_block) => &full_block.block, - EthereumBlockType::Full(full_block) => &full_block, - EthereumBlockType::Light(light_block) => light_block, - }; + let block: LightEthereumBlock = block.into(); EthereumBlockData { hash: block.hash.unwrap(), @@ -816,12 +740,8 @@ impl<'a> From<&'a EthereumBlock> for EthereumBlockPointer { } impl<'a> From<&'a EthereumBlockType> for EthereumBlockPointer { - fn from(b: &'a EthereumBlockType) -> EthereumBlockPointer { - match b { - EthereumBlockType::Light(block) => EthereumBlockPointer::from(block), - EthereumBlockType::Full(block) => EthereumBlockPointer::from(block), - EthereumBlockType::FullWithReceipts(block) => EthereumBlockPointer::from(block), - } + fn from(block_type: &'a EthereumBlockType) -> EthereumBlockPointer { + EthereumBlockPointer::from(LightEthereumBlock::from(block_type)) } } From f5398c560889708fc77f7d8e9175c79f5ed6bc45 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Sun, 5 Jul 2020 20:22:43 -0700 Subject: [PATCH 13/18] runtime/wasm: Impl and use From>> for AscPtr --- runtime/wasm/src/asc_abi/asc_ptr.rs | 9 ++++++++ runtime/wasm/src/to_from/external.rs | 34 +++++++++------------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/runtime/wasm/src/asc_abi/asc_ptr.rs b/runtime/wasm/src/asc_abi/asc_ptr.rs index c3c62438249..c22821e3e4d 100644 --- a/runtime/wasm/src/asc_abi/asc_ptr.rs +++ b/runtime/wasm/src/asc_abi/asc_ptr.rs @@ -74,6 +74,15 @@ impl AscPtr { } } +impl From>> for AscPtr { + fn from(option: Option>) -> Self { + match option { + Some(ptr) => ptr, + None => AscPtr::null(), + } + } +} + impl From for AscPtr { fn from(ptr: u32) -> Self { AscPtr(ptr, PhantomData) diff --git a/runtime/wasm/src/to_from/external.rs b/runtime/wasm/src/to_from/external.rs index b14d1446282..6d72113f5b2 100644 --- a/runtime/wasm/src/to_from/external.rs +++ b/runtime/wasm/src/to_from/external.rs @@ -320,7 +320,7 @@ impl ToAscObj for EthereumBlockData { size: self .size .map(|size| heap.asc_new(&BigInt::from_unsigned_u256(&size))) - .unwrap_or_else(|| AscPtr::null()), + .into(), } } } @@ -344,7 +344,7 @@ impl ToAscObj for FullEthereumBlockData { size: self .size .map(|size| heap.asc_new(&BigInt::from_unsigned_u256(&size))) - .unwrap_or_else(|| AscPtr::null()), + .into(), transactions: heap.asc_new(self.transactions.as_slice()), } } @@ -369,7 +369,7 @@ impl ToAscObj for FullEthereumBlockDataWithRec size: self .size .map(|size| heap.asc_new(&BigInt::from_unsigned_u256(&size))) - .unwrap_or_else(|| AscPtr::null()), + .into(), transaction_receipts: heap.asc_new(self.transaction_receipts.as_slice()), } } @@ -386,26 +386,20 @@ impl ToAscObj for EthereumTransactionReceiptData gas_used: self .gas_used .map(|gas_used| heap.asc_new(&BigInt::from_unsigned_u256(&gas_used))) - .unwrap_or_else(|| AscPtr::null()), + .into(), contract_address: self .contract_address .map(|contract_address| heap.asc_new(&contract_address)) - .unwrap_or_else(|| AscPtr::null()), + .into(), status: self .status .map(|status| heap.asc_new(&BigInt::from(status))) - .unwrap_or_else(|| AscPtr::null()), - root: self - .root - .map(|root| heap.asc_new(&root)) - .unwrap_or_else(|| AscPtr::null()), + .into(), + root: self.root.map(|root| heap.asc_new(&root)).into(), // // from txs from: heap.asc_new(&self.from), - to: self - .to - .map(|to| heap.asc_new(&to)) - .unwrap_or_else(|| AscPtr::null()), + to: self.to.map(|to| heap.asc_new(&to)).into(), value: heap.asc_new(&BigInt::from_unsigned_u256(&self.value)), gas_price: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_price)), gas: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas)), @@ -420,10 +414,7 @@ impl ToAscObj for EthereumTransactionData { hash: heap.asc_new(&self.hash), index: heap.asc_new(&BigInt::from(self.index)), from: heap.asc_new(&self.from), - to: self - .to - .map(|to| heap.asc_new(&to)) - .unwrap_or_else(|| AscPtr::null()), + to: self.to.map(|to| heap.asc_new(&to)).into(), value: heap.asc_new(&BigInt::from_unsigned_u256(&self.value)), gas_used: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_used)), gas_price: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_price)), @@ -437,10 +428,7 @@ impl ToAscObj for EthereumTransactionData { hash: heap.asc_new(&self.hash), index: heap.asc_new(&BigInt::from(self.index)), from: heap.asc_new(&self.from), - to: self - .to - .map(|to| heap.asc_new(&to)) - .unwrap_or_else(|| AscPtr::null()), + to: self.to.map(|to| heap.asc_new(&to)).into(), value: heap.asc_new(&BigInt::from_unsigned_u256(&self.value)), gas_used: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_used)), gas_price: heap.asc_new(&BigInt::from_unsigned_u256(&self.gas_price)), @@ -463,7 +451,7 @@ where .log_type .clone() .map(|log_type| heap.asc_new(&log_type)) - .unwrap_or_else(|| AscPtr::null()), + .into(), block: heap.asc_new(&self.block), transaction: heap.asc_new::(&self.transaction), params: heap.asc_new(self.params.as_slice()), From daf740f1fef83e850032839143c0ef624d781089 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Sun, 5 Jul 2020 20:23:25 -0700 Subject: [PATCH 14/18] runtime/wasm: Do not clone block from context in handle_ethereum_block() --- runtime/wasm/src/module/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index 3a665cf8486..53704bc7807 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -161,7 +161,7 @@ impl WasmInstance { mut self, handler_name: &str, ) -> Result { - let block = self.instance_ctx().ctx.block.clone(); + let block: &Arc = &self.take_ctx().ctx.block; // Prepare an Ethereum Block for the WASM runtime let arg = match block.as_ref() { From e6288d109d0b33518bb6c7195f86d87c82a7e422 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Mon, 13 Jul 2020 08:46:47 -0700 Subject: [PATCH 15/18] graph,runtime: Remove root field on transaction receipt (no on Geth) --- graph/src/components/ethereum/types.rs | 3 +-- runtime/wasm/src/asc_abi/class.rs | 1 - runtime/wasm/src/to_from/external.rs | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index 2b11a47ef09..9cea3915ef9 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -347,13 +347,13 @@ impl PartialOrd for EthereumTrigger { pub struct EthereumTransactionReceiptData { // from receipts + // Geth nodes do not support `root` so it is not included pub hash: H256, pub index: Index, pub cumulative_gas_used: U256, pub gas_used: Option, pub contract_address: Option, pub status: Option, - pub root: Option, // from txs pub from: H160, @@ -402,7 +402,6 @@ impl<'a> From<&'a EthereumBlock> for FullEthereumBlockDataWithReceipts { gas_used: transaction_and_receipt.1.gas_used, contract_address: transaction_and_receipt.1.contract_address, status: transaction_and_receipt.1.status, - root: transaction_and_receipt.1.root, // from txs from: transaction_and_receipt.0.from, diff --git a/runtime/wasm/src/asc_abi/class.rs b/runtime/wasm/src/asc_abi/class.rs index 10feaaec548..52ad5d8fe71 100644 --- a/runtime/wasm/src/asc_abi/class.rs +++ b/runtime/wasm/src/asc_abi/class.rs @@ -479,7 +479,6 @@ pub(crate) struct AscEthereumTransactionReceipt { pub gas_used: AscPtr, pub contract_address: AscPtr, pub status: AscPtr, - pub root: AscPtr, pub from: AscPtr, pub to: AscPtr, diff --git a/runtime/wasm/src/to_from/external.rs b/runtime/wasm/src/to_from/external.rs index 6d72113f5b2..f501787b2f8 100644 --- a/runtime/wasm/src/to_from/external.rs +++ b/runtime/wasm/src/to_from/external.rs @@ -395,7 +395,6 @@ impl ToAscObj for EthereumTransactionReceiptData .status .map(|status| heap.asc_new(&BigInt::from(status))) .into(), - root: self.root.map(|root| heap.asc_new(&root)).into(), // // from txs from: heap.asc_new(&self.from), From 9609561608d4e013654366a2b0c9ca15889d6494 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Mon, 13 Jul 2020 08:51:20 -0700 Subject: [PATCH 16/18] core,graph: Ethereum struct improvements, impl copy, make clone explicit --- core/src/subgraph/instance.rs | 5 ++- graph/src/components/ethereum/adapter.rs | 4 +-- graph/src/components/ethereum/types.rs | 45 +++++++++++------------- graph/src/data/subgraph/schema.rs | 10 +++--- 4 files changed, 29 insertions(+), 35 deletions(-) diff --git a/core/src/subgraph/instance.rs b/core/src/subgraph/instance.rs index 3ce29774182..b54942a924e 100644 --- a/core/src/subgraph/instance.rs +++ b/core/src/subgraph/instance.rs @@ -165,12 +165,11 @@ where let transaction = block .transaction_for_log(&log) - .map(Arc::new) + .map(|tx| Arc::new(tx.clone())) .context("Found no transaction for event")?; let matching_hosts = hosts.iter().filter(|host| host.matches_log(&log)); // Process the log in each host in the same order the corresponding data // sources appear in the subgraph manifest - let transaction = Arc::new(transaction); for host in matching_hosts { state = host .process_log( @@ -189,8 +188,8 @@ where let transaction = block .transaction_for_call(&call) + .map(|tx| Arc::new(tx.clone())) .context("Found no transaction for call")?; - let transaction = Arc::new(transaction); let matching_hosts = hosts.iter().filter(|host| host.matches_call(&call)); for host in matching_hosts { diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index a4d447ff86c..ceaa4ede9ab 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -438,9 +438,9 @@ impl EthereumBlockFilter { .mapping .block_handlers .iter() - .map(|handler| &handler.input) + .map(|handler| handler.input) .max() - .unwrap_or(&EthereumBlockHandlerData::Block) + .unwrap_or(EthereumBlockHandlerData::Block) .into(); filter_opt.extend(Self { diff --git a/graph/src/components/ethereum/types.rs b/graph/src/components/ethereum/types.rs index 9cea3915ef9..359c132790e 100644 --- a/graph/src/components/ethereum/types.rs +++ b/graph/src/components/ethereum/types.rs @@ -200,6 +200,7 @@ pub enum EthereumBlockType { FullWithReceipts(EthereumBlock), } + impl EthereumBlockType { pub fn light_block(&self) -> &LightEthereumBlock { match self { @@ -217,26 +218,22 @@ impl EthereumBlockType { self.light_block().number.unwrap().as_u64() } - pub fn transaction_for_log(&self, log: &Log) -> Option { - log.transaction_hash - .and_then(|hash| { - self.light_block() - .transactions - .iter() - .find(|tx| tx.hash == hash) - }) - .cloned() + pub fn transaction_for_log(&self, log: &Log) -> Option<&Transaction> { + log.transaction_hash.and_then(|hash| { + self.light_block() + .transactions + .iter() + .find(|tx| tx.hash == hash) + }) } - pub fn transaction_for_call(&self, call: &EthereumCall) -> Option { - call.transaction_hash - .and_then(|hash| { - self.light_block() - .transactions - .iter() - .find(|tx| tx.hash == hash) - }) - .cloned() + pub fn transaction_for_call(&self, call: &EthereumCall) -> Option<&Transaction> { + call.transaction_hash.and_then(|hash| { + self.light_block() + .transactions + .iter() + .find(|tx| tx.hash == hash) + }) } } @@ -259,8 +256,8 @@ impl Default for BlockType { } } -impl<'a> From<&'a EthereumBlockHandlerData> for BlockType { - fn from(block: &'a EthereumBlockHandlerData) -> BlockType { +impl From for BlockType { + fn from(block: EthereumBlockHandlerData) -> BlockType { match block { EthereumBlockHandlerData::Block => BlockType::Light, EthereumBlockHandlerData::FullBlock => BlockType::Full, @@ -364,6 +361,7 @@ pub struct EthereumTransactionReceiptData { pub input: Bytes, } +/// Ethereum block data with transactions and their receipts. pub struct FullEthereumBlockDataWithReceipts { pub hash: H256, pub parent_hash: H256, @@ -443,18 +441,15 @@ impl<'a> TryFrom<&'a EthereumBlockType> for FullEthereumBlockDataWithReceipts { ) -> Result { let fullblock = match block { EthereumBlockType::FullWithReceipts(full_block) => full_block, - EthereumBlockType::Full(_) => return Err(anyhow::anyhow!( + EthereumBlockType::Full(_) | EthereumBlockType::Light(_) => return Err(anyhow::anyhow!( "Failed to convert EthereumBlockType to FullEthereumBlockDataWithReceipts, requires an EthereumBlockType::FullWithReceipts()" )), - EthereumBlockType::Light(_) => return Err(anyhow::anyhow!( - "Failed to convert EthereumBlockType to FullEthereumBlockDataWithReceipts, requires an EthereumBlockType::FullWithReceipts()" - )) }; Ok(fullblock.into()) } } -/// Ethereum block data. +/// Ethereum block data with transactions. #[derive(Clone, Debug, Default)] pub struct FullEthereumBlockData { pub hash: H256, diff --git a/graph/src/data/subgraph/schema.rs b/graph/src/data/subgraph/schema.rs index b5fd3ba188f..cdfde345d33 100644 --- a/graph/src/data/subgraph/schema.rs +++ b/graph/src/data/subgraph/schema.rs @@ -1026,7 +1026,7 @@ impl WriteOperations for EthereumBlockHandlerEntity { if let Some(filter_id) = filter_id { entity.set("filter", filter_id); } - entity.set("input", String::from(self.input)); + entity.set("input", self.input); ops.add(Self::TYPENAME, id.to_owned(), entity); } } @@ -1110,7 +1110,7 @@ impl TryFromValue for EthereumBlockHandlerFilterEntity { } } -#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Deserialize)] pub enum EthereumBlockHandlerData { Block, FullBlock, @@ -1149,9 +1149,9 @@ impl From for String { } } -impl From for q::Value { - fn from(data: EthereumBlockHandlerData) -> q::Value { - q::Value::Enum(data.into()) +impl From for Value { + fn from(data: EthereumBlockHandlerData) -> Value { + Value::String(data.into()) } } From 1a31f8363539b683ce58fe933f28c0bd384213bf Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Mon, 13 Jul 2020 08:52:59 -0700 Subject: [PATCH 17/18] runtime: Reference the mapping context safely in handle_ethereum_block --- runtime/wasm/src/module/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index 53704bc7807..fdef3833c15 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -161,13 +161,13 @@ impl WasmInstance { mut self, handler_name: &str, ) -> Result { - let block: &Arc = &self.take_ctx().ctx.block; + let context = self.take_ctx(); // Prepare an Ethereum Block for the WASM runtime - let arg = match block.as_ref() { + let arg = match context.ctx.block.as_ref() { EthereumBlockType::FullWithReceipts(block) => self .asc_new::( - &FullEthereumBlockDataWithReceipts::from(block), + &FullEthereumBlockDataWithReceipts::try_from(block).unwrap(), ) .erase(), EthereumBlockType::Full(block) => self @@ -180,7 +180,7 @@ impl WasmInstance { self.invoke_handler(handler_name, arg)?; - Ok(self.take_ctx().ctx.state) + Ok(context.ctx.state) } pub(crate) fn take_ctx(&mut self) -> WasmInstanceContext { From b0530228f821fa8eca87fb7e64d2ed3936047986 Mon Sep 17 00:00:00 2001 From: Ford Nickels Date: Thu, 16 Jul 2020 10:57:48 -0700 Subject: [PATCH 18/18] graph: Rename BlockHandler input -> blockFormat --- graph/src/components/ethereum/adapter.rs | 2 +- graph/src/data/subgraph/mod.rs | 4 ++-- graph/src/data/subgraph/schema.rs | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/graph/src/components/ethereum/adapter.rs b/graph/src/components/ethereum/adapter.rs index ceaa4ede9ab..aa1a409ef44 100644 --- a/graph/src/components/ethereum/adapter.rs +++ b/graph/src/components/ethereum/adapter.rs @@ -438,7 +438,7 @@ impl EthereumBlockFilter { .mapping .block_handlers .iter() - .map(|handler| handler.input) + .map(|handler| handler.block_format) .max() .unwrap_or(EthereumBlockHandlerData::Block) .into(); diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index 59c1032622b..2a70f2eef3a 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -515,7 +515,7 @@ impl UnresolvedMappingABI { pub struct MappingBlockHandler { pub handler: String, pub filter: Option, - pub input: EthereumBlockHandlerData, + pub block_format: EthereumBlockHandlerData, } #[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] @@ -531,7 +531,7 @@ impl From for MappingBlockHandler { Self { handler: entity.handler, filter: None, - input: entity.input, + block_format: entity.block_format, } } } diff --git a/graph/src/data/subgraph/schema.rs b/graph/src/data/subgraph/schema.rs index cdfde345d33..1ff4627263b 100644 --- a/graph/src/data/subgraph/schema.rs +++ b/graph/src/data/subgraph/schema.rs @@ -1009,7 +1009,7 @@ impl TryFromValue for EthereumContractAbiEntity { pub struct EthereumBlockHandlerEntity { pub handler: String, pub filter: Option, - pub input: EthereumBlockHandlerData, + pub block_format: EthereumBlockHandlerData, } impl WriteOperations for EthereumBlockHandlerEntity { @@ -1026,7 +1026,7 @@ impl WriteOperations for EthereumBlockHandlerEntity { if let Some(filter_id) = filter_id { entity.set("filter", filter_id); } - entity.set("input", self.input); + entity.set("input", self.block_format); ops.add(Self::TYPENAME, id.to_owned(), entity); } } @@ -1050,7 +1050,7 @@ impl From for EthereumBlockHandlerEntity { EthereumBlockHandlerEntity { handler: block_handler.handler, filter, - input: EthereumBlockHandlerData::from(block_handler.input), + block_format: EthereumBlockHandlerData::from(block_handler.block_format), } } } @@ -1068,7 +1068,7 @@ impl TryFromValue for EthereumBlockHandlerEntity { Ok(EthereumBlockHandlerEntity { handler: map.get_required("handler")?, filter: map.get_optional("filter")?, - input: map.get_optional("input")?.unwrap_or_default(), + block_format: map.get_optional("blockFormat")?.unwrap_or_default(), }) } }