Skip to content
This repository has been archived by the owner on Nov 6, 2020. It is now read-only.

New Transaction Queue implementation #8074

Merged
merged 97 commits into from
Apr 13, 2018
Merged
Show file tree
Hide file tree
Changes from 52 commits
Commits
Show all changes
97 commits
Select commit Hold shift + click to select a range
c467c7d
Implementation of Verifier, Scoring and Ready.
tomusdrw Jan 3, 2018
15a52b7
Queue in progress.
tomusdrw Jan 29, 2018
d358050
TransactionPool.
tomusdrw Jan 30, 2018
34e1eb1
Prepare for txpool release.
tomusdrw Jan 30, 2018
1b5ff18
Merge branch 'td-txpool-releas' into td-txpool2
tomusdrw Jan 30, 2018
f77a46d
Miner refactor [WiP]
tomusdrw Feb 2, 2018
b176521
Merge remote-tracking branch 'origin/master' into td-txpool2
tomusdrw Feb 6, 2018
d966a4b
Merge branch 'master' into td-txpool2
tomusdrw Feb 9, 2018
43e8289
WiP reworking miner.
tomusdrw Feb 12, 2018
5e783eb
Make it compile.
tomusdrw Feb 14, 2018
71fa691
Add some docs.
tomusdrw Feb 14, 2018
a70f1c3
Split blockchain access to a separate file.
tomusdrw Feb 15, 2018
afae75b
Work on miner API.
tomusdrw Feb 16, 2018
47e8928
Fix ethcore tests.
tomusdrw Feb 20, 2018
4b25b86
Refactor miner interface for sealing/work packages.
tomusdrw Feb 20, 2018
81a96e4
Implement next nonce.
tomusdrw Feb 20, 2018
b538413
RPC compiles.
tomusdrw Feb 21, 2018
3368324
Implement couple of missing methdods for RPC.
tomusdrw Feb 22, 2018
7a24420
Merge branch 'master' into td-txpool2
tomusdrw Feb 22, 2018
fc4a0aa
Add transaction queue listeners.
tomusdrw Feb 23, 2018
e595e83
Compiles!
tomusdrw Feb 23, 2018
620e6be
Merge remote-tracking branch 'origin/master' into td-txpool2
tomusdrw Feb 23, 2018
5a6cdf5
Clean-up and parallelize.
tomusdrw Feb 23, 2018
0f2424c
Get rid of RefCell in header.
tomusdrw Feb 23, 2018
a1cb18c
Revert "Get rid of RefCell in header."
tomusdrw Feb 23, 2018
e902bec
Override Sync requirement.
tomusdrw Feb 23, 2018
fede1ea
Fix status display.
tomusdrw Feb 23, 2018
5075cec
Unify logging.
tomusdrw Feb 23, 2018
449facf
Extract some cheap checks.
tomusdrw Feb 23, 2018
e2be467
Measurements and optimizations.
tomusdrw Feb 27, 2018
c07ea86
Merge remote-tracking branch 'origin/master' into td-txpool2
tomusdrw Feb 27, 2018
b720077
Fix scoring bug, heap size of bug and add cache
tomusdrw Feb 28, 2018
fe74263
Merge remote-tracking branch 'origin/master' into td-txpool2
tomusdrw Feb 28, 2018
8312e4a
Disable tx queueing and parallel verification.
tomusdrw Feb 28, 2018
51e2f43
Merge remote-tracking branch 'origin/master' into td-txpool2
tomusdrw Mar 5, 2018
54965d6
Make ethcore and ethcore-miner compile again.
tomusdrw Mar 5, 2018
43e9358
Make RPC compile again.
tomusdrw Mar 5, 2018
236ac24
Bunch of txpool tests.
tomusdrw Mar 5, 2018
d698145
Migrate transaction queue tests.
tomusdrw Mar 6, 2018
e97e453
Nonce Cap
tomusdrw Mar 6, 2018
f079b91
Nonce cap cache and tests.
tomusdrw Mar 7, 2018
41a5acc
Remove stale future transactions from the queue.
tomusdrw Mar 7, 2018
35c7783
Optimize scoring and write some tests.
tomusdrw Mar 7, 2018
886224a
Simple penalization.
tomusdrw Mar 8, 2018
b4d9341
Clean up and support for different scoring algorithms.
tomusdrw Mar 8, 2018
03e89ec
Add CLI parameters for the new queue.
tomusdrw Mar 8, 2018
900b446
Remove banning queue.
tomusdrw Mar 8, 2018
7dab842
Merge branch 'master' into td-txpool2
tomusdrw Mar 8, 2018
8046ef9
Disable debug build.
tomusdrw Mar 8, 2018
83c92bc
Change per_sender limit to be 1% instead of 5%
tomusdrw Mar 8, 2018
eb025db
Avoid cloning when propagating transactions.
tomusdrw Mar 9, 2018
1f1694b
Remove old todo.
tomusdrw Mar 9, 2018
893131f
Merge branch 'master' into td-txpool2
tomusdrw Mar 12, 2018
660b6b0
Post-review fixes.
tomusdrw Mar 12, 2018
39c6bb4
Merge branch 'master' into td-txpool2
tomusdrw Mar 12, 2018
006d635
Fix miner options default.
tomusdrw Mar 12, 2018
a6b9324
Implement back ready transactions for light client.
tomusdrw Mar 12, 2018
cfc2200
Get rid of from_pending_block
tomusdrw Mar 12, 2018
ce02494
Pass rejection reason.
tomusdrw Mar 12, 2018
68b3f2a
Add more details to drop.
tomusdrw Mar 12, 2018
72f49d4
Rollback heap size of.
tomusdrw Mar 12, 2018
8ab2912
Avoid cloning hashes when propagating and include more details on rej…
tomusdrw Mar 13, 2018
901e532
Fix tests.
tomusdrw Mar 13, 2018
7f15140
Introduce nonces cache.
tomusdrw Mar 13, 2018
1dcc47e
Remove uneccessary hashes allocation.
tomusdrw Mar 13, 2018
a74f18a
Lower the mem limit.
tomusdrw Mar 13, 2018
6d7d16b
Merge branch 'master' into td-txpool2
tomusdrw Mar 13, 2018
bdfdb3d
Re-enable parallel verification.
tomusdrw Mar 13, 2018
2d1da26
Add miner log. Don't check the type if not below min_gas_price.
tomusdrw Mar 13, 2018
cca0db7
Add more traces, fix disabling miner.
tomusdrw Mar 14, 2018
281e54d
Fix creating pending blocks twice on AuRa authorities.
tomusdrw Mar 14, 2018
6a304c4
Fix tests.
tomusdrw Mar 14, 2018
383a908
re-use pending blocks in AuRa
tomusdrw Mar 15, 2018
7aaff4f
Merge branch 'master' into td-txpool2
tomusdrw Mar 16, 2018
93f93a0
Use reseal_min_period to prevent too frequent update_sealing.
tomusdrw Mar 16, 2018
7f290e7
Fix log to contain hash not sender.
tomusdrw Mar 16, 2018
bed88ee
Optimize local transactions.
tomusdrw Mar 16, 2018
4a7940e
Fix aura tests.
tomusdrw Mar 16, 2018
6dcea90
Merge remote-tracking branch 'origin/master' into td-txpool2
tomusdrw Mar 20, 2018
c36d39d
Update locks comments.
tomusdrw Mar 20, 2018
c92812b
Get rid of unsafe Sync impl.
tomusdrw Mar 20, 2018
be75720
Review fixes.
tomusdrw Mar 21, 2018
357fcf3
Remove excessive matches.
tomusdrw Mar 21, 2018
f41b0f2
Merge branch 'master' into td-txpool2
tomusdrw Apr 9, 2018
da0f845
Fix compilation errors.
tomusdrw Apr 9, 2018
296cc42
Merge branch 'master' into td-txpool2
tomusdrw Apr 9, 2018
e199b86
Merge branch 'master' into td-txpool2
tomusdrw Apr 10, 2018
285f4fd
Use new pool in private transactions.
tomusdrw Apr 10, 2018
af81bee
Fix private-tx test.
tomusdrw Apr 11, 2018
53f328c
Merge branch 'master' into td-txpool2
tomusdrw Apr 11, 2018
5e012ae
Merge branch 'master' into td-txpool2
tomusdrw Apr 11, 2018
0e2bf52
Fix secret store tests.
tomusdrw Apr 11, 2018
7f15c30
Actually use gas_floor_target
tomusdrw Apr 11, 2018
483b70c
Fix config tests.
tomusdrw Apr 11, 2018
b7789e2
Fix pool tests.
tomusdrw Apr 11, 2018
1c757e0
Merge branch 'master' into td-txpool2
tomusdrw Apr 12, 2018
d7c8b94
Address grumbles.
tomusdrw Apr 13, 2018
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 86 additions & 39 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -131,5 +131,5 @@ members = [
"miner",
"transaction-pool",
"whisper",
"util/rlp_compress"
"util/rlp_compress",
]
4 changes: 1 addition & 3 deletions ethcore/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@ num = "0.1"
num_cpus = "1.2"
parity-machine = { path = "../machine" }
parking_lot = "0.5"
price-info = { path = "../price-info" }
rayon = "0.8"
rayon = "1.0"
rand = "0.4"
rlp = { path = "../util/rlp" }
rlp_compress = { path = "../util/rlp_compress" }
Expand All @@ -62,7 +61,6 @@ stats = { path = "../util/stats" }
time = "0.1"
trace-time = { path = "../util/trace-time" }
using_queue = { path = "../util/using_queue" }
table = { path = "../util/table" }
vm = { path = "vm" }
wasm = { path = "wasm" }
keccak-hash = { path = "../util/hash" }
Expand Down
2 changes: 1 addition & 1 deletion ethcore/light/src/net/tests/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ impl Provider for TestProvider {
})
}

fn ready_transactions(&self) -> Vec<PendingTransaction> {
fn ready_transactions(&self) -> Vec<SignedTransaction> {
self.0.client.ready_transactions()
}
}
Expand Down
3 changes: 2 additions & 1 deletion ethcore/light/src/provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,8 @@ impl<T: ProvingBlockChainClient + ?Sized> Provider for T {
}

fn ready_transactions(&self) -> Vec<PendingTransaction> {
BlockChainClient::ready_transactions(self)
unimplemented!()
// BlockChainClient::ready_transactions(self)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

needs to be reimplemented

}

fn epoch_signal(&self, req: request::CompleteSignalRequest) -> Option<request::SignalResponse> {
Expand Down
24 changes: 18 additions & 6 deletions ethcore/light/src/transaction_queue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,18 @@ impl AccountTransactions {
}
}

/// Transaction import result.
pub enum ImportResult {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe ImportResult is not very idiomatic, since in Rust Result usually means stuff that may fail, so the second outcome is treated as an error of some kind. However that's not always the case even in standard library. For example, some search methods return Result<usize, usize>.

In our case Future is definitely not an error, so I think it would be better to avoid Result in its name. Maybe something like Status would work?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree, but worth noting that this is a type that existed before, only moved.

/// Transaction has been imported to the current queue.
///
/// It's going to be propagated to peers.
Current,
/// Transaction has been imported to future queue.
///
/// It means it won't be propagated until the gap is filled.
Future
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not very clear for me why these options were selected. Current is not an action/result for me, it's more like status.

Maybe adopt something like Imported and Deferred? At first I wanted to use Ok for the first option, but it would clash with standard Result::Ok.

}

type Listener = Box<Fn(&[H256]) + Send + Sync>;

/// Light transaction queue. See module docs for more details.
Expand All @@ -142,7 +154,7 @@ impl fmt::Debug for TransactionQueue {

impl TransactionQueue {
/// Import a pending transaction to be queued.
pub fn import(&mut self, tx: PendingTransaction) -> Result<transaction::ImportResult, transaction::Error> {
pub fn import(&mut self, tx: PendingTransaction) -> Result<ImportResult, transaction::Error> {
let sender = tx.sender();
let hash = tx.hash();
let nonce = tx.nonce;
Expand All @@ -158,7 +170,7 @@ impl TransactionQueue {
future: BTreeMap::new(),
});

(transaction::ImportResult::Current, vec![hash])
(ImportResult::Current, vec![hash])
}
Entry::Occupied(mut entry) => {
let acct_txs = entry.get_mut();
Expand All @@ -180,7 +192,7 @@ impl TransactionQueue {
let old = ::std::mem::replace(&mut acct_txs.current[idx], tx_info);
self.by_hash.remove(&old.hash);

(transaction::ImportResult::Current, vec![hash])
(ImportResult::Current, vec![hash])
}
Err(idx) => {
let cur_len = acct_txs.current.len();
Expand All @@ -202,13 +214,13 @@ impl TransactionQueue {
acct_txs.future.insert(future_nonce, future);
}

(transaction::ImportResult::Current, vec![hash])
(ImportResult::Current, vec![hash])
} else if idx == cur_len && acct_txs.current.last().map_or(false, |f| f.nonce + 1.into() != nonce) {
trace!(target: "txqueue", "Queued future transaction for {}, nonce={}", sender, nonce);
let future_nonce = nonce;
acct_txs.future.insert(future_nonce, tx_info);

(transaction::ImportResult::Future, vec![])
(ImportResult::Future, vec![])
} else {
trace!(target: "txqueue", "Queued current transaction for {}, nonce={}", sender, nonce);

Expand All @@ -217,7 +229,7 @@ impl TransactionQueue {
let mut promoted = acct_txs.adjust_future();
promoted.insert(0, hash);

(transaction::ImportResult::Current, promoted)
(ImportResult::Current, promoted)
}
}
}
Expand Down
3 changes: 0 additions & 3 deletions ethcore/src/account_provider/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,6 @@ pub enum SignError {
Hardware(HardwareError),
/// Low-level error from store
SStore(SSError),
/// Inappropriate chain
InappropriateChain,
}

impl fmt::Display for SignError {
Expand All @@ -77,7 +75,6 @@ impl fmt::Display for SignError {
SignError::NotFound => write!(f, "Account does not exist"),
SignError::Hardware(ref e) => write!(f, "{}", e),
SignError::SStore(ref e) => write!(f, "{}", e),
SignError::InappropriateChain => write!(f, "Inappropriate chain"),
}
}
}
Expand Down
81 changes: 39 additions & 42 deletions ethcore/src/block.rs
Original file line number Diff line number Diff line change
Expand Up @@ -336,8 +336,33 @@ impl<'x> OpenBlock<'x> {
}

/// Push transactions onto the block.
pub fn push_transactions(&mut self, transactions: &[SignedTransaction]) -> Result<(), Error> {
push_transactions(self, transactions)
#[cfg(not(feature = "slow-blocks"))]
fn push_transactions(&mut self, transactions: Vec<SignedTransaction>) -> Result<(), Error> {
for t in transactions {
self.push_transaction(t, None)?;
}
Ok(())
}

/// Push transactions onto the block.
#[cfg(feature = "slow-blocks")]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd love to see a comment explaining what slow-blocks feature intends.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added documentation in ethcore/Cargo.toml

fn push_transactions(&mut self, transactions: Vec<SignedTransaction>) -> Result<(), Error> {
use std::time;

let slow_tx = option_env!("SLOW_TX_DURATION").and_then(|v| v.parse().ok()).unwrap_or(100);
for t in transactions {
let hash = t.hash();
let start = time::Instant::now();
self.push_transaction(t, None)?;
let took = start.elapsed();
let took_ms = took.as_secs() * 1000 + took.subsec_nanos() as u64 / 1000000;
if took > time::Duration::from_millis(slow_tx) {
warn!("Heavy ({} ms) transaction in block {:?}: {:?}", took_ms, block.header().number(), hash);
}
debug!(target: "tx", "Transaction {:?} took: {} ms", hash, took_ms);
}

Ok(())
}

/// Populate self from a header.
Expand Down Expand Up @@ -534,10 +559,10 @@ impl IsBlock for SealedBlock {
}

/// Enact the block given by block header, transactions and uncles
pub fn enact(
header: &Header,
transactions: &[SignedTransaction],
uncles: &[Header],
fn enact(
header: Header,
transactions: Vec<SignedTransaction>,
uncles: Vec<Header>,
engine: &EthEngine,
tracing: bool,
db: StateDB,
Expand Down Expand Up @@ -567,48 +592,19 @@ pub fn enact(
is_epoch_begin,
)?;

b.populate_from(header);
b.populate_from(&header);
b.push_transactions(transactions)?;

for u in uncles {
b.push_uncle(u.clone())?;
b.push_uncle(u)?;
}

Ok(b.close_and_lock())
}

#[inline]
#[cfg(not(feature = "slow-blocks"))]
fn push_transactions(block: &mut OpenBlock, transactions: &[SignedTransaction]) -> Result<(), Error> {
for t in transactions {
block.push_transaction(t.clone(), None)?;
}
Ok(())
}

#[cfg(feature = "slow-blocks")]
fn push_transactions(block: &mut OpenBlock, transactions: &[SignedTransaction]) -> Result<(), Error> {
use std::time;

let slow_tx = option_env!("SLOW_TX_DURATION").and_then(|v| v.parse().ok()).unwrap_or(100);
for t in transactions {
let hash = t.hash();
let start = time::Instant::now();
block.push_transaction(t.clone(), None)?;
let took = start.elapsed();
let took_ms = took.as_secs() * 1000 + took.subsec_nanos() as u64 / 1000000;
if took > time::Duration::from_millis(slow_tx) {
warn!("Heavy ({} ms) transaction in block {:?}: {:?}", took_ms, block.header().number(), hash);
}
debug!(target: "tx", "Transaction {:?} took: {} ms", hash, took_ms);
}
Ok(())
}

// TODO [ToDr] Pass `PreverifiedBlock` by move, this will avoid unecessary allocation
/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
pub fn enact_verified(
block: &PreverifiedBlock,
block: PreverifiedBlock,
engine: &EthEngine,
tracing: bool,
db: StateDB,
Expand All @@ -620,9 +616,9 @@ pub fn enact_verified(
let view = BlockView::new(&block.bytes);

enact(
&block.header,
&block.transactions,
&view.uncles(),
block.header,
block.transactions,
view.uncles(),
engine,
tracing,
db,
Expand Down Expand Up @@ -690,7 +686,7 @@ mod tests {
)?;

b.populate_from(&header);
b.push_transactions(&transactions)?;
b.push_transactions(transactions)?;

for u in &block.uncles() {
b.push_uncle(u.clone())?;
Expand Down Expand Up @@ -783,3 +779,4 @@ mod tests {
assert!(orig_db.journal_db().keys().iter().filter(|k| orig_db.journal_db().get(k.0) != db.journal_db().get(k.0)).next() == None);
}
}

Loading