Merge branch 'master' into ao-lmdb

* master:
  CI improvements (#10579)
  Watch transactions pool (#10558)
  fix(evmbin): make benches compile again (#10586)
  fix issue with compilation when 'slow-blocks' feature enabled (#10585)
  Reject crazy timestamps instead of truncating. (#10574)
  Node table limiting and cache for node filter (#10288)

.gitlab-ci.yml

@@ -31,116 +31,164 @@ variables:
     paths:
       - artifacts/
 
+.collect_logs:                     &collect_logs
+  artifacts:
+    name:                          "${CI_JOB_NAME}_${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}"
+    when:                          always
+    expire_in:                     7 days
+    paths:
+      - artifacts/
+
 .docker-cache-status:              &docker-cache-status
   variables:
     CARGO_HOME:                    "/ci-cache/parity-ethereum/cargo/${CI_JOB_NAME}"
+  dependencies:                    []
   before_script:
-    - SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_error.log RUST_LOG=sccache::server=debug sccache --start-server
+    - SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_debug.log
+        RUST_LOG=sccache::server=debug
+        SCCACHE_CACHE_SIZE=50G
+        SCCACHE_DIR=/ci-cache/parity-ethereum/sccache/
+        sccache --start-server
     - sccache -s
   after_script:
-    - echo "All crate-types:"
-    - grep 'parse_arguments.*--crate-type' sccache_error.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c
-    - echo "Non-cacheable reasons:"
-    - grep CannotCache sccache_error.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c
+    # sccache debug info
+    - if test -e sccache_debug.log;
+      then
+        echo "All crate-types:";
+        grep 'parse_arguments.*--crate-type' sccache_debug.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c;
+        echo "Non-cacheable reasons:";
+        grep CannotCache sccache_debug.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c;
+      else
+        echo "No logs from sccache";
+        exit 0;
+      fi
+    # collect log files
+    - mkdir -p ./artifacts
+    - find . -name "*.log"
+    - find . -name "*.log" | xargs tar --append -f ./artifacts/logs_"${CI_JOB_NAME}_${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}".tar
   tags:
     - linux-docker
 
+.build-on-linux:                   &build-on-linux
+  stage:                           build
+  <<:                              *docker-cache-status
+  <<:                              *collect_artifacts
+  script:
+    - scripts/gitlab/build-linux.sh
+    - sccache -s
+
 
 cargo-check 0 3:
   stage:                           test
   <<:                              *docker-cache-status
+  <<:                              *collect_logs
   script:
-    - time cargo check --target $CARGO_TARGET --locked --no-default-features
+    - time cargo check --target $CARGO_TARGET --locked --no-default-features --verbose --color=always
     - sccache -s
 
 cargo-check 1 3:
   stage:                           test
   <<:                              *docker-cache-status
+  <<:                              *collect_logs
   script:
-    - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features
+    - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features --verbose --color=always
     - sccache -s
 
 cargo-check 2 3:
   stage:                           test
   <<:                              *docker-cache-status
+  <<:                              *collect_logs
   script:
-    - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio"
+    - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio" --verbose --color=always
     - sccache -s
 
 cargo-audit:
   stage:                           test
   <<:                              *docker-cache-status
   script:
     - cargo audit
-    - sccache -s
 
 validate-chainspecs:
   stage:                           test
   <<:                              *docker-cache-status
+  <<:                              *collect_logs
   script:
     - ./scripts/gitlab/validate-chainspecs.sh
     - sccache -s
 
 test-cpp:
   stage:                           build
   <<:                              *docker-cache-status
+  <<:                              *collect_logs
   script:
     - ./scripts/gitlab/test-cpp.sh
     - sccache -s
 
 test-linux:
   stage:                           build
   <<:                              *docker-cache-status
+  <<:                              *collect_logs
   script:
-    - ./scripts/gitlab/test-linux.sh
+    - ./scripts/gitlab/test-linux.sh stable
     - sccache -s
 
-build-android:
+test-linux-beta:
   stage:                           build
-  image:                           parity/rust-parity-ethereum-android-build:stretch
-  variables:
-    CARGO_TARGET:                  armv7-linux-androideabi
+  only:                            *releaseable_branches
   <<:                              *docker-cache-status
-  <<:                              *collect_artifacts
+  <<:                              *collect_logs
   script:
-    - scripts/gitlab/build-linux.sh
-  tags:
-    - linux-docker
+    - ./scripts/gitlab/test-linux.sh beta
+    - sccache -s
 
-build-linux:                       &build-linux
+test-linux-nightly:
   stage:                           build
   only:                            *releaseable_branches
   <<:                              *docker-cache-status
-  <<:                              *collect_artifacts
+  <<:                              *collect_logs
   script:
-    - scripts/gitlab/build-linux.sh
+    - ./scripts/gitlab/test-linux.sh nightly
     - sccache -s
+  allow_failure:                   true
+
+build-android:
+  <<:                              *build-on-linux
+  image:                           parity/rust-parity-ethereum-android-build:stretch
+  variables:
+    CARGO_TARGET:                  armv7-linux-androideabi
+
+build-linux:
+  <<:                              *build-on-linux
+  only:                            *releaseable_branches
 
 build-linux-i386:
-  <<:                              *build-linux
+  <<:                              *build-on-linux
+  only:                            *releaseable_branches
   image:                           parity/rust-parity-ethereum-build:i386
   variables:
     CARGO_TARGET:                  i686-unknown-linux-gnu
 
 build-linux-arm64:
-  <<:                              *build-linux
+  <<:                              *build-on-linux
+  only:                            *releaseable_branches
   image:                           parity/rust-parity-ethereum-build:arm64
   variables:
     CARGO_TARGET:                  aarch64-unknown-linux-gnu
 
 build-linux-armhf:
-  <<:                              *build-linux
+  <<:                              *build-on-linux
+  only:                            *releaseable_branches
   image:                           parity/rust-parity-ethereum-build:armhf
   variables:
     CARGO_TARGET:                  armv7-unknown-linux-gnueabihf
 
 build-darwin:
   stage:                           build
-  only:                            *releaseable_branches
   <<:                              *collect_artifacts
+  only:                            *releaseable_branches
   variables:
     CARGO_TARGET:                  x86_64-apple-darwin
-    CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
+    CARGO_HOME:                    "${CI_PROJECT_DIR}/.cargo"
     CC:                            gcc
     CXX:                           g++
   script:

ethcore/Cargo.toml

@@ -30,6 +30,7 @@ ethereum-types = "0.4"
 ethjson = { path = "../json" }
 ethkey = { path = "../accounts/ethkey" }
 evm = { path = "evm" }
+futures = "0.1"
 hash-db = "0.11.0"
 heapsize = "0.4"
 itertools = "0.5"

ethcore/light/Cargo.toml

@@ -18,6 +18,7 @@ memory-db = "0.11.0"
 trie-db = "0.11.0"
 patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
 ethcore-network = { path = "../../util/network" }
+ethcore-miner = { path = "../../miner" }
 ethcore-io = { path = "../../util/io" }
 hash-db = "0.11.0"
 heapsize = "0.4"

ethcore/light/src/lib.rs

@@ -61,6 +61,7 @@ extern crate ethcore_io as io;
 extern crate ethcore_network as network;
 extern crate parity_bytes as bytes;
 extern crate ethereum_types;
+extern crate ethcore_miner as miner;
 extern crate ethcore;
 extern crate hash_db;
 extern crate heapsize;

ethcore/light/src/transaction_queue.rs

@@ -24,12 +24,15 @@
 //! address-wise manner.
 
 use std::fmt;
+use std::sync::Arc;
 use std::collections::{BTreeMap, HashMap};
 use std::collections::hash_map::Entry;
 
 use common_types::transaction::{self, Condition, PendingTransaction, SignedTransaction};
 use ethereum_types::{H256, U256, Address};
 use fastmap::H256FastMap;
+use futures::sync::mpsc;
+use miner::pool::TxStatus;
 
 // Knowledge of an account's current nonce.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -134,6 +137,7 @@ pub struct TransactionQueue {
 	by_account: HashMap<Address, AccountTransactions>,
 	by_hash: H256FastMap<PendingTransaction>,
 	listeners: Vec<Listener>,
+	tx_statuses_listeners: Vec<mpsc::UnboundedSender<Arc<Vec<(H256, TxStatus)>>>>,
 }
 
 impl fmt::Debug for TransactionQueue {
@@ -231,7 +235,7 @@ impl TransactionQueue {
 		};
 
 		self.by_hash.insert(hash, tx);
-		self.notify(&promoted);
+		self.notify(&promoted, TxStatus::Added);
 		Ok(res)
 	}
 
@@ -343,6 +347,8 @@ impl TransactionQueue {
 		trace!(target: "txqueue", "Culled {} old transactions from sender {} (nonce={})",
 			removed_hashes.len(), address, cur_nonce);
 
+		self.notify(&removed_hashes, TxStatus::Culled);
+
 		for hash in removed_hashes {
 			self.by_hash.remove(&hash);
 		}
@@ -358,11 +364,26 @@ impl TransactionQueue {
 		self.listeners.push(f);
 	}
 
+	/// Add a transaction queue listener.
+	pub fn tx_statuses_receiver(&mut self) -> mpsc::UnboundedReceiver<Arc<Vec<(H256, TxStatus)>>> {
+		let (sender, receiver) = mpsc::unbounded();
+		self.tx_statuses_listeners.push(sender);
+		receiver
+	}
+
 	/// Notifies all listeners about new pending transaction.
-	fn notify(&self, hashes: &[H256]) {
+	fn notify(&mut self, hashes: &[H256], status: TxStatus) {
 		for listener in &self.listeners {
 			listener(hashes)
 		}
+
+		let to_send: Arc<Vec<(H256, TxStatus)>> = Arc::new(
+			hashes
+				.into_iter()
+				.map(|hash| (hash.clone(), status)).collect()
+		);
+
+		self.tx_statuses_listeners.retain(| listener| listener.unbounded_send(to_send.clone()).is_ok());
 	}
 }
 

ethcore/node-filter/src/lib.rs

@@ -37,28 +37,45 @@ extern crate tempdir;
 #[macro_use]
 extern crate log;
 
+use std::collections::{HashMap, VecDeque};
 use std::sync::Weak;
 
-use ethcore::client::{BlockChainClient, BlockId};
+use ethcore::client::{BlockChainClient, BlockId, ChainNotify, NewBlocks};
+
 use ethereum_types::{H256, Address};
 use ethabi::FunctionOutputDecoder;
 use network::{ConnectionFilter, ConnectionDirection};
 use devp2p::NodeId;
+use devp2p::MAX_NODES_IN_TABLE;
+use parking_lot::RwLock;
 
 use_contract!(peer_set, "res/peer_set.json");
 
 /// Connection filter that uses a contract to manage permissions.
 pub struct NodeFilter {
 	client: Weak<BlockChainClient>,
 	contract_address: Address,
+	cache: RwLock<Cache>
+}
+
+struct Cache {
+	cache: HashMap<NodeId, bool>,
+	order: VecDeque<NodeId>
 }
 
+// Increase cache size due to possible reserved peers, which do not count in the node table size
+pub const CACHE_SIZE: usize = MAX_NODES_IN_TABLE + 1024;
+
 impl NodeFilter {
 	/// Create a new instance. Accepts a contract address.
 	pub fn new(client: Weak<BlockChainClient>, contract_address: Address) -> NodeFilter {
 		NodeFilter {
 			client,
 			contract_address,
+			cache: RwLock::new(Cache{
+				cache: HashMap::with_capacity(CACHE_SIZE),
+				order: VecDeque::with_capacity(CACHE_SIZE)
+			})
 		}
 	}
 }
@@ -70,6 +87,10 @@ impl ConnectionFilter for NodeFilter {
 			None => return false,
 		};
 
+		if let Some(allowed) = self.cache.read().cache.get(connecting_id) {
+			return *allowed;
+		}
+
 		let address = self.contract_address;
 		let own_low = H256::from_slice(&own_id[0..32]);
 		let own_high = H256::from_slice(&own_id[32..64]);
@@ -83,11 +104,26 @@ impl ConnectionFilter for NodeFilter {
 				debug!("Error callling peer set contract: {:?}", e);
 				false
 			});
-
+		let mut cache = self.cache.write();
+		if cache.cache.len() == CACHE_SIZE {
+			let poped = cache.order.pop_front().unwrap();
+			cache.cache.remove(&poped).is_none();
+		};
+		if cache.cache.insert(*connecting_id, allowed).is_none() {
+			cache.order.push_back(*connecting_id);
+		}
 		allowed
 	}
 }
 
+impl ChainNotify for NodeFilter {
+	fn new_blocks(&self, _new_blocks: NewBlocks)	{
+		let mut cache = self.cache.write();
+		cache.cache.clear();
+		cache.order.clear();
+	}
+}
+
 #[cfg(test)]
 mod test {
 	use std::sync::{Arc, Weak};