Backports for 2.1.0 beta (#9518)

* parity-version: mark 2.1.0 track beta

* ci: update branch version references

* docker: release master to latest

* Fix checkpointing when creating contract failed (#9514)

* ci: fix json docs generation (#9515)

* fix typo in version string (#9516)

* Update patricia trie to 0.2.2 crates. Default dependencies on minor
version only.

* Putting back ethereum tests to the right commit

* Enable all Constantinople hard fork changes in constantinople_test.json (#9505)

* Enable all Constantinople hard fork changes in constantinople_test.json

* Address grumbles

* Remove EIP-210 activation

* 8m -> 5m

* Temporarily add back eip210 transition so we can get test passed

* Add eip210_test and remove eip210 transition from const_test

* In create memory calculation is the same for create2 because the additional parameter was popped before. (#9522)

* deps: bump fs-swap and kvdb-rocksdb

* Multithreaded snapshot creation (#9239)

* Add Progress to Snapshot Secondary chunks creation

* Use half of CPUs to multithread snapshot creation

* Use env var to define number of threads

* info to debug logs

* Add Snapshot threads as CLI option

* Randomize chunks per thread

* Remove randomness, add debugging

* Add warning

* Add tracing

* Use parity-common fix seek branch

* Fix log

* Fix tests

* Fix tests

* PR Grumbles

* PR Grumble II

* Update Cargo.lock

* PR Grumbles

* Default snapshot threads to half number of CPUs

* Fix default snapshot threads // min 1

* correct before_script for nightly build versions (#9543)

- fix gitlab array of strings syntax error
- get proper commit id
- avoid colon in stings

* Remove initial token for WS. (#9545)

* version: mark release critical

* ci: fix rpc docs generation 2 (#9550)

* Improve P2P discovery (#9526)

* Add `target` to Rust traces

* network-devp2p: Don't remove discovery peer in main sync

* network-p2p: Refresh discovery more often

* Update Peer discovery protocol

* Run discovery more often when not enough nodes connected

* Start the first discovery early

* Update fast discovery rate

* Fix tests

* Fix `ping` tests

* Fixing remote Node address ; adding PingPong round

* Fix tests: update new +1 PingPong round

* Increase slow Discovery rate
Check in flight FindNode before pings

* Add `deprecated` to deprecated_echo_hash

* Refactor `discovery_round` branching

* net_version caches network_id to avoid redundant aquire of sync read lock (#9544)

* net_version caches network_id to avoid redundant aquire of sync read lock, #8746

* use lower_hex display formatting for net_peerCount rpc method

* Increase Gas-floor-target and Gas Cap (#9564)

+ Gas-floor-target increased to 8M by default

+ Gas-cap increased to 10M by default

* Revert to old parity-tokio-ipc.

* Downgrade named pipes.

.gitlab-ci.yml

@@ -41,17 +41,14 @@ cache:
     paths:
     - artifacts/
 
-.determine_version:
-  before_script:                   &determine_version
-    - >
-      VERSION="$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' < Cargo.toml)";
-      if [ "${CI_COMMIT_REF_NAME}" = "nightly" ]; then
-        COMMIT_REF_SHORT="echo ${CI_COMMIT_REF} | grep -oE '^.{7}')";
-        DATE_STRING="$(date +%Y%m%d)";
-        export VERSION="${VERSION}-${COMMIT_REF_SHORT}-${DATE_STRING}";
-      fi;
-      export VERSION;
-      echo "Version: $VERSION"
+.determine_version:                &determine_version
+  - VERSION="$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)"
+  - DATE_STR="$(date +%Y%m%d)"
+  - ID_SHORT="$(echo ${CI_COMMIT_SHA} | cut -c 1-7)"
+  - test "${CI_COMMIT_REF_NAME}" = "nightly" && VERSION="${VERSION}-${ID_SHORT}-${DATE_STR}"
+  - export VERSION
+  - echo "Version = ${VERSION}"
+
 
 
 #### stage:                        test
@@ -256,7 +253,7 @@ publish-linux-snap-armhf:
 
 publish-docker-parity-amd64: &publish_docker
   stage:                           publish
-  only:                            *publishable_branches
+  only:                            *releaseable_branches
   cache: {}
   dependencies:
     - build-linux-ubuntu-amd64

ethcore/Cargo.toml

@@ -18,7 +18,7 @@ ethcore-bloom-journal = { path = "../util/bloom" }
 parity-bytes = "0.1"
 hashdb = "0.2.1"
 memorydb = "0.2.1"
-patricia-trie = "0.2.1"
+patricia-trie = "0.2"
 patricia-trie-ethereum = { path = "../util/patricia-trie-ethereum" }
 parity-crypto = "0.1"
 error-chain = { version = "0.12", default-features = false }

ethcore/evm/src/interpreter/gasometer.rs

@@ -233,11 +233,7 @@ impl<Gas: evm::CostType> Gasometer<Gas> {
 			},
 			instructions::CREATE | instructions::CREATE2 => {
 				let gas = Gas::from(schedule.create_gas);
-				let mem = match instruction {
-					instructions::CREATE => mem_needed(stack.peek(1), stack.peek(2))?,
-					instructions::CREATE2 => mem_needed(stack.peek(2), stack.peek(3))?,
-					_ => unreachable!("instruction can only be CREATE/CREATE2 checked above; qed"),
-				};
+				let mem = mem_needed(stack.peek(1), stack.peek(2))?;
 
 				Request::GasMemProvide(gas, mem, None)
 			},

ethcore/light/Cargo.toml

@@ -13,7 +13,7 @@ parity-bytes = "0.1"
 ethcore-transaction = { path = "../transaction" }
 ethereum-types = "0.4"
 memorydb = "0.2.1"
-patricia-trie = "0.2.1"
+patricia-trie = "0.2"
 patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
 ethcore-network = { path = "../../util/network" }
 ethcore-io = { path = "../../util/io" }

ethcore/private-tx/Cargo.toml

@@ -26,7 +26,7 @@ heapsize = "0.4"
 keccak-hash = "0.1.2"
 log = "0.4"
 parking_lot = "0.6"
-patricia-trie = "0.2.1"
+patricia-trie = "0.2"
 patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
 rand = "0.3"
 rlp = { version = "0.2.4", features = ["ethereum"] }

ethcore/res/ethereum/constantinople_test.json

@@ -1,16 +1,16 @@
 {
-	"name": "Byzantium (Test)",
+	"name": "Constantinople (test)",
 	"engine": {
 		"Ethash": {
 			"params": {
 				"minimumDifficulty": "0x020000",
 				"difficultyBoundDivisor": "0x0800",
 				"durationLimit": "0x0d",
-				"blockReward": "0x29A2241AF62C0000",
+				"blockReward": "0x1BC16D674EC80000",
 				"homesteadTransition": "0x0",
 				"eip100bTransition": "0x0",
 				"difficultyBombDelays": {
-					"0": 3000000
+					"0": 5000000
 				}
 			}
 		}
@@ -30,11 +30,13 @@
 		"eip161abcTransition": "0x0",
 		"eip161dTransition": "0x0",
 		"eip140Transition": "0x0",
-		"eip210Transition": "0x0",
 		"eip211Transition": "0x0",
 		"eip214Transition": "0x0",
 		"eip155Transition": "0x0",
 		"eip658Transition": "0x0",
+		"eip145Transition": "0x0",
+		"eip1014Transition": "0x0",
+		"eip1052Transition": "0x0",
 		"eip1283Transition": "0x0"
 	},
 	"genesis": {

ethcore/res/ethereum/eip210_test.json

@@ -0,0 +1,54 @@
+{
+	"name": "EIP210 (test)",
+	"engine": {
+		"Ethash": {
+			"params": {
+				"minimumDifficulty": "0x020000",
+				"difficultyBoundDivisor": "0x0800",
+				"durationLimit": "0x0d",
+				"blockReward": "0x4563918244F40000",
+				"homesteadTransition": "0x0"
+			}
+		}
+	},
+	"params": {
+		"gasLimitBoundDivisor": "0x0400",
+		"registrar" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b",
+		"accountStartNonce": "0x00",
+		"maximumExtraDataSize": "0x20",
+		"minGasLimit": "0x1388",
+		"networkID" : "0x1",
+		"maxCodeSize": 24576,
+		"maxCodeSizeTransition": "0x0",
+		"eip98Transition": "0xffffffffffffffff",
+		"eip150Transition": "0x0",
+		"eip160Transition": "0x0",
+		"eip161abcTransition": "0x0",
+		"eip161dTransition": "0x0",
+		"eip210Transition": "0x0"
+	},
+	"genesis": {
+		"seal": {
+			"ethereum": {
+				"nonce": "0x0000000000000042",
+				"mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000"
+			}
+		},
+		"difficulty": "0x400000000",
+		"author": "0x0000000000000000000000000000000000000000",
+		"timestamp": "0x00",
+		"parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+		"extraData": "0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa",
+		"gasLimit": "0x1388"
+	},
+	"accounts": {
+		"0000000000000000000000000000000000000001": { "balance": "1", "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } },
+		"0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } },
+		"0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } },
+		"0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } },
+		"0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 100 } } } },
+		"0000000000000000000000000000000000000006": { "builtin": { "name": "alt_bn128_add", "activate_at": "0x00", "pricing": { "linear": { "base": 500, "word": 0 } } } },
+		"0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 2000, "word": 0 } } } },
+		"0000000000000000000000000000000000000008": { "builtin": { "name": "alt_bn128_pairing", "activate_at": "0x00", "pricing": { "alt_bn128_pairing": { "base": 100000, "pair": 80000 } } } }
+	}
+}

ethcore/src/client/client.rs

@@ -1153,7 +1153,8 @@ impl Client {
 			},
 		};
 
-		snapshot::take_snapshot(&*self.engine, &self.chain.read(), start_hash, db.as_hashdb(), writer, p)?;
+		let processing_threads = self.config.snapshot.processing_threads;
+		snapshot::take_snapshot(&*self.engine, &self.chain.read(), start_hash, db.as_hashdb(), writer, p, processing_threads)?;
 
 		Ok(())
 	}

ethcore/src/client/config.rs

@@ -19,6 +19,7 @@ use std::fmt::{Display, Formatter, Error as FmtError};
 
 use verification::{VerifierType, QueueConfig};
 use journaldb;
+use snapshot::SnapshotConfiguration;
 
 pub use std::time::Duration;
 pub use blockchain::Config as BlockChainConfig;
@@ -120,6 +121,8 @@ pub struct ClientConfig {
 	pub check_seal: bool,
 	/// Maximal number of transactions queued for verification in a separate thread.
 	pub transaction_verification_queue_size: usize,
+	/// Snapshot configuration
+	pub snapshot: SnapshotConfiguration,
 }
 
 impl Default for ClientConfig {
@@ -144,6 +147,7 @@ impl Default for ClientConfig {
 			history_mem: 32 * mb,
 			check_seal: true,
 			transaction_verification_queue_size: 8192,
+			snapshot: Default::default(),
 		}
 	}
 }

ethcore/src/ethereum/mod.rs

@@ -152,6 +152,9 @@ pub fn new_frontier_test_machine() -> EthereumMachine { load_machine(include_byt
 /// Create a new Foundation Homestead-era chain spec as though it never changed from Frontier.
 pub fn new_homestead_test_machine() -> EthereumMachine { load_machine(include_bytes!("../../res/ethereum/homestead_test.json")) }
 
+/// Create a new Foundation Homestead-EIP210-era chain spec as though it never changed from Homestead/Frontier.
+pub fn new_eip210_test_machine() -> EthereumMachine { load_machine(include_bytes!("../../res/ethereum/eip210_test.json")) }
+
 /// Create a new Foundation Byzantium era spec.
 pub fn new_byzantium_test_machine() -> EthereumMachine { load_machine(include_bytes!("../../res/ethereum/byzantium_test.json")) }
 

ethcore/src/snapshot/consensus/authority.rs

@@ -30,7 +30,7 @@ use machine::EthereumMachine;
 use ids::BlockId;
 use header::Header;
 use receipt::Receipt;
-use snapshot::{Error, ManifestData};
+use snapshot::{Error, ManifestData, Progress};
 
 use itertools::{Position, Itertools};
 use rlp::{RlpStream, Rlp};
@@ -59,6 +59,7 @@ impl SnapshotComponents for PoaSnapshot {
 		chain: &BlockChain,
 		block_at: H256,
 		sink: &mut ChunkSink,
+		_progress: &Progress,
 		preferred_size: usize,
 	) -> Result<(), Error> {
 		let number = chain.block_number(&block_at)

ethcore/src/snapshot/consensus/mod.rs

@@ -22,7 +22,7 @@ use std::sync::Arc;
 
 use blockchain::{BlockChain, BlockChainDB};
 use engines::EthEngine;
-use snapshot::{Error, ManifestData};
+use snapshot::{Error, ManifestData, Progress};
 
 use ethereum_types::H256;
 
@@ -49,6 +49,7 @@ pub trait SnapshotComponents: Send {
 		chain: &BlockChain,
 		block_at: H256,
 		chunk_sink: &mut ChunkSink,
+		progress: &Progress,
 		preferred_size: usize,
 	) -> Result<(), Error>;
 

ethcore/src/snapshot/consensus/work.rs

@@ -28,7 +28,7 @@ use std::sync::Arc;
 
 use blockchain::{BlockChain, BlockChainDB, BlockProvider};
 use engines::EthEngine;
-use snapshot::{Error, ManifestData};
+use snapshot::{Error, ManifestData, Progress};
 use snapshot::block::AbridgedBlock;
 use ethereum_types::H256;
 use kvdb::KeyValueDB;
@@ -65,13 +65,15 @@ impl SnapshotComponents for PowSnapshot {
 		chain: &BlockChain,
 		block_at: H256,
 		chunk_sink: &mut ChunkSink,
+		progress: &Progress,
 		preferred_size: usize,
 	) -> Result<(), Error> {
 		PowWorker {
 			chain: chain,
 			rlps: VecDeque::new(),
 			current_hash: block_at,
 			writer: chunk_sink,
+			progress: progress,
 			preferred_size: preferred_size,
 		}.chunk_all(self.blocks)
 	}
@@ -96,6 +98,7 @@ struct PowWorker<'a> {
 	rlps: VecDeque<Bytes>,
 	current_hash: H256,
 	writer: &'a mut ChunkSink<'a>,
+	progress: &'a Progress,
 	preferred_size: usize,
 }
 
@@ -138,6 +141,7 @@ impl<'a> PowWorker<'a> {
 
 			last = self.current_hash;
 			self.current_hash = block.header_view().parent_hash();
+			self.progress.blocks.fetch_add(1, Ordering::SeqCst);
 		}
 
 		if loaded_size != 0 {