Add constantinople conf to EvmTestClient. (#9570)

* Add constantinople conf to EvmTestClient.

* Skip some test to update submodule etheureum/tests submodule to latest.

* Put skipping 'under issue' test behind a feature.

* Change blockReward for const-test to pass ethereum/tests

* Update tests to new constantinple definition (change of reward at block
5).
Switch 'reference' to string, that way we can include issues from others
repo (more flexible)Update tests to new constantinple definition (change
of reward at block 5).
Switch 'reference' to string, that way we can include issues from others
repo (more flexible).

* Fix modexp and bn128_mul gas prices in chain config

* Changes `run_test_path` method to append its directory results (without
that it stop testing at the first file failure).
Add some missing tests.
Add skip for those (block create2 is one hundred percent false but on
hive we can see that geth and aleth got similar issue for this item).

* retab current.json

* Update reference to parity issue for failing tests.

Cargo.toml

@@ -89,6 +89,7 @@ daemonize = { git = "https://github.com/paritytech/daemonize" }
 [features]
 miner-debug = ["ethcore/miner-debug"]
 json-tests = ["ethcore/json-tests"]
+ci-skip-issue = ["ethcore/ci-skip-issue"]
 test-heavy = ["ethcore/test-heavy"]
 evm-debug = ["ethcore/evm-debug"]
 evm-debug-tests = ["ethcore/evm-debug-tests"]

ethcore/Cargo.toml

@@ -102,6 +102,8 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 slow-blocks = []
 # Run JSON consensus tests.
 json-tests = ["ethcore-transaction/json-tests", "test-helpers", "tempdir"]
+# Skip JSON consensus tests with pending issues.
+ci-skip-issue = []
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile benches

ethcore/res/ethereum/constantinople_test.json

@@ -6,12 +6,12 @@
 				"minimumDifficulty": "0x020000",
 				"difficultyBoundDivisor": "0x0800",
 				"durationLimit": "0x0d",
-				"blockReward": "0x1BC16D674EC80000",
+				"blockReward": "0x29A2241AF62C0000",
 				"homesteadTransition": "0x0",
 				"eip100bTransition": "0x0",
-				"eip649Transition": "0x0",
+				"eip649Transition": "0x5",
 				"eip649Reward": "0x1BC16D674EC80000",
-				"eip1234Transition": "0x0",
+				"eip1234Transition": "0x5",
 				"eip1234Reward": "0x1BC16D674EC80000"
 			}
 		}
@@ -59,9 +59,9 @@
 		"0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } },
 		"0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } },
 		"0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } },
-		"0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 100 } } } },
+		"0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 20 } } } },
 		"0000000000000000000000000000000000000006": { "builtin": { "name": "alt_bn128_add", "activate_at": "0x00", "pricing": { "linear": { "base": 500, "word": 0 } } } },
-		"0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 2000, "word": 0 } } } },
+		"0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 40000, "word": 0 } } } },
 		"0000000000000000000000000000000000000008": { "builtin": { "name": "alt_bn128_pairing", "activate_at": "0x00", "pricing": { "alt_bn128_pairing": { "base": 100000, "pair": 80000 } } } }
 	}
 }

ethcore/res/ethereum/tests-issues/currents.json

@@ -365,104 +365,6 @@
 					"chain": "Constantinople (test)"
 				}
 			}
-		},
-		{
-			"reference": "9590",
-			"failing": "stCreate2Test",
-			"subtests": {
-				"call_then_create2_successful_then_returndatasize": {
-					"subnumbers": ["1"],
-					"chain": "Constantinople (test)"
-				},
-				"returndatacopy_afterFailing_create": {
-					"subnumbers": ["1"],
-					"chain": "Constantinople (test)"
-				},
-				"create2checkFieldsInInitcode": {
-					"subnumbers": ["1","2","3","5","6","7"],
-					"chain": "Constantinople (test)"
-				},
-				"Create2Recursive": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"create2collisionBalance": {
-					"subnumbers": ["2","3"],
-					"chain": "Constantinople (test)"
-				},
-				"create2InitCodes": {
-					"subnumbers": ["1","5","6","7","8","9"],
-					"chain": "Constantinople (test)"
-				},
-				"Create2OOGafterInitCode": {
-					"subnumbers": ["2"],
-					"chain": "Constantinople (test)"
-				},
-				"CreateMessageRevertedOOGInInit": {
-					"subnumbers": ["2"],
-					"chain": "Constantinople (test)"
-				},
-				"returndatacopy_following_revert_in_create": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"create2collisionSelfdestructed": {
-					"subnumbers": ["2"],
-					"chain": "Constantinople (test)"
-				},
-				"returndatacopy_0_0_following_successful_create": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"Create2OnDepth1023": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"Create2OOGafterInitCodeReturndata2": {
-					"subnumbers": ["2"],
-					"chain": "Constantinople (test)"
-				},
-				"RevertOpcodeInCreateReturns": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"CREATE2_ContractSuicideDuringInit_ThenStoreThenReturn": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"returndatasize_following_successful_create": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"call_outsize_then_create2_successful_then_returndatasize": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"CreateMessageReverted": {
-					"subnumbers": ["2"],
-					"chain": "Constantinople (test)"
-				},
-				"CREATE2_Suicide": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"Create2OOGafterInitCodeRevert": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"Create2OnDepth1024": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				},
-				"create2collisionStorage": {
-					"subnumbers": ["2","3"],
-					"chain": "Constantinople (test)"
-				},
-				"create2callPrecompiles": {
-					"subnumbers": ["*"],
-					"chain": "Constantinople (test)"
-				}
-			}
 		}
 	]
 }

ethcore/src/client/evm_test_client.rs

@@ -86,9 +86,9 @@ impl<'a> EvmTestClient<'a> {
 			ForkSpec::EIP150 => Some(ethereum::new_eip150_test()),
 			ForkSpec::EIP158 => Some(ethereum::new_eip161_test()),
 			ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()),
+			ForkSpec::Constantinople => Some(ethereum::new_constantinople_test()),
 			ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()),
 			ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None,
-			_ => None,
 		}
 	}
 

ethcore/src/json_tests/chain.rs

@@ -23,6 +23,7 @@ use ethjson;
 use miner::Miner;
 use io::IoChannel;
 use test_helpers;
+use super::SKIP_TEST_STATE;
 
 use super::HookType;
 
@@ -36,12 +37,20 @@ pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
 	::json_tests::test_common::run_test_file(p, json_chain_test, h)
 }
 
+fn skip_test(name: &String) -> bool {
+	SKIP_TEST_STATE.block.iter().any(|block_test|block_test.subtests.contains(name))
+}
+
 pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::blockchain::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 
 	for (name, blockchain) in tests.into_iter() {
+		if skip_test(&name) {
+			println!("   - {} | {:?} Ignoring tests because in skip list", name, blockchain.network);
+			continue;
+		}
 		start_stop_hook(&name, HookType::OnStart);
 
 		let mut fail = false;
@@ -122,19 +131,24 @@ mod block_tests {
 	declare_test!{BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"}
 	declare_test!{BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"}
 	declare_test!{BlockchainTests_bcRandomBlockhashTest, "BlockchainTests/bcRandomBlockhashTest"}
+	declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTests"}
 	declare_test!{BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"}
 	declare_test!{BlockchainTests_bcUncleHeaderValidity, "BlockchainTests/bcUncleHeaderValidity"}
 	declare_test!{BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"}
 	declare_test!{BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"}
 	declare_test!{BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"}
 
+	declare_test!{BlockchainTests_GeneralStateTest_stArgsZeroOneBalance, "BlockchainTests/GeneralStateTests/stArgsZeroOneBalance/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stAttackTest, "BlockchainTests/GeneralStateTests/stAttackTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stBadOpcodeTest, "BlockchainTests/GeneralStateTests/stBadOpcode/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stBugsTest, "BlockchainTests/GeneralStateTests/stBugs/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCallCodes, "BlockchainTests/GeneralStateTests/stCallCodes/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stCallCreateCallCodeTest, "BlockchainTests/GeneralStateTests/stCallCreateCallCodeTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesHomestead/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stChangedEIP150, "BlockchainTests/GeneralStateTests/stChangedEIP150/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCodeSizeLimit, "BlockchainTests/GeneralStateTests/stCodeSizeLimit/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stCreate2, "BlockchainTests/GeneralStateTests/stCreate2/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCreateTest, "BlockchainTests/GeneralStateTests/stCreateTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stDelegatecallTestHomestead, "BlockchainTests/GeneralStateTests/stDelegatecallTestHomestead/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stEIP150singleCodeGasPrices, "BlockchainTests/GeneralStateTests/stEIP150singleCodeGasPrices/"}
@@ -149,12 +163,15 @@ mod block_tests {
 	declare_test!{BlockchainTests_GeneralStateTest_stMemoryTest, "BlockchainTests/GeneralStateTests/stMemoryTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stNonZeroCallsTest, "BlockchainTests/GeneralStateTests/stNonZeroCallsTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts, "BlockchainTests/GeneralStateTests/stPreCompiledContracts/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts2, "BlockchainTests/GeneralStateTests/stPreCompiledContracts2/"}
 	declare_test!{heavy => BlockchainTests_GeneralStateTest_stQuadraticComplexityTest, "BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRandom, "BlockchainTests/GeneralStateTests/stRandom/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stRandom2, "BlockchainTests/GeneralStateTests/stRandom2/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRecursiveCreate, "BlockchainTests/GeneralStateTests/stRecursiveCreate/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRefundTest, "BlockchainTests/GeneralStateTests/stRefundTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRevertTest, "BlockchainTests/GeneralStateTests/stRevertTest/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stShift, "BlockchainTests/GeneralStateTests/stShift/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stSolidityTest, "BlockchainTests/GeneralStateTests/stSolidityTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stSpecialTest, "BlockchainTests/GeneralStateTests/stSpecialTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stStackTests, "BlockchainTests/GeneralStateTests/stStackTests/"}
@@ -166,6 +183,7 @@ mod block_tests {
 	declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsRevert, "BlockchainTests/GeneralStateTests/stZeroCallsRevert/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsTest, "BlockchainTests/GeneralStateTests/stZeroCallsTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge, "BlockchainTests/GeneralStateTests/stZeroKnowledge/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge2, "BlockchainTests/GeneralStateTests/stZeroKnowledge2/"}
 
 	declare_test!{BlockchainTests_TransitionTests_bcEIP158ToByzantium, "BlockchainTests/TransitionTests/bcEIP158ToByzantium/"}
 	declare_test!{BlockchainTests_TransitionTests_bcFrontierToHomestead, "BlockchainTests/TransitionTests/bcFrontierToHomestead/"}

ethcore/src/json_tests/mod.rs

@@ -24,10 +24,12 @@ mod executive;
 mod state;
 mod chain;
 mod trie;
+mod skip;
 
 #[cfg(test)]
 mod difficulty;
 
+
 pub use self::test_common::HookType;
 
 pub use self::transaction::run_test_path as run_transaction_test_path;
@@ -42,3 +44,4 @@ pub use self::trie::run_generic_test_path as run_generic_trie_test_path;
 pub use self::trie::run_generic_test_file as run_generic_trie_test_file;
 pub use self::trie::run_secure_test_path as run_secure_trie_test_path;
 pub use self::trie::run_secure_test_file as run_secure_trie_test_file;
+use self::skip::SKIP_TEST_STATE;

ethcore/src/json_tests/skip.rs

@@ -0,0 +1,37 @@
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! State tests to skip.
+
+use ethjson;
+
+#[cfg(all(not(test), feature = "ci-skip-tests"))]
+compile_error!("ci-skip-tests can only be enabled for testing builds.");
+
+#[cfg(feature="ci-skip-issue")]
+lazy_static!{
+	pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = {
+		let skip_data = include_bytes!("../../res/ethereum/tests-issues/currents.json");
+		ethjson::test::SkipStates::load(&skip_data[..]).expect("No invalid json allowed")
+	};
+}
+
+#[cfg(not(feature="ci-skip-issue"))]
+lazy_static!{
+	pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = {
+		ethjson::test::SkipStates::empty()
+	};
+}

ethcore/src/json_tests/state.rs

@@ -22,7 +22,7 @@ use client::{EvmTestClient, EvmTestError, TransactResult};
 use ethjson;
 use transaction::SignedTransaction;
 use vm::EnvInfo;
-
+use super::SKIP_TEST_STATE;
 use super::HookType;
 
 /// Run state jsontests on a given folder.
@@ -35,6 +35,18 @@ pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
 	::json_tests::test_common::run_test_file(p, json_chain_test, h)
 }
 
+fn skip_test(subname: &str, chain: &String, number: usize) -> bool {
+	SKIP_TEST_STATE.state.iter().any(|state_test|{
+		if let Some(subtest) = state_test.subtests.get(subname) {
+			chain == &subtest.chain &&
+			(subtest.subnumbers[0] == "*"
+				 || subtest.subnumbers.contains(&number.to_string()))
+		} else {
+			false
+		}
+	})
+}
+
 pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::state::test::Test::load(json_data).unwrap();
@@ -60,6 +72,10 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
 
 				for (i, state) in states.into_iter().enumerate() {
 					let info = format!("   - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total);
+					if skip_test(&name, &spec.name, i + 1) {
+						println!("{} in skip list : SKIPPED", info);
+						continue;
+					}
 
 					let post_root: H256 = state.hash.into();
 					let transaction: SignedTransaction = multitransaction.select(&state.indexes).into();
@@ -125,11 +141,13 @@ mod state_tests {
 	declare_test!{GeneralStateTest_stChangedEIP150, "GeneralStateTests/stChangedEIP150/"}
 	declare_test!{GeneralStateTest_stCodeCopyTest, "GeneralStateTests/stCodeCopyTest/"}
 	declare_test!{GeneralStateTest_stCodeSizeLimit, "GeneralStateTests/stCodeSizeLimit/"}
+	declare_test!{GeneralStateTest_stCreate2Test, "GeneralStateTests/stCreate2/"}
 	declare_test!{GeneralStateTest_stCreateTest, "GeneralStateTests/stCreateTest/"}
 	declare_test!{GeneralStateTest_stDelegatecallTestHomestead, "GeneralStateTests/stDelegatecallTestHomestead/"}
 	declare_test!{GeneralStateTest_stEIP150singleCodeGasPrices, "GeneralStateTests/stEIP150singleCodeGasPrices/"}
 	declare_test!{GeneralStateTest_stEIP150Specific, "GeneralStateTests/stEIP150Specific/"}
 	declare_test!{GeneralStateTest_stEIP158Specific, "GeneralStateTests/stEIP158Specific/"}
+	declare_test!{GeneralStateTest_stEWASMTests, "GeneralStateTests/stEWASMTests/"}
 	declare_test!{GeneralStateTest_stExample, "GeneralStateTests/stExample/"}
 	declare_test!{GeneralStateTest_stHomesteadSpecific, "GeneralStateTests/stHomesteadSpecific/"}
 	declare_test!{GeneralStateTest_stInitCodeTest, "GeneralStateTests/stInitCodeTest/"}