diff --git a/Cargo.toml b/Cargo.toml index cb2880377e4..787e336615a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,7 @@ daemonize = { git = "https://github.com/paritytech/daemonize" } [features] miner-debug = ["ethcore/miner-debug"] json-tests = ["ethcore/json-tests"] +ci-skip-issue = ["ethcore/ci-skip-issue"] test-heavy = ["ethcore/test-heavy"] evm-debug = ["ethcore/evm-debug"] evm-debug-tests = ["ethcore/evm-debug-tests"] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index e21a5bf0870..e452f4b7ca0 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -102,6 +102,8 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"] slow-blocks = [] # Run JSON consensus tests. json-tests = ["ethcore-transaction/json-tests", "test-helpers", "tempdir"] +# Skip JSON consensus tests with pending issues. +ci-skip-issue = [] # Run memory/cpu heavy tests. test-heavy = [] # Compile benches diff --git a/ethcore/res/ethereum/constantinople_test.json b/ethcore/res/ethereum/constantinople_test.json index 6169b743eff..888eda466f9 100644 --- a/ethcore/res/ethereum/constantinople_test.json +++ b/ethcore/res/ethereum/constantinople_test.json @@ -6,7 +6,10 @@ "minimumDifficulty": "0x020000", "difficultyBoundDivisor": "0x0800", "durationLimit": "0x0d", - "blockReward": "0x1BC16D674EC80000", + "blockReward": { + "0": "0x29A2241AF62C0000", + "5": "0x1BC16D674EC80000" + }, "homesteadTransition": "0x0", "eip100bTransition": "0x0", "difficultyBombDelays": { @@ -58,9 +61,9 @@ "0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } }, "0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } }, "0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }, - "0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 100 } } } }, + "0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 20 } } } }, "0000000000000000000000000000000000000006": { "builtin": { "name": "alt_bn128_add", "activate_at": "0x00", "pricing": { "linear": { "base": 500, "word": 0 } } } }, - "0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 2000, "word": 0 } } } }, + "0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 40000, "word": 0 } } } }, "0000000000000000000000000000000000000008": { "builtin": { "name": "alt_bn128_pairing", "activate_at": "0x00", "pricing": { "alt_bn128_pairing": { "base": 100000, "pair": 80000 } } } } } } diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index b8a21c19369..3f5febc9019 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit b8a21c193696976ca3b33b6d82107601063a5d26 +Subproject commit 3f5febc901913ef698f1b09dda8705babd729e4a diff --git a/ethcore/res/ethereum/tests-issues/currents.json b/ethcore/res/ethereum/tests-issues/currents.json new file mode 100644 index 00000000000..371211cab82 --- /dev/null +++ b/ethcore/res/ethereum/tests-issues/currents.json @@ -0,0 +1,370 @@ +{ "block": + [ + { + "reference": "9590", + "failing": "stCreateTest", + "subtests": [ + "CreateOOGafterInitCodeReturndata2_d0g1v0_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "stCreate2", + "subtests": [ + "RevertDepthCreateAddressCollision_d0g1v0_Constantinople", + "RevertDepthCreateAddressCollision_d1g1v1_Constantinople", + "CREATE2_Suicide_d5g0v0_Constantinople", + "CREATE2_Suicide_d7g0v0_Constantinople", + "create2collisionSelfdestructedOOG_d2g0v0_Byzantium", + "create2collisionSelfdestructedOOG_d2g0v0_Constantinople", + "create2collisionNonce_d1g0v0_Byzantium", + "create2collisionNonce_d1g0v0_Constantinople", + "CreateMessageRevertedOOGInInit_d0g1v0_Constantinople", + "create2callPrecompiles_d3g0v0_Constantinople", + "create2collisionCode_d1g0v0_Byzantium", + "create2collisionCode_d1g0v0_Constantinople", + "create2collisionStorage_d0g0v0_Byzantium", + "create2collisionStorage_d0g0v0_Constantinople", + "create2callPrecompiles_d4g0v0_Constantinople", + "create2collisionSelfdestructedRevert_d0g0v0_Byzantium", + "create2collisionSelfdestructedRevert_d0g0v0_Constantinople", + "CreateMessageReverted_d0g1v0_Constantinople", + "RevertOpcodeCreate_d0g1v0_Constantinople", + "CREATE2_Suicide_d11g0v0_Constantinople", + "create2checkFieldsInInitcode_d5g0v0_Constantinople", + "create2collisionSelfdestructedOOG_d1g0v0_Byzantium", + "create2collisionSelfdestructedOOG_d1g0v0_Constantinople", + "returndatacopy_following_create_d1g0v0_Constantinople", + "RevertDepthCreate2OOG_d1g1v1_Constantinople", + "create2collisionSelfdestructed_d2g0v0_Byzantium", + "create2collisionSelfdestructed_d2g0v0_Constantinople", + "create2callPrecompiles_d2g0v0_Constantinople", + "create2InitCodes_d2g0v0_Constantinople", + "create2collisionNonce_d2g0v0_Byzantium", + "create2collisionNonce_d2g0v0_Constantinople", + "create2collisionCode_d0g0v0_Byzantium", + "create2collisionCode_d0g0v0_Constantinople", + "CREATE2_Bounds_d0g0v0_Constantinople", + "RevertDepthCreate2OOG_d0g0v0_Constantinople", + "CREATE2_Suicide_d1g0v0_Constantinople", + "CREATE2_Bounds3_d0g1v0_Constantinople", + "create2collisionStorage_d2g0v0_Byzantium", + "create2collisionStorage_d2g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d0g0v1_Constantinople", + "create2callPrecompiles_d5g0v0_Constantinople", + "create2collisionCode2_d0g0v0_Byzantium", + "create2collisionCode2_d0g0v0_Constantinople", + "create2noCash_d0g0v0_Byzantium", + "create2noCash_d0g0v0_Constantinople", + "create2checkFieldsInInitcode_d7g0v0_Constantinople", + "create2SmartInitCode_d1g0v0_Constantinople", + "create2InitCodes_d6g0v0_Constantinople", + "create2noCash_d1g0v0_Byzantium", + "create2noCash_d1g0v0_Constantinople", + "CREATE2_ContractSuicideDuringInit_ThenStoreThenReturn_d0g0v0_Constantinople", + "RevertOpcodeInCreateReturns_d0g0v0_Constantinople", + "create2collisionStorage_d1g0v0_Byzantium", + "create2collisionStorage_d1g0v0_Constantinople", + "create2checkFieldsInInitcode_d3g0v0_Constantinople", + "create2collisionBalance_d0g0v0_Byzantium", + "create2collisionBalance_d0g0v0_Constantinople", + "create2collisionSelfdestructed2_d0g0v0_Constantinople", + "create2InitCodes_d3g0v0_Constantinople", + "create2collisionCode2_d1g0v0_Byzantium", + "create2collisionCode2_d1g0v0_Constantinople", + "create2checkFieldsInInitcode_d1g0v0_Constantinople", + "create2collisionBalance_d1g0v0_Byzantium", + "create2collisionBalance_d1g0v0_Constantinople", + "CREATE2_Bounds3_d0g2v0_Constantinople", + "create2callPrecompiles_d6g0v0_Constantinople", + "Create2Recursive_d0g0v0_Constantinople", + "create2collisionSelfdestructedOOG_d0g0v0_Byzantium", + "create2collisionSelfdestructedOOG_d0g0v0_Constantinople", + "CREATE2_Suicide_d3g0v0_Constantinople", + "returndatacopy_following_create_d0g0v0_Constantinople", + "create2InitCodes_d8g0v0_Constantinople", + "RevertDepthCreate2OOG_d0g0v1_Constantinople", + "create2checkFieldsInInitcode_d2g0v0_Constantinople", + "RevertDepthCreate2OOG_d1g0v1_Constantinople", + "Create2OnDepth1024_d0g0v0_Constantinople", + "create2collisionSelfdestructed2_d1g0v0_Constantinople", + "create2collisionSelfdestructedRevert_d2g0v0_Byzantium", + "create2collisionSelfdestructedRevert_d2g0v0_Constantinople", + "create2callPrecompiles_d0g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d0g1v1_Constantinople", + "create2collisionSelfdestructed_d1g0v0_Byzantium", + "create2collisionSelfdestructed_d1g0v0_Constantinople", + "call_outsize_then_create2_successful_then_returndatasize_d0g0v0_Byzantium", + "call_outsize_then_create2_successful_then_returndatasize_d0g0v0_Constantinople", + "Create2OOGafterInitCodeRevert_d0g0v0_Constantinople", + "Create2OOGafterInitCodeReturndata3_d0g0v0_Constantinople", + "Create2OOGafterInitCodeReturndataSize_d0g0v0_Constantinople", + "create2InitCodes_d7g0v0_Constantinople", + "CREATE2_Suicide_d10g0v0_Constantinople", + "RevertDepthCreate2OOG_d0g1v0_Constantinople", + "create2InitCodes_d5g0v0_Constantinople", + "create2collisionSelfdestructedRevert_d1g0v0_Byzantium", + "create2collisionSelfdestructedRevert_d1g0v0_Constantinople", + "RevertDepthCreate2OOG_d1g1v0_Constantinople", + "create2collisionSelfdestructed_d0g0v0_Byzantium", + "create2collisionSelfdestructed_d0g0v0_Constantinople", + "create2noCash_d2g0v0_Byzantium", + "create2noCash_d2g0v0_Constantinople", + "CREATE2_Bounds3_d0g0v0_Constantinople", + "create2collisionNonce_d0g0v0_Byzantium", + "create2collisionNonce_d0g0v0_Constantinople", + "CREATE2_Suicide_d2g0v0_Constantinople", + "Create2OOGafterInitCode_d0g0v0_Constantinople", + "call_then_create2_successful_then_returndatasize_d0g0v0_Byzantium", + "call_then_create2_successful_then_returndatasize_d0g0v0_Constantinople", + "create2collisionBalance_d2g0v0_Byzantium", + "create2collisionBalance_d2g0v0_Constantinople", + "create2checkFieldsInInitcode_d6g0v0_Constantinople", + "RevertDepthCreate2OOG_d0g1v1_Constantinople", + "returndatacopy_afterFailing_create_d0g0v0_Constantinople", + "returndatacopy_following_revert_in_create_d0g0v0_Constantinople", + "CREATE2_Suicide_d9g0v0_Constantinople", + "create2callPrecompiles_d7g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d1g0v1_Constantinople", + "create2InitCodes_d1g0v0_Constantinople", + "CREATE2_Bounds_d0g1v0_Constantinople", + "Create2OOGafterInitCodeReturndata_d0g0v0_Constantinople", + "create2checkFieldsInInitcode_d4g0v0_Constantinople", + "CreateMessageRevertedOOGInInit_d0g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d1g1v0_Constantinople", + "returndatacopy_following_successful_create_d0g0v0_Constantinople", + "create2checkFieldsInInitcode_d0g0v0_Constantinople", + "CreateMessageReverted_d0g0v0_Constantinople", + "create2SmartInitCode_d0g0v0_Constantinople", + "CREATE2_Bounds2_d0g0v0_Constantinople", + "returndatasize_following_successful_create_d0g0v0_Constantinople", + "CREATE2_Bounds2_d0g1v0_Constantinople", + "returndatacopy_0_0_following_successful_create_d0g0v0_Constantinople", + "RevertDepthCreateAddressCollision_d0g0v0_Constantinople", + "CREATE2_Suicide_d0g0v0_Constantinople", + "create2InitCodes_d0g0v0_Constantinople", + "Create2OnDepth1023_d0g0v0_Constantinople", + "create2InitCodes_d4g0v0_Constantinople", + "Create2OOGafterInitCodeReturndata2_d0g0v0_Constantinople", + "create2collisionBalance_d3g0v0_Byzantium", + "create2collisionBalance_d3g0v0_Constantinople", + "CREATE2_Suicide_d4g0v0_Constantinople", + "Create2OOGafterInitCode_d0g1v0_Constantinople", + "RevertDepthCreateAddressCollision_d1g0v0_Constantinople", + "Create2OOGafterInitCodeRevert2_d0g0v0_Constantinople", + "Create2OOGafterInitCodeReturndata_d0g1v0_Constantinople", + "Create2Recursive_d0g1v0_Constantinople", + "create2collisionCode_d2g0v0_Byzantium", + "create2collisionCode_d2g0v0_Constantinople", + "CREATE2_Suicide_d6g0v0_Constantinople", + "CREATE2_Suicide_d8g0v0_Constantinople", + "RevertOpcodeCreate_d0g0v0_Constantinople", + "Create2OOGafterInitCodeReturndata2_d0g1v0_Constantinople", + "create2callPrecompiles_d1g0v0_Constantinople", + "RevertInCreateInInit_d0g0v0_Constantinople", + "RevertDepthCreate2OOG_d1g0v0_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "bcStateTest", + "subtests": [ + "suicideStorageCheck_Byzantium", + "suicideStorageCheck_Constantinople", + "suicideStorageCheckVCreate2_Byzantium", + "suicideStorageCheckVCreate2_Constantinople", + "create2collisionwithSelfdestructSameBlock_Constantinople", + "blockhashNonConstArg_Constantinople", + "suicideThenCheckBalance_Constantinople", + "suicideThenCheckBalance_Homestead", + "suicideStorageCheckVCreate_Byzantium", + "suicideStorageCheckVCreate_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "stEIP158Specific", + "subtests": [ + "callToEmptyThenCallError_d0g0v0_Byzantium", + "callToEmptyThenCallError_d0g0v0_Constantinople", + "callToEmptyThenCallError_d0g0v0_EIP158" + ] + }, + { + "reference": "9590", + "failing": "stPreCompiledContracts", + "subtests": [ + "identity_to_smaller_d0g0v0_Constantinople", + "identity_to_bigger_d0g0v0_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "stReturnDataTest", + "subtests": [ + "modexp_modsize0_returndatasize_d0g1v0_Constantinople", + "modexp_modsize0_returndatasize_d0g2v0_Constantinople", + "modexp_modsize0_returndatasize_d0g3v0_Constantinople" + ] + }, + { + "reference": "9590", + "failing": "stSpecialTest", + "subtests": [ + "push32withoutByte_d0g0v0_Constantinople" + ] + } + + ], + "state": + [ + { + "reference": "9590", + "failing": "stCreateTest", + "subtests": { + "CreateOOGafterInitCodeReturndata2": { + "subnumbers": ["2"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stCreate2Test", + "subtests": { + "RevertInCreateInInit": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stEIP150Specific", + "subtests": { + "NewGasPriceForCodes": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stInitCodeTest", + "subtests": { + "OutOfGasContractCreation": { + "subnumbers": ["4"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stPreCompiledContracts", + "subtests": { + "modexp": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stRevertTest", + "subtests": { + "LoopCallsDepthThenRevert3": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + }, + "RevertOpcodeCreate": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + }, + "RevertSubCallStorageOOG2": { + "subnumbers": ["1","3"], + "chain": "Constantinople (test)" + }, + "RevertDepthCreateOOG": { + "subnumbers": ["3","4"], + "chain": "Constantinople (test)" + }, + "RevertOpcodeMultipleSubCalls": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + }, + "RevertOpcodeDirectCall": { + "subnumbers": ["1","2"], + "chain": "Constantinople (test)" + }, + "LoopCallsDepthThenRevert2": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + }, + "RevertDepth2": { + "subnumbers": ["1"], + "chain": "Constantinople (test)" + }, + "RevertRemoteSubCallStorageOOG2": { + "subnumbers": ["1","2"], + "chain": "Constantinople (test)" + }, + "RevertDepthCreateAddressCollision": { + "subnumbers": ["3","4"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "9590", + "failing": "stStaticCall", + "subtests": { + "static_RevertDepth2": { + "subnumbers": ["1","3"], + "chain": "Constantinople (test)" + }, + "static_CheckOpcodes4": { + "subnumbers": ["3"], + "chain": "Constantinople (test)" + }, + "static_CheckOpcodes3": { + "subnumbers": ["5","6","7","8"], + "chain": "Constantinople (test)" + }, + "static_callBasic": { + "subnumbers": ["1","2"], + "chain": "Constantinople (test)" + }, + "static_CheckOpcodes2": { + "subnumbers": ["5","6","7","8"], + "chain": "Constantinople (test)" + }, + "static_callCreate": { + "subnumbers": ["2"], + "chain": "Constantinople (test)" + } + } + }, + { + "reference": "https://github.com/ethereum/tests/issues/512", + "failing": "stZeroKnowledge", + "subtests": { + "pointAddTrunc": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + }, + "pointAdd": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + }, + "pointMulAdd": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + }, + "pointMulAdd2": { + "subnumbers": ["*"], + "chain": "Constantinople (test)" + } + } + } + ] +} diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index 1d827dea626..74f2b2fc133 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -86,9 +86,9 @@ impl<'a> EvmTestClient<'a> { ForkSpec::EIP150 => Some(ethereum::new_eip150_test()), ForkSpec::EIP158 => Some(ethereum::new_eip161_test()), ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()), + ForkSpec::Constantinople => Some(ethereum::new_constantinople_test()), ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()), ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None, - _ => None, } } diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index 83a940fcb64..120bcce8393 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -23,6 +23,7 @@ use miner::Miner; use io::IoChannel; use test_helpers; use verification::queue::kind::blocks::Unverified; +use super::SKIP_TEST_STATE; use super::HookType; @@ -36,12 +37,20 @@ pub fn run_test_file(p: &Path, h: &mut H) { ::json_tests::test_common::run_test_file(p, json_chain_test, h) } +fn skip_test(name: &String) -> bool { + SKIP_TEST_STATE.block.iter().any(|block_test|block_test.subtests.contains(name)) +} + pub fn json_chain_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::blockchain::Test::load(json_data).unwrap(); let mut failed = Vec::new(); for (name, blockchain) in tests.into_iter() { + if skip_test(&name) { + println!(" - {} | {:?} Ignoring tests because in skip list", name, blockchain.network); + continue; + } start_stop_hook(&name, HookType::OnStart); let mut fail = false; @@ -122,19 +131,24 @@ mod block_tests { declare_test!{BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"} declare_test!{BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"} declare_test!{BlockchainTests_bcRandomBlockhashTest, "BlockchainTests/bcRandomBlockhashTest"} + declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTests"} declare_test!{BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"} declare_test!{BlockchainTests_bcUncleHeaderValidity, "BlockchainTests/bcUncleHeaderValidity"} declare_test!{BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"} declare_test!{BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"} declare_test!{BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"} + declare_test!{BlockchainTests_GeneralStateTest_stArgsZeroOneBalance, "BlockchainTests/GeneralStateTests/stArgsZeroOneBalance/"} declare_test!{BlockchainTests_GeneralStateTest_stAttackTest, "BlockchainTests/GeneralStateTests/stAttackTest/"} declare_test!{BlockchainTests_GeneralStateTest_stBadOpcodeTest, "BlockchainTests/GeneralStateTests/stBadOpcode/"} + declare_test!{BlockchainTests_GeneralStateTest_stBugsTest, "BlockchainTests/GeneralStateTests/stBugs/"} declare_test!{BlockchainTests_GeneralStateTest_stCallCodes, "BlockchainTests/GeneralStateTests/stCallCodes/"} + declare_test!{BlockchainTests_GeneralStateTest_stCallCreateCallCodeTest, "BlockchainTests/GeneralStateTests/stCallCreateCallCodeTest/"} declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"} declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesHomestead/"} declare_test!{BlockchainTests_GeneralStateTest_stChangedEIP150, "BlockchainTests/GeneralStateTests/stChangedEIP150/"} declare_test!{BlockchainTests_GeneralStateTest_stCodeSizeLimit, "BlockchainTests/GeneralStateTests/stCodeSizeLimit/"} + declare_test!{BlockchainTests_GeneralStateTest_stCreate2, "BlockchainTests/GeneralStateTests/stCreate2/"} declare_test!{BlockchainTests_GeneralStateTest_stCreateTest, "BlockchainTests/GeneralStateTests/stCreateTest/"} declare_test!{BlockchainTests_GeneralStateTest_stDelegatecallTestHomestead, "BlockchainTests/GeneralStateTests/stDelegatecallTestHomestead/"} declare_test!{BlockchainTests_GeneralStateTest_stEIP150singleCodeGasPrices, "BlockchainTests/GeneralStateTests/stEIP150singleCodeGasPrices/"} @@ -149,12 +163,15 @@ mod block_tests { declare_test!{BlockchainTests_GeneralStateTest_stMemoryTest, "BlockchainTests/GeneralStateTests/stMemoryTest/"} declare_test!{BlockchainTests_GeneralStateTest_stNonZeroCallsTest, "BlockchainTests/GeneralStateTests/stNonZeroCallsTest/"} declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts, "BlockchainTests/GeneralStateTests/stPreCompiledContracts/"} + declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts2, "BlockchainTests/GeneralStateTests/stPreCompiledContracts2/"} declare_test!{heavy => BlockchainTests_GeneralStateTest_stQuadraticComplexityTest, "BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/"} declare_test!{BlockchainTests_GeneralStateTest_stRandom, "BlockchainTests/GeneralStateTests/stRandom/"} + declare_test!{BlockchainTests_GeneralStateTest_stRandom2, "BlockchainTests/GeneralStateTests/stRandom2/"} declare_test!{BlockchainTests_GeneralStateTest_stRecursiveCreate, "BlockchainTests/GeneralStateTests/stRecursiveCreate/"} declare_test!{BlockchainTests_GeneralStateTest_stRefundTest, "BlockchainTests/GeneralStateTests/stRefundTest/"} declare_test!{BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"} declare_test!{BlockchainTests_GeneralStateTest_stRevertTest, "BlockchainTests/GeneralStateTests/stRevertTest/"} + declare_test!{BlockchainTests_GeneralStateTest_stShift, "BlockchainTests/GeneralStateTests/stShift/"} declare_test!{BlockchainTests_GeneralStateTest_stSolidityTest, "BlockchainTests/GeneralStateTests/stSolidityTest/"} declare_test!{BlockchainTests_GeneralStateTest_stSpecialTest, "BlockchainTests/GeneralStateTests/stSpecialTest/"} declare_test!{BlockchainTests_GeneralStateTest_stStackTests, "BlockchainTests/GeneralStateTests/stStackTests/"} @@ -166,6 +183,7 @@ mod block_tests { declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsRevert, "BlockchainTests/GeneralStateTests/stZeroCallsRevert/"} declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsTest, "BlockchainTests/GeneralStateTests/stZeroCallsTest/"} declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge, "BlockchainTests/GeneralStateTests/stZeroKnowledge/"} + declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge2, "BlockchainTests/GeneralStateTests/stZeroKnowledge2/"} declare_test!{BlockchainTests_TransitionTests_bcEIP158ToByzantium, "BlockchainTests/TransitionTests/bcEIP158ToByzantium/"} declare_test!{BlockchainTests_TransitionTests_bcFrontierToHomestead, "BlockchainTests/TransitionTests/bcFrontierToHomestead/"} diff --git a/ethcore/src/json_tests/mod.rs b/ethcore/src/json_tests/mod.rs index fa1c822ce2a..35d5034806a 100644 --- a/ethcore/src/json_tests/mod.rs +++ b/ethcore/src/json_tests/mod.rs @@ -24,10 +24,12 @@ mod executive; mod state; mod chain; mod trie; +mod skip; #[cfg(test)] mod difficulty; + pub use self::test_common::HookType; pub use self::transaction::run_test_path as run_transaction_test_path; @@ -42,3 +44,4 @@ pub use self::trie::run_generic_test_path as run_generic_trie_test_path; pub use self::trie::run_generic_test_file as run_generic_trie_test_file; pub use self::trie::run_secure_test_path as run_secure_trie_test_path; pub use self::trie::run_secure_test_file as run_secure_trie_test_file; +use self::skip::SKIP_TEST_STATE; diff --git a/ethcore/src/json_tests/skip.rs b/ethcore/src/json_tests/skip.rs new file mode 100644 index 00000000000..784e2975d73 --- /dev/null +++ b/ethcore/src/json_tests/skip.rs @@ -0,0 +1,37 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! State tests to skip. + +use ethjson; + +#[cfg(all(not(test), feature = "ci-skip-tests"))] +compile_error!("ci-skip-tests can only be enabled for testing builds."); + +#[cfg(feature="ci-skip-issue")] +lazy_static!{ + pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = { + let skip_data = include_bytes!("../../res/ethereum/tests-issues/currents.json"); + ethjson::test::SkipStates::load(&skip_data[..]).expect("No invalid json allowed") + }; +} + +#[cfg(not(feature="ci-skip-issue"))] +lazy_static!{ + pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = { + ethjson::test::SkipStates::empty() + }; +} diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index 5837f191c7d..04ca947bf19 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -22,7 +22,7 @@ use client::{EvmTestClient, EvmTestError, TransactResult}; use ethjson; use transaction::SignedTransaction; use vm::EnvInfo; - +use super::SKIP_TEST_STATE; use super::HookType; /// Run state jsontests on a given folder. @@ -35,6 +35,18 @@ pub fn run_test_file(p: &Path, h: &mut H) { ::json_tests::test_common::run_test_file(p, json_chain_test, h) } +fn skip_test(subname: &str, chain: &String, number: usize) -> bool { + SKIP_TEST_STATE.state.iter().any(|state_test|{ + if let Some(subtest) = state_test.subtests.get(subname) { + chain == &subtest.chain && + (subtest.subnumbers[0] == "*" + || subtest.subnumbers.contains(&number.to_string())) + } else { + false + } + }) +} + pub fn json_chain_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::state::test::Test::load(json_data).unwrap(); @@ -60,6 +72,10 @@ pub fn json_chain_test(json_data: &[u8], start_stop_ho for (i, state) in states.into_iter().enumerate() { let info = format!(" - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total); + if skip_test(&name, &spec.name, i + 1) { + println!("{} in skip list : SKIPPED", info); + continue; + } let post_root: H256 = state.hash.into(); let transaction: SignedTransaction = multitransaction.select(&state.indexes).into(); @@ -125,11 +141,13 @@ mod state_tests { declare_test!{GeneralStateTest_stChangedEIP150, "GeneralStateTests/stChangedEIP150/"} declare_test!{GeneralStateTest_stCodeCopyTest, "GeneralStateTests/stCodeCopyTest/"} declare_test!{GeneralStateTest_stCodeSizeLimit, "GeneralStateTests/stCodeSizeLimit/"} + declare_test!{GeneralStateTest_stCreate2Test, "GeneralStateTests/stCreate2/"} declare_test!{GeneralStateTest_stCreateTest, "GeneralStateTests/stCreateTest/"} declare_test!{GeneralStateTest_stDelegatecallTestHomestead, "GeneralStateTests/stDelegatecallTestHomestead/"} declare_test!{GeneralStateTest_stEIP150singleCodeGasPrices, "GeneralStateTests/stEIP150singleCodeGasPrices/"} declare_test!{GeneralStateTest_stEIP150Specific, "GeneralStateTests/stEIP150Specific/"} declare_test!{GeneralStateTest_stEIP158Specific, "GeneralStateTests/stEIP158Specific/"} + declare_test!{GeneralStateTest_stEWASMTests, "GeneralStateTests/stEWASMTests/"} declare_test!{GeneralStateTest_stExample, "GeneralStateTests/stExample/"} declare_test!{GeneralStateTest_stHomesteadSpecific, "GeneralStateTests/stHomesteadSpecific/"} declare_test!{GeneralStateTest_stInitCodeTest, "GeneralStateTests/stInitCodeTest/"} diff --git a/ethcore/src/json_tests/test_common.rs b/ethcore/src/json_tests/test_common.rs index ce5c68a73a0..ce16395d228 100644 --- a/ethcore/src/json_tests/test_common.rs +++ b/ethcore/src/json_tests/test_common.rs @@ -34,6 +34,18 @@ pub fn run_test_path( p: &Path, skip: &[&'static str], runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, start_stop_hook: &mut H +) { + let mut errors = Vec::new(); + run_test_path_inner(p, skip, runner, start_stop_hook, &mut errors); + let empty: [String; 0] = []; + assert_eq!(errors, empty); +} + +fn run_test_path_inner( + p: &Path, skip: &[&'static str], + runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, + start_stop_hook: &mut H, + errors: &mut Vec ) { let path = Path::new(p); let s: HashSet = skip.iter().map(|s| { @@ -50,17 +62,32 @@ pub fn run_test_path( } else { Some(e.path()) }}) { - run_test_path(&p, skip, runner, start_stop_hook) + run_test_path_inner(&p, skip, runner, start_stop_hook, errors); } } else if extension == Some("swp") || extension == None { // Ignore junk } else { let mut path = p.to_path_buf(); path.set_extension("json"); - run_test_file(&path, runner, start_stop_hook) + run_test_file_append(&path, runner, start_stop_hook, errors) } } +fn run_test_file_append( + path: &Path, + runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, + start_stop_hook: &mut H, + errors: &mut Vec +) { + let mut data = Vec::new(); + let mut file = match File::open(&path) { + Ok(file) => file, + Err(_) => panic!("Error opening test file at: {:?}", path), + }; + file.read_to_end(&mut data).expect("Error reading test file"); + errors.append(&mut runner(&data, start_stop_hook)); +} + pub fn run_test_file( path: &Path, runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, diff --git a/json/src/test/mod.rs b/json/src/test/mod.rs index 8f95a9aec49..e6148e0d08c 100644 --- a/json/src/test/mod.rs +++ b/json/src/test/mod.rs @@ -45,6 +45,7 @@ pub struct DifficultyTestCase { #[serde(rename="currentBlockNumber")] pub current_block_number: Uint, } + /// Blockchain test deserializer. #[derive(Debug, PartialEq, Deserialize)] pub struct DifficultyTest(BTreeMap); @@ -64,3 +65,59 @@ impl DifficultyTest { serde_json::from_reader(reader) } } + +/// Test to skip (only if issue ongoing) +#[derive(Debug, PartialEq, Deserialize)] +pub struct SkipStates { + /// Block tests + pub block: Vec, + /// State tests + pub state: Vec, + +} + +/// Block test to skip. +#[derive(Debug, PartialEq, Deserialize)] +pub struct BlockSkipStates { + /// Issue reference. + pub reference: String, + /// Test failing name. + pub failing: String, + /// Items failing for the test. + pub subtests: Vec, +} + +/// State test to skip. +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateSkipStates { + /// Issue reference. + pub reference: String, + /// Test failing name. + pub failing: String, + /// Items failing for the test. + pub subtests: BTreeMap +} + +/// State subtest to skip. +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateSkipSubStates { + /// State test number of this item. Or '*' for all state. + pub subnumbers: Vec, + /// Chain for this items. + pub chain: String, +} + +impl SkipStates { + /// Loads skip states from json. + pub fn load(reader: R) -> Result where R: Read { + serde_json::from_reader(reader) + } + + /// Empty skip states. + pub fn empty() -> Self { + SkipStates { + block: Vec::new(), + state: Vec::new(), + } + } +} diff --git a/test.sh b/test.sh index 83d07b78bc9..a3ed39e8d2c 100755 --- a/test.sh +++ b/test.sh @@ -1,7 +1,7 @@ #!/bin/sh # Running Parity Full Test Suite -FEATURES="json-tests" +FEATURES="json-tests,ci-skip-issue" OPTIONS="--release" VALIDATE=1