From ac5d367b89885577c3b5bb84a86832d4a5109da4 Mon Sep 17 00:00:00 2001 From: evalir Date: Wed, 28 Jun 2023 11:23:10 -0400 Subject: [PATCH] feat(cheatcodes): `vm.skip(bool)` for skipping tests (#5205) * feat(abi): add skip(bool) * feat: skip impl * feat: make skip only work at test level * feat: rewrite test runner to use status enum instead of bool * feat: simple tests * feat: works with fuzz tests * feat: works for invariant * chore: remove println * chore: clippy * chore: clippy * chore: prioritize skip decoding over abi decoding * chore: handle skips on invariant & fuzz tests more gracefully * feat: add skipped to test results * chore: clippy * fix: fixtures --- abi/abi/HEVM.sol | 1 + abi/src/bindings/hevm.rs | 52 +++++++++++++ cli/src/cmd/forge/test/mod.rs | 25 +++++-- cli/tests/fixtures/can_check_snapshot.stdout | 2 +- .../can_run_test_in_custom_test_folder.stdout | 2 +- cli/tests/fixtures/can_test_repeatedly.stdout | 2 +- .../can_use_libs_in_multi_fork.stdout | 2 +- ...y_once_with_changed_versions.0.8.10.stdout | 2 +- ...y_once_with_changed_versions.0.8.13.stdout | 2 +- evm/src/decode.rs | 6 +- evm/src/executor/inspector/cheatcodes/mod.rs | 13 +++- evm/src/executor/inspector/cheatcodes/util.rs | 20 ++++- evm/src/executor/mod.rs | 10 ++- forge/src/result.rs | 23 ++++-- forge/src/runner.rs | 75 +++++++++++++++++-- forge/tests/it/cheats.rs | 2 +- forge/tests/it/config.rs | 13 +++- forge/tests/it/fuzz.rs | 6 +- testdata/cheats/Cheats.sol | 3 + testdata/cheats/Skip.t.sol | 34 +++++++++ 20 files changed, 254 insertions(+), 41 deletions(-) create mode 100644 testdata/cheats/Skip.t.sol diff --git a/abi/abi/HEVM.sol b/abi/abi/HEVM.sol index 98845926f8cb..8433fafbbeac 100644 --- a/abi/abi/HEVM.sol +++ b/abi/abi/HEVM.sol @@ -71,6 +71,7 @@ expectRevert(bytes) expectRevert(bytes4) record() accesses(address)(bytes32[],bytes32[]) +skip(bool) recordLogs() getRecordedLogs()(Log[]) diff --git a/abi/src/bindings/hevm.rs b/abi/src/bindings/hevm.rs index f4e7bbfc038c..d04125eb72e0 100644 --- a/abi/src/bindings/hevm.rs +++ b/abi/src/bindings/hevm.rs @@ -4210,6 +4210,24 @@ pub mod hevm { }, ], ), + ( + ::std::borrow::ToOwned::to_owned("skip"), + ::std::vec![ + ::ethers_core::abi::ethabi::Function { + name: ::std::borrow::ToOwned::to_owned("skip"), + inputs: ::std::vec![ + ::ethers_core::abi::ethabi::Param { + name: ::std::string::String::new(), + kind: ::ethers_core::abi::ethabi::ParamType::Bool, + internal_type: ::core::option::Option::None, + }, + ], + outputs: ::std::vec![], + constant: ::core::option::Option::None, + state_mutability: ::ethers_core::abi::ethabi::StateMutability::NonPayable, + }, + ], + ), ( ::std::borrow::ToOwned::to_owned("snapshot"), ::std::vec![ @@ -6397,6 +6415,15 @@ pub mod hevm { .method_hash([227, 65, 234, 164], (p0, p1)) .expect("method not found (this should never happen)") } + ///Calls the contract's `skip` (0xdd82d13e) function + pub fn skip( + &self, + p0: bool, + ) -> ::ethers_contract::builders::ContractCall { + self.0 + .method_hash([221, 130, 209, 62], p0) + .expect("method not found (this should never happen)") + } ///Calls the contract's `snapshot` (0x9711715a) function pub fn snapshot( &self, @@ -9016,6 +9043,19 @@ pub mod hevm { )] #[ethcall(name = "sign", abi = "sign(uint256,bytes32)")] pub struct SignCall(pub ::ethers_core::types::U256, pub [u8; 32]); + ///Container type for all input parameters for the `skip` function with signature `skip(bool)` and selector `0xdd82d13e` + #[derive( + Clone, + ::ethers_contract::EthCall, + ::ethers_contract::EthDisplay, + Default, + Debug, + PartialEq, + Eq, + Hash + )] + #[ethcall(name = "skip", abi = "skip(bool)")] + pub struct SkipCall(pub bool); ///Container type for all input parameters for the `snapshot` function with signature `snapshot()` and selector `0x9711715a` #[derive( Clone, @@ -9506,6 +9546,7 @@ pub mod hevm { SetNonce(SetNonceCall), SetNonceUnsafe(SetNonceUnsafeCall), Sign(SignCall), + Skip(SkipCall), Snapshot(SnapshotCall), StartBroadcast0(StartBroadcast0Call), StartBroadcast1(StartBroadcast1Call), @@ -10236,6 +10277,10 @@ pub mod hevm { = ::decode(data) { return Ok(Self::Sign(decoded)); } + if let Ok(decoded) + = ::decode(data) { + return Ok(Self::Skip(decoded)); + } if let Ok(decoded) = ::decode(data) { return Ok(Self::Snapshot(decoded)); @@ -10729,6 +10774,7 @@ pub mod hevm { ::ethers_core::abi::AbiEncode::encode(element) } Self::Sign(element) => ::ethers_core::abi::AbiEncode::encode(element), + Self::Skip(element) => ::ethers_core::abi::AbiEncode::encode(element), Self::Snapshot(element) => ::ethers_core::abi::AbiEncode::encode(element), Self::StartBroadcast0(element) => { ::ethers_core::abi::AbiEncode::encode(element) @@ -10980,6 +11026,7 @@ pub mod hevm { Self::SetNonce(element) => ::core::fmt::Display::fmt(element, f), Self::SetNonceUnsafe(element) => ::core::fmt::Display::fmt(element, f), Self::Sign(element) => ::core::fmt::Display::fmt(element, f), + Self::Skip(element) => ::core::fmt::Display::fmt(element, f), Self::Snapshot(element) => ::core::fmt::Display::fmt(element, f), Self::StartBroadcast0(element) => ::core::fmt::Display::fmt(element, f), Self::StartBroadcast1(element) => ::core::fmt::Display::fmt(element, f), @@ -11832,6 +11879,11 @@ pub mod hevm { Self::Sign(value) } } + impl ::core::convert::From for HEVMCalls { + fn from(value: SkipCall) -> Self { + Self::Skip(value) + } + } impl ::core::convert::From for HEVMCalls { fn from(value: SnapshotCall) -> Self { Self::Snapshot(value) diff --git a/cli/src/cmd/forge/test/mod.rs b/cli/src/cmd/forge/test/mod.rs index 517e48893e58..13dabfeac019 100644 --- a/cli/src/cmd/forge/test/mod.rs +++ b/cli/src/cmd/forge/test/mod.rs @@ -13,7 +13,7 @@ use forge::{ decode::decode_console_logs, executor::inspector::CheatsConfig, gas_report::GasReport, - result::{SuiteResult, TestKind, TestResult}, + result::{SuiteResult, TestKind, TestResult, TestStatus}, trace::{ identifier::{EtherscanIdentifier, LocalTraceIdentifier, SignaturesIdentifier}, CallTraceDecoderBuilder, TraceKind, @@ -348,12 +348,16 @@ impl TestOutcome { /// Iterator over all succeeding tests and their names pub fn successes(&self) -> impl Iterator { - self.tests().filter(|(_, t)| t.success) + self.tests().filter(|(_, t)| t.status == TestStatus::Success) } /// Iterator over all failing tests and their names pub fn failures(&self) -> impl Iterator { - self.tests().filter(|(_, t)| !t.success) + self.tests().filter(|(_, t)| t.status == TestStatus::Failure) + } + + pub fn skips(&self) -> impl Iterator { + self.tests().filter(|(_, t)| t.status == TestStatus::Skipped) } /// Iterator over all tests and their names @@ -418,18 +422,21 @@ impl TestOutcome { let failed = self.failures().count(); let result = if failed == 0 { Paint::green("ok") } else { Paint::red("FAILED") }; format!( - "Test result: {}. {} passed; {} failed; finished in {:.2?}", + "Test result: {}. {} passed; {} failed; {} skipped; finished in {:.2?}", result, self.successes().count(), failed, + self.skips().count(), self.duration() ) } } fn short_test_result(name: &str, result: &TestResult) { - let status = if result.success { + let status = if result.status == TestStatus::Success { Paint::green("[PASS]".to_string()) + } else if result.status == TestStatus::Skipped { + Paint::yellow("[SKIP]".to_string()) } else { let reason = result .reason @@ -553,7 +560,7 @@ fn test( short_test_result(name, result); // If the test failed, we want to stop processing the rest of the tests - if fail_fast && !result.success { + if fail_fast && result.status == TestStatus::Failure { break 'outer } @@ -596,10 +603,12 @@ fn test( // tests At verbosity level 5, we display // all traces for all tests TraceKind::Setup => { - (verbosity >= 5) || (verbosity == 4 && !result.success) + (verbosity >= 5) || + (verbosity == 4 && result.status == TestStatus::Failure) } TraceKind::Execution => { - verbosity > 3 || (verbosity == 3 && !result.success) + verbosity > 3 || + (verbosity == 3 && result.status == TestStatus::Failure) } _ => false, }; diff --git a/cli/tests/fixtures/can_check_snapshot.stdout b/cli/tests/fixtures/can_check_snapshot.stdout index feba216a6952..be81bad9505d 100644 --- a/cli/tests/fixtures/can_check_snapshot.stdout +++ b/cli/tests/fixtures/can_check_snapshot.stdout @@ -4,4 +4,4 @@ Compiler run successful! Running 1 test for src/ATest.t.sol:ATest [PASS] testExample() (gas: 168) -Test result: ok. 1 passed; 0 failed; finished in 4.42ms +Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 4.42ms diff --git a/cli/tests/fixtures/can_run_test_in_custom_test_folder.stdout b/cli/tests/fixtures/can_run_test_in_custom_test_folder.stdout index 049b5827c5fd..3be7abdfbbcd 100644 --- a/cli/tests/fixtures/can_run_test_in_custom_test_folder.stdout +++ b/cli/tests/fixtures/can_run_test_in_custom_test_folder.stdout @@ -4,4 +4,4 @@ Compiler run successful! Running 1 test for src/nested/forge-tests/MyTest.t.sol:MyTest [PASS] testTrue() (gas: 168) -Test result: ok. 1 passed; 0 failed; finished in 2.93ms +Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 2.93ms diff --git a/cli/tests/fixtures/can_test_repeatedly.stdout b/cli/tests/fixtures/can_test_repeatedly.stdout index dcfed6e57722..46719cc883d7 100644 --- a/cli/tests/fixtures/can_test_repeatedly.stdout +++ b/cli/tests/fixtures/can_test_repeatedly.stdout @@ -3,4 +3,4 @@ No files changed, compilation skipped Running 2 tests for test/Counter.t.sol:CounterTest [PASS] testIncrement() (gas: 28334) [PASS] testSetNumber(uint256) (runs: 256, μ: 26521, ~: 28387) -Test result: ok. 2 passed; 0 failed; finished in 9.42ms +Test result: ok. 2 passed; 0 failed; 0 skipped; finished in 9.42ms diff --git a/cli/tests/fixtures/can_use_libs_in_multi_fork.stdout b/cli/tests/fixtures/can_use_libs_in_multi_fork.stdout index 7be9472c3f71..f07dae93406d 100644 --- a/cli/tests/fixtures/can_use_libs_in_multi_fork.stdout +++ b/cli/tests/fixtures/can_use_libs_in_multi_fork.stdout @@ -4,4 +4,4 @@ Compiler run successful! Running 1 test for test/Contract.t.sol:ContractTest [PASS] test() (gas: 70373) -Test result: ok. 1 passed; 0 failed; finished in 3.21s +Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 3.21s diff --git a/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.10.stdout b/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.10.stdout index df4f80d68221..342564e4d897 100644 --- a/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.10.stdout +++ b/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.10.stdout @@ -4,4 +4,4 @@ Compiler run successful! Running 1 test for src/Contract.t.sol:ContractTest [PASS] testExample() (gas: 190) -Test result: ok. 1 passed; 0 failed; finished in 1.89ms +Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 1.89ms diff --git a/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.13.stdout b/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.13.stdout index 2c349e66e5fb..9f9f0614ebac 100644 --- a/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.13.stdout +++ b/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.13.stdout @@ -4,4 +4,4 @@ Compiler run successful! Running 1 test for src/Contract.t.sol:ContractTest [PASS] testExample() (gas: 190) -Test result: ok. 1 passed; 0 failed; finished in 1.89ms +Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 1.89ms diff --git a/evm/src/decode.rs b/evm/src/decode.rs index f2476fae9f47..2c0ae73fd102 100644 --- a/evm/src/decode.rs +++ b/evm/src/decode.rs @@ -2,6 +2,7 @@ use crate::{ abi::ConsoleEvents::{self, *}, error::ERROR_PREFIX, + executor::inspector::cheatcodes::util::MAGIC_SKIP_BYTES, }; use ethers::{ abi::{decode, AbiDecode, Contract as Abi, ParamType, RawLog, Token}, @@ -161,6 +162,10 @@ pub fn decode_revert( eyre::bail!("Unknown error selector") } _ => { + // See if the revert is caused by a skip() call. + if err == MAGIC_SKIP_BYTES { + return Ok("SKIPPED".to_string()) + } // try to decode a custom error if provided an abi if let Some(abi) = maybe_abi { for abi_error in abi.errors() { @@ -178,7 +183,6 @@ pub fn decode_revert( } } } - // optimistically try to decode as string, unknown selector or `CheatcodeError` String::decode(err) .ok() diff --git a/evm/src/executor/inspector/cheatcodes/mod.rs b/evm/src/executor/inspector/cheatcodes/mod.rs index a169ad6eb975..5f7e9d8af5e3 100644 --- a/evm/src/executor/inspector/cheatcodes/mod.rs +++ b/evm/src/executor/inspector/cheatcodes/mod.rs @@ -1,7 +1,7 @@ use self::{ env::Broadcast, expect::{handle_expect_emit, handle_expect_revert, ExpectedCallType}, - util::{check_if_fixed_gas_limit, process_create, BroadcastableTransactions}, + util::{check_if_fixed_gas_limit, process_create, BroadcastableTransactions, MAGIC_SKIP_BYTES}, }; use crate::{ abi::HEVMCalls, @@ -115,6 +115,9 @@ pub struct Cheatcodes { /// Rememebered private keys pub script_wallets: Vec, + /// Whether the skip cheatcode was activated + pub skip: bool, + /// Prank information pub prank: Option, @@ -744,6 +747,14 @@ where return (status, remaining_gas, retdata) } + if data.journaled_state.depth() == 0 && self.skip { + return ( + InstructionResult::Revert, + remaining_gas, + Error::custom_bytes(MAGIC_SKIP_BYTES).encode_error().0, + ) + } + // Clean up pranks if let Some(prank) = &self.prank { if data.journaled_state.depth() == prank.depth { diff --git a/evm/src/executor/inspector/cheatcodes/util.rs b/evm/src/executor/inspector/cheatcodes/util.rs index 1bb471e41a7a..abb9aad05aba 100644 --- a/evm/src/executor/inspector/cheatcodes/util.rs +++ b/evm/src/executor/inspector/cheatcodes/util.rs @@ -1,4 +1,4 @@ -use super::{ensure, fmt_err, Cheatcodes, Result}; +use super::{ensure, fmt_err, Cheatcodes, Error, Result}; use crate::{ abi::HEVMCalls, executor::backend::{ @@ -40,6 +40,8 @@ pub const DEFAULT_CREATE2_DEPLOYER: H160 = H160([ 78, 89, 180, 72, 71, 179, 121, 87, 133, 136, 146, 12, 167, 143, 191, 38, 192, 180, 149, 108, ]); +pub const MAGIC_SKIP_BYTES: &[u8] = b"FOUNDRY::SKIP"; + /// Helps collecting transactions from different forks. #[derive(Debug, Clone, Default)] pub struct BroadcastableTransaction { @@ -197,6 +199,21 @@ pub fn parse(s: &str, ty: &ParamType) -> Result { .map_err(|e| fmt_err!("Failed to parse `{s}` as type `{ty}`: {e}")) } +pub fn skip(state: &mut Cheatcodes, depth: u64, skip: bool) -> Result { + if !skip { + return Ok(b"".into()) + } + + // Skip should not work if called deeper than at test level. + // As we're not returning the magic skip bytes, this will cause a test failure. + if depth > 1 { + return Err(Error::custom("The skip cheatcode can only be used at test level")) + } + + state.skip = true; + Err(Error::custom_bytes(MAGIC_SKIP_BYTES)) +} + #[instrument(level = "error", name = "util", target = "evm::cheatcodes", skip_all)] pub fn apply( state: &mut Cheatcodes, @@ -253,6 +270,7 @@ pub fn apply( HEVMCalls::ParseInt(inner) => parse(&inner.0, &ParamType::Int(256)), HEVMCalls::ParseBytes32(inner) => parse(&inner.0, &ParamType::FixedBytes(32)), HEVMCalls::ParseBool(inner) => parse(&inner.0, &ParamType::Bool), + HEVMCalls::Skip(inner) => skip(state, data.journaled_state.depth(), inner.0), _ => return None, }) } diff --git a/evm/src/executor/mod.rs b/evm/src/executor/mod.rs index e024bc771146..c8b4591c71f9 100644 --- a/evm/src/executor/mod.rs +++ b/evm/src/executor/mod.rs @@ -320,7 +320,6 @@ impl Executor { value, ); let call_result = self.call_raw_with_env(env)?; - convert_call_result(abi, &func, call_result) } @@ -642,6 +641,9 @@ pub enum EvmError { /// Error which occurred during ABI encoding/decoding #[error(transparent)] AbiError(#[from] ethers::contract::AbiError), + /// Error caused which occurred due to calling the skip() cheatcode. + #[error("Skipped")] + SkipError, /// Any other error. #[error(transparent)] Eyre(#[from] eyre::Error), @@ -669,6 +671,7 @@ pub struct DeployResult { /// The result of a call. #[derive(Debug)] pub struct CallResult { + pub skipped: bool, /// Whether the call reverted or not pub reverted: bool, /// The decoded result of the call @@ -798,7 +801,6 @@ fn convert_executed_result( (halt_to_instruction_result(reason), 0_u64, gas_used, None) } }; - let stipend = calc_stipend(&env.tx.data, env.cfg.spec_id); let result = match out { @@ -897,11 +899,15 @@ fn convert_call_result( script_wallets, env, breakpoints, + skipped: false, }) } _ => { let reason = decode::decode_revert(result.as_ref(), abi, Some(status)) .unwrap_or_else(|_| format!("{status:?}")); + if reason == "SKIPPED" { + return Err(EvmError::SkipError) + } Err(EvmError::Execution(Box::new(ExecutionErr { reverted, reason, diff --git a/forge/src/result.rs b/forge/src/result.rs index 5bdafc3ab27e..5cbc2296b48c 100644 --- a/forge/src/result.rs +++ b/forge/src/result.rs @@ -34,12 +34,12 @@ impl SuiteResult { /// Iterator over all succeeding tests and their names pub fn successes(&self) -> impl Iterator { - self.tests().filter(|(_, t)| t.success) + self.tests().filter(|(_, t)| t.status == TestStatus::Success) } /// Iterator over all failing tests and their names pub fn failures(&self) -> impl Iterator { - self.tests().filter(|(_, t)| !t.success) + self.tests().filter(|(_, t)| t.status == TestStatus::Failure) } /// Iterator over all tests and their names @@ -58,13 +58,22 @@ impl SuiteResult { } } +#[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq, Eq)] +pub enum TestStatus { + Success, + #[default] + Failure, + Skipped, +} + /// The result of an executed solidity test #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct TestResult { - /// Whether the test case was successful. This means that the transaction executed - /// properly, or that there was a revert and that the test was expected to fail - /// (prefixed with `testFail`) - pub success: bool, + /// The test status, indicating whether the test case succeeded, failed, or was marked as + /// skipped. This means that the transaction executed properly, the test was marked as + /// skipped with vm.skip(), or that there was a revert and that the test was expected to + /// fail (prefixed with `testFail`) + pub status: TestStatus, /// If there was a revert, this field will be populated. Note that the test can /// still be successful (i.e self.success == true) when it's expected to fail. @@ -99,7 +108,7 @@ pub struct TestResult { impl TestResult { pub fn fail(reason: String) -> Self { - Self { success: false, reason: Some(reason), ..Default::default() } + Self { status: TestStatus::Failure, reason: Some(reason), ..Default::default() } } /// Returns `true` if this is the result of a fuzz test diff --git a/forge/src/runner.rs b/forge/src/runner.rs index 6a83c0156e3b..a233d3b67e45 100644 --- a/forge/src/runner.rs +++ b/forge/src/runner.rs @@ -1,5 +1,5 @@ use crate::{ - result::{SuiteResult, TestKind, TestResult, TestSetup}, + result::{SuiteResult, TestKind, TestResult, TestSetup, TestStatus}, TestFilter, TestOptions, }; use ethers::{ @@ -225,7 +225,7 @@ impl<'a> ContractRunner<'a> { [( "setUp()".to_string(), TestResult { - success: false, + status: TestStatus::Failure, reason: setup.reason, counterexample: None, decoded_logs: decode_console_logs(&setup.logs), @@ -288,7 +288,8 @@ impl<'a> ContractRunner<'a> { let duration = start.elapsed(); if !test_results.is_empty() { - let successful = test_results.iter().filter(|(_, tst)| tst.success).count(); + let successful = + test_results.iter().filter(|(_, tst)| tst.status == TestStatus::Success).count(); info!( duration = ?duration, "done. {}/{} successful", @@ -347,9 +348,20 @@ impl<'a> ContractRunner<'a> { HashMap::new(), ) } + Err(EvmError::SkipError) => { + return TestResult { + status: TestStatus::Skipped, + reason: None, + decoded_logs: decode_console_logs(&logs), + traces, + labeled_addresses, + kind: TestKind::Standard(0), + ..Default::default() + } + } Err(err) => { return TestResult { - success: false, + status: TestStatus::Failure, reason: Some(err.to_string()), decoded_logs: decode_console_logs(&logs), traces, @@ -375,7 +387,10 @@ impl<'a> ContractRunner<'a> { ); TestResult { - success, + status: match success { + true => TestStatus::Success, + false => TestStatus::Failure, + }, reason, counterexample: None, decoded_logs: decode_console_logs(&logs), @@ -403,6 +418,26 @@ impl<'a> ContractRunner<'a> { let project_contracts = known_contracts.unwrap_or(&empty); let TestSetup { address, logs, traces, labeled_addresses, .. } = setup; + // First, run the test normally to see if it needs to be skipped. + if let Err(EvmError::SkipError) = self.executor.execute_test::<(), _, _>( + self.sender, + address, + functions[0].clone(), + (), + 0.into(), + self.errors, + ) { + return vec![TestResult { + status: TestStatus::Skipped, + reason: None, + decoded_logs: decode_console_logs(&logs), + traces, + labeled_addresses, + kind: TestKind::Standard(0), + ..Default::default() + }] + }; + let mut evm = InvariantExecutor::new( &mut self.executor, runner, @@ -473,7 +508,10 @@ impl<'a> ContractRunner<'a> { }; TestResult { - success, + status: match success { + true => TestStatus::Success, + false => TestStatus::Failure, + }, reason, counterexample, decoded_logs: decode_console_logs(&logs), @@ -499,6 +537,26 @@ impl<'a> ContractRunner<'a> { ) -> TestResult { let TestSetup { address, mut logs, mut traces, mut labeled_addresses, .. } = setup; + let skip_fuzz_config = FuzzConfig { runs: 1, ..Default::default() }; + + // Fuzz the test with only 1 run to check if it needs to be skipped. + let result = + FuzzedExecutor::new(&self.executor, runner.clone(), self.sender, skip_fuzz_config) + .fuzz(func, address, should_fail, self.errors); + if let Some(reason) = result.reason { + if matches!(reason.as_str(), "SKIPPED") { + return TestResult { + status: TestStatus::Skipped, + reason: None, + decoded_logs: decode_console_logs(&logs), + traces, + labeled_addresses, + kind: TestKind::Standard(0), + ..Default::default() + } + } + } + // Run fuzz test let start = Instant::now(); let mut result = FuzzedExecutor::new(&self.executor, runner, self.sender, fuzz_config) @@ -523,7 +581,10 @@ impl<'a> ContractRunner<'a> { ); TestResult { - success: result.success, + status: match result.success { + true => TestStatus::Success, + false => TestStatus::Failure, + }, reason: result.reason, counterexample: result.counterexample, decoded_logs: decode_console_logs(&logs), diff --git a/forge/tests/it/cheats.rs b/forge/tests/it/cheats.rs index c0253cb0e001..81d593ac5b49 100644 --- a/forge/tests/it/cheats.rs +++ b/forge/tests/it/cheats.rs @@ -9,7 +9,7 @@ use crate::{ #[test] fn test_cheats_local() { let filter = - Filter::new(".*", ".*", &format!(".*cheats{RE_PATH_SEPARATOR}*")).exclude_paths("Fork"); + Filter::new(".*", "Skip*", &format!(".*cheats{RE_PATH_SEPARATOR}*")).exclude_paths("Fork"); // on windows exclude ffi tests since no echo and file test that expect a certain file path #[cfg(windows)] diff --git a/forge/tests/it/config.rs b/forge/tests/it/config.rs index 6b4ff28a592c..333f0c06238b 100644 --- a/forge/tests/it/config.rs +++ b/forge/tests/it/config.rs @@ -3,7 +3,10 @@ use crate::test_helpers::{ filter::Filter, COMPILED, COMPILED_WITH_LIBS, EVM_OPTS, LIBS_PROJECT, PROJECT, }; -use forge::{result::SuiteResult, MultiContractRunner, MultiContractRunnerBuilder, TestOptions}; +use forge::{ + result::{SuiteResult, TestStatus}, + MultiContractRunner, MultiContractRunnerBuilder, TestOptions, +}; use foundry_config::{ fs_permissions::PathPermission, Config, FsPermissions, FuzzConfig, FuzzDictionaryConfig, InvariantConfig, RpcEndpoint, RpcEndpoints, @@ -75,7 +78,9 @@ impl TestConfig { } for (_, SuiteResult { test_results, .. }) in suite_result { for (test_name, result) in test_results { - if self.should_fail != !result.success { + if self.should_fail && (result.status == TestStatus::Success) || + !self.should_fail && (result.status == TestStatus::Failure) + { let logs = decode_console_logs(&result.logs); let outcome = if self.should_fail { "fail" } else { "pass" }; @@ -239,7 +244,7 @@ pub fn assert_multiple( if *should_pass { assert!( - actuals[*contract_name].test_results[*test_name].success, + actuals[*contract_name].test_results[*test_name].status == TestStatus::Success, "Test {} did not pass as expected.\nReason: {:?}\nLogs:\n{}", test_name, actuals[*contract_name].test_results[*test_name].reason, @@ -247,7 +252,7 @@ pub fn assert_multiple( ); } else { assert!( - !actuals[*contract_name].test_results[*test_name].success, + actuals[*contract_name].test_results[*test_name].status == TestStatus::Failure, "Test {} did not fail as expected.\nLogs:\n{}", test_name, logs.join("\n") diff --git a/forge/tests/it/fuzz.rs b/forge/tests/it/fuzz.rs index 7cdb75d992c3..a3c75c2683ff 100644 --- a/forge/tests/it/fuzz.rs +++ b/forge/tests/it/fuzz.rs @@ -2,7 +2,7 @@ use crate::{config::*, test_helpers::filter::Filter}; use ethers::types::U256; -use forge::result::SuiteResult; +use forge::result::{SuiteResult, TestStatus}; use std::collections::BTreeMap; #[test] @@ -26,14 +26,14 @@ fn test_fuzz() { "testPositive(int256)" | "testSuccessfulFuzz(uint128,uint128)" | "testToStringFuzz(bytes32)" => assert!( - result.success, + result.status == TestStatus::Success, "Test {} did not pass as expected.\nReason: {:?}\nLogs:\n{}", test_name, result.reason, result.decoded_logs.join("\n") ), _ => assert!( - !result.success, + result.status == TestStatus::Failure, "Test {} did not fail as expected.\nReason: {:?}\nLogs:\n{}", test_name, result.reason, diff --git a/testdata/cheats/Cheats.sol b/testdata/cheats/Cheats.sol index cbb44603bfde..540dc31cd662 100644 --- a/testdata/cheats/Cheats.sol +++ b/testdata/cheats/Cheats.sol @@ -180,6 +180,9 @@ interface Cheats { // Sets an address' code, (who, newCode) function etch(address, bytes calldata) external; + // Skips a test. + function skip(bool) external; + // Expects an error on next call function expectRevert() external; diff --git a/testdata/cheats/Skip.t.sol b/testdata/cheats/Skip.t.sol new file mode 100644 index 000000000000..34a2d76d0916 --- /dev/null +++ b/testdata/cheats/Skip.t.sol @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity 0.8.18; + +import "ds-test/test.sol"; +import "./Cheats.sol"; + +contract SkipTest is DSTest { + Cheats constant cheats = Cheats(HEVM_ADDRESS); + + function testSkip() public { + cheats.skip(true); + revert("Should not reach this revert"); + } + + function testFailNotSkip() public { + cheats.skip(false); + revert("This test should fail"); + } + + function testFuzzSkip(uint256 x) public { + cheats.skip(true); + revert("Should not reach revert"); + } + + function testFailFuzzSkip(uint256 x) public { + cheats.skip(false); + revert("This test should fail"); + } + + function statefulFuzzSkip() public { + cheats.skip(true); + require(true == false, "Test should not reach invariant"); + } +}