Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(revme): add --keep-going to statetest command #1277

Merged
merged 1 commit into from
Apr 9, 2024
Merged

feat(revme): add --keep-going to statetest command #1277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion bins/revme/src/cmd/statetest.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ pub struct Cmd {
/// It will stop second run of evm on failure.
#[structopt(short = "o", long)]
json_outcome: bool,
#[structopt(long, alias = "no-fail-fast")]
keep_going: bool,
}

impl Cmd {
Expand All @@ -37,7 +39,13 @@ impl Cmd {
for path in &self.path {
println!("\nRunning tests in {}...", path.display());
let test_files = find_all_json_tests(path);
run(test_files, self.single_thread, self.json, self.json_outcome)?
run(
test_files,
self.single_thread,
self.json,
self.json_outcome,
self.keep_going,
)?
}
Ok(())
}
Expand Down
105 changes: 66 additions & 39 deletions bins/revme/src/cmd/statetest/runner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@ use std::{
convert::Infallible,
io::{stderr, stdout},
path::{Path, PathBuf},
sync::atomic::Ordering,
sync::{atomic::AtomicBool, Arc, Mutex},
sync::{
atomic::{AtomicBool, AtomicUsize, Ordering},
Arc, Mutex,
},
time::{Duration, Instant},
};
use thiserror::Error;
Expand All @@ -36,33 +38,35 @@ pub struct TestError {

#[derive(Debug, Error)]
pub enum TestErrorKind {
#[error("logs root mismatch: expected {expected:?}, got {got:?}")]
#[error("logs root mismatch: got {got}, expected {expected}")]
LogsRootMismatch { got: B256, expected: B256 },
#[error("state root mismatch: expected {expected:?}, got {got:?}")]
#[error("state root mismatch: got {got}, expected {expected}")]
StateRootMismatch { got: B256, expected: B256 },
#[error("Unknown private key: {0:?}")]
#[error("unknown private key: {0:?}")]
UnknownPrivateKey(B256),
#[error("Unexpected exception: {got_exception:?} but test expects:{expected_exception:?}")]
#[error("unexpected exception: got {got_exception:?}, expected {expected_exception:?}")]
UnexpectedException {
expected_exception: Option<String>,
got_exception: Option<String>,
},
#[error("Unexpected output: {got_output:?} but test expects:{expected_output:?}")]
#[error("unexpected output: got {got_output:?}, expected {expected_output:?}")]
UnexpectedOutput {
expected_output: Option<Bytes>,
got_output: Option<Bytes>,
},
#[error(transparent)]
SerdeDeserialize(#[from] serde_json::Error),
#[error("thread panicked")]
Panic,
}

pub fn find_all_json_tests(path: &Path) -> Vec<PathBuf> {
WalkDir::new(path)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_name().to_string_lossy().ends_with(".json"))
.filter_map(Result::ok)
.filter(|e| e.path().extension() == Some("json".as_ref()))
.map(DirEntry::into_path)
.collect::<Vec<PathBuf>>()
.collect()
}

fn skip_test(path: &Path) -> bool {
Expand Down Expand Up @@ -121,19 +125,26 @@ fn check_evm_execution<EXT>(
let print_json_output = |error: Option<String>| {
if print_json_outcome {
let json = json!({
"stateRoot": state_root,
"logsRoot": logs_root,
"output": exec_result.as_ref().ok().and_then(|r| r.output().cloned()).unwrap_or_default(),
"gasUsed": exec_result.as_ref().ok().map(|r| r.gas_used()).unwrap_or_default(),
"pass": error.is_none(),
"errorMsg": error.unwrap_or_default(),
"evmResult": exec_result.as_ref().err().map(|e| e.to_string()).unwrap_or("Ok".to_string()),
"postLogsHash": logs_root,
"fork": evm.handler.cfg().spec_id,
"test": test_name,
"d": test.indexes.data,
"g": test.indexes.gas,
"v": test.indexes.value,
"stateRoot": state_root,
"logsRoot": logs_root,
"output": exec_result.as_ref().ok().and_then(|r| r.output().cloned()).unwrap_or_default(),
"gasUsed": exec_result.as_ref().ok().map(|r| r.gas_used()).unwrap_or_default(),
"pass": error.is_none(),
"errorMsg": error.unwrap_or_default(),
"evmResult": match exec_result {
Ok(r) => match r {
ExecutionResult::Success { reason, .. } => format!("Success: {reason:?}"),
ExecutionResult::Revert { .. } => "Revert".to_string(),
ExecutionResult::Halt { reason, .. } => format!("Halt: {reason:?}"),
},
Err(e) => e.to_string(),
},
"postLogsHash": logs_root,
"fork": evm.handler.cfg().spec_id,
"test": test_name,
"d": test.indexes.data,
"g": test.indexes.gas,
"v": test.indexes.value,
});
eprintln!("{json}");
}
Expand Down Expand Up @@ -443,6 +454,7 @@ pub fn run(
mut single_thread: bool,
trace: bool,
mut print_outcome: bool,
keep_going: bool,
) -> Result<(), TestError> {
// trace implies print_outcome
if trace {
Expand All @@ -454,7 +466,7 @@ pub fn run(
}
let n_files = test_files.len();

let endjob = Arc::new(AtomicBool::new(false));
let n_errors = Arc::new(AtomicUsize::new(0));
let console_bar = Arc::new(ProgressBar::with_draw_target(
Some(n_files as u64),
ProgressDrawTarget::stdout(),
Expand All @@ -470,14 +482,14 @@ pub fn run(
let mut handles = Vec::with_capacity(num_threads);
for i in 0..num_threads {
let queue = queue.clone();
let endjob = endjob.clone();
let n_errors = n_errors.clone();
let console_bar = console_bar.clone();
let elapsed = elapsed.clone();

let thread = std::thread::Builder::new().name(format!("runner-{i}"));

let f = move || loop {
if endjob.load(Ordering::SeqCst) {
if !keep_going && n_errors.load(Ordering::SeqCst) > 0 {
return Ok(());
}

Expand All @@ -491,20 +503,27 @@ pub fn run(
(prev_idx, test_path)
};

console_bar.inc(1);
if let Err(err) = execute_test_suite(&test_path, &elapsed, trace, print_outcome) {
endjob.store(true, Ordering::SeqCst);
return Err(err);
n_errors.fetch_add(1, Ordering::SeqCst);
if !keep_going {
return Err(err);
}
}
console_bar.inc(1);
};
handles.push(thread.spawn(f).unwrap());
}

// join all threads before returning an error
let mut errors = Vec::new();
for handle in handles {
if let Err(e) = handle.join().unwrap() {
errors.push(e);
let mut thread_errors = Vec::new();
for (i, handle) in handles.into_iter().enumerate() {
match handle.join() {
Ok(Ok(())) => {}
Ok(Err(e)) => thread_errors.push(e),
Err(_) => thread_errors.push(TestError {
name: format!("thread {i} panicked"),
kind: TestErrorKind::Panic,
}),
}
}
console_bar.finish();
Expand All @@ -513,17 +532,25 @@ pub fn run(
"Finished execution. Total CPU time: {:.6}s",
elapsed.lock().unwrap().as_secs_f64()
);
if errors.is_empty() {

let n_errors = n_errors.load(Ordering::SeqCst);
let n_thread_errors = thread_errors.len();
if n_errors == 0 && n_thread_errors == 0 {
println!("All tests passed!");
Ok(())
} else {
let n = errors.len();
if n > 1 {
println!("{n} threads returned an error, out of {num_threads} total:");
for error in &errors {
println!("Encountered {n_errors} errors out of {n_files} total tests");

if n_thread_errors == 0 {
std::process::exit(1);
}

if n_thread_errors > 1 {
println!("{n_thread_errors} threads returned an error, out of {num_threads} total:");
for error in &thread_errors {
println!("{error}");
}
}
Err(errors.swap_remove(0))
Err(thread_errors.swap_remove(0))
}
}
Loading