diff --git a/CHANGELOG.md b/CHANGELOG.md index a0599f1bc..0d8354a97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # unreleased - Add command exit code to output if it fails, see #342 (@KaindlJulian) +- Export command exit code to JSON output, see #371 (@JordiChauzi) ## Features diff --git a/src/hyperfine/benchmark.rs b/src/hyperfine/benchmark.rs index 446d75de5..60ac8f5de 100644 --- a/src/hyperfine/benchmark.rs +++ b/src/hyperfine/benchmark.rs @@ -1,6 +1,6 @@ use std::cmp; use std::io; -use std::process::Stdio; +use std::process::{ExitStatus, Stdio}; use colored::*; use statistical::{mean, median, standard_deviation}; @@ -46,7 +46,7 @@ pub fn time_shell_command( show_output: bool, failure_action: CmdFailureAction, shell_spawning_time: Option, -) -> io::Result<(TimingResult, bool)> { +) -> io::Result<(TimingResult, ExitStatus)> { let (stdout, stderr) = if show_output { (Stdio::inherit(), Stdio::inherit()) } else { @@ -88,7 +88,7 @@ pub fn time_shell_command( time_user, time_system, }, - result.status.success(), + result.status, )) } @@ -201,6 +201,27 @@ fn run_cleanup_command( run_intermediate_command(shell, command, show_output, error_output) } +#[cfg(unix)] +fn extract_exit_code(status: ExitStatus) -> Option { + use std::os::unix::process::ExitStatusExt; + + /* From the ExitStatus::code documentation: + "On Unix, this will return None if the process was terminated by a signal." + In that case, ExitStatusExt::signal should never return None. + */ + status.code().or_else(|| + /* To differentiate between "normal" exit codes and signals, we are using + something similar to bash exit codes (https://tldp.org/LDP/abs/html/exitcodes.html) + by adding 128 to a signal integer value. + */ + status.signal().map(|s| 128 + s)) +} + +#[cfg(not(unix))] +fn extract_exit_code(status: ExitStatus) -> Option { + status.code() +} + /// Run the benchmark for a single shell command pub fn run_benchmark( num: usize, @@ -228,6 +249,7 @@ pub fn run_benchmark( let mut times_real: Vec = vec![]; let mut times_user: Vec = vec![]; let mut times_system: Vec = vec![]; + let mut exit_codes: Vec> = vec![]; let mut all_succeeded = true; // Run init command @@ -280,13 +302,14 @@ pub fn run_benchmark( let prepare_res = run_preparation_command(&options.shell, &prepare_cmd, options.show_output)?; // Initial timing run - let (res, success) = time_shell_command( + let (res, status) = time_shell_command( &options.shell, cmd, options.show_output, options.failure_action, Some(shell_spawning_time), )?; + let success = status.success(); // Determine number of benchmark runs let runs_in_min_time = (options.min_time_sec @@ -310,6 +333,7 @@ pub fn run_benchmark( times_real.push(res.time_real); times_user.push(res.time_user); times_system.push(res.time_system); + exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; @@ -328,17 +352,19 @@ pub fn run_benchmark( progress_bar.as_ref().map(|bar| bar.set_message(&msg)); - let (res, success) = time_shell_command( + let (res, status) = time_shell_command( &options.shell, cmd, options.show_output, options.failure_action, Some(shell_spawning_time), )?; + let success = status.success(); times_real.push(res.time_real); times_user.push(res.time_user); times_system.push(res.time_system); + exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; @@ -438,6 +464,7 @@ pub fn run_benchmark( t_min, t_max, times_real, + exit_codes, cmd.get_parameters() .iter() .map(|(name, value)| ((*name).to_string(), value.to_string())) diff --git a/src/hyperfine/export/asciidoc.rs b/src/hyperfine/export/asciidoc.rs index cae6075f9..08623bfe6 100644 --- a/src/hyperfine/export/asciidoc.rs +++ b/src/hyperfine/export/asciidoc.rs @@ -99,7 +99,8 @@ fn test_asciidoc_table_row() { 0.10745223440000001, 0.10697327940000001, ], - BTreeMap::new(), // param + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // param ); let expms = format!( @@ -151,7 +152,8 @@ fn test_asciidoc_table_row_command_escape() { 0.10745223440000001, 0.10697327940000001, ], - BTreeMap::new(), // param + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // param ); let exps = format!( "| `sleep 1\\|`\n\ @@ -185,6 +187,7 @@ fn test_asciidoc() { 5.0, 6.0, vec![7.0, 8.0, 9.0], + vec![Some(0), Some(0), Some(0)], { let mut params = BTreeMap::new(); params.insert("foo".into(), "1".into()); @@ -202,6 +205,7 @@ fn test_asciidoc() { 15.0, 16.0, vec![17.0, 18.0, 19.0], + vec![Some(0), Some(0), Some(0)], { let mut params = BTreeMap::new(); params.insert("foo".into(), "1".into()); diff --git a/src/hyperfine/export/csv.rs b/src/hyperfine/export/csv.rs index 28593a44a..ef2cc2328 100644 --- a/src/hyperfine/export/csv.rs +++ b/src/hyperfine/export/csv.rs @@ -17,7 +17,7 @@ impl Exporter for CsvExporter { { let mut headers: Vec> = [ - // The list of times cannot be exported to the CSV file - omit it. + // The list of times and exit codes cannot be exported to the CSV file - omit them. "command", "mean", "stddev", "median", "user", "system", "min", "max", ] .iter() @@ -68,6 +68,7 @@ fn test_csv() { 5.0, 6.0, vec![7.0, 8.0, 9.0], + vec![Some(0), Some(0), Some(0)], { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); @@ -85,6 +86,7 @@ fn test_csv() { 15.0, 16.5, vec![17.0, 18.0, 19.0], + vec![Some(0), Some(0), Some(0)], { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); diff --git a/src/hyperfine/export/markdown.rs b/src/hyperfine/export/markdown.rs index 40b3a6abe..3d7a2e25e 100644 --- a/src/hyperfine/export/markdown.rs +++ b/src/hyperfine/export/markdown.rs @@ -94,28 +94,30 @@ fn test_markdown_format_ms() { timing_results.push(BenchmarkResult::new( String::from("sleep 0.1"), - 0.1057, // mean - 0.0016, // std dev - 0.1057, // median - 0.0009, // user_mean - 0.0011, // system_mean - 0.1023, // min - 0.1080, // max - vec![0.1, 0.1, 0.1], // times - BTreeMap::new(), // parameter + 0.1057, // mean + 0.0016, // std dev + 0.1057, // median + 0.0009, // user_mean + 0.0011, // system_mean + 0.1023, // min + 0.1080, // max + vec![0.1, 0.1, 0.1], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); timing_results.push(BenchmarkResult::new( String::from("sleep 2"), - 2.0050, // mean - 0.0020, // std dev - 2.0050, // median - 0.0009, // user_mean - 0.0012, // system_mean - 2.0020, // min - 2.0080, // max - vec![2.0, 2.0, 2.0], // times - BTreeMap::new(), // parameter + 2.0050, // mean + 0.0020, // std dev + 2.0050, // median + 0.0009, // user_mean + 0.0012, // system_mean + 2.0020, // min + 2.0080, // max + vec![2.0, 2.0, 2.0], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); let formatted = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap(); @@ -142,28 +144,30 @@ fn test_markdown_format_s() { timing_results.push(BenchmarkResult::new( String::from("sleep 2"), - 2.0050, // mean - 0.0020, // std dev - 2.0050, // median - 0.0009, // user_mean - 0.0012, // system_mean - 2.0020, // min - 2.0080, // max - vec![2.0, 2.0, 2.0], // times - BTreeMap::new(), // parameter + 2.0050, // mean + 0.0020, // std dev + 2.0050, // median + 0.0009, // user_mean + 0.0012, // system_mean + 2.0020, // min + 2.0080, // max + vec![2.0, 2.0, 2.0], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); timing_results.push(BenchmarkResult::new( String::from("sleep 0.1"), - 0.1057, // mean - 0.0016, // std dev - 0.1057, // median - 0.0009, // user_mean - 0.0011, // system_mean - 0.1023, // min - 0.1080, // max - vec![0.1, 0.1, 0.1], // times - BTreeMap::new(), // parameter + 0.1057, // mean + 0.0016, // std dev + 0.1057, // median + 0.0009, // user_mean + 0.0011, // system_mean + 0.1023, // min + 0.1080, // max + vec![0.1, 0.1, 0.1], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); let formatted = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap(); @@ -189,28 +193,30 @@ fn test_markdown_format_time_unit_s() { timing_results.push(BenchmarkResult::new( String::from("sleep 0.1"), - 0.1057, // mean - 0.0016, // std dev - 0.1057, // median - 0.0009, // user_mean - 0.0011, // system_mean - 0.1023, // min - 0.1080, // max - vec![0.1, 0.1, 0.1], // times - BTreeMap::new(), // parameter + 0.1057, // mean + 0.0016, // std dev + 0.1057, // median + 0.0009, // user_mean + 0.0011, // system_mean + 0.1023, // min + 0.1080, // max + vec![0.1, 0.1, 0.1], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); timing_results.push(BenchmarkResult::new( String::from("sleep 2"), - 2.0050, // mean - 0.0020, // std dev - 2.0050, // median - 0.0009, // user_mean - 0.0012, // system_mean - 2.0020, // min - 2.0080, // max - vec![2.0, 2.0, 2.0], // times - BTreeMap::new(), // parameter + 2.0050, // mean + 0.0020, // std dev + 2.0050, // median + 0.0009, // user_mean + 0.0012, // system_mean + 2.0020, // min + 2.0080, // max + vec![2.0, 2.0, 2.0], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); let formatted = String::from_utf8( @@ -242,28 +248,30 @@ fn test_markdown_format_time_unit_ms() { timing_results.push(BenchmarkResult::new( String::from("sleep 2"), - 2.0050, // mean - 0.0020, // std dev - 2.0050, // median - 0.0009, // user_mean - 0.0012, // system_mean - 2.0020, // min - 2.0080, // max - vec![2.0, 2.0, 2.0], // times - BTreeMap::new(), // parameter + 2.0050, // mean + 0.0020, // std dev + 2.0050, // median + 0.0009, // user_mean + 0.0012, // system_mean + 2.0020, // min + 2.0080, // max + vec![2.0, 2.0, 2.0], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); timing_results.push(BenchmarkResult::new( String::from("sleep 0.1"), - 0.1057, // mean - 0.0016, // std dev - 0.1057, // median - 0.0009, // user_mean - 0.0011, // system_mean - 0.1023, // min - 0.1080, // max - vec![0.1, 0.1, 0.1], // times - BTreeMap::new(), // parameter + 0.1057, // mean + 0.0016, // std dev + 0.1057, // median + 0.0009, // user_mean + 0.0011, // system_mean + 0.1023, // min + 0.1080, // max + vec![0.1, 0.1, 0.1], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); let formatted = String::from_utf8( diff --git a/src/hyperfine/internal.rs b/src/hyperfine/internal.rs index adeff63b3..72ea70033 100644 --- a/src/hyperfine/internal.rs +++ b/src/hyperfine/internal.rs @@ -157,6 +157,7 @@ fn create_result(name: &str, mean: Scalar) -> BenchmarkResult { min: mean, max: mean, times: None, + exit_codes: Vec::new(), parameters: BTreeMap::new(), } } diff --git a/src/hyperfine/types.rs b/src/hyperfine/types.rs index 98e5ea0b3..850a5415b 100644 --- a/src/hyperfine/types.rs +++ b/src/hyperfine/types.rs @@ -268,6 +268,9 @@ pub struct BenchmarkResult { #[serde(skip_serializing_if = "Option::is_none")] pub times: Option>, + /// All run exit codes + pub exit_codes: Vec>, + /// Any parameter values used #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, @@ -285,6 +288,7 @@ impl BenchmarkResult { min: Second, max: Second, times: Vec, + exit_codes: Vec>, parameters: BTreeMap, ) -> Self { BenchmarkResult { @@ -297,6 +301,7 @@ impl BenchmarkResult { min, max, times: Some(times), + exit_codes, parameters, } }