diff --git a/compiler/fm/src/file_map.rs b/compiler/fm/src/file_map.rs index ba552fe5156..857c7460fb9 100644 --- a/compiler/fm/src/file_map.rs +++ b/compiler/fm/src/file_map.rs @@ -80,6 +80,19 @@ impl FileMap { pub fn all_file_ids(&self) -> impl Iterator { self.name_to_id.values() } + + pub fn get_name(&self, file_id: FileId) -> Result { + let name = self.files.get(file_id.as_usize())?.name().clone(); + + // See if we can make the file name a bit shorter/easier to read if it starts with the current directory + if let Some(current_dir) = &self.current_dir { + if let Ok(name_without_prefix) = name.0.strip_prefix(current_dir) { + return Ok(PathString::from_path(name_without_prefix.to_path_buf())); + } + } + + Ok(name) + } } impl Default for FileMap { fn default() -> Self { @@ -97,16 +110,7 @@ impl<'a> Files<'a> for FileMap { type Source = &'a str; fn name(&self, file_id: Self::FileId) -> Result { - let name = self.files.get(file_id.as_usize())?.name().clone(); - - // See if we can make the file name a bit shorter/easier to read if it starts with the current directory - if let Some(current_dir) = &self.current_dir { - if let Ok(name_without_prefix) = name.0.strip_prefix(current_dir) { - return Ok(PathString::from_path(name_without_prefix.to_path_buf())); - } - } - - Ok(name) + self.get_name(file_id) } fn source(&'a self, file_id: Self::FileId) -> Result { diff --git a/compiler/noirc_errors/src/reporter.rs b/compiler/noirc_errors/src/reporter.rs index f029b4e6de8..e57775d9a7f 100644 --- a/compiler/noirc_errors/src/reporter.rs +++ b/compiler/noirc_errors/src/reporter.rs @@ -272,7 +272,7 @@ fn convert_diagnostic( diagnostic.with_message(&cd.message).with_labels(secondary_labels).with_notes(notes) } -fn stack_trace<'files>( +pub fn stack_trace<'files>( files: &'files impl Files<'files, FileId = fm::FileId>, call_stack: &[Location], ) -> String { diff --git a/tooling/nargo_cli/src/cli/test_cmd.rs b/tooling/nargo_cli/src/cli/test_cmd.rs index 32512eba5bb..1fd4ed2d873 100644 --- a/tooling/nargo_cli/src/cli/test_cmd.rs +++ b/tooling/nargo_cli/src/cli/test_cmd.rs @@ -12,7 +12,7 @@ use acvm::{BlackBoxFunctionSolver, FieldElement}; use bn254_blackbox_solver::Bn254BlackBoxSolver; use clap::Args; use fm::FileManager; -use formatters::{Formatter, PrettyFormatter, TerseFormatter}; +use formatters::{Formatter, JsonFormatter, PrettyFormatter, TerseFormatter}; use nargo::{ insert_all_files_for_workspace_into_file_manager, ops::TestStatus, package::Package, parse_all, prepare_package, workspace::Workspace, PrintOutput, @@ -71,6 +71,8 @@ enum Format { Pretty, /// Display one character per test Terse, + /// Output a JSON Lines document + Json, } impl Format { @@ -78,6 +80,7 @@ impl Format { match self { Format::Pretty => Box::new(PrettyFormatter), Format::Terse => Box::new(TerseFormatter), + Format::Json => Box::new(JsonFormatter), } } } @@ -87,6 +90,7 @@ impl Display for Format { match self { Format::Pretty => write!(f, "pretty"), Format::Terse => write!(f, "terse"), + Format::Json => write!(f, "json"), } } } @@ -211,6 +215,12 @@ impl<'a> TestRunner<'a> { ) -> bool { let mut all_passed = true; + for (package_name, total_test_count) in test_count_per_package { + self.formatter + .package_start_async(package_name, *total_test_count) + .expect("Could not display package start"); + } + let (sender, receiver) = mpsc::channel(); let iter = &Mutex::new(tests.into_iter()); thread::scope(|scope| { @@ -228,6 +238,10 @@ impl<'a> TestRunner<'a> { break; }; + self.formatter + .test_start_async(&test.name, &test.package_name) + .expect("Could not display test start"); + let time_before_test = std::time::Instant::now(); let (status, output) = match catch_unwind(test.runner) { Ok((status, output)) => (status, output), @@ -255,6 +269,16 @@ impl<'a> TestRunner<'a> { time_to_run, }; + self.formatter + .test_end_async( + &test_result, + self.file_manager, + self.args.show_output, + self.args.compile_options.deny_warnings, + self.args.compile_options.silence_warnings, + ) + .expect("Could not display test start"); + if thread_sender.send(test_result).is_err() { break; } @@ -275,7 +299,7 @@ impl<'a> TestRunner<'a> { let total_test_count = *total_test_count; self.formatter - .package_start(package_name, total_test_count) + .package_start_sync(package_name, total_test_count) .expect("Could not display package start"); // Check if we have buffered test results for this package @@ -485,7 +509,7 @@ impl<'a> TestRunner<'a> { current_test_count: usize, total_test_count: usize, ) -> std::io::Result<()> { - self.formatter.test_end( + self.formatter.test_end_sync( test_result, current_test_count, total_test_count, diff --git a/tooling/nargo_cli/src/cli/test_cmd/formatters.rs b/tooling/nargo_cli/src/cli/test_cmd/formatters.rs index 2a791930f60..1b9b2d50378 100644 --- a/tooling/nargo_cli/src/cli/test_cmd/formatters.rs +++ b/tooling/nargo_cli/src/cli/test_cmd/formatters.rs @@ -2,15 +2,47 @@ use std::{io::Write, panic::RefUnwindSafe, time::Duration}; use fm::FileManager; use nargo::ops::TestStatus; +use noirc_errors::{reporter::stack_trace, FileDiagnostic}; +use serde_json::{json, Map}; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, StandardStreamLock, WriteColor}; use super::TestResult; +/// A formatter for showing test results. +/// +/// The order of events is: +/// 1. Compilation of all packages happen (in parallel). There's no formatter method for this. +/// 2. If compilation is successful, one `package_start_async` for each package. +/// 3. For each test, one `test_start_async` event +/// (there's no `test_start_sync` event because it would happen right before `test_end_sync`) +/// 4. For each package, sequentially: +/// a. A `package_start_sync` event +/// b. One `test_end` event for each test +/// a. A `package_end` event +/// +/// The reason we have some `sync` and `async` events is that formatters that show output +/// to humans rely on the `sync` events to show a more predictable output (package by package), +/// and formatters that output to a machine-readable format (like JSON) rely on the `async` +/// events to show things as soon as they happen, regardless of a package ordering. pub(super) trait Formatter: Send + Sync + RefUnwindSafe { - fn package_start(&self, package_name: &str, test_count: usize) -> std::io::Result<()>; + fn package_start_async(&self, package_name: &str, test_count: usize) -> std::io::Result<()>; + + fn package_start_sync(&self, package_name: &str, test_count: usize) -> std::io::Result<()>; + + fn test_start_async(&self, name: &str, package_name: &str) -> std::io::Result<()>; + + #[allow(clippy::too_many_arguments)] + fn test_end_async( + &self, + test_result: &TestResult, + file_manager: &FileManager, + show_output: bool, + deny_warnings: bool, + silence_warnings: bool, + ) -> std::io::Result<()>; #[allow(clippy::too_many_arguments)] - fn test_end( + fn test_end_sync( &self, test_result: &TestResult, current_test_count: usize, @@ -35,11 +67,30 @@ pub(super) trait Formatter: Send + Sync + RefUnwindSafe { pub(super) struct PrettyFormatter; impl Formatter for PrettyFormatter { - fn package_start(&self, package_name: &str, test_count: usize) -> std::io::Result<()> { + fn package_start_async(&self, _package_name: &str, _test_count: usize) -> std::io::Result<()> { + Ok(()) + } + + fn package_start_sync(&self, package_name: &str, test_count: usize) -> std::io::Result<()> { package_start(package_name, test_count) } - fn test_end( + fn test_start_async(&self, _name: &str, _package_name: &str) -> std::io::Result<()> { + Ok(()) + } + + fn test_end_async( + &self, + _test_result: &TestResult, + _file_manager: &FileManager, + _show_output: bool, + _deny_warnings: bool, + _silence_warnings: bool, + ) -> std::io::Result<()> { + Ok(()) + } + + fn test_end_sync( &self, test_result: &TestResult, _current_test_count: usize, @@ -173,11 +224,30 @@ impl Formatter for PrettyFormatter { pub(super) struct TerseFormatter; impl Formatter for TerseFormatter { - fn package_start(&self, package_name: &str, test_count: usize) -> std::io::Result<()> { + fn package_start_async(&self, _package_name: &str, _test_count: usize) -> std::io::Result<()> { + Ok(()) + } + + fn package_start_sync(&self, package_name: &str, test_count: usize) -> std::io::Result<()> { package_start(package_name, test_count) } - fn test_end( + fn test_start_async(&self, _name: &str, _package_name: &str) -> std::io::Result<()> { + Ok(()) + } + + fn test_end_async( + &self, + _test_result: &TestResult, + _file_manager: &FileManager, + _show_output: bool, + _deny_warnings: bool, + _silence_warnings: bool, + ) -> std::io::Result<()> { + Ok(()) + } + + fn test_end_sync( &self, test_result: &TestResult, current_test_count: usize, @@ -317,8 +387,153 @@ impl Formatter for TerseFormatter { } } +pub(super) struct JsonFormatter; + +impl Formatter for JsonFormatter { + fn package_start_async(&self, package_name: &str, test_count: usize) -> std::io::Result<()> { + let json = json!({"type": "suite", "event": "started", "name": package_name, "test_count": test_count}); + println!("{json}"); + Ok(()) + } + + fn package_start_sync(&self, _package_name: &str, _test_count: usize) -> std::io::Result<()> { + Ok(()) + } + + fn test_start_async(&self, name: &str, package_name: &str) -> std::io::Result<()> { + let json = json!({"type": "test", "event": "started", "name": name, "suite": package_name}); + println!("{json}"); + Ok(()) + } + + fn test_end_async( + &self, + test_result: &TestResult, + file_manager: &FileManager, + show_output: bool, + _deny_warnings: bool, + silence_warnings: bool, + ) -> std::io::Result<()> { + let mut json = Map::new(); + json.insert("type".to_string(), json!("test")); + json.insert("name".to_string(), json!(&test_result.name)); + json.insert("exec_time".to_string(), json!(test_result.time_to_run.as_secs_f64())); + + let mut stdout = String::new(); + if show_output && !test_result.output.is_empty() { + stdout.push_str(test_result.output.trim()); + } + + match &test_result.status { + TestStatus::Pass => { + json.insert("event".to_string(), json!("ok")); + } + TestStatus::Fail { message, error_diagnostic } => { + json.insert("event".to_string(), json!("failed")); + + if !stdout.is_empty() { + stdout.push('\n'); + } + stdout.push_str(message.trim()); + + if let Some(diagnostic) = error_diagnostic { + if !(diagnostic.diagnostic.is_warning() && silence_warnings) { + stdout.push('\n'); + stdout.push_str(&diagnostic_to_string(diagnostic, file_manager)); + } + } + } + TestStatus::Skipped => { + json.insert("event".to_string(), json!("ignored")); + } + TestStatus::CompileError(diagnostic) => { + json.insert("event".to_string(), json!("failed")); + + if !(diagnostic.diagnostic.is_warning() && silence_warnings) { + if !stdout.is_empty() { + stdout.push('\n'); + } + stdout.push_str(&diagnostic_to_string(diagnostic, file_manager)); + } + } + } + + if !stdout.is_empty() { + json.insert("stdout".to_string(), json!(stdout)); + } + + let json = json!(json); + println!("{json}"); + + Ok(()) + } + + fn test_end_sync( + &self, + _test_result: &TestResult, + _current_test_count: usize, + _total_test_count: usize, + _file_manager: &FileManager, + _show_output: bool, + _deny_warnings: bool, + _silence_warnings: bool, + ) -> std::io::Result<()> { + Ok(()) + } + + fn package_end( + &self, + _package_name: &str, + test_results: &[TestResult], + _file_manager: &FileManager, + _show_output: bool, + _deny_warnings: bool, + _silence_warnings: bool, + ) -> std::io::Result<()> { + let mut passed = 0; + let mut failed = 0; + let mut ignored = 0; + for test_result in test_results { + match &test_result.status { + TestStatus::Pass => passed += 1, + TestStatus::Fail { .. } | TestStatus::CompileError(..) => failed += 1, + TestStatus::Skipped => ignored += 1, + } + } + let event = if failed == 0 { "ok" } else { "failed" }; + let json = json!({"type": "suite", "event": event, "passed": passed, "failed": failed, "ignored": ignored}); + println!("{json}"); + Ok(()) + } +} + fn package_start(package_name: &str, test_count: usize) -> std::io::Result<()> { let plural = if test_count == 1 { "" } else { "s" }; println!("[{package_name}] Running {test_count} test function{plural}"); Ok(()) } + +fn diagnostic_to_string(file_diagnostic: &FileDiagnostic, file_manager: &FileManager) -> String { + let file_map = file_manager.as_file_map(); + + let custom_diagnostic = &file_diagnostic.diagnostic; + let mut message = String::new(); + message.push_str(custom_diagnostic.message.trim()); + + for note in &custom_diagnostic.notes { + message.push('\n'); + message.push_str(note.trim()); + } + + if let Ok(name) = file_map.get_name(file_diagnostic.file_id) { + message.push('\n'); + message.push_str(&format!("at {name}")); + } + + if !custom_diagnostic.call_stack.is_empty() { + message.push('\n'); + message.push_str(&stack_trace(file_map, &custom_diagnostic.call_stack)); + } + + message +}