feat: add nargo test --format json

compiler/fm/src/file_map.rs

@@ -80,6 +80,19 @@ impl FileMap {
     pub fn all_file_ids(&self) -> impl Iterator<Item = &FileId> {
         self.name_to_id.values()
     }
+
+    pub fn get_name(&self, file_id: FileId) -> Result<PathString, Error> {
+        let name = self.files.get(file_id.as_usize())?.name().clone();
+
+        // See if we can make the file name a bit shorter/easier to read if it starts with the current directory
+        if let Some(current_dir) = &self.current_dir {
+            if let Ok(name_without_prefix) = name.0.strip_prefix(current_dir) {
+                return Ok(PathString::from_path(name_without_prefix.to_path_buf()));
+            }
+        }
+
+        Ok(name)
+    }
 }
 impl Default for FileMap {
     fn default() -> Self {
@@ -97,16 +110,7 @@ impl<'a> Files<'a> for FileMap {
     type Source = &'a str;
 
     fn name(&self, file_id: Self::FileId) -> Result<Self::Name, Error> {
-        let name = self.files.get(file_id.as_usize())?.name().clone();
-
-        // See if we can make the file name a bit shorter/easier to read if it starts with the current directory
-        if let Some(current_dir) = &self.current_dir {
-            if let Ok(name_without_prefix) = name.0.strip_prefix(current_dir) {
-                return Ok(PathString::from_path(name_without_prefix.to_path_buf()));
-            }
-        }
-
-        Ok(name)
+        self.get_name(file_id)
     }
 
     fn source(&'a self, file_id: Self::FileId) -> Result<Self::Source, Error> {

compiler/noirc_errors/src/reporter.rs

@@ -272,7 +272,7 @@ fn convert_diagnostic(
     diagnostic.with_message(&cd.message).with_labels(secondary_labels).with_notes(notes)
 }
 
-fn stack_trace<'files>(
+pub fn stack_trace<'files>(
     files: &'files impl Files<'files, FileId = fm::FileId>,
     call_stack: &[Location],
 ) -> String {

tooling/nargo_cli/src/cli/test_cmd.rs

@@ -12,7 +12,7 @@ use acvm::{BlackBoxFunctionSolver, FieldElement};
 use bn254_blackbox_solver::Bn254BlackBoxSolver;
 use clap::Args;
 use fm::FileManager;
-use formatters::{Formatter, PrettyFormatter, TerseFormatter};
+use formatters::{Formatter, JsonFormatter, PrettyFormatter, TerseFormatter};
 use nargo::{
     insert_all_files_for_workspace_into_file_manager, ops::TestStatus, package::Package, parse_all,
     prepare_package, workspace::Workspace, PrintOutput,
@@ -71,13 +71,16 @@ enum Format {
     Pretty,
     /// Display one character per test
     Terse,
+    /// Output a JSON Lines document
+    Json,
 }
 
 impl Format {
     fn formatter(&self) -> Box<dyn Formatter> {
         match self {
             Format::Pretty => Box::new(PrettyFormatter),
             Format::Terse => Box::new(TerseFormatter),
+            Format::Json => Box::new(JsonFormatter),
         }
     }
 }
@@ -87,6 +90,7 @@ impl Display for Format {
         match self {
             Format::Pretty => write!(f, "pretty"),
             Format::Terse => write!(f, "terse"),
+            Format::Json => write!(f, "json"),
         }
     }
 }

tooling/nargo_cli/src/cli/test_cmd/formatters.rs

@@ -2,6 +2,8 @@ use std::{io::Write, panic::RefUnwindSafe, time::Duration};
 
 use fm::FileManager;
 use nargo::ops::TestStatus;
+use noirc_errors::{reporter::stack_trace, FileDiagnostic};
+use serde_json::{json, Map};
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, StandardStreamLock, WriteColor};
 
 use super::TestResult;
@@ -317,8 +319,132 @@ impl Formatter for TerseFormatter {
     }
 }
 
+pub(super) struct JsonFormatter;
+
+impl Formatter for JsonFormatter {
+    fn package_start(&self, package_name: &str, test_count: usize) -> std::io::Result<()> {
+        let json = json!({"type": "suite", "event": "started", "name": package_name, "test_count": test_count});
+        println!("{json}");
+        Ok(())
+    }
+
+    fn test_end(
+        &self,
+        test_result: &TestResult,
+        _current_test_count: usize,
+        _total_test_count: usize,
+        file_manager: &FileManager,
+        show_output: bool,
+        _deny_warnings: bool,
+        silence_warnings: bool,
+    ) -> std::io::Result<()> {
+        let mut json = Map::new();
+        json.insert("type".to_string(), json!("test"));
+        json.insert("name".to_string(), json!(&test_result.name));
+        json.insert("exec_time".to_string(), json!(test_result.time_to_run.as_secs_f64()));
+
+        let mut stdout = String::new();
+        if show_output && !test_result.output.is_empty() {
+            stdout.push_str(test_result.output.trim());
+        }
+
+        match &test_result.status {
+            TestStatus::Pass => {
+                json.insert("event".to_string(), json!("ok"));
+            }
+            TestStatus::Fail { message, error_diagnostic } => {
+                json.insert("event".to_string(), json!("failed"));
+
+                if !stdout.is_empty() {
+                    stdout.push('\n');
+                }
+                stdout.push_str(message.trim());
+
+                if let Some(diagnostic) = error_diagnostic {
+                    if !(diagnostic.diagnostic.is_warning() && silence_warnings) {
+                        stdout.push('\n');
+                        stdout.push_str(&diagnostic_to_string(diagnostic, file_manager));
+                    }
+                }
+            }
+            TestStatus::Skipped => {
+                json.insert("event".to_string(), json!("skipped"));
+            }
+            TestStatus::CompileError(diagnostic) => {
+                json.insert("event".to_string(), json!("failed"));
+
+                if !(diagnostic.diagnostic.is_warning() && silence_warnings) {
+                    if !stdout.is_empty() {
+                        stdout.push('\n');
+                    }
+                    stdout.push_str(&diagnostic_to_string(diagnostic, file_manager));
+                }
+            }
+        }
+
+        if !stdout.is_empty() {
+            json.insert("stdout".to_string(), json!(stdout));
+        }
+
+        let json = json!(json);
+        println!("{json}");
+
+        Ok(())
+    }
+
+    fn package_end(
+        &self,
+        _package_name: &str,
+        test_results: &[TestResult],
+        _file_manager: &FileManager,
+        _show_output: bool,
+        _deny_warnings: bool,
+        _silence_warnings: bool,
+    ) -> std::io::Result<()> {
+        let mut passed = 0;
+        let mut failed = 0;
+        let mut skipped = 0;
+        for test_result in test_results {
+            match &test_result.status {
+                TestStatus::Pass => passed += 1,
+                TestStatus::Fail { .. } | TestStatus::CompileError(..) => failed += 1,
+                TestStatus::Skipped => skipped += 1,
+            }
+        }
+        let event = if failed == 0 { "ok" } else { "failed" };
+        let json = json!({"type": "suite", "event": event, "passed": passed, "failed": failed, "skipped": skipped});
+        println!("{json}");
+        Ok(())
+    }
+}
+
 fn package_start(package_name: &str, test_count: usize) -> std::io::Result<()> {
     let plural = if test_count == 1 { "" } else { "s" };
     println!("[{package_name}] Running {test_count} test function{plural}");
     Ok(())
 }
+
+fn diagnostic_to_string(file_diagnostic: &FileDiagnostic, file_manager: &FileManager) -> String {
+    let file_map = file_manager.as_file_map();
+
+    let custom_diagnostic = &file_diagnostic.diagnostic;
+    let mut message = String::new();
+    message.push_str(custom_diagnostic.message.trim());
+
+    for note in &custom_diagnostic.notes {
+        message.push('\n');
+        message.push_str(note.trim());
+    }
+
+    if let Ok(name) = file_map.get_name(file_diagnostic.file_id) {
+        message.push('\n');
+        message.push_str(&format!("at {name}"));
+    }
+
+    if !custom_diagnostic.call_stack.is_empty() {
+        message.push('\n');
+        message.push_str(&stack_trace(file_map, &custom_diagnostic.call_stack));
+    }
+
+    message
+}