feat: add main/subprocess mode to test harness

This sandboxes the executions so we can survive process aborts

include/harness/harness_main.rs

@@ -0,0 +1,75 @@
+//! This is the primary file for the abi-cafe harness main that all tests are compiled into.
+//!
+//! This will be statically linked into a cdylib with two other static libraries:
+//! the caller and callee. The caller is expected to define the function `do_test`,
+//! and call a bunch of functions defined by the callee. The cdylib
+//! is run by the harness `dlopen`ing it and running `test_start`, passing in various
+//! buffers and callbacks for instrumenting the result of the execution.
+//!
+//! This instrumentation is only used in the default mode of `WriteImpl::HarnessCallback`.
+//! Otherwise the caller/callee may use things like asserts/prints.
+
+/// Tests write back the raw bytes of their values to a WriteBuffer.
+pub struct WriteBuffer {
+    pub identity: &'static str,
+}
+
+impl WriteBuffer {
+    fn new(identity: &'static str) -> Self {
+        // Preload the hierarchy for the first test.
+        WriteBuffer {
+            identity,
+        }
+    }
+}
+
+// The signatures of the interface from our perspective.
+// From the test's perspective the WriteBuffers are totally opaque.
+pub type SetFuncCallback = unsafe extern "C" fn(&mut WriteBuffer, u32) -> ();
+pub type WriteValCallback = unsafe extern "C" fn(&mut WriteBuffer, u32, *const u8, u32) -> ();
+pub type TestInit =
+    unsafe extern "C" fn(SetFuncCallback, WriteValCallback, &mut WriteBuffer, &mut WriteBuffer) -> ();
+
+pub unsafe extern "C" fn set_func(test: &mut WriteBuffer, func: u32) {
+    let ident = &test.identity;
+    println!(r#"{{ "info": "func", "id": "{ident}", "func": {func} }}"#);
+}
+
+pub unsafe extern "C" fn write_val(
+    test: &mut WriteBuffer,
+    val_idx: u32,
+    input: *const u8,
+    size: u32,
+) {
+    let data = std::slice::from_raw_parts(input, size as usize);
+    let ident = &test.identity;
+    println!(r#"{{ "info": "val", "id": "{ident}", "val": {val_idx}, "bytes": {data:?} }}"#);
+}
+
+
+#[no_mangle]
+pub static mut CALLER_VALS: *mut () = core::ptr::null_mut();
+#[no_mangle]
+pub static mut CALLEE_VALS: *mut () = core::ptr::null_mut();
+#[no_mangle]
+pub static mut SET_FUNC: Option<SetFuncCallback> = None;
+#[no_mangle]
+pub static mut WRITE_VAL: Option<WriteValCallback> = None;
+
+extern {
+    fn do_test();
+}
+
+pub fn main() {
+    unsafe {
+        let mut caller_vals = WriteBuffer::new("caller");
+        let mut callee_vals = WriteBuffer::new("callee");
+        CALLER_VALS = &mut caller_vals as *mut _ as *mut _;
+        CALLEE_VALS = &mut callee_vals as *mut _ as *mut _;
+        SET_FUNC = Some(set_func);
+        WRITE_VAL = Some(write_val);
+
+        do_test();
+        println!(r#"{{ "info": "done" }}"#);
+    }
+}

include/harness/main.rs

@@ -8,6 +8,7 @@
 //!
 //! In theory this could be replaced with just making `caller::do_test` into `main`
 //! but this might be a bit easier..?
+
 extern {
     fn do_test();
 }

src/error.rs

@@ -1,6 +1,6 @@
 use miette::Diagnostic;
 
-use crate::harness::test::TestId;
+use crate::{harness::test::TestId, TestBuffer};
 
 #[derive(Debug, thiserror::Error, Diagnostic)]
 pub enum CliParseError {
@@ -132,12 +132,37 @@ pub enum LinkError {
 
 #[derive(Debug, thiserror::Error, Diagnostic)]
 pub enum RunError {
-    #[error("test loading error (dynamic linking failed)\n{0}")]
+    #[error("test loading error (dynamic linking failed)\n  {0}")]
     LoadError(#[from] libloading::Error),
+    #[error("failed to spawn test bin: {bin}\n  {e}")]
+    ExecError {
+        bin: camino::Utf8PathBuf,
+        e: std::io::Error,
+    },
     #[error("test impl didn't call set_func before calling write_val")]
     MissingSetFunc,
     #[error("test impl called write_val on func {func} val {val} twice")]
     DoubleWrite { func: usize, val: usize },
+    #[error(
+        "test impl exited with bad status (crashed?): {status}
+    caller last reported: fn {caller_func} value {caller_val_idx}
+    callee last reported: fn {callee_func} value {callee_val_idx}
+    "
+    )]
+    BadExit {
+        status: std::process::ExitStatus,
+        caller_func_idx: usize,
+        caller_val_idx: usize,
+        caller_func: String,
+        callee_func_idx: usize,
+        callee_val_idx: usize,
+        callee_func: String,
+    },
+    #[error("test impl sent invalid messages to harness (executed some kind of UB?)")]
+    InvalidMessages {
+        caller_funcs: TestBuffer,
+        callee_funcs: TestBuffer,
+    },
 }
 
 fn fmt_bytes(bytes: &[u8]) -> String {

src/files.rs

@@ -15,8 +15,14 @@ pub struct Paths {
     pub runtime_test_input_dir: Option<Utf8PathBuf>,
 }
 impl Paths {
-    pub fn harness_main_file(&self) -> Utf8PathBuf {
-        self.out_dir.join("harness.rs")
+    pub fn harness_dylib_main_file(&self) -> Utf8PathBuf {
+        self.out_dir.join("harness_lib.rs")
+    }
+    pub fn harness_bin_main_file(&self) -> Utf8PathBuf {
+        self.out_dir.join("harness_main.rs")
+    }
+    pub fn freestanding_bin_main_file(&self) -> Utf8PathBuf {
+        self.out_dir.join("main.rs")
     }
 
     /// Delete and recreate the build dir
@@ -27,12 +33,28 @@ impl Paths {
 
         // Initialize harness.rs
         {
-            let harness_file_contents = get_file("harness/harness.rs");
-            let harness_file_path = self.harness_main_file();
+            let harness_file_contents = get_file("harness/harness_lib.rs");
+            let harness_file_path = self.harness_dylib_main_file();
+            let mut file = std::fs::File::create_new(harness_file_path)
+                .expect("failed to create harness_lib.rs");
+            file.write_all(harness_file_contents.as_bytes())
+                .expect("failed to initialize harness_lib.rs");
+        }
+        {
+            let harness_file_contents = get_file("harness/harness_main.rs");
+            let harness_file_path = self.harness_bin_main_file();
+            let mut file = std::fs::File::create_new(harness_file_path)
+                .expect("failed to create harness_main.rs");
+            file.write_all(harness_file_contents.as_bytes())
+                .expect("failed to initialize harness_main.rs");
+        }
+        {
+            let harness_file_contents = get_file("harness/main.rs");
+            let harness_file_path = self.freestanding_bin_main_file();
             let mut file =
-                std::fs::File::create_new(harness_file_path).expect("failed to create harness.rs");
+                std::fs::File::create_new(harness_file_path).expect("failed to create main.rs");
             file.write_all(harness_file_contents.as_bytes())
-                .expect("failed to initialize harness.rs");
+                .expect("failed to initialize main.rs");
         }
 
         // Set up env vars for CC

src/harness/build.rs

@@ -14,14 +14,13 @@ impl TestHarness {
         &self,
         key: &TestKey,
         src: &GenerateOutput,
-        out_dir: &Utf8Path,
     ) -> Result<BuildOutput, BuildError> {
         // FIXME: these two could be done concurrently
         let caller_lib = self
-            .build_static_lib(key, CallSide::Caller, &src.caller_src, out_dir)
+            .build_static_lib(key, CallSide::Caller, &src.caller_src)
             .await?;
         let callee_lib = self
-            .build_static_lib(key, CallSide::Callee, &src.callee_src, out_dir)
+            .build_static_lib(key, CallSide::Callee, &src.callee_src)
             .await?;
         Ok(BuildOutput {
             caller_lib,
@@ -34,7 +33,6 @@ impl TestHarness {
         key: &TestKey,
         call_side: CallSide,
         src_path: &Utf8Path,
-        out_dir: &Utf8Path,
     ) -> Result<String, BuildError> {
         let toolchain = self.toolchain_by_test_key(key, call_side);
         let lib_name = self.static_lib_name(key, call_side);
@@ -55,18 +53,17 @@ impl TestHarness {
                     .await
                     .expect("failed to acquire concurrency limit semaphore");
                 info!("compiling   {lib_name}");
-                build_static_lib(src_path, toolchain, call_side, out_dir, &lib_name).await
+                build_static_lib(&self.paths, src_path, toolchain, call_side, &lib_name).await
             })
             .await?
             .clone();
         Ok(real_lib_name)
     }
 
-    pub async fn link_dynamic_lib(
+    pub async fn link_dylib(
         &self,
         key: &TestKey,
         build: &BuildOutput,
-        out_dir: &Utf8Path,
     ) -> Result<LinkOutput, LinkError> {
         let _token = self
             .concurrency_limiter
@@ -75,7 +72,27 @@ impl TestHarness {
             .expect("failed to acquire concurrency limit semaphore");
         let dynamic_lib_name = self.dynamic_lib_name(key);
         info!("linking     {dynamic_lib_name}");
-        link_dynamic_lib(build, out_dir, &dynamic_lib_name)
+        build_harness_dylib(&self.paths, build, &dynamic_lib_name)
+    }
+
+    pub async fn link_bin(
+        &self,
+        key: &TestKey,
+        build: &BuildOutput,
+    ) -> Result<LinkOutput, LinkError> {
+        let _token = self
+            .concurrency_limiter
+            .acquire()
+            .await
+            .expect("failed to acquire concurrency limit semaphore");
+        let bin_name = self.bin_name(key);
+        info!("linking     {bin_name}");
+        let bin_main = if let WriteImpl::HarnessCallback = key.options.val_writer {
+            self.paths.harness_bin_main_file()
+        } else {
+            self.paths.freestanding_bin_main_file()
+        };
+        build_harness_main(&self.paths, build, &bin_name, &bin_main)
     }
 
     fn static_lib_name(&self, key: &TestKey, call_side: CallSide) -> String {
@@ -87,35 +104,43 @@ impl TestHarness {
         output.push_str(".dll");
         output
     }
+
+    fn bin_name(&self, key: &TestKey) -> String {
+        let mut output = self.base_id(key, None, "_");
+        if cfg!(target_os = "windows") {
+            output.push_str(".exe");
+        }
+        output
+    }
 }
 
 async fn build_static_lib(
+    paths: &Paths,
     src_path: &Utf8Path,
     toolchain: Arc<dyn Toolchain + Send + Sync>,
     call_side: CallSide,
-    out_dir: &Utf8Path,
     static_lib_name: &str,
 ) -> Result<String, BuildError> {
     let lib_name = match call_side {
-        CallSide::Callee => toolchain.compile_callee(src_path, out_dir, static_lib_name)?,
-        CallSide::Caller => toolchain.compile_caller(src_path, out_dir, static_lib_name)?,
+        CallSide::Callee => toolchain.compile_callee(src_path, &paths.out_dir, static_lib_name)?,
+        CallSide::Caller => toolchain.compile_caller(src_path, &paths.out_dir, static_lib_name)?,
     };
 
     Ok(lib_name)
 }
 
 /// Compile and link the test harness with the two sides of the FFI boundary.
-fn link_dynamic_lib(
+fn build_harness_dylib(
+    paths: &Paths,
     build: &BuildOutput,
-    out_dir: &Utf8Path,
     dynamic_lib_name: &str,
 ) -> Result<LinkOutput, LinkError> {
-    let src = out_dir.join("harness.rs");
-    let output = out_dir.join(dynamic_lib_name);
+    let src = paths.harness_dylib_main_file();
+    let output = paths.out_dir.join(dynamic_lib_name);
     let mut cmd = Command::new("rustc");
     cmd.arg("-v")
         .arg("-L")
-        .arg(out_dir)
+        .arg(&paths.out_dir)
         .arg("-l")
         .arg(&build.caller_lib)
         .arg("-l")
@@ -140,3 +165,40 @@ fn link_dynamic_lib(
         Ok(LinkOutput { test_bin: output })
     }
 }
+
+/// Compile and link the test harness with the two sides of the FFI boundary.
+fn build_harness_main(
+    paths: &Paths,
+    build: &BuildOutput,
+    bin_name: &str,
+    bin_main: &Utf8Path,
+) -> Result<LinkOutput, LinkError> {
+    let output = paths.out_dir.join(bin_name);
+    let mut cmd = Command::new("rustc");
+    cmd.arg("-v")
+        .arg("-L")
+        .arg(&paths.out_dir)
+        .arg("-l")
+        .arg(&build.caller_lib)
+        .arg("-l")
+        .arg(&build.callee_lib)
+        .arg("--crate-type")
+        .arg("bin")
+        .arg("--target")
+        .arg(built_info::TARGET)
+        // .arg("-Csave-temps=y")
+        // .arg("--out-dir")
+        // .arg("target/temp/")
+        .arg("-o")
+        .arg(&output)
+        .arg(bin_main);
+
+    debug!("running: {:?}", cmd);
+    let out = cmd.output()?;
+
+    if !out.status.success() {
+        Err(LinkError::RustLink(out))
+    } else {
+        Ok(LinkOutput { test_bin: output })
+    }
+}

src/harness/mod.rs

@@ -131,7 +131,6 @@ impl TestHarness {
     pub async fn do_test(&self, test_key: TestKey, test_rules: TestRules) -> TestRunResults {
         use TestRunMode::*;
 
-        let out_dir = self.paths.out_dir.clone();
         let mut res = TestRunResults::new(test_key, test_rules);
         if res.rules.run <= Skip {
             return res;
@@ -159,7 +158,7 @@ impl TestHarness {
         }
 
         res.ran_to = Build;
-        res.build = Some(self.build_test(&res.key, source, &out_dir).await);
+        res.build = Some(self.build_test(&res.key, source).await);
         let build = match res.build.as_ref().unwrap() {
             Ok(v) => v,
             Err(e) => {
@@ -172,7 +171,7 @@ impl TestHarness {
         }
 
         res.ran_to = Link;
-        res.link = Some(self.link_dynamic_lib(&res.key, build, &out_dir).await);
+        res.link = Some(self.link_bin(&res.key, build).await);
         let link = match res.link.as_ref().unwrap() {
             Ok(v) => v,
             Err(e) => {
@@ -185,7 +184,7 @@ impl TestHarness {
         }
 
         res.ran_to = Run;
-        res.run = Some(self.run_dynamic_test(&res.key, link).await);
+        res.run = Some(self.run_bin_test(&res.key, link).await);
         let run = match res.run.as_ref().unwrap() {
             Ok(v) => v,
             Err(e) => {