Various improvements to differential fuzzing

crates/fuzzing/src/generators/config.rs

@@ -36,12 +36,6 @@ impl Config {
     pub fn set_differential_config(&mut self) {
         let config = &mut self.module_config.config;
 
-        // Disable the start function for now.
-        //
-        // TODO: should probably allow this after testing it works with the new
-        // differential setup in all engines.
-        config.allow_start_export = false;
-
         // Make it more likely that there are types available to generate a
         // function with.
         config.min_types = 1;

crates/fuzzing/src/oracles/diff_wasmi.rs

@@ -3,7 +3,7 @@
 use crate::generators::{DiffValue, DiffValueType, ModuleConfig};
 use crate::oracles::engine::{DiffEngine, DiffInstance};
 use anyhow::{bail, Context, Error, Result};
-use wasmtime::Trap;
+use wasmtime::{Trap, TrapCode};
 
 /// A wrapper for `wasmi` as a [`DiffEngine`].
 pub struct WasmiEngine;
@@ -36,6 +36,9 @@ impl WasmiEngine {
         if config.config.threads_enabled {
             bail!("wasmi does not support threads");
         }
+        if config.config.max_memories > 1 {
+            bail!("wasmi does not support multi-memory");
+        }
         Ok(Self)
     }
 }
@@ -49,13 +52,69 @@ impl DiffEngine for WasmiEngine {
         let module = wasmi::Module::from_buffer(wasm).context("unable to validate Wasm module")?;
         let instance = wasmi::ModuleInstance::new(&module, &wasmi::ImportsBuilder::default())
             .context("unable to instantiate module in wasmi")?;
-        let instance = instance.assert_no_start();
+        let instance = instance.run_start(&mut wasmi::NopExternals)?;
         Ok(Box::new(WasmiInstance { module, instance }))
     }
 
     fn assert_error_match(&self, trap: &Trap, err: Error) {
-        // TODO: should implement this for `wasmi`
-        drop((trap, err));
+        // Acquire a `wasmi::Trap` from the wasmi error which we'll use to
+        // assert that it has the same kind of trap as the wasmtime-based trap.
+        let wasmi = match err.downcast::<wasmi::Error>() {
+            Ok(wasmi::Error::Trap(trap)) => trap,
+
+            // Out-of-bounds data segments turn into this category which
+            // Wasmtime reports as a `MemoryOutOfBounds`.
+            Ok(wasmi::Error::Memory(msg)) => {
+                assert_eq!(
+                    trap.trap_code(),
+                    Some(TrapCode::MemoryOutOfBounds),
+                    "wasmtime error did not match wasmi: {msg}"
+                );
+                return;
+            }
+
+            // Ignore this for now, looks like "elements segment does not fit"
+            // falls into this category and to avoid doing string matching this
+            // is just ignored.
+            Ok(wasmi::Error::Instantiation(msg)) => {
+                log::debug!("ignoring wasmi instantiation error: {msg}");
+                return;
+            }
+
+            Ok(other) => panic!("unexpected wasmi error: {}", other),
+
+            Err(err) => err.downcast::<wasmi::Trap>().unwrap(),
+        };
+        match wasmi.kind() {
+            wasmi::TrapKind::StackOverflow => {
+                assert_eq!(trap.trap_code(), Some(TrapCode::StackOverflow))
+            }
+            wasmi::TrapKind::MemoryAccessOutOfBounds => {
+                assert_eq!(trap.trap_code(), Some(TrapCode::MemoryOutOfBounds))
+            }
+            wasmi::TrapKind::Unreachable => {
+                assert_eq!(trap.trap_code(), Some(TrapCode::UnreachableCodeReached))
+            }
+            wasmi::TrapKind::TableAccessOutOfBounds => {
+                assert_eq!(trap.trap_code(), Some(TrapCode::TableOutOfBounds))
+            }
+            wasmi::TrapKind::ElemUninitialized => {
+                assert_eq!(trap.trap_code(), Some(TrapCode::IndirectCallToNull))
+            }
+            wasmi::TrapKind::DivisionByZero => {
+                assert_eq!(trap.trap_code(), Some(TrapCode::IntegerDivisionByZero))
+            }
+            wasmi::TrapKind::IntegerOverflow => {
+                assert_eq!(trap.trap_code(), Some(TrapCode::IntegerOverflow))
+            }
+            wasmi::TrapKind::InvalidConversionToInt => {
+                assert_eq!(trap.trap_code(), Some(TrapCode::BadConversionToInteger))
+            }
+            wasmi::TrapKind::UnexpectedSignature => {
+                assert_eq!(trap.trap_code(), Some(TrapCode::BadSignature))
+            }
+            wasmi::TrapKind::Host(_) => unreachable!(),
+        }
     }
 }
 

crates/fuzzing/src/oracles/engine.rs

@@ -13,7 +13,7 @@ pub fn choose(
     u: &mut Unstructured<'_>,
     existing_config: &Config,
     allowed: &[&str],
-) -> arbitrary::Result<Box<dyn DiffEngine>> {
+) -> arbitrary::Result<Option<Box<dyn DiffEngine>>> {
     // Filter out any engines that cannot match the `existing_config` or are not
     // `allowed`.
     let mut engines: Vec<Box<dyn DiffEngine>> = vec![];
@@ -54,13 +54,16 @@ pub fn choose(
         }
     }
 
+    if engines.is_empty() {
+        return Ok(None);
+    }
+
     // Use the input of the fuzzer to pick an engine that we'll be fuzzing
     // Wasmtime against.
-    assert!(!engines.is_empty());
     let index: usize = u.int_in_range(0..=engines.len() - 1)?;
     let engine = engines.swap_remove(index);
     log::debug!("selected engine: {}", engine.name());
-    Ok(engine)
+    Ok(Some(engine))
 }
 
 /// Provide a way to instantiate Wasm modules.

fuzz/fuzz_targets/differential.rs

@@ -86,8 +86,13 @@ fn run(data: &[u8]) -> Result<()> {
     };
     log_wasm(&wasm);
 
-    // Choose a left-hand side Wasm engine.
-    let mut lhs = engine::choose(&mut u, &config, unsafe { &ALLOWED_ENGINES })?;
+    // Choose a left-hand side Wasm engine. If no engine could be chosen then
+    // that means the configuration selected above doesn't match any allowed
+    // engine (configured via an env var) so the test case is thrown out.
+    let mut lhs = match engine::choose(&mut u, &config, unsafe { &ALLOWED_ENGINES })? {
+        Some(engine) => engine,
+        None => return Ok(()),
+    };
     let lhs_instance = lhs.instantiate(&wasm);
     STATS.bump_engine(lhs.name());