wasmtime: Implement funcref with NonNull<VMCallerCheckedAnyfunc>

This should be more efficient than using a `VMExternRef` that points at a
`VMCallerCheckedAnyfunc` because it gets rid of an indirection, dynamic
allocation, and some reference counting.

Note that the null function reference is *NOT* a null pointer; it is a
`VMCallerCheckedAnyfunc` that has a null `func_ptr` member.

cranelift/wasm/src/sections_translator.rs

@@ -97,6 +97,7 @@ pub fn parse_import_section<'data>(
             ImportSectionEntryType::Global(ref ty) => {
                 environ.declare_global_import(
                     Global {
+                        wasm_ty: ty.content_type,
                         ty: type_to_type(ty.content_type, environ).unwrap(),
                         mutability: ty.mutable,
                         initializer: GlobalInit::Import,
@@ -227,6 +228,7 @@ pub fn parse_global_section(
             }
         };
         let global = Global {
+            wasm_ty: content_type,
             ty: type_to_type(content_type, environ).unwrap(),
             mutability: mutable,
             initializer,

cranelift/wasm/src/translation_utils.rs

@@ -67,10 +67,18 @@ entity_impl!(DataIndex);
 pub struct ElemIndex(u32);
 entity_impl!(ElemIndex);
 
-/// WebAssembly global.
+/// A WebAssembly global.
+///
+/// Note that we record both the original Wasm type and the Cranelift IR type
+/// used to represent it. This is because multiple different kinds of Wasm types
+/// might be represented with the same Cranelift IR type. For example, both a
+/// Wasm `i64` and a `funcref` might be represented with a Cranelift `i64` on
+/// 64-bit architectures, and when GC is not required for func refs.
 #[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)]
 pub struct Global {
-    /// The type of the value stored in the global.
+    /// The Wasm type of the value stored in the global.
+    pub wasm_ty: crate::WasmType,
+    /// The Cranelift IR type of the value stored in the global.
     pub ty: ir::Type,
     /// A flag indicating whether the value may change at runtime.
     pub mutability: bool,

crates/c-api/src/extern.rs

@@ -1,7 +1,7 @@
 use crate::host_ref::HostRef;
 use crate::wasm_externkind_t;
 use crate::{wasm_externtype_t, wasm_func_t, wasm_global_t, wasm_memory_t, wasm_table_t};
-use wasmtime::{ExternType, Func, Global, Memory, Table};
+use wasmtime::{ExternType, Func, FuncType, Global, Memory, Table};
 
 #[derive(Clone)]
 pub struct wasm_extern_t {
@@ -12,7 +12,7 @@ wasmtime_c_api_macros::declare_ref!(wasm_extern_t);
 
 #[derive(Clone)]
 pub(crate) enum ExternHost {
-    Func(HostRef<Func>),
+    Func(HostRef<Option<Func>>),
     Global(HostRef<Global>),
     Memory(HostRef<Memory>),
     Table(HostRef<Table>),
@@ -42,7 +42,14 @@ pub extern "C" fn wasm_extern_kind(e: &wasm_extern_t) -> wasm_externkind_t {
 #[no_mangle]
 pub extern "C" fn wasm_extern_type(e: &wasm_extern_t) -> Box<wasm_externtype_t> {
     let ty = match &e.which {
-        ExternHost::Func(f) => ExternType::Func(f.borrow().ty()),
+        ExternHost::Func(f) => ExternType::Func({
+            let f = f.borrow();
+            if f.is_none() {
+                FuncType::new(Default::default(), Default::default())
+            } else {
+                f.as_ref().unwrap().ty()
+            }
+        }),
         ExternHost::Global(f) => ExternType::Global(f.borrow().ty()),
         ExternHost::Table(f) => ExternType::Table(f.borrow().ty()),
         ExternHost::Memory(f) => ExternType::Memory(f.borrow().ty()),

crates/c-api/src/func.rs

@@ -6,7 +6,7 @@ use std::ffi::c_void;
 use std::panic::{self, AssertUnwindSafe};
 use std::ptr;
 use std::str;
-use wasmtime::{Caller, Extern, Func, Trap};
+use wasmtime::{Caller, Extern, Func, FuncType, Trap};
 
 #[derive(Clone)]
 #[repr(transparent)]
@@ -64,7 +64,7 @@ impl wasm_func_t {
         }
     }
 
-    pub(crate) fn func(&self) -> &HostRef<Func> {
+    pub(crate) fn func(&self) -> &HostRef<Option<Func>> {
         match &self.ext.which {
             ExternHost::Func(f) => f,
             _ => unsafe { std::hint::unreachable_unchecked() },
@@ -76,8 +76,8 @@ impl wasm_func_t {
     }
 }
 
-impl From<HostRef<Func>> for wasm_func_t {
-    fn from(func: HostRef<Func>) -> wasm_func_t {
+impl From<HostRef<Option<Func>>> for wasm_func_t {
+    fn from(func: HostRef<Option<Func>>) -> wasm_func_t {
         wasm_func_t {
             ext: wasm_extern_t {
                 which: ExternHost::Func(func),
@@ -108,7 +108,7 @@ fn create_function(
         }
         Ok(())
     });
-    Box::new(HostRef::new(store, func).into())
+    Box::new(HostRef::new(store, Some(func)).into())
 }
 
 #[no_mangle]
@@ -172,19 +172,26 @@ pub unsafe extern "C" fn wasm_func_call(
     args: *const wasm_val_t,
     results: *mut wasm_val_t,
 ) -> *mut wasm_trap_t {
-    let func = wasm_func.func().borrow();
-    let mut trap = ptr::null_mut();
-    let error = wasmtime_func_call(
-        wasm_func,
-        args,
-        func.param_arity(),
-        results,
-        func.result_arity(),
-        &mut trap,
-    );
-    match error {
-        Some(err) => Box::into_raw(err.to_trap(&wasm_func.ext.externref().store().unwrap())),
-        None => trap,
+    if let Some(func) = &*wasm_func.func().borrow() {
+        let mut trap = ptr::null_mut();
+        let error = wasmtime_func_call(
+            wasm_func,
+            args,
+            func.param_arity(),
+            results,
+            func.result_arity(),
+            &mut trap,
+        );
+        match error {
+            Some(err) => Box::into_raw(err.to_trap(&wasm_func.ext.externref().store().unwrap())),
+            None => trap,
+        }
+    } else {
+        let store = wasm_func.ext.externref().store().unwrap();
+        Box::into_raw(Box::new(wasm_trap_t::new(
+            &store,
+            Trap::new("attempt to call null func ref"),
+        )))
     }
 }
 
@@ -213,6 +220,10 @@ fn _wasmtime_func_call(
 ) -> Option<Box<wasmtime_error_t>> {
     let store = &func.ext.externref().store().unwrap();
     let func = func.func().borrow();
+    let func = match func.as_ref() {
+        Some(f) => f,
+        None => return Some(Box::new(anyhow!("attempt to call null func ref").into())),
+    };
     if results.len() != func.result_arity() {
         return Some(Box::new(anyhow!("wrong number of results provided").into()));
     }
@@ -254,17 +265,32 @@ fn _wasmtime_func_call(
 
 #[no_mangle]
 pub extern "C" fn wasm_func_type(f: &wasm_func_t) -> Box<wasm_functype_t> {
-    Box::new(wasm_functype_t::new(f.func().borrow().ty()))
+    if let Some(f) = &*f.func().borrow() {
+        Box::new(wasm_functype_t::new(f.ty()))
+    } else {
+        Box::new(wasm_functype_t::new(FuncType::new(
+            Default::default(),
+            Default::default(),
+        )))
+    }
 }
 
 #[no_mangle]
 pub extern "C" fn wasm_func_param_arity(f: &wasm_func_t) -> usize {
-    f.func().borrow().param_arity()
+    if let Some(f) = &*f.func().borrow() {
+        f.param_arity()
+    } else {
+        0
+    }
 }
 
 #[no_mangle]
 pub extern "C" fn wasm_func_result_arity(f: &wasm_func_t) -> usize {
-    f.func().borrow().result_arity()
+    if let Some(f) = &*f.func().borrow() {
+        f.result_arity()
+    } else {
+        0
+    }
 }
 
 #[no_mangle]
@@ -281,7 +307,8 @@ pub unsafe extern "C" fn wasmtime_caller_export_get(
     let export = caller.caller.get_export(name)?;
     let store = caller.caller.store();
     let which = match export {
-        Extern::Func(f) => ExternHost::Func(HostRef::new(&store, f)),
+        Extern::Func(None) => ExternHost::Func(HostRef::new(&store, None)),
+        Extern::Func(Some(f)) => ExternHost::Func(HostRef::new(&store, Some(f))),
         Extern::Global(g) => ExternHost::Global(HostRef::new(&store, g)),
         Extern::Memory(m) => ExternHost::Memory(HostRef::new(&store, m)),
         Extern::Table(t) => ExternHost::Table(HostRef::new(&store, t)),

crates/c-api/src/linker.rs

@@ -117,6 +117,6 @@ pub unsafe extern "C" fn wasmtime_linker_get_default(
         Err(_) => return bad_utf8(),
     };
     handle_result(linker.get_default(name), |f| {
-        *func = Box::into_raw(Box::new(HostRef::new(linker.store(), f).into()))
+        *func = Box::into_raw(Box::new(HostRef::new(linker.store(), Some(f)).into()))
     })
 }

crates/c-api/src/wasi.rs

@@ -359,7 +359,7 @@ pub extern "C" fn wasi_instance_bind_import<'a>(
         .entry(name.to_string())
         .or_insert_with(|| {
             Box::new(wasm_extern_t {
-                which: ExternHost::Func(HostRef::new(store, export.clone())),
+                which: ExternHost::Func(HostRef::new(store, Some(export.clone()))),
             })
         });
     Some(entry)

crates/environ/src/func_environ.rs

@@ -10,8 +10,8 @@ use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Si
 use cranelift_codegen::isa::{self, TargetFrontendConfig};
 use cranelift_entity::EntityRef;
 use cranelift_wasm::{
-    self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableIndex,
-    TargetEnvironment, WasmError, WasmResult,
+    self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableElementType,
+    TableIndex, TargetEnvironment, WasmError, WasmResult,
 };
 #[cfg(feature = "lightbeam")]
 use cranelift_wasm::{DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex};
@@ -68,6 +68,10 @@ macro_rules! declare_builtin_functions {
                 AbiParam::new(self.reference_type)
             }
 
+            fn pointer(&self) -> AbiParam {
+                AbiParam::new(self.pointer_type)
+            }
+
             fn i32(&self) -> AbiParam {
                 AbiParam::new(I32)
             }
@@ -161,10 +165,10 @@ declare_builtin_functions! {
     memory_init(vmctx, i32, i32, i32, i32, i32) -> ();
     /// Returns an index for wasm's `data.drop` instruction.
     data_drop(vmctx, i32) -> ();
-    /// Returns an index for Wasm's `table.grow` instruction.
-    table_grow(vmctx, i32, i32, reference) -> (i32);
-    /// Returns an index for Wasm's `ref.func` instruction.
-    ref_func(vmctx, i32) -> (reference);
+    /// Returns an index for Wasm's `table.grow` instruction for `funcref`s.
+    table_grow_funcref(vmctx, i32, i32, pointer) -> (i32);
+    /// Returns an index for Wasm's `table.grow` instruction for `externref`s.
+    table_grow_externref(vmctx, i32, i32, reference) -> (i32);
 }
 
 impl BuiltinFunctionIndex {
@@ -586,9 +590,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         });
 
         let element_size = match self.module.table_plans[index].style {
-            TableStyle::CallerChecksSignature => {
-                u64::from(self.offsets.size_of_vmcaller_checked_anyfunc())
-            }
+            TableStyle::CallerChecksSignature => self.pointer_type().bytes() as _,
         };
 
         Ok(func.create_table(ir::TableData {
@@ -607,12 +609,25 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         delta: ir::Value,
         init_value: ir::Value,
     ) -> WasmResult<ir::Value> {
-        let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64);
+        let (func_idx, func_sig) = match self.module.table_plans[table_index].table.ty {
+            TableElementType::Func => (
+                BuiltinFunctionIndex::table_grow_funcref(),
+                self.builtin_function_signatures
+                    .table_grow_funcref(&mut pos.func),
+            ),
+            TableElementType::Val(ty) => {
+                debug_assert_eq!(ty, self.reference_type());
+                (
+                    BuiltinFunctionIndex::table_grow_externref(),
+                    self.builtin_function_signatures
+                        .table_grow_externref(&mut pos.func),
+                )
+            }
+        };
 
-        let func_idx = BuiltinFunctionIndex::table_grow();
         let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
 
-        let func_sig = self.builtin_function_signatures.table_grow(&mut pos.func);
+        let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64);
         let call_inst = pos.ins().call_indirect(
             func_sig,
             func_addr,
@@ -663,17 +678,10 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         mut pos: cranelift_codegen::cursor::FuncCursor<'_>,
         func_index: FuncIndex,
     ) -> WasmResult<ir::Value> {
-        let func_sig = self.builtin_function_signatures.ref_func(&mut pos.func);
-        let ref_func_index = BuiltinFunctionIndex::ref_func();
-        let func_index_arg = pos.ins().iconst(I32, func_index.as_u32() as i64);
-
-        let (vmctx, func_addr) =
-            self.translate_load_builtin_function_address(&mut pos, ref_func_index);
-        let call_inst = pos
-            .ins()
-            .call_indirect(func_sig, func_addr, &[vmctx, func_index_arg]);
-
-        Ok(pos.func.dfg.first_result(call_inst))
+        let vmctx = self.vmctx(&mut pos.func);
+        let vmctx = pos.ins().global_value(self.pointer_type(), vmctx);
+        let offset = self.offsets.vmctx_anyfunc(func_index);
+        Ok(pos.ins().iadd_imm(vmctx, i64::from(offset)))
     }
 
     fn translate_custom_global_get(
@@ -843,12 +851,21 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
 
         let table_entry_addr = pos.ins().table_addr(pointer_type, table, callee, 0);
 
-        // Dereference table_entry_addr to get the function address.
+        // Dereference the table entry to get the pointer to the
+        // `VMCallerCheckedAnyfunc`.
+        let anyfunc_ptr =
+            pos.ins()
+                .load(pointer_type, ir::MemFlags::trusted(), table_entry_addr, 0);
+        // Note: `anyfunc_ptr` is always non-null, even for null func refs! In
+        // that case, the pointed-to `VMCallerCheckedAnyfunc`'s `func_ptr`
+        // member is null, which is checked below.
+
+        // Dereference anyfunc pointer to get the function address.
         let mem_flags = ir::MemFlags::trusted();
         let func_addr = pos.ins().load(
             pointer_type,
             mem_flags,
-            table_entry_addr,
+            anyfunc_ptr,
             i32::from(self.offsets.vmcaller_checked_anyfunc_func_ptr()),
         );
 
@@ -875,7 +892,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
                 let callee_sig_id = pos.ins().load(
                     sig_id_type,
                     mem_flags,
-                    table_entry_addr,
+                    anyfunc_ptr,
                     i32::from(self.offsets.vmcaller_checked_anyfunc_type_index()),
                 );
 
@@ -892,7 +909,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         let vmctx = pos.ins().load(
             pointer_type,
             mem_flags,
-            table_entry_addr,
+            anyfunc_ptr,
             i32::from(self.offsets.vmcaller_checked_anyfunc_vmctx()),
         );
         real_call_args.push(vmctx);