wasmtime: Add support for func.ref and table.grow with funcrefs

`funcref`s are implemented as `NonNull<VMCallerCheckedAnyfunc>`.

This should be more efficient than using a `VMExternRef` that points at a
`VMCallerCheckedAnyfunc` because it gets rid of an indirection, dynamic
allocation, and some reference counting.

Note that the null function reference is *NOT* a null pointer; it is a
`VMCallerCheckedAnyfunc` that has a null `func_ptr` member.

Part of #929

build.rs

@@ -215,7 +215,8 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
 
             ("reference_types", "table_copy_on_imported_tables")
             | ("reference_types", "externref_id_function")
-            | ("reference_types", "table_size") => {
+            | ("reference_types", "table_size")
+            | ("reference_types", "table_grow_with_funcref") => {
                 // TODO(#1886): Ignore if this isn't x64, because Cranelift only
                 // supports reference types on x64.
                 return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "x86_64";

cranelift/wasm/src/sections_translator.rs

@@ -97,6 +97,7 @@ pub fn parse_import_section<'data>(
             ImportSectionEntryType::Global(ref ty) => {
                 environ.declare_global_import(
                     Global {
+                        wasm_ty: ty.content_type,
                         ty: type_to_type(ty.content_type, environ).unwrap(),
                         mutability: ty.mutable,
                         initializer: GlobalInit::Import,
@@ -229,6 +230,7 @@ pub fn parse_global_section(
             }
         };
         let global = Global {
+            wasm_ty: content_type,
             ty: type_to_type(content_type, environ).unwrap(),
             mutability: mutable,
             initializer,

cranelift/wasm/src/translation_utils.rs

@@ -67,10 +67,18 @@ entity_impl!(DataIndex);
 pub struct ElemIndex(u32);
 entity_impl!(ElemIndex);
 
-/// WebAssembly global.
+/// A WebAssembly global.
+///
+/// Note that we record both the original Wasm type and the Cranelift IR type
+/// used to represent it. This is because multiple different kinds of Wasm types
+/// might be represented with the same Cranelift IR type. For example, both a
+/// Wasm `i64` and a `funcref` might be represented with a Cranelift `i64` on
+/// 64-bit architectures, and when GC is not required for func refs.
 #[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)]
 pub struct Global {
-    /// The type of the value stored in the global.
+    /// The Wasm type of the value stored in the global.
+    pub wasm_ty: crate::WasmType,
+    /// The Cranelift IR type of the value stored in the global.
     pub ty: ir::Type,
     /// A flag indicating whether the value may change at runtime.
     pub mutability: bool,

crates/c-api/src/table.rs

@@ -42,7 +42,7 @@ pub extern "C" fn wasm_table_new(
 ) -> Option<Box<wasm_table_t>> {
     let init: Val = match init {
         Some(init) => init.r.into(),
-        None => Val::ExternRef(None),
+        None => Val::FuncRef(None),
     };
     let table = Table::new(&store.store, tt.ty().ty.clone(), init).ok()?;
     Some(Box::new(wasm_table_t {
@@ -60,8 +60,8 @@ pub extern "C" fn wasmtime_funcref_table_new(
     out: &mut *mut wasm_table_t,
 ) -> Option<Box<wasmtime_error_t>> {
     let init: Val = match init {
-        Some(val) => Val::FuncRef(val.func().borrow().clone()),
-        None => Val::ExternRef(None),
+        Some(val) => Val::FuncRef(Some(val.func().borrow().clone())),
+        None => Val::FuncRef(None),
     };
     handle_result(
         Table::new(&store.store, tt.ty().ty.clone(), init),
@@ -85,7 +85,7 @@ pub extern "C" fn wasm_table_type(t: &wasm_table_t) -> Box<wasm_tabletype_t> {
 pub extern "C" fn wasm_table_get(t: &wasm_table_t, index: wasm_table_size_t) -> *mut wasm_ref_t {
     match t.table().borrow().get(index) {
         Some(val) => into_funcref(val),
-        None => into_funcref(Val::ExternRef(None)),
+        None => into_funcref(Val::FuncRef(None)),
     }
 }
 
@@ -99,14 +99,14 @@ pub extern "C" fn wasmtime_funcref_table_get(
         Some(val) => {
             *ptr = match val {
                 // TODO: what do do about creating new `HostRef` handles here?
-                Val::FuncRef(f) => {
+                Val::FuncRef(None) => ptr::null_mut(),
+                Val::FuncRef(Some(f)) => {
                     let store = match t.table().as_ref().store() {
                         None => return false,
                         Some(store) => store,
                     };
                     Box::into_raw(Box::new(HostRef::new(&store, f).into()))
                 }
-                Val::ExternRef(None) => ptr::null_mut(),
                 _ => return false,
             };
         }
@@ -133,14 +133,14 @@ pub extern "C" fn wasmtime_funcref_table_set(
     val: Option<&wasm_func_t>,
 ) -> Option<Box<wasmtime_error_t>> {
     let val = match val {
-        Some(val) => Val::FuncRef(val.func().borrow().clone()),
-        None => Val::ExternRef(None),
+        Some(val) => Val::FuncRef(Some(val.func().borrow().clone())),
+        None => Val::FuncRef(None),
     };
     handle_result(t.table().borrow().set(index, val), |()| {})
 }
 
 fn into_funcref(val: Val) -> *mut wasm_ref_t {
-    if let Val::ExternRef(None) = val {
+    if let Val::FuncRef(None) = val {
         return ptr::null_mut();
     }
     let externref = match val.externref() {
@@ -155,7 +155,7 @@ unsafe fn from_funcref(r: *mut wasm_ref_t) -> Val {
     if !r.is_null() {
         Box::from_raw(r).r.into()
     } else {
-        Val::ExternRef(None)
+        Val::FuncRef(None)
     }
 }
 
@@ -182,8 +182,8 @@ pub extern "C" fn wasmtime_funcref_table_grow(
     prev_size: Option<&mut wasm_table_size_t>,
 ) -> Option<Box<wasmtime_error_t>> {
     let val = match init {
-        Some(val) => Val::FuncRef(val.func().borrow().clone()),
-        None => Val::ExternRef(None),
+        Some(val) => Val::FuncRef(Some(val.func().borrow().clone())),
+        None => Val::FuncRef(None),
     };
     handle_result(t.table().borrow().grow(delta, val), |prev| {
         if let Some(ptr) = prev_size {

crates/environ/src/func_environ.rs

@@ -10,8 +10,8 @@ use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Si
 use cranelift_codegen::isa::{self, TargetFrontendConfig};
 use cranelift_entity::EntityRef;
 use cranelift_wasm::{
-    self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableElementType,
-    TableIndex, TargetEnvironment, WasmError, WasmResult,
+    self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableIndex,
+    TargetEnvironment, WasmError, WasmResult, WasmType,
 };
 #[cfg(feature = "lightbeam")]
 use cranelift_wasm::{DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex};
@@ -68,6 +68,10 @@ macro_rules! declare_builtin_functions {
                 AbiParam::new(self.reference_type)
             }
 
+            fn pointer(&self) -> AbiParam {
+                AbiParam::new(self.pointer_type)
+            }
+
             fn i32(&self) -> AbiParam {
                 AbiParam::new(I32)
             }
@@ -161,8 +165,10 @@ declare_builtin_functions! {
     memory_init(vmctx, i32, i32, i32, i32, i32) -> ();
     /// Returns an index for wasm's `data.drop` instruction.
     data_drop(vmctx, i32) -> ();
+    /// Returns an index for Wasm's `table.grow` instruction for `funcref`s.
+    table_grow_funcref(vmctx, i32, i32, pointer) -> (i32);
     /// Returns an index for Wasm's `table.grow` instruction for `externref`s.
-    table_grow_extern_ref(vmctx, i32, i32, reference) -> (i32);
+    table_grow_externref(vmctx, i32, i32, reference) -> (i32);
 }
 
 impl BuiltinFunctionIndex {
@@ -280,26 +286,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         }
     }
 
-    fn get_table_grow_func(
-        &mut self,
-        func: &mut Function,
-        table_index: TableIndex,
-    ) -> WasmResult<(ir::SigRef, BuiltinFunctionIndex, usize)> {
-        match self.module.table_plans[table_index].table.ty {
-            TableElementType::Func => Err(WasmError::Unsupported(
-                "the `table.grow` instruction is not supported with `funcref` yet".into(),
-            )),
-            TableElementType::Val(ty) => {
-                assert_eq!(ty, self.reference_type());
-                Ok((
-                    self.builtin_function_signatures.table_grow_extern_ref(func),
-                    BuiltinFunctionIndex::table_grow_extern_ref(),
-                    table_index.as_u32() as usize,
-                ))
-            }
-        }
-    }
-
     fn get_table_copy_func(
         &mut self,
         func: &mut Function,
@@ -552,6 +538,10 @@ impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environm
     fn target_config(&self) -> TargetFrontendConfig {
         self.target_config
     }
+
+    fn reference_type(&self, ty: WasmType) -> ir::Type {
+        crate::reference_type(ty, self.pointer_type())
+    }
 }
 
 impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'module_environment> {
@@ -604,9 +594,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         });
 
         let element_size = match self.module.table_plans[index].style {
-            TableStyle::CallerChecksSignature => {
-                u64::from(self.offsets.size_of_vmcaller_checked_anyfunc())
-            }
+            TableStyle::CallerChecksSignature => u64::from(self.pointer_type().bytes()),
         };
 
         Ok(func.create_table(ir::TableData {
@@ -626,24 +614,34 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         delta: ir::Value,
         init_value: ir::Value,
     ) -> WasmResult<ir::Value> {
-        let (func_sig, func_idx, table_index_arg) =
-            self.get_table_grow_func(&mut pos.func, table_index)?;
-
-        let table_index_arg = pos.ins().iconst(I32, table_index_arg as i64);
+        let (func_idx, func_sig) =
+            match self.module.table_plans[table_index].table.wasm_ty {
+                WasmType::FuncRef => (
+                    BuiltinFunctionIndex::table_grow_funcref(),
+                    self.builtin_function_signatures
+                        .table_grow_funcref(&mut pos.func),
+                ),
+                WasmType::ExternRef => (
+                    BuiltinFunctionIndex::table_grow_externref(),
+                    self.builtin_function_signatures
+                        .table_grow_externref(&mut pos.func),
+                ),
+                _ => return Err(WasmError::Unsupported(
+                    "`table.grow` with a table element type that is not `funcref` or `externref`"
+                        .into(),
+                )),
+            };
 
         let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+
+        let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64);
         let call_inst = pos.ins().call_indirect(
             func_sig,
             func_addr,
             &[vmctx, table_index_arg, delta, init_value],
         );
-        Ok(pos
-            .func
-            .dfg
-            .inst_results(call_inst)
-            .first()
-            .copied()
-            .unwrap())
+
+        Ok(pos.func.dfg.first_result(call_inst))
     }
 
     fn translate_table_get(
@@ -684,14 +682,50 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         ))
     }
 
+    fn translate_ref_null(
+        &mut self,
+        mut pos: cranelift_codegen::cursor::FuncCursor,
+        ty: WasmType,
+    ) -> WasmResult<ir::Value> {
+        Ok(match ty {
+            WasmType::FuncRef => pos.ins().iconst(self.pointer_type(), 0),
+            WasmType::ExternRef => pos.ins().null(self.reference_type(ty)),
+            _ => {
+                return Err(WasmError::Unsupported(
+                    "`ref.null T` that is not a `funcref` or an `externref`".into(),
+                ));
+            }
+        })
+    }
+
+    fn translate_ref_is_null(
+        &mut self,
+        mut pos: cranelift_codegen::cursor::FuncCursor,
+        value: ir::Value,
+    ) -> WasmResult<ir::Value> {
+        let bool_is_null = match pos.func.dfg.value_type(value) {
+            // `externref`
+            ty if ty.is_ref() => pos.ins().is_null(value),
+            // `funcref`
+            ty if ty == self.pointer_type() => {
+                pos.ins()
+                    .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0)
+            }
+            _ => unreachable!(),
+        };
+
+        Ok(pos.ins().bint(ir::types::I32, bool_is_null))
+    }
+
     fn translate_ref_func(
         &mut self,
-        _: cranelift_codegen::cursor::FuncCursor<'_>,
-        _: FuncIndex,
+        mut pos: cranelift_codegen::cursor::FuncCursor<'_>,
+        func_index: FuncIndex,
     ) -> WasmResult<ir::Value> {
-        Err(WasmError::Unsupported(
-            "the `ref.func` instruction is not supported yet".into(),
-        ))
+        let vmctx = self.vmctx(&mut pos.func);
+        let vmctx = pos.ins().global_value(self.pointer_type(), vmctx);
+        let offset = self.offsets.vmctx_anyfunc(func_index);
+        Ok(pos.ins().iadd_imm(vmctx, i64::from(offset)))
     }
 
     fn translate_custom_global_get(
@@ -861,18 +895,25 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
 
         let table_entry_addr = pos.ins().table_addr(pointer_type, table, callee, 0);
 
-        // Dereference table_entry_addr to get the function address.
+        // Dereference the table entry to get the pointer to the
+        // `VMCallerCheckedAnyfunc`.
+        let anyfunc_ptr =
+            pos.ins()
+                .load(pointer_type, ir::MemFlags::trusted(), table_entry_addr, 0);
+
+        // Check for whether the table element is null, and trap if so.
+        pos.ins()
+            .trapz(anyfunc_ptr, ir::TrapCode::IndirectCallToNull);
+
+        // Dereference anyfunc pointer to get the function address.
         let mem_flags = ir::MemFlags::trusted();
         let func_addr = pos.ins().load(
             pointer_type,
             mem_flags,
-            table_entry_addr,
+            anyfunc_ptr,
             i32::from(self.offsets.vmcaller_checked_anyfunc_func_ptr()),
         );
 
-        // Check whether `func_addr` is null.
-        pos.ins().trapz(func_addr, ir::TrapCode::IndirectCallToNull);
-
         // If necessary, check the signature.
         match self.module.table_plans[table_index].style {
             TableStyle::CallerChecksSignature => {
@@ -893,7 +934,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
                 let callee_sig_id = pos.ins().load(
                     sig_id_type,
                     mem_flags,
-                    table_entry_addr,
+                    anyfunc_ptr,
                     i32::from(self.offsets.vmcaller_checked_anyfunc_type_index()),
                 );
 
@@ -910,7 +951,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         let vmctx = pos.ins().load(
             pointer_type,
             mem_flags,
-            table_entry_addr,
+            anyfunc_ptr,
             i32::from(self.offsets.vmcaller_checked_anyfunc_vmctx()),
         );
         real_call_args.push(vmctx);

crates/environ/src/lib.rs

@@ -72,3 +72,18 @@ pub const WASM_MAX_PAGES: u32 = 0x10000;
 
 /// Version number of this crate.
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+
+pub(crate) fn reference_type(
+    wasm_ty: cranelift_wasm::WasmType,
+    pointer_type: ir::Type,
+) -> ir::Type {
+    match wasm_ty {
+        cranelift_wasm::WasmType::FuncRef => pointer_type,
+        cranelift_wasm::WasmType::ExternRef => match pointer_type {
+            ir::types::I32 => ir::types::R32,
+            ir::types::I64 => ir::types::R64,
+            _ => panic!("unsupported pointer type"),
+        },
+        _ => panic!("unsupported Wasm reference type"),
+    }
+}

crates/environ/src/module_environ.rs

@@ -92,6 +92,10 @@ impl<'data> TargetEnvironment for ModuleEnvironment<'data> {
     fn target_config(&self) -> TargetFrontendConfig {
         self.result.target_config
     }
+
+    fn reference_type(&self, ty: cranelift_wasm::WasmType) -> ir::Type {
+        crate::reference_type(ty, self.pointer_type())
+    }
 }
 
 /// This trait is useful for `translate_module` because it tells how to translate