From 61845ce0a883b743ea127defa51b076f0212c37c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hillerstr=C3=B6m?= Date: Tue, 20 Aug 2024 14:27:02 +0200 Subject: [PATCH 1/8] Linking tags This patch implements support for importing and exporting tags in the virtual machine. As a consequence tags are now unique to their origin instance, meaning that two different instantiations of the same module yields distinct tags. The runtime representation of a tag has changed from an `u32` to a `usize` (or native pointer). The identity of a tag is given by its address in the vmcontext. Resolves https://github.com/wasmfx/wasmfxtime/issues/25. --- cranelift/wasm/src/code_translator.rs | 3 +- cranelift/wasm/src/environ/spec.rs | 2 +- cranelift/wasm/src/sections_translator.rs | 4 +- crates/continuations/src/lib.rs | 205 +++++------- crates/cranelift/src/func_environ.rs | 12 +- crates/cranelift/src/wasmfx/baseline.rs | 5 +- crates/cranelift/src/wasmfx/optimized.rs | 308 +++++++++--------- crates/cranelift/src/wasmfx/shared.rs | 81 +++++ crates/environ/src/builtin.rs | 4 +- crates/environ/src/compile/module_environ.rs | 29 +- .../environ/src/component/translate/inline.rs | 1 + crates/environ/src/module.rs | 37 ++- crates/environ/src/vmoffsets.rs | 89 ++++- crates/types/src/lib.rs | 58 +++- crates/wasmtime/src/runtime/externals.rs | 13 + crates/wasmtime/src/runtime/externals/tag.rs | 44 +++ crates/wasmtime/src/runtime/instance.rs | 19 +- crates/wasmtime/src/runtime/linker.rs | 3 + crates/wasmtime/src/runtime/store/data.rs | 3 + crates/wasmtime/src/runtime/types.rs | 37 ++- crates/wasmtime/src/runtime/types/matching.rs | 8 +- crates/wasmtime/src/runtime/vm.rs | 2 +- .../wasmtime/src/runtime/vm/continuation.rs | 10 +- crates/wasmtime/src/runtime/vm/export.rs | 27 +- crates/wasmtime/src/runtime/vm/fibre/mod.rs | 12 +- crates/wasmtime/src/runtime/vm/fibre/unix.rs | 12 +- .../wasmtime/src/runtime/vm/gc/enabled/drc.rs | 6 + crates/wasmtime/src/runtime/vm/imports.rs | 5 +- crates/wasmtime/src/runtime/vm/instance.rs | 62 +++- crates/wasmtime/src/runtime/vm/libcalls.rs | 6 +- crates/wasmtime/src/runtime/vm/vmcontext.rs | 70 ++++ tests/all/typed_continuations.rs | 13 +- .../typed-continuations/linking_tags.wast | 164 +++++++++- tests/wast.rs | 20 +- 34 files changed, 1004 insertions(+), 370 deletions(-) create mode 100644 crates/wasmtime/src/runtime/externals/tag.rs diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs index 9adde5afe507..dc9ec461c343 100644 --- a/cranelift/wasm/src/code_translator.rs +++ b/cranelift/wasm/src/code_translator.rs @@ -2579,9 +2579,8 @@ pub fn translate_operator( let params = state.peekn(param_types.len()); let param_count = params.len(); - let tag_index_val = builder.ins().iconst(I32, *tag_index as i64); let return_values = - environ.translate_suspend(builder, tag_index_val, params, &return_types); + environ.translate_suspend(builder, *tag_index, params, &return_types); state.popn(param_count); state.pushn(&return_values); diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index 6b5a89eb3d20..60991c638065 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -647,7 +647,7 @@ pub trait FuncEnvironment: TargetEnvironment { fn translate_suspend( &mut self, builder: &mut FunctionBuilder, - tag_index: ir::Value, + tag_index: u32, suspend_args: &[ir::Value], tag_return_types: &[wasmtime_types::WasmValType], ) -> Vec; diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index 08f7eebf9ef4..e1700257c1b2 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -27,9 +27,7 @@ use wasmtime_types::ConstExpr; fn tag(e: TagType) -> Tag { match e.kind { - wasmparser::TagKind::Exception => Tag { - ty: TypeIndex::from_u32(e.func_type_idx), - }, + wasmparser::TagKind::Exception => Tag::partial(e.func_type_idx), } } diff --git a/crates/continuations/src/lib.rs b/crates/continuations/src/lib.rs index ccda5ea189b9..a0c2e6486f4a 100644 --- a/crates/continuations/src/lib.rs +++ b/crates/continuations/src/lib.rs @@ -41,15 +41,6 @@ pub mod types { /// Type of the entries in the actual buffer pub type DataEntries = u128; } - - /// Types used by `wasmtime_fibre::SwitchDirection` struct - pub mod switch_reason { - /// Type of `discriminant` field - pub type Discriminant = u32; - - /// Type of `data` field - pub type Data = u32; - } } /// Runtime configuration options for WasmFX that can be set via the command @@ -165,114 +156,6 @@ impl From for i32 { } } -// Runtime representation of tags -pub type TagId = u32; - -/// See SwitchDirection below for overall use of this type. -#[repr(u32)] -#[derive(Debug, Clone)] -pub enum SwitchDirectionEnum { - // Used to indicate that the contination has returned normally. - Return = 0, - - // Indicates that we are suspendinga continuation due to invoking suspend. - // The payload is the tag to suspend with - Suspend = 1, - - // Indicates that we are resuming a continuation via resume. - Resume = 2, -} - -impl SwitchDirectionEnum { - pub fn discriminant_val(&self) -> u32 { - // This is well-defined for an enum with repr(u32). - unsafe { *(self as *const SwitchDirectionEnum as *const u32) } - } -} - -/// Values of this type are passed to `wasmtime_fibre_switch` to indicate why we -/// are switching. A nicer way of representing this type would be the following -/// enum: -/// -///``` -/// #[repr(C, u32)] -/// pub enum SwitchDirection { -/// // Used to indicate that the contination has returned normally. -/// Return = 0, -/// -/// // Indicates that we are suspendinga continuation due to invoking suspend. -/// // The payload is the tag to suspend with -/// Suspend(u32) = 1, -/// -/// // Indicates that we are resuming a continuation via resume. -/// Resume = 2, -/// } -///``` -/// -/// However, we want to convert values of type `SwitchDirection` to and from u64 -/// easily, which is why we need to ensure that it contains no uninitialised -/// memory, to avoid undefined behavior. -/// -/// We allow converting values of this type to and from u64. -/// In that representation, bits 0 to 31 (where 0 is the LSB) contain the -/// discriminant (as u32), while bits 32 to 63 contain the `data`. -#[repr(C)] -#[derive(Debug, Clone)] -pub struct SwitchDirection { - pub discriminant: SwitchDirectionEnum, - - // Stores tag value if `discriminant` is `suspend`, 0 otherwise. - pub data: u32, -} - -impl SwitchDirection { - pub fn return_() -> SwitchDirection { - SwitchDirection { - discriminant: SwitchDirectionEnum::Return, - data: 0, - } - } - - pub fn resume() -> SwitchDirection { - SwitchDirection { - discriminant: SwitchDirectionEnum::Resume, - data: 0, - } - } - - pub fn suspend(tag: u32) -> SwitchDirection { - SwitchDirection { - discriminant: SwitchDirectionEnum::Suspend, - data: tag, - } - } -} - -impl From for u64 { - fn from(val: SwitchDirection) -> u64 { - // TODO(frank-emrich) This assumes little endian data layout. Should - // make this more explicit. - unsafe { core::mem::transmute::(val) } - } -} - -impl From for SwitchDirection { - fn from(val: u64) -> SwitchDirection { - #[cfg(debug_assertions)] - { - let discriminant = val as u32; - debug_assert!(discriminant <= 2); - if discriminant != SwitchDirectionEnum::Suspend.discriminant_val() { - let data = val >> 32; - debug_assert_eq!(data, 0); - } - } - // TODO(frank-emrich) This assumes little endian data layout. Should - // make this more explicit. - unsafe { core::mem::transmute::(val) } - } -} - /// Defines offsets of the fields in the continuation-related types pub mod offsets { /// Offsets of fields in `Payloads` @@ -327,3 +210,91 @@ pub mod offsets { /// We test there that this value is correct. pub const STACK_CHAIN_SIZE: usize = 2 * core::mem::size_of::(); } + +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct TaggedPointer(usize); + +impl TaggedPointer { + const LOW_TAG_MASK: usize = 0b11; + + pub fn untagged(val: usize) -> Self { + Self(val) + } + + pub fn low_tag(self, tag: usize) -> Self { + assert!(tag <= Self::LOW_TAG_MASK); + Self(self.0 | tag) + } + + pub fn get_low_tag(self) -> usize { + self.0 & Self::LOW_TAG_MASK + } + + pub fn low_untag(self) -> Self { + Self(self.0 & !Self::LOW_TAG_MASK) + } +} + +impl From for usize { + fn from(val: TaggedPointer) -> usize { + val.0 + } +} + +impl From for TaggedPointer { + fn from(val: usize) -> TaggedPointer { + TaggedPointer::untagged(val) + } +} + +/// Universal control effect. This structure encodes return signal, +/// resume signal, suspension signal, and suspension tags into a +/// pointer. This instance is used at runtime. There is a codegen +/// counterpart in `cranelift/src/wasmfx/shared.rs`. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ControlEffect(TaggedPointer); + +impl ControlEffect { + pub fn suspend(ptr: *const u8) -> Self { + let tptr = TaggedPointer::untagged(ptr as usize); + Self(TaggedPointer::low_tag(tptr, 0b01)) + } + + pub fn return_() -> Self { + Self((0b00 as usize).into()) + } + + pub fn resume() -> Self { + Self((0b11 as usize).into()) + } + + fn new(raw: usize) -> Self { + Self(TaggedPointer::untagged(raw)) + } + + pub fn is_suspend(self) -> bool { + TaggedPointer::get_low_tag(self.0) == 0b01 + } +} + +impl From for ControlEffect { + fn from(val: u64) -> ControlEffect { + ControlEffect::new(val as usize) + } +} + +impl From for u64 { + fn from(val: ControlEffect) -> u64 { + let raw: usize = val.0.into(); + raw as u64 + } +} + +impl From for *mut u8 { + fn from(val: ControlEffect) -> *mut u8 { + let raw: usize = val.0.into(); + raw as *mut u8 + } +} diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 20b0993a65c3..5f73f31a9ab3 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -93,7 +93,7 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures); pub struct FuncEnvironment<'module_environment> { /// NOTE(frank-emrich) pub for use in crate::wasmfx::* modules pub(crate) isa: &'module_environment (dyn TargetIsa + 'module_environment), - module: &'module_environment Module, + pub(crate) module: &'module_environment Module, types: &'module_environment ModuleTypesBuilder, wasm_func_ty: &'module_environment WasmFuncType, sig_ref_to_ty: SecondaryMap>, @@ -2810,7 +2810,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m fn translate_suspend( &mut self, builder: &mut FunctionBuilder, - tag_index: ir::Value, + tag_index: u32, suspend_args: &[ir::Value], tag_return_types: &[WasmValType], ) -> Vec { @@ -2833,12 +2833,16 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m fn tag_params(&self, tag_index: u32) -> &[WasmValType] { let idx = self.module.tags[TagIndex::from_u32(tag_index)].signature; - self.types[idx].unwrap_func().params() + self.types[idx.unwrap_module_type_index()] + .unwrap_func() + .params() } fn tag_returns(&self, tag_index: u32) -> &[WasmValType] { let idx = self.module.tags[TagIndex::from_u32(tag_index)].signature; - self.types[idx].unwrap_func().returns() + self.types[idx.unwrap_module_type_index()] + .unwrap_func() + .returns() } fn use_x86_blendv_for_relaxed_laneselect(&self, ty: Type) -> bool { diff --git a/crates/cranelift/src/wasmfx/baseline.rs b/crates/cranelift/src/wasmfx/baseline.rs index 37abbf24bd4b..89c1b520f664 100644 --- a/crates/cranelift/src/wasmfx/baseline.rs +++ b/crates/cranelift/src/wasmfx/baseline.rs @@ -443,12 +443,13 @@ pub(crate) fn translate_cont_new<'a>( pub(crate) fn translate_suspend<'a>( env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, - tag_index: ir::Value, + tag_index: u32, suspend_args: &[ir::Value], tag_return_types: &[WasmValType], ) -> Vec { + let tag_index_val = builder.ins().iconst(I32, tag_index as i64); typed_continuations_store_payloads(env, builder, suspend_args); - call_builtin!(builder, env, tc_baseline_suspend(tag_index)); + call_builtin!(builder, env, tc_baseline_suspend(tag_index_val)); let contref = typed_continuations_load_continuation_reference(env, builder); let return_values = diff --git a/crates/cranelift/src/wasmfx/optimized.rs b/crates/cranelift/src/wasmfx/optimized.rs index 17cf782c15ef..d10df4b38701 100644 --- a/crates/cranelift/src/wasmfx/optimized.rs +++ b/crates/cranelift/src/wasmfx/optimized.rs @@ -5,13 +5,13 @@ use cranelift_codegen::ir; use cranelift_codegen::ir::condcodes::*; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::InstBuilder; -use cranelift_frontend::{FunctionBuilder, Switch}; +use cranelift_frontend::FunctionBuilder; use cranelift_wasm::FuncEnvironment; use cranelift_wasm::{FuncTranslationState, WasmResult, WasmValType}; use wasmtime_environ::PtrSize; #[cfg_attr(feature = "wasmfx_baseline", allow(unused_imports))] -pub(crate) use shared::{assemble_contobj, disassemble_contobj, vm_contobj_type}; +pub(crate) use shared::{assemble_contobj, disassemble_contobj, vm_contobj_type, ControlEffect}; #[macro_use] pub(crate) mod typed_continuation_helpers { @@ -1369,16 +1369,63 @@ pub(crate) fn translate_resume<'a>( resume_args: &[ir::Value], resumetable: &[(u32, ir::Block)], ) -> Vec { + // The resume instruction is the most involved instruction to + // compile as it is responsible for both continuation application + // and control tag dispatch. + // + // We store the continuation arguments, continuation return + // values, and suspension payloads on the vm context. + // + // Here we translate a resume instruction into several basic + // blocks as follows: + // + // prelude_block + // | + // | + // resume_block <-----------\ + // | | + // | | + // control_block | + // / \ | + // | | | + // return_block dispatch_block | + // | | + // | | + // forward_block --/ + // + // * prelude_block pushes the continuation arguments onto the + // buffer in the libcall context. + // * resume_block continues a given `contref`. It jumps to + // the `control_block`. + // * control_block handles the control effect of resume, i.e. on + // ordinary return from resume, it jumps to the `return_block`, + // whereas on suspension it jumps to the `dispatch_block`. + // * return_block reads the return values from the libcall + // context. + // * dispatch_block(NOTE1) dispatches on a tag provided by the + // control_block to an associated user-defined block. If + // there is no suitable user-defined block, then it jumps to + // the forward_block. + // * forward_block dispatches the handling of a given tag to + // the ambient context. Once control returns it jumps to the + // resume_block to continue continuation at the suspension + // site. + // + // NOTE1: The dispatch block is the head of a collection of blocks + // which encodes a right-leaning (almost binary) decision tree, + // that is a series of nested if-then-else. The `then` branch + // contains a "leaf" node which setups the jump to a user-defined + // handler block, whilst the `else` branch contains another + // decision tree or the forward_block. let resume_block = builder.create_block(); let return_block = builder.create_block(); - let suspend_block = builder.create_block(); - let switch_block = builder.create_block(); + let control_block = builder.create_block(); + let dispatch_block = builder.create_block(); let forwarding_block = builder.create_block(); let vmctx = env.vmctx_val(&mut builder.cursor()); - // Preamble: Part of previously active block - + // Preamble: Part of previously active block. let (next_revision, resume_contref, parent_stack_chain) = { let (witness, resume_contref) = shared::disassemble_contobj(env, builder, contobj); @@ -1468,47 +1515,35 @@ pub(crate) fn translate_resume<'a>( // Now the parent contref (or main stack) is active again vmctx.store_stack_chain(env, builder, &parent_stack_chain); + let vm_runtime_limits_ptr = vmctx.load_vm_runtime_limits_ptr(env, builder); - // The `result` is a value of type wasmtime_continuations::SwitchDirection, - // using the encoding described at its definition. - // Thus, the first 32 bit encode the discriminant, and the - // subsequent 32 bit encode the tag if suspending, or 0 otherwise. - // Thus, when returning, the overall u64 should be zero. - let return_discriminant = - wasmtime_continuations::SwitchDirectionEnum::Return.discriminant_val(); - debug_assert_eq!(return_discriminant, 0); - - // If these two assumptions don't hold anymore, the code here becomes invalid. - debug_assert_eq!( - std::mem::size_of::(), - 4 - ); - debug_assert_eq!( - std::mem::size_of::(), - 4 - ); + // Extract the result and signal bit. + let result = ControlEffect::new(result); + let signal = ControlEffect::signal(result, env, builder); - if cfg!(debug_assertions) { - let discriminant = builder.ins().ireduce(I32, result); - emit_debug_println!(env, builder, "[resume] discriminant is {}", discriminant); - } - - let vm_runtime_limits_ptr = vmctx.load_vm_runtime_limits_ptr(env, builder); + emit_debug_println!( + env, + builder, + "[resume] in resume block, signal is {}", + signal + ); - // Jump to the return block if the result is 0, otherwise jump to + // Jump to the return block if the result signal is 0, otherwise jump to // the suspend block. builder .ins() - .brif(result, suspend_block, &[], return_block, &[]); + .brif(signal, control_block, &[], return_block, &[]); // We do not seal this block, yet, because the effect forwarding block has a back edge to it (result, vm_runtime_limits_ptr) }; - // Suspend block. - let tag = { - builder.switch_to_block(suspend_block); - builder.seal_block(suspend_block); + // The control block; here we extract the tag of the suspension + // (regardless of whether an actual suspension occurred... the + // return_block never reads `tag`). + let suspend_tag_addr = { + builder.switch_to_block(control_block); + builder.seal_block(control_block); // We store parts of the VMRuntimeLimits into the continuation that just suspended. let suspended_chain = @@ -1519,71 +1554,96 @@ pub(crate) fn translate_resume<'a>( // parent of the suspended continuation (which is now active). parent_stack_chain.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); - let discriminant = builder.ins().ireduce(I32, resume_result); - let discriminant_size_bytes = - std::mem::size_of::(); - - if cfg!(debug_assertions) { - let suspend_discriminant = - wasmtime_continuations::SwitchDirectionEnum::Suspend.discriminant_val(); - let suspend_discriminant = builder.ins().iconst(I32, suspend_discriminant as i64); - emit_debug_assert_eq!(env, builder, discriminant, suspend_discriminant); - } - - let tag = builder - .ins() - .ushr_imm(resume_result, discriminant_size_bytes as i64 * 8); - let tag = builder.ins().ireduce(I32, tag); - - emit_debug_println!(env, builder, "[resume] in suspend block, tag is {}", tag); + // Extract the tag + let tag = ControlEffect::value(resume_result, env, builder); + emit_debug_println!(env, builder, "[resume] in suspend block, tag is {:p}", tag); // We need to terminate this block before being allowed to switch to another one - builder.ins().jump(switch_block, &[]); + builder.ins().jump(dispatch_block, &[]); + builder.seal_block(dispatch_block); tag }; - // Now, construct blocks for the three continuations: - // 1) `resume` returned normally. - // 2) `resume` returned via a suspend. - // 3) `resume` is forwarding + // Forwarding block: The last block in the if-then-else dispatch + // chain. Control flows to this block when the table on (resume + // ...) does not have a matching mapping (on ...). + { + builder.switch_to_block(forwarding_block); - // Strategy: - // - // Translate each each `(tag, label)` pair in the resume table - // to a switch-case of the form "case tag: br label". NOTE: - // `tag` may appear multiple times in resume table, only the - // first appearance should be processed as it shadows the - // subsequent entries. The switching logic then ensures that - // we jump to the block handling the corresponding tag. - // - // The fallback/default case performs effect forwarding (TODO). - // - // First, initialise the switch structure. - let mut switch = Switch::new(); - // Second, we consume the resume table entry-wise. - let mut case_blocks = vec![]; + let parent_contref = parent_stack_chain.unwrap_continuation_or_trap( + env, + builder, + ir::TrapCode::UnhandledTag, + ); + + // We suspend, thus deferring handling to the parent. We do + // nothing about tag *parameters*, these remain unchanged + // within the payload buffer associated with the whole + // VMContext. + call_builtin!(builder, env, tc_suspend(suspend_tag_addr)); + + // "Tag return values" (i.e., values provided by cont.bind or + // resume to the continuation) are actually stored in + // `VMContRef`s, and we need to move them down the chain back + // to the `VMContRef` where we originally suspended. + typed_continuations_forward_tag_return_values(env, builder, parent_contref, resume_contref); + + // We create a back edge to the resume block. Note that both + // `resume_contobj` and `parent_stack_chain` remain unchanged: + // In the current design, where forwarding is implemented by + // suspending up the chain of parent continuations and + // subsequently resume-ing back down the chain, both the + // continuation being resumed and its parent stay the same. + builder.ins().jump(resume_block, &[]); + builder.seal_block(resume_block); + } + + // Dispatch block. Now we create the nested if-then-else chain, + // which attempts to find a suitable handler for + // `suspend_tag_addr`. let mut tag_seen = std::collections::HashSet::new(); // Used to keep track of tags - for &(tag, target_block) in resumetable { - // Skip if this `tag` has been seen previously. - if !tag_seen.insert(tag) { + let mut tail_block = dispatch_block; + for &(handle_tag, target_block) in resumetable { + // Skip if this tag has been seen previously. + if !tag_seen.insert(handle_tag) { continue; } - let case = builder.create_block(); - switch.set_entry(tag as u128, case); - builder.switch_to_block(case); + // Switch to the current tail of the dispatch chain. + builder.switch_to_block(tail_block); + // Generate the test for whether `handle_tag` matches the suspension tag. + let tag_addr = shared::tag_address(env, builder, handle_tag); + emit_debug_println!( + env, + builder, + "[resume] comparing handle_tag_addr = {:p} and suspend_tag_addr = {:p}", + tag_addr, + suspend_tag_addr + ); + let cond = builder.ins().icmp(IntCC::Equal, suspend_tag_addr, tag_addr); + // Create landing sites: + // 1. If the tags match, then jump to the preamble block to load data + // 2. Otherwise jump to the next tail block + let target_preamble_block = builder.create_block(); + tail_block = builder.create_block(); + builder + .ins() + .brif(cond, target_preamble_block, &[], tail_block, &[]); + builder.seal_block(tail_block); + builder.seal_block(target_preamble_block); + // Fill the preamble block. + builder.switch_to_block(target_preamble_block); // Load and push arguments. - let param_types = env.tag_params(tag); + let param_types = env.tag_params(handle_tag); // TODO(dhil): Check whether this function behaves correctly for imported tags. let param_types: Vec = param_types .iter() .map(|wty| crate::value_type(env.isa, *wty)) .collect(); let mut args = typed_continuations_load_payloads(env, builder, ¶m_types); - // We have an actual handling block for this tag, rather than just - // forwarding. Detatch the `VMContRef` by setting its parent - // link to `StackChain::Absent`. + // Detatch the `VMContRef` by setting its parent link to + // `StackChain::Absent`. let pointer_type = env.pointer_type(); let chain = tc::StackChain::absent(builder, pointer_type); let mut vmcontref = tc::VMContRef::new(resume_contref, pointer_type); @@ -1599,75 +1659,26 @@ pub(crate) fn translate_resume<'a>( next_revision, resume_contref ); - args.push(contobj); - // Now jump to the actual user-defined block handling - // this tag, as given by the resumetable. + // Now jump to the actual user-defined block handling this + // tag, as given by the resume table. builder.ins().jump(target_block, &args); - case_blocks.push(case); } - - // Note that at this point we haven't actually emitted any - // code for the switching logic itself, but only filled - // the Switch structure and created the blocks it jumps - // to. - - // Forwarding block: Default case for the switching logic on the - // tag. Used when the (resume ...) clause we currently translate - // does not have a matching (tag ...) entry. - { - builder.switch_to_block(forwarding_block); - - let parent_contref = parent_stack_chain.unwrap_continuation_or_trap( - env, - builder, - ir::TrapCode::UnhandledTag, - ); - - // We suspend, thus deferring handling to the parent. - // We do nothing about tag *parameters*, these remain unchanged within the - // payload buffer associated with the whole VMContext. - call_builtin!(builder, env, tc_suspend(tag)); - - // "Tag return values" (i.e., values provided by cont.bind or - // resume to the continuation) are actually stored in - // `VMContRef`s, and we need to move them down the chain - // back to the `VMContRef` where we originally - // suspended. - typed_continuations_forward_tag_return_values(env, builder, parent_contref, resume_contref); - - // We create a back edge to the resume block. - // Note that both `resume_cotobj` and `parent_stack_chain` remain unchanged: - // In the current design, where forwarding is implemented by suspending - // up the chain of parent continuations and subsequently resume-ing back - // down the chain, both the continuation being resumed and its parent - // stay the same. - builder.ins().jump(resume_block, &[]); - builder.seal_block(resume_block); - } - - // Switch block: actual switching logic is emitted here. - { - builder.switch_to_block(switch_block); - switch.emit(builder, tag, forwarding_block); - builder.seal_block(switch_block); - builder.seal_block(forwarding_block); - - // We can only seal the blocks we generated for each - // tag now, after switch.emit ran. - for case_block in case_blocks { - builder.seal_block(case_block); - } - } - - // Return block: Jumped to by resume block if continuation returned normally. + // The last tail_block unconditionally jumps to the forwarding + // block. + builder.switch_to_block(tail_block); + builder.ins().jump(forwarding_block, &[]); + builder.seal_block(forwarding_block); + + // Return block: Jumped to by resume block if continuation + // returned normally. { builder.switch_to_block(return_block); builder.seal_block(return_block); - // Restore parts of the VMRuntimeLimits from the - // parent of the returned continuation (which is now active). + // Restore parts of the VMRuntimeLimits from the parent of the + // returned continuation (which is now active). parent_stack_chain.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); let co = tc::VMContRef::new(resume_contref, env.pointer_type()); @@ -1677,8 +1688,8 @@ pub(crate) fn translate_resume<'a>( let returns = env.continuation_returns(type_index).to_vec(); let values = typed_continuations_load_return_values(env, builder, &returns, resume_contref); - // The continuation has returned and all `VMContObjs` - // to it should have been be invalidated. We may safely deallocate + // The continuation has returned and all `VMContObjs` to it + // should have been be invalidated. We may safely deallocate // it. shared::typed_continuations_drop_cont_ref(env, builder, resume_contref); @@ -1689,13 +1700,14 @@ pub(crate) fn translate_resume<'a>( pub(crate) fn translate_suspend<'a>( env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, - tag_index: ir::Value, + tag_index: u32, suspend_args: &[ir::Value], tag_return_types: &[WasmValType], ) -> Vec { typed_continuations_store_payloads(env, builder, suspend_args); - call_builtin!(builder, env, tc_suspend(tag_index)); + let tag_addr = shared::tag_address(env, builder, tag_index); + call_builtin!(builder, env, tc_suspend(tag_addr)); let contref = typed_continuations_load_continuation_reference(env, builder); diff --git a/crates/cranelift/src/wasmfx/shared.rs b/crates/cranelift/src/wasmfx/shared.rs index 99f81a121e31..5ec163cb0132 100644 --- a/crates/cranelift/src/wasmfx/shared.rs +++ b/crates/cranelift/src/wasmfx/shared.rs @@ -140,3 +140,84 @@ pub(crate) fn typed_continuations_drop_cont_ref<'a>( ) { call_builtin!(builder, env, tc_drop_cont_ref(contref)); } + +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct TaggedPointer(pub ir::Value); + +impl TaggedPointer { + const LOW_TAG_MASK: i64 = 0b11; + const LOW_TAG_INVERSE_MASK: i64 = !0b11; + + pub fn new(val: ir::Value) -> Self { + Self(val) + } + + pub fn get_low_tag<'a>( + self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().band_imm(self.0, Self::LOW_TAG_MASK) + } + + pub fn unmask<'a>( + self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().band_imm(self.0, Self::LOW_TAG_INVERSE_MASK) + } +} + +/// Universal control effect. This structure encodes return signal, +/// resume signal, suspension signal, and suspension tags into a +/// pointer. This instance is used at compile time. There is a runtime +/// counterpart in `continuations/src/lib.rs`. +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct ControlEffect(pub TaggedPointer); + +impl ControlEffect { + pub fn new(val: ir::Value) -> Self { + Self(TaggedPointer::new(val)) + } + + pub fn signal<'a>( + self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + TaggedPointer::get_low_tag(self.0, env, builder) + } + + pub fn value<'a>( + self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + TaggedPointer::unmask(self.0, env, builder) + } +} + +pub(crate) fn tag_address<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + index: u32, +) -> ir::Value { + let vmctx = env.vmctx_val(&mut builder.cursor()); + let tag_index = wasmtime_environ::TagIndex::from_u32(index); + let pointer_type = env.pointer_type(); + if let Some(def_index) = env.module.defined_tag_index(tag_index) { + let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap(); + builder.ins().iadd_imm(vmctx, offset as i64) + } else { + let offset = i32::try_from(env.offsets.vmctx_vmtag_import_from(tag_index)).unwrap(); + builder.ins().load( + pointer_type, + ir::MemFlags::trusted().with_readonly(), + vmctx, + ir::immediates::Offset32::new(offset), + ) + } +} diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index bfd96db90eb5..a988f640f473 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -102,9 +102,9 @@ macro_rules! foreach_builtin_function { tc_cont_new(vmctx: vmctx, r: pointer, param_count: i32, result_count: i32) -> pointer; // Resumes a continuation. The result value is of type // wasmtime_continuations::SwitchDirection. - tc_resume(vmctx: vmctx, contref: pointer, parent_stack_limits: pointer) -> i64; + tc_resume(vmctx: vmctx, contref: pointer, parent_stack_limits: pointer) -> pointer; // Suspends a continuation. - tc_suspend(vmctx: vmctx, tag: i32); + tc_suspend(vmctx: vmctx, tag: pointer); // Sets the tag return values of `child_contref` to those of `parent_contref`. // This is implemented by exchanging the pointers to the underlying buffers. diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index a11aab922fcf..894b6c6ed068 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -6,8 +6,8 @@ use crate::prelude::*; use crate::{ DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, FuncIndex, GlobalIndex, InitMemory, MemoryIndex, ModuleTypesBuilder, PrimaryMap, StaticMemoryInitializer, TableIndex, - TableInitialValue, Tunables, TypeConvert, TypeIndex, Unsigned, WasmError, WasmHeapType, - WasmResult, WasmValType, WasmparserTypeConverter, + TableInitialValue, TagIndex, Tunables, TypeConvert, TypeIndex, Unsigned, WasmError, + WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter, }; use anyhow::{bail, Result}; use cranelift_entity::packed_option::ReservedValue; @@ -322,8 +322,19 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { EntityType::Table(self.convert_table_type(&ty)?) } - // doesn't get past validation - TypeRef::Tag(_) => unreachable!(), + TypeRef::Tag(ty) => { + let index = TypeIndex::from_u32(ty.func_type_idx); + let interned_index = self.result.module.types[index]; + let signature = + wasmtime_types::EngineOrModuleTypeIndex::Module(interned_index); + let tag = wasmtime_types::Tag { + ty: index, + signature, + }; + self.result.module.num_imported_tags += 1; + // TODO(dhil): debug info? + EntityType::Tag(tag) + } }; self.declare_import(import.module, import.name, ty); } @@ -395,8 +406,8 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { for entry in tags { let sigindex = entry?.func_type_idx; let ty = TypeIndex::from_u32(sigindex); - let sig_index = self.result.module.types[ty]; - self.result.module.push_tag(sig_index); + let interned_index = self.result.module.types[ty]; + self.result.module.push_tag(ty, interned_index); } // This feature isn't enabled at this time, so we should @@ -439,9 +450,7 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(index)), ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(index)), ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(index)), - - // this never gets past validation - ExternalKind::Tag => unreachable!(), + ExternalKind::Tag => EntityIndex::Tag(TagIndex::from_u32(index)), }; self.result .module @@ -789,7 +798,7 @@ and for re-adding support for interface types you can see this issue: EntityIndex::Memory(self.result.module.memory_plans.push(plan)) } EntityType::Global(ty) => EntityIndex::Global(self.result.module.globals.push(ty)), - EntityType::Tag(_) => unimplemented!(), + EntityType::Tag(ty) => EntityIndex::Tag(self.result.module.tags.push(ty)), } } diff --git a/crates/environ/src/component/translate/inline.rs b/crates/environ/src/component/translate/inline.rs index d3a84da55ae2..0f4dfd9bdcc5 100644 --- a/crates/environ/src/component/translate/inline.rs +++ b/crates/environ/src/component/translate/inline.rs @@ -943,6 +943,7 @@ impl<'a> Inliner<'a> { EntityIndex::Table(i) => frame.tables[i].clone().into(), EntityIndex::Global(i) => frame.globals[i].clone().into(), EntityIndex::Memory(i) => frame.memories[i].clone().into(), + EntityIndex::Tag(_) => todo!(), // TODO(dhil): Revisit later }, } } diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 85e20fdae4e3..4d162267c86b 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -467,6 +467,9 @@ pub struct Module { /// Number of imported or aliased globals in the module. pub num_imported_globals: usize, + /// Number of imported or aliased tags in the module. + pub num_imported_tags: usize, + /// Number of functions that "escape" from this module may need to have a /// `VMFuncRef` constructed for them. /// @@ -490,7 +493,7 @@ pub struct Module { pub globals: PrimaryMap, /// WebAssembly exceptions and typed control tags. - pub tags: PrimaryMap, + pub tags: PrimaryMap, /// WebAssembly global initializers for locally-defined globals. pub global_initializers: PrimaryMap, @@ -634,6 +637,29 @@ impl Module { } } + /// Test whether the given tag index is for an imported tag. + #[inline] + pub fn is_imported_tag(&self, index: TagIndex) -> bool { + index.index() < self.num_imported_tags + } + + /// Convert a `DefinedTagIndex` into a `TagIndex`. + #[inline] + pub fn tag_index(&self, defined_tag: DefinedTagIndex) -> TagIndex { + TagIndex::new(self.num_imported_tags + defined_tag.index()) + } + + /// Convert a `TagIndex` into a `DefinedTagIndex`. Returns None if the + /// index is an imported tag. + #[inline] + pub fn defined_tag_index(&self, tag: TagIndex) -> Option { + if tag.index() < self.num_imported_tags { + None + } else { + Some(DefinedTagIndex::new(tag.index() - self.num_imported_tags)) + } + } + /// Test whether the given global index is for an imported global. #[inline] pub fn is_imported_global(&self, index: GlobalIndex) -> bool { @@ -659,6 +685,7 @@ impl Module { EntityIndex::Function(i) => { EntityType::Function(EngineOrModuleTypeIndex::Module(self.functions[i].signature)) } + EntityIndex::Tag(i) => EntityType::Tag(self.tags[i]), } } @@ -673,11 +700,9 @@ impl Module { } /// TODO - pub fn push_tag(&mut self, signature: ModuleInternedTypeIndex) -> TagIndex { - self.tags.push(FunctionType { - signature, - func_ref: FuncRefIndex::reserved_value(), - }) + pub fn push_tag(&mut self, idx: TypeIndex, signature: ModuleInternedTypeIndex) -> TagIndex { + self.tags + .push(Tag::new(idx, EngineOrModuleTypeIndex::Module(signature))) } /// Returns an iterator over all of the defined function indices in this diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index b636e35b9480..aa978d0752b1 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -19,16 +19,18 @@ // imported_tables: [VMTableImport; module.num_imported_tables], // imported_memories: [VMMemoryImport; module.num_imported_memories], // imported_globals: [VMGlobalImport; module.num_imported_globals], +// imported_tags: [VMTagImport; module.num_imported_tags], // tables: [VMTableDefinition; module.num_defined_tables], // memories: [*mut VMMemoryDefinition; module.num_defined_memories], // owned_memories: [VMMemoryDefinition; module.num_owned_memories], // globals: [VMGlobalDefinition; module.num_defined_globals], +// tags: [VMTagDefinition; module.num_defined_tags], // func_refs: [VMFuncRef; module.num_escaped_funcs], // } use crate::{ - DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, FuncRefIndex, - GlobalIndex, MemoryIndex, Module, TableIndex, + DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex, + FuncRefIndex, GlobalIndex, MemoryIndex, Module, TableIndex, TagIndex, }; use cranelift_entity::packed_option::ReservedValue; use wasmtime_types::OwnedMemoryIndex; @@ -62,6 +64,8 @@ pub struct VMOffsets

{ pub num_imported_memories: u32, /// The number of imported globals in the module. pub num_imported_globals: u32, + /// The number of imported tags in the module. + pub num_imported_tags: u32, /// The number of defined tables in the module. pub num_defined_tables: u32, /// The number of defined memories in the module. @@ -70,6 +74,8 @@ pub struct VMOffsets

{ pub num_owned_memories: u32, /// The number of defined globals in the module. pub num_defined_globals: u32, + /// The number of defined tags in the module. + pub num_defined_tags: u32, /// The number of escaped functions in the module, the size of the func_refs /// array. pub num_escaped_funcs: u32, @@ -79,10 +85,12 @@ pub struct VMOffsets

{ imported_tables: u32, imported_memories: u32, imported_globals: u32, + imported_tags: u32, defined_tables: u32, defined_memories: u32, owned_memories: u32, defined_globals: u32, + defined_tags: u32, defined_func_refs: u32, size: u32, @@ -150,6 +158,12 @@ pub trait PtrSize { 16 } + /// Return the size of `VMTagDefinition`. + #[inline] + fn size_of_vmtag_definition(&self) -> u8 { + 8 + } + /// This is the size of the largest value type (i.e. a V128). #[inline] fn maximum_value_size(&self) -> u8 { @@ -335,6 +349,8 @@ pub struct VMOffsetsFields

{ pub num_imported_memories: u32, /// The number of imported globals in the module. pub num_imported_globals: u32, + /// The number of imported tags in the module. + pub num_imported_tags: u32, /// The number of defined tables in the module. pub num_defined_tables: u32, /// The number of defined memories in the module. @@ -343,6 +359,8 @@ pub struct VMOffsetsFields

{ pub num_owned_memories: u32, /// The number of defined globals in the module. pub num_defined_globals: u32, + /// The number of defined tags in the module. + pub num_defined_tags: u32, /// The number of escaped functions in the module, the size of the function /// references array. pub num_escaped_funcs: u32, @@ -364,6 +382,7 @@ impl VMOffsets

{ num_imported_functions: cast_to_u32(module.num_imported_funcs), num_imported_tables: cast_to_u32(module.num_imported_tables), num_imported_memories: cast_to_u32(module.num_imported_memories), + num_imported_tags: cast_to_u32(module.num_imported_tags), num_imported_globals: cast_to_u32(module.num_imported_globals), num_defined_tables: cast_to_u32(module.table_plans.len() - module.num_imported_tables), num_defined_memories: cast_to_u32( @@ -371,6 +390,7 @@ impl VMOffsets

{ ), num_owned_memories, num_defined_globals: cast_to_u32(module.globals.len() - module.num_imported_globals), + num_defined_tags: cast_to_u32(module.tags.len() - module.num_imported_tags), num_escaped_funcs: cast_to_u32(module.num_escaped_funcs), }) } @@ -396,8 +416,10 @@ impl VMOffsets

{ num_imported_tables: _, num_imported_memories: _, num_imported_globals: _, + num_imported_tags: _, num_defined_tables: _, num_defined_globals: _, + num_defined_tags: _, num_defined_memories: _, num_owned_memories: _, num_escaped_funcs: _, @@ -432,10 +454,12 @@ impl VMOffsets

{ typed_continuations_payloads: "typed continuations payloads object", typed_continuations_stack_chain: "typed continuations stack chain", defined_func_refs: "module functions", + defined_tags: "defined tags", defined_globals: "defined globals", owned_memories: "owned memories", defined_memories: "defined memories", defined_tables: "defined tables", + imported_tags: "imported tags", imported_globals: "imported globals", imported_memories: "imported memories", imported_tables: "imported tables", @@ -452,19 +476,23 @@ impl From> for VMOffsets

{ num_imported_tables: fields.num_imported_tables, num_imported_memories: fields.num_imported_memories, num_imported_globals: fields.num_imported_globals, + num_imported_tags: fields.num_imported_tags, num_defined_tables: fields.num_defined_tables, num_defined_memories: fields.num_defined_memories, num_owned_memories: fields.num_owned_memories, num_defined_globals: fields.num_defined_globals, + num_defined_tags: fields.num_defined_tags, num_escaped_funcs: fields.num_escaped_funcs, imported_functions: 0, imported_tables: 0, imported_memories: 0, imported_globals: 0, + imported_tags: 0, defined_tables: 0, defined_memories: 0, owned_memories: 0, defined_globals: 0, + defined_tags: 0, defined_func_refs: 0, size: 0, typed_continuations_stack_chain: 0, @@ -509,6 +537,8 @@ impl From> for VMOffsets

{ = cmul(ret.num_imported_memories, ret.size_of_vmmemory_import()), size(imported_globals) = cmul(ret.num_imported_globals, ret.size_of_vmglobal_import()), + size(imported_tags) + = cmul(ret.num_imported_tags, ret.size_of_vmtag_import()), size(defined_tables) = cmul(ret.num_defined_tables, ret.size_of_vmtable_definition()), size(defined_memories) @@ -518,6 +548,8 @@ impl From> for VMOffsets

{ align(16), size(defined_globals) = cmul(ret.num_defined_globals, ret.ptr.size_of_vmglobal_definition()), + size(defined_tags) + = cmul(ret.num_defined_tags, ret.ptr.size_of_vmtag_definition()), size(defined_func_refs) = cmul( ret.num_escaped_funcs, ret.ptr.size_of_vm_func_ref(), @@ -656,6 +688,27 @@ impl VMOffsets

{ } } +/// Offsets for `VMTagImport`. +impl VMOffsets

{ + /// The offset of the `from` field. + #[inline] + pub fn vmtag_import_from(&self) -> u8 { + 0 * self.pointer_size() + } + + /// The offset of the `vmctx` field. + #[inline] + pub fn vmtag_import_vmctx(&self) -> u8 { + 1 * self.pointer_size() + } + + /// Return the size of `VMTagImport`. + #[inline] + pub fn size_of_vmtag_import(&self) -> u8 { + 2 * self.pointer_size() + } +} + /// Offsets for `VMSharedTypeIndex`. impl VMOffsets

{ /// Return the size of `VMSharedTypeIndex`. @@ -691,6 +744,12 @@ impl VMOffsets

{ self.imported_globals } + /// The offset of the `tags` array. + #[inline] + pub fn vmctx_imported_tags_begin(&self) -> u32 { + self.imported_tags + } + /// The offset of the `tables` array. #[inline] pub fn vmctx_tables_begin(&self) -> u32 { @@ -715,6 +774,12 @@ impl VMOffsets

{ self.defined_globals } + /// The offset of the `tags` array. + #[inline] + pub fn vmctx_tags_begin(&self) -> u32 { + self.defined_tags + } + /// The offset of the `func_refs` array. #[inline] pub fn vmctx_func_refs_begin(&self) -> u32 { @@ -773,6 +838,13 @@ impl VMOffsets

{ + index.as_u32() * u32::from(self.size_of_vmglobal_import()) } + /// Return the offset to `VMTagImport` index `index`. + #[inline] + pub fn vmctx_vmtag_import(&self, index: TagIndex) -> u32 { + assert!(index.as_u32() < self.num_imported_tags); + self.vmctx_imported_tags_begin() + index.as_u32() * u32::from(self.size_of_vmtag_import()) + } + /// Return the offset to `VMTableDefinition` index `index`. #[inline] pub fn vmctx_vmtable_definition(&self, index: DefinedTableIndex) -> u32 { @@ -804,6 +876,13 @@ impl VMOffsets

{ + index.as_u32() * u32::from(self.ptr.size_of_vmglobal_definition()) } + /// Return the offset to the `VMTagDefinition` index `index`. + #[inline] + pub fn vmctx_vmtag_definition(&self, index: DefinedTagIndex) -> u32 { + assert!(index.as_u32() < self.num_defined_tags); + self.vmctx_tags_begin() + index.as_u32() * u32::from(self.ptr.size_of_vmtag_definition()) + } + /// Return the offset to the `VMFuncRef` for the given function /// index (either imported or defined). #[inline] @@ -837,6 +916,12 @@ impl VMOffsets

{ self.vmctx_vmtable_import(index) + u32::from(self.vmtable_import_from()) } + /// Return the offset to the `from` field in `VMTagImport` index `index`. + #[inline] + pub fn vmctx_vmtag_import_from(&self, index: TagIndex) -> u32 { + self.vmctx_vmtag_import(index) + u32::from(self.vmtag_import_from()) + } + /// Return the offset to the `base` field in `VMTableDefinition` index `index`. #[inline] pub fn vmctx_vmtable_definition_base(&self, index: DefinedTableIndex) -> u32 { diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index f1cf80f19ca3..98c73e9d0081 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -1307,6 +1307,11 @@ entity_impl!(ElemIndex); pub struct TagIndex(u32); entity_impl!(TagIndex); +/// Index type of a defined tag inside the WebAssembly module. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)] +pub struct DefinedTagIndex(u32); +entity_impl!(DefinedTagIndex); + /// Index into the global list of modules found within an entire component. /// /// Module translations are saved on the side to get fully compiled after @@ -1326,6 +1331,8 @@ pub enum EntityIndex { Memory(MemoryIndex), /// Global index. Global(GlobalIndex), + /// Tag index. + Tag(TagIndex), } impl From for EntityIndex { @@ -1352,6 +1359,12 @@ impl From for EntityIndex { } } +impl From for EntityIndex { + fn from(idx: TagIndex) -> EntityIndex { + EntityIndex::Tag(idx) + } +} + /// A type of an item in a wasm module where an item is typically something that /// can be exported. #[allow(missing_docs)] @@ -1379,7 +1392,8 @@ impl TypeTrace for EntityType { Self::Global(g) => g.trace(func), Self::Table(t) => t.trace(func), Self::Function(idx) => func(*idx), - Self::Memory(_) | Self::Tag(_) => Ok(()), + Self::Tag(t) => t.trace(func), + Self::Memory(_) => Ok(()), } } @@ -1391,7 +1405,8 @@ impl TypeTrace for EntityType { Self::Global(g) => g.trace_mut(func), Self::Table(t) => t.trace_mut(func), Self::Function(idx) => func(idx), - Self::Memory(_) | Self::Tag(_) => Ok(()), + Self::Tag(t) => t.trace_mut(func), + Self::Memory(_) => Ok(()), } } } @@ -1414,9 +1429,9 @@ impl EntityType { } /// Assert that this entity is a tag - pub fn unwrap_tag(&self) -> &Tag { + pub fn unwrap_tag(&self) -> Tag { match self { - EntityType::Tag(g) => g, + EntityType::Tag(g) => *g, _ => panic!("not a tag"), } } @@ -1786,14 +1801,43 @@ impl From for Memory { pub struct Tag { /// The event signature type. pub ty: TypeIndex, + /// The tag signature type. + pub signature: EngineOrModuleTypeIndex, +} + +impl Tag { + pub fn new(idx: TypeIndex, signature: EngineOrModuleTypeIndex) -> Self { + Self { ty: idx, signature } + } + + pub fn partial(idx: u32) -> Self { + Tag::new( + TypeIndex::from_u32(idx), + EngineOrModuleTypeIndex::Module(ModuleInternedTypeIndex(u32::MAX - 1)), + ) + } +} + +impl TypeTrace for Tag { + fn trace(&self, func: &mut F) -> Result<(), E> + where + F: FnMut(EngineOrModuleTypeIndex) -> Result<(), E>, + { + func(self.signature) + } + + fn trace_mut(&mut self, func: &mut F) -> Result<(), E> + where + F: FnMut(&mut EngineOrModuleTypeIndex) -> Result<(), E>, + { + func(&mut self.signature) + } } impl From for Tag { fn from(ty: wasmparser::TagType) -> Tag { match ty.kind { - wasmparser::TagKind::Exception => Tag { - ty: TypeIndex::from_u32(ty.func_type_idx), - }, + wasmparser::TagKind::Exception => Tag::partial(ty.func_type_idx), } } } diff --git a/crates/wasmtime/src/runtime/externals.rs b/crates/wasmtime/src/runtime/externals.rs index d457a13a568b..933b0c9c2406 100644 --- a/crates/wasmtime/src/runtime/externals.rs +++ b/crates/wasmtime/src/runtime/externals.rs @@ -3,9 +3,11 @@ use crate::{AsContext, Engine, ExternType, Func, Memory, SharedMemory}; mod global; mod table; +mod tag; pub use global::Global; pub use table::Table; +pub use tag::Tag; // Externals @@ -30,6 +32,8 @@ pub enum Extern { /// A WebAssembly shared memory; these are handled separately from /// [`Memory`]. SharedMemory(SharedMemory), + /// A WebAssembly `tag`. + Tag(Tag), } impl Extern { @@ -100,6 +104,7 @@ impl Extern { Extern::SharedMemory(ft) => ExternType::Memory(ft.ty()), Extern::Table(tt) => ExternType::Table(tt.ty(store)), Extern::Global(gt) => ExternType::Global(gt.ty(store)), + Extern::Tag(tt) => ExternType::Tag(tt.ty(store)), } } @@ -124,6 +129,7 @@ impl Extern { crate::runtime::vm::Export::Table(t) => { Extern::Table(Table::from_wasmtime_table(t, store)) } + crate::runtime::vm::Export::Tag(t) => Extern::Tag(Tag::from_wasmtime_tag(t, store)), } } @@ -134,6 +140,7 @@ impl Extern { Extern::Memory(m) => m.comes_from_same_store(store), Extern::SharedMemory(m) => Engine::same(m.engine(), store.engine()), Extern::Table(t) => store.store_data().contains(t.0), + Extern::Tag(t) => store.store_data().contains(t.0), } } } @@ -168,6 +175,12 @@ impl From for Extern { } } +impl From for Extern { + fn from(r: Tag) -> Self { + Extern::Tag(r) + } +} + // Exports /// An exported WebAssembly value. diff --git a/crates/wasmtime/src/runtime/externals/tag.rs b/crates/wasmtime/src/runtime/externals/tag.rs new file mode 100644 index 000000000000..4623d0217217 --- /dev/null +++ b/crates/wasmtime/src/runtime/externals/tag.rs @@ -0,0 +1,44 @@ +use crate::runtime::types::TagType; +use crate::{ + store::{StoreData, StoreOpaque, Stored}, + AsContext, +}; + +/// A WebAssembly `tag`. +#[derive(Copy, Clone, Debug)] +#[repr(transparent)] // here for the C API +pub struct Tag(pub(super) Stored); + +impl Tag { + pub(crate) unsafe fn from_wasmtime_tag( + mut wasmtime_export: crate::runtime::vm::ExportTag, + store: &mut StoreOpaque, + ) -> Self { + use wasmtime_environ::TypeTrace; + wasmtime_export + .tag + .canonicalize_for_runtime_usage(&mut |module_index| { + crate::runtime::vm::Instance::from_vmctx(wasmtime_export.vmctx, |instance| { + instance.engine_type_index(module_index) + }) + }); + + Tag(store.store_data_mut().insert(wasmtime_export)) + } + + pub(crate) fn ty(&self, _store: impl AsContext) -> TagType { + todo!() + } + + pub(crate) fn wasmtime_ty<'a>(&self, data: &'a StoreData) -> &'a wasmtime_environ::Tag { + &data[self.0].tag + } + + pub(crate) fn vmimport(&self, store: &StoreOpaque) -> crate::runtime::vm::VMTagImport { + let export = &store[self.0]; + crate::runtime::vm::VMTagImport { + from: export.definition, + vmctx: export.vmctx, + } + } +} diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs index a36a2fd0c2c3..53a733be8ce6 100644 --- a/crates/wasmtime/src/runtime/instance.rs +++ b/crates/wasmtime/src/runtime/instance.rs @@ -2,7 +2,7 @@ use crate::linker::{Definition, DefinitionType}; use crate::prelude::*; use crate::runtime::vm::{ Imports, InstanceAllocationRequest, ModuleRuntimeInfo, StorePtr, VMFuncRef, VMFunctionImport, - VMGlobalImport, VMMemoryImport, VMOpaqueContext, VMTableImport, + VMGlobalImport, VMMemoryImport, VMOpaqueContext, VMTableImport, VMTagImport, }; use crate::store::{InstanceId, StoreOpaque, Stored}; use crate::types::matching; @@ -14,7 +14,8 @@ use alloc::sync::Arc; use core::ptr::NonNull; use wasmparser::WasmFeatures; use wasmtime_environ::{ - EntityIndex, EntityType, FuncIndex, GlobalIndex, MemoryIndex, PrimaryMap, TableIndex, TypeTrace, + EntityIndex, EntityType, FuncIndex, GlobalIndex, MemoryIndex, PrimaryMap, TableIndex, TagIndex, + TypeTrace, }; /// An instantiated WebAssembly module. @@ -651,6 +652,7 @@ pub(crate) struct OwnedImports { tables: PrimaryMap, memories: PrimaryMap, globals: PrimaryMap, + tags: PrimaryMap, } impl OwnedImports { @@ -666,6 +668,7 @@ impl OwnedImports { tables: PrimaryMap::new(), memories: PrimaryMap::new(), globals: PrimaryMap::new(), + tags: PrimaryMap::new(), } } @@ -675,6 +678,7 @@ impl OwnedImports { self.tables.reserve(raw.num_imported_tables); self.memories.reserve(raw.num_imported_memories); self.globals.reserve(raw.num_imported_globals); + self.tags.reserve(raw.num_imported_tags); } #[cfg(feature = "component-model")] @@ -683,6 +687,7 @@ impl OwnedImports { self.tables.clear(); self.memories.clear(); self.globals.clear(); + self.tags.clear(); } fn push(&mut self, item: &Extern, store: &mut StoreOpaque, module: &Module) { @@ -702,6 +707,9 @@ impl OwnedImports { Extern::SharedMemory(i) => { self.memories.push(i.vmimport(store)); } + Extern::Tag(i) => { + self.tags.push(i.vmimport(store)); + } } } @@ -734,6 +742,12 @@ impl OwnedImports { index: m.index, }); } + crate::runtime::vm::Export::Tag(t) => { + self.tags.push(VMTagImport { + from: t.definition, + vmctx: t.vmctx, + }); + } } } @@ -743,6 +757,7 @@ impl OwnedImports { globals: self.globals.values().as_slice(), memories: self.memories.values().as_slice(), functions: self.functions.values().as_slice(), + tags: self.tags.values().as_slice(), } } } diff --git a/crates/wasmtime/src/runtime/linker.rs b/crates/wasmtime/src/runtime/linker.rs index c0a8517a704b..316613270b16 100644 --- a/crates/wasmtime/src/runtime/linker.rs +++ b/crates/wasmtime/src/runtime/linker.rs @@ -131,6 +131,7 @@ pub(crate) enum DefinitionType { // no longer be the current size of the table/memory. Table(wasmtime_environ::Table, u32), Memory(wasmtime_environ::Memory, u64), + Tag(wasmtime_environ::Tag), } impl Linker { @@ -1391,6 +1392,7 @@ impl DefinitionType { DefinitionType::Memory(*t.wasmtime_ty(data), t.internal_size(store)) } Extern::SharedMemory(t) => DefinitionType::Memory(*t.ty().wasmtime_memory(), t.size()), + Extern::Tag(t) => DefinitionType::Tag(*t.wasmtime_ty(data)), } } @@ -1400,6 +1402,7 @@ impl DefinitionType { DefinitionType::Table(..) => "table", DefinitionType::Memory(..) => "memory", DefinitionType::Global(_) => "global", + DefinitionType::Tag(_) => "tag", } } } diff --git a/crates/wasmtime/src/runtime/store/data.rs b/crates/wasmtime/src/runtime/store/data.rs index 4a119bcd80db..0acf18b7341d 100644 --- a/crates/wasmtime/src/runtime/store/data.rs +++ b/crates/wasmtime/src/runtime/store/data.rs @@ -27,6 +27,7 @@ pub struct StoreData { globals: Vec, instances: Vec, memories: Vec, + tags: Vec, #[cfg(feature = "component-model")] pub(crate) components: crate::component::ComponentStoreData, } @@ -53,6 +54,7 @@ impl_store_data! { globals => crate::runtime::vm::ExportGlobal, instances => crate::instance::InstanceData, memories => crate::runtime::vm::ExportMemory, + tags => crate::runtime::vm::ExportTag, } impl StoreData { @@ -64,6 +66,7 @@ impl StoreData { globals: Vec::new(), instances: Vec::new(), memories: Vec::new(), + tags: Vec::new(), #[cfg(feature = "component-model")] components: Default::default(), } diff --git a/crates/wasmtime/src/runtime/types.rs b/crates/wasmtime/src/runtime/types.rs index 7de2c44b2736..a4412bbdd0fb 100644 --- a/crates/wasmtime/src/runtime/types.rs +++ b/crates/wasmtime/src/runtime/types.rs @@ -1306,6 +1306,8 @@ pub enum ExternType { Table(TableType), /// This external type is the type of a WebAssembly memory. Memory(MemoryType), + /// This external type is the type of a WebAssembly tag. + Tag(TagType), } macro_rules! extern_type_accessors { @@ -1338,6 +1340,7 @@ impl ExternType { (Global(GlobalType) global unwrap_global) (Table(TableType) table unwrap_table) (Memory(MemoryType) memory unwrap_memory) + (Tag(TagType) tag unwrap_tag) } pub(crate) fn from_wasmtime( @@ -1365,7 +1368,26 @@ impl ExternType { EntityType::Global(ty) => GlobalType::from_wasmtime_global(engine, ty).into(), EntityType::Memory(ty) => MemoryType::from_wasmtime_memory(ty).into(), EntityType::Table(ty) => TableType::from_wasmtime_table(engine, ty).into(), - EntityType::Tag(_) => unimplemented!("wasm tag support"), + EntityType::Tag(_idx) => { + todo!() + // let ty = match idx { + // EngineOrModuleTypeIndex::Engine(e) => { + // FuncType::from_shared_type_index(engine, *e).into() + // } + // EngineOrModuleTypeIndex::Module(m) => { + // let subty = &types[*m]; + // FuncType::from_wasm_func_type( + // engine, + // subty.is_final, + // subty.supertype, + // subty.unwrap_func().clone(), + // ) + // .into() + // } + // EngineOrModuleTypeIndex::RecGroup(_) => unreachable!(), + // }; + // TagType::from_wasmtime_tag(engine, ty).into() + } } } } @@ -1394,6 +1416,12 @@ impl From for ExternType { } } +impl From for ExternType { + fn from(ty: TagType) -> ExternType { + ExternType::Tag(ty) + } +} + /// The storage type of a `struct` field or `array` element. /// /// This is either a packed 8- or -16 bit integer, or else it is some unpacked @@ -2982,6 +3010,13 @@ impl MemoryType { } } +// Tag types +/// A descriptor for a tag in a WebAssembly module. +#[derive(Debug, Clone, Hash)] +pub struct TagType { + ty: WasmFuncType, +} + // Import Types /// A descriptor for an imported value into a wasm module. diff --git a/crates/wasmtime/src/runtime/types/matching.rs b/crates/wasmtime/src/runtime/types/matching.rs index 340ff49f9c61..d821e647c9e0 100644 --- a/crates/wasmtime/src/runtime/types/matching.rs +++ b/crates/wasmtime/src/runtime/types/matching.rs @@ -79,7 +79,13 @@ impl MatchCx<'_> { } _ => bail!("expected func, but found {}", actual.desc()), }, - EntityType::Tag(_) => unimplemented!(), + EntityType::Tag(expected) => match actual { + DefinitionType::Tag(actual) => self.type_reference( + expected.signature.unwrap_engine_type_index(), + actual.signature.unwrap_engine_type_index(), + ), + _ => bail!("expected tag, but found {}", actual.desc()), + }, } } } diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 52ec9a443212..dbe248079142 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -74,7 +74,7 @@ pub use crate::runtime::vm::traphandlers::*; pub use crate::runtime::vm::vmcontext::{ VMArrayCallFunction, VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionBody, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport, - VMOpaqueContext, VMRuntimeLimits, VMTableImport, VMWasmCallFunction, ValRaw, + VMOpaqueContext, VMRuntimeLimits, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw, }; pub use send_sync_ptr::SendSyncPtr; diff --git a/crates/wasmtime/src/runtime/vm/continuation.rs b/crates/wasmtime/src/runtime/vm/continuation.rs index eb565713dd2d..638b45d30289 100644 --- a/crates/wasmtime/src/runtime/vm/continuation.rs +++ b/crates/wasmtime/src/runtime/vm/continuation.rs @@ -80,8 +80,8 @@ pub mod optimized { }; use core::cmp; use core::mem; - use wasmtime_continuations::{debug_println, ENABLE_DEBUG_PRINTING}; - pub use wasmtime_continuations::{Payloads, StackLimits, State, SwitchDirection}; + use wasmtime_continuations::{debug_println, ControlEffect, ENABLE_DEBUG_PRINTING}; + pub use wasmtime_continuations::{Payloads, StackLimits, State}; use wasmtime_environ::prelude::*; /// Fibers used for continuations @@ -243,7 +243,7 @@ pub mod optimized { instance: &mut Instance, contref: *mut VMContRef, parent_stack_limits: *mut StackLimits, - ) -> Result { + ) -> Result { let cont = unsafe { contref.as_ref().ok_or_else(|| { TrapReason::user_without_backtrace(anyhow::anyhow!( @@ -300,7 +300,7 @@ pub mod optimized { /// TODO #[inline(always)] - pub fn suspend(instance: &mut Instance, tag_index: u32) -> Result<(), TrapReason> { + pub fn suspend(instance: &mut Instance, tag_addr: *mut u8) -> Result<(), TrapReason> { let chain_ptr = instance.typed_continuations_stack_chain(); // TODO(dhil): This should be handled in generated code. @@ -335,7 +335,7 @@ pub mod optimized { ); let suspend = crate::runtime::vm::fibre::unix::Suspend::from_top_ptr(stack_ptr); - let payload = SwitchDirection::suspend(tag_index); + let payload = ControlEffect::suspend(tag_addr as *const u8); Ok(suspend.switch(payload)) } diff --git a/crates/wasmtime/src/runtime/vm/export.rs b/crates/wasmtime/src/runtime/vm/export.rs index aebe55b49ca4..2a0edec49ae8 100644 --- a/crates/wasmtime/src/runtime/vm/export.rs +++ b/crates/wasmtime/src/runtime/vm/export.rs @@ -1,8 +1,9 @@ use crate::runtime::vm::vmcontext::{ VMContext, VMFuncRef, VMGlobalDefinition, VMMemoryDefinition, VMTableDefinition, + VMTagDefinition, }; use core::ptr::NonNull; -use wasmtime_environ::{DefinedMemoryIndex, Global, MemoryPlan, TablePlan}; +use wasmtime_environ::{DefinedMemoryIndex, Global, MemoryPlan, TablePlan, Tag}; /// The value of an export passed from one instance to another. pub enum Export { @@ -17,6 +18,9 @@ pub enum Export { /// A global export value. Global(ExportGlobal), + + /// A tag export value. + Tag(ExportTag), } /// A function export value. @@ -106,3 +110,24 @@ impl From for Export { Export::Global(func) } } + +#[derive(Debug, Clone)] +pub struct ExportTag { + /// The address of the tag. + pub definition: *mut VMTagDefinition, + /// Pointer to the containing `VMContext`. May be null for + /// host-created tags. + pub vmctx: *mut VMContext, + /// The tag declaration, used for compatibility checking. + pub tag: Tag, +} + +// See docs on send/sync for `ExportFunction` above. +unsafe impl Send for ExportTag {} +unsafe impl Sync for ExportTag {} + +impl From for Export { + fn from(e: ExportTag) -> Export { + Export::Tag(e) + } +} diff --git a/crates/wasmtime/src/runtime/vm/fibre/mod.rs b/crates/wasmtime/src/runtime/vm/fibre/mod.rs index 55e5f934e1bc..7486cb06b843 100644 --- a/crates/wasmtime/src/runtime/vm/fibre/mod.rs +++ b/crates/wasmtime/src/runtime/vm/fibre/mod.rs @@ -9,7 +9,7 @@ cfg_if::cfg_if! { use std::cell::Cell; use std::io; use std::ops::Range; - use wasmtime_continuations::{SwitchDirection, SwitchDirectionEnum}; + use wasmtime_continuations::ControlEffect; use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; @@ -111,16 +111,12 @@ cfg_if::cfg_if! { /// /// Note that if the fiber itself panics during execution then the panic /// will be propagated to this caller. - pub fn resume(&self) -> SwitchDirection { + pub fn resume(&self) -> ControlEffect { assert!(!self.done.replace(true), "cannot resume a finished fiber"); let reason = self.inner.resume(&self.stack.0); - if let SwitchDirection { - discriminant: SwitchDirectionEnum::Suspend, - data: _, - } = reason - { + if ControlEffect::is_suspend(reason) { self.done.set(false) - }; + } reason } diff --git a/crates/wasmtime/src/runtime/vm/fibre/unix.rs b/crates/wasmtime/src/runtime/vm/fibre/unix.rs index 325215e4617e..b9b9bc1707e3 100644 --- a/crates/wasmtime/src/runtime/vm/fibre/unix.rs +++ b/crates/wasmtime/src/runtime/vm/fibre/unix.rs @@ -104,7 +104,7 @@ use std::alloc::{alloc, dealloc, Layout}; use std::io; use std::ops::Range; use std::ptr; -use wasmtime_continuations::SwitchDirection; +use wasmtime_continuations::ControlEffect; use crate::runtime::vm::{VMContext, VMFuncRef, VMOpaqueContext, ValRaw}; @@ -263,7 +263,7 @@ extern "C" fn fiber_start( // Switch back to parent, indicating that the continuation returned. let inner = Suspend(top_of_stack); - let reason = SwitchDirection::return_(); + let reason = ControlEffect::return_(); inner.switch(reason); } } @@ -290,16 +290,16 @@ impl Fiber { Ok(Self) } - pub(crate) fn resume(&self, stack: &FiberStack) -> SwitchDirection { + pub(crate) fn resume(&self, stack: &FiberStack) -> ControlEffect { unsafe { - let reason = SwitchDirection::resume().into(); - SwitchDirection::from(wasmtime_fibre_switch(stack.top, reason)) + let reason = ControlEffect::resume().into(); + ControlEffect::from(wasmtime_fibre_switch(stack.top, reason)) } } } impl Suspend { - pub fn switch(&self, payload: SwitchDirection) { + pub fn switch(&self, payload: ControlEffect) { unsafe { let arg = payload.into(); wasmtime_fibre_switch(self.0, arg); diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs index abc1a799bf4f..2d3c729d8d3c 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs @@ -1094,10 +1094,12 @@ mod tests { num_imported_tables: 0, num_imported_memories: 0, num_imported_globals: 0, + num_imported_tags: 0, num_defined_tables: 0, num_defined_memories: 0, num_owned_memories: 0, num_defined_globals: 0, + num_defined_tags: 0, num_escaped_funcs: 0, }); @@ -1122,10 +1124,12 @@ mod tests { num_imported_tables: 0, num_imported_memories: 0, num_imported_globals: 0, + num_imported_tags: 0, num_defined_tables: 0, num_defined_memories: 0, num_owned_memories: 0, num_defined_globals: 0, + num_defined_tags: 0, num_escaped_funcs: 0, }); assert_eq!( @@ -1149,10 +1153,12 @@ mod tests { num_imported_tables: 0, num_imported_memories: 0, num_imported_globals: 0, + num_imported_tags: 0, num_defined_tables: 0, num_defined_memories: 0, num_owned_memories: 0, num_defined_globals: 0, + num_defined_tags: 0, num_escaped_funcs: 0, }); assert_eq!( diff --git a/crates/wasmtime/src/runtime/vm/imports.rs b/crates/wasmtime/src/runtime/vm/imports.rs index a1b47e17d9f4..61082b673b58 100644 --- a/crates/wasmtime/src/runtime/vm/imports.rs +++ b/crates/wasmtime/src/runtime/vm/imports.rs @@ -1,5 +1,5 @@ use crate::runtime::vm::vmcontext::{ - VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport, + VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport, VMTagImport, }; /// Resolved import pointers. @@ -26,4 +26,7 @@ pub struct Imports<'a> { /// Resolved addresses for imported globals. pub globals: &'a [VMGlobalImport], + + /// Resolved addresses for imported tags. + pub tags: &'a [VMTagImport], } diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs index c64a012b0a95..e75b4c7e8489 100644 --- a/crates/wasmtime/src/runtime/vm/instance.rs +++ b/crates/wasmtime/src/runtime/vm/instance.rs @@ -11,11 +11,11 @@ use crate::runtime::vm::table::{Table, TableElement, TableElementType}; use crate::runtime::vm::vmcontext::{ VMBuiltinFunctionsArray, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMRuntimeLimits, - VMTableDefinition, VMTableImport, + VMTableDefinition, VMTableImport, VMTagDefinition, VMTagImport, }; use crate::runtime::vm::{ - ExportFunction, ExportGlobal, ExportMemory, ExportTable, GcStore, Imports, ModuleRuntimeInfo, - SendSyncPtr, Store, VMFunctionBody, VMGcRef, WasmFault, + ExportFunction, ExportGlobal, ExportMemory, ExportTable, ExportTag, GcStore, Imports, + ModuleRuntimeInfo, SendSyncPtr, Store, VMFunctionBody, VMGcRef, WasmFault, }; use alloc::sync::Arc; use core::alloc::Layout; @@ -27,10 +27,10 @@ use core::{mem, ptr}; use sptr::Strict; use wasmtime_environ::{ packed_option::ReservedValue, DataIndex, DefinedGlobalIndex, DefinedMemoryIndex, - DefinedTableIndex, ElemIndex, EntityIndex, EntityRef, EntitySet, FuncIndex, GlobalIndex, - HostPtr, MemoryIndex, MemoryPlan, Module, ModuleInternedTypeIndex, PrimaryMap, PtrSize, - TableIndex, TableInitialValue, TableSegmentElements, Trap, VMOffsets, VMSharedTypeIndex, - WasmHeapTopType, VMCONTEXT_MAGIC, + DefinedTableIndex, DefinedTagIndex, ElemIndex, EntityIndex, EntityRef, EntitySet, FuncIndex, + GlobalIndex, HostPtr, MemoryIndex, MemoryPlan, Module, ModuleInternedTypeIndex, PrimaryMap, + PtrSize, TableIndex, TableInitialValue, TableSegmentElements, TagIndex, Trap, VMOffsets, + VMSharedTypeIndex, WasmHeapTopType, VMCONTEXT_MAGIC, }; #[cfg(feature = "wmemcheck")] use wasmtime_wmemcheck::Wmemcheck; @@ -301,6 +301,11 @@ impl Instance { unsafe { &*self.vmctx_plus_offset(self.offsets().vmctx_vmglobal_import(index)) } } + /// Return the indexed `VMTagImport`. + fn imported_tag(&self, index: TagIndex) -> &VMTagImport { + unsafe { &*self.vmctx_plus_offset(self.offsets().vmctx_vmtag_import(index)) } + } + /// Return the indexed `VMTableDefinition`. #[allow(dead_code)] fn table(&mut self, index: DefinedTableIndex) -> VMTableDefinition { @@ -424,6 +429,11 @@ impl Instance { }) } + /// Return the indexed `VMTagDefinition`. + fn tag_ptr(&mut self, index: DefinedTagIndex) -> *mut VMTagDefinition { + unsafe { self.vmctx_plus_offset_mut(self.offsets().vmctx_vmtag_definition(index)) } + } + /// Return a pointer to the interrupts structure #[inline] pub fn runtime_limits(&mut self) -> *mut *const VMRuntimeLimits { @@ -561,6 +571,18 @@ impl Instance { } } + fn get_exported_tag(&mut self, index: TagIndex) -> ExportTag { + ExportTag { + definition: if let Some(def_index) = self.module().defined_tag_index(index) { + self.tag_ptr(def_index) + } else { + self.imported_tag(index).from + }, + vmctx: self.vmctx(), + tag: self.module().tags[index], + } + } + fn get_exported_memory(&mut self, index: MemoryIndex) -> ExportMemory { let (definition, vmctx, def_index) = if let Some(def_index) = self.module().defined_memory_index(index) { @@ -1239,6 +1261,12 @@ impl Instance { self.vmctx_plus_offset_mut(offsets.vmctx_imported_globals_begin()), imports.globals.len(), ); + debug_assert_eq!(imports.tags.len(), module.num_imported_tags); + ptr::copy_nonoverlapping( + imports.tags.as_ptr(), + self.vmctx_plus_offset_mut(offsets.vmctx_imported_tags_begin()), + imports.tags.len(), + ); // N.B.: there is no need to initialize the funcrefs array because we // eagerly construct each element in it whenever asked for a reference @@ -1280,6 +1308,20 @@ impl Instance { // Initialize the defined globals let mut const_evaluator = ConstExprEvaluator::default(); self.initialize_vmctx_globals(&mut const_evaluator, module); + + // Initialize the defined tags + for index in 0..module.tags.len() - module.num_imported_tags { + let defined_index = DefinedTagIndex::new(index); + let tag_index = module.tag_index(defined_index); + let tag = module.tags[tag_index]; + let to = self.tag_ptr(defined_index); + ptr::write( + to, + VMTagDefinition::new( + self.engine_type_index(tag.signature.unwrap_module_type_index()), + ), + ); + } } unsafe fn initialize_vmctx_globals( @@ -1413,6 +1455,11 @@ impl InstanceHandle { self.instance_mut().get_exported_table(export) } + /// Lookup a tag by index. + pub fn get_exported_tag(&mut self, export: TagIndex) -> ExportTag { + self.instance_mut().get_exported_tag(export) + } + /// Lookup an item with the given index. pub fn get_export_by_index(&mut self, export: EntityIndex) -> Export { match export { @@ -1420,6 +1467,7 @@ impl InstanceHandle { EntityIndex::Global(i) => Export::Global(self.get_exported_global(i)), EntityIndex::Table(i) => Export::Table(self.get_exported_table(i)), EntityIndex::Memory(i) => Export::Memory(self.get_exported_memory(i)), + EntityIndex::Tag(i) => Export::Tag(self.get_exported_tag(i)), } } diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index a396d11859f1..472201ebaeb5 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -935,7 +935,7 @@ fn tc_resume( instance: &mut Instance, contref: *mut u8, parent_stack_limits: *mut u8, -) -> Result { +) -> Result<*mut u8, TrapReason> { crate::vm::continuation::optimized::resume( instance, contref.cast::(), @@ -944,8 +944,8 @@ fn tc_resume( .map(|reason| reason.into()) } -fn tc_suspend(instance: &mut Instance, tag_index: u32) -> Result<(), TrapReason> { - crate::vm::continuation::optimized::suspend(instance, tag_index) +fn tc_suspend(instance: &mut Instance, tag_addr: *mut u8) -> Result<(), TrapReason> { + crate::vm::continuation::optimized::suspend(instance, tag_addr) } fn tc_cont_ref_forward_tag_return_values_buffer( diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index c065276148cc..7c255c8f1bfa 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -251,6 +251,44 @@ mod test_vmglobal_import { } } +/// The fields compiled code needs to access to utilize a WebAssembly +/// tag imported from another instance. +#[derive(Debug, Copy, Clone)] +#[repr(C)] +pub struct VMTagImport { + /// A pointer to the imported tag description. + pub from: *mut VMTagDefinition, + /// A pointer to the `VMContext` that owns the tag description. + pub vmctx: *mut VMContext, +} + +// Declare that this type is send/sync, it's the responsibility of users of +// `VMGlobalImport` to uphold this guarantee. +unsafe impl Send for VMTagImport {} +unsafe impl Sync for VMTagImport {} + +#[cfg(test)] +mod test_vmtag_import { + use super::VMTagImport; + use memoffset::offset_of; + use std::mem::size_of; + use wasmtime_environ::{Module, VMOffsets}; + + #[test] + fn check_vmtag_import_offsets() { + let module = Module::new(); + let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); + assert_eq!( + size_of::(), + usize::from(offsets.size_of_vmtag_import()) + ); + assert_eq!( + offset_of!(VMTagImport, from), + usize::from(offsets.vmtag_import_from()) + ); + } +} + /// The fields compiled code needs to access to utilize a WebAssembly linear /// memory defined within the instance, namely the start address and the /// size in bytes. @@ -623,6 +661,38 @@ mod test_vmshared_type_index { } } +/// A WebAssembly tag defined within the instance. +/// +#[derive(Debug)] +#[repr(C, align(8))] +pub struct VMTagDefinition { + /// Function signature's type id. + pub type_index: VMSharedTypeIndex, +} + +impl VMTagDefinition { + pub fn new(type_index: VMSharedTypeIndex) -> Self { + Self { type_index } + } +} + +#[cfg(test)] +mod test_vmtag_definition { + use super::VMTagDefinition; + use std::mem::size_of; + use wasmtime_environ::{Module, PtrSize, VMOffsets}; + + #[test] + fn check_vmtag_definition_offsets() { + let module = Module::new(); + let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmtag_definition()) + ); + } +} + /// The VM caller-checked "funcref" record, for caller-side signature checking. /// /// It consists of function pointer(s), a type id to be checked by the diff --git a/tests/all/typed_continuations.rs b/tests/all/typed_continuations.rs index d0f013271f7e..c20e92179413 100644 --- a/tests/all/typed_continuations.rs +++ b/tests/all/typed_continuations.rs @@ -279,10 +279,8 @@ mod wasi { } } -/// Test that we can handle a `suspend` from another instance. Note that this -/// test is working around the fact that wasmtime does not support exporting -/// tags at the moment. Thus, instead of sharing a tag between two modules, we -/// instantiate the same module twice to share a tag. +/// Test that two distinct instantiations of the same module yield +/// different control tag identities. #[test] fn inter_instance_suspend() -> Result<()> { let mut config = Config::default(); @@ -339,7 +337,8 @@ fn inter_instance_suspend() -> Result<()> { let other_inst1 = Instance::new(&mut store, &module_other, &[])?; let other_inst2 = Instance::new(&mut store, &module_other, &[])?; - // Crucially, suspend and resume are from two instances of the same module. + // Crucially, suspend and resume are from two separate instances + // of the same module. let suspend = other_inst1.get_func(&mut store, "suspend").unwrap(); let resume = other_inst2.get_func(&mut store, "resume").unwrap(); @@ -347,8 +346,8 @@ fn inter_instance_suspend() -> Result<()> { let main_instance = Instance::new(&mut store, &module_main, &[suspend.into(), resume.into()])?; let entry_func = main_instance.get_func(&mut store, "entry").unwrap(); - entry_func.call(&mut store, &[], &mut [])?; - + let result = entry_func.call(&mut store, &[], &mut []); + assert!(result.is_err()); Ok(()) } diff --git a/tests/misc_testsuite/typed-continuations/linking_tags.wast b/tests/misc_testsuite/typed-continuations/linking_tags.wast index 1f55cc7f5a90..28ecd5be251d 100644 --- a/tests/misc_testsuite/typed-continuations/linking_tags.wast +++ b/tests/misc_testsuite/typed-continuations/linking_tags.wast @@ -1,23 +1,169 @@ +(module $alien + (tag $alien_tag (export "alien_tag")) +) +(register "alien") + +(module $mine + (type $ft (func)) + (type $ct (cont $ft)) + (tag $alien_tag (import "alien" "alien_tag")) + (tag $my_tag) + (func $do_alien_tag + (suspend $alien_tag)) + + ;; Don't handle the imported alien. + (func (export "main-1") + (block $on_my_tag (result (ref $ct)) + (resume $ct (on $my_tag $on_my_tag) (cont.new $ct (ref.func $do_alien_tag))) + (unreachable) + ) + (unreachable)) + + ;; Handle the imported alien. + (func (export "main-2") + (block $on_alien_tag (result (ref $ct)) + (resume $ct (on $alien_tag $on_alien_tag) (cont.new $ct (ref.func $do_alien_tag))) + (unreachable) + ) + (drop)) + + (elem declare func $do_alien_tag) +) +(register "mine") +(assert_suspension (invoke "main-1") "unhandled") +(assert_return (invoke "main-2")) + (module $foo - (tag $foo (export "foo")) + (type $ft (func (result i32))) + (type $ct (cont $ft)) + (type $ft-2 (func (param i32) (result i32))) + (type $ct-2 (cont $ft-2)) + + (tag $foo (export "foo") (result i32)) ;; occupies defined tag entry 0 + + (func $do_foo (export "do_foo") (result i32) + (suspend $foo)) + (func $handle_foo (export "handle_foo") (param $f (ref $ft)) (result i32) + (block $on_foo (result (ref $ct-2)) + (resume $ct (on $foo $on_foo) (cont.new $ct (local.get $f))) + (return) + ) ;; on_foo + (drop) + (return (i32.const 1)) + ) + (func (export "test_foo") (result i32) + (call $handle_foo (ref.func $do_foo))) + (elem declare func $do_foo) ) (register "foo") +(assert_return (invoke "test_foo") (i32.const 1)) (module $bar - (type $ft (func)) + (type $ft (func (result i32))) (type $ct (cont $ft)) - (tag $foo (import "foo" "foo")) - (tag $bar) - (func $do_foo + + (type $ft-2 (func (param i32) (result i32))) + (type $ct-2 (cont $ft-2)) + + (tag $foo (import "foo" "foo") (result i32)) + (tag $bar (result i32)) + (func $do_foo (result i32) (suspend $foo)) - (func $main (export "main") - (block $on_bar (result (ref $ct)) - (resume $ct (tag $bar $on_bar) (cont.new $ct (ref.func $do_foo))) + ;; Don't handle the imported foo. + (func (export "skip-imported-foo") (result i32) + (block $on_bar (result (ref $ct-2)) + (resume $ct (on $bar $on_bar) (cont.new $ct (ref.func $do_foo))) (unreachable) ) (unreachable)) + + ;; Handle the imported foo. + (func (export "handle-imported-foo") (result i32) + (block $on_foo (result (ref $ct-2)) + (resume $ct (on $foo $on_foo) (cont.new $ct (ref.func $do_foo))) + (unreachable) + ) + (drop) + (return (i32.const 2)) + ) + (elem declare func $do_foo) ) (register "bar") -(assert_suspension (invoke "main") "unhandled") \ No newline at end of file +(assert_suspension (invoke "skip-imported-foo") "unhandled") +(assert_return (invoke "handle-imported-foo") (i32.const 2)) + + +(module $baz + (type $ft (func (result i32))) + (type $ct (cont $ft)) + + (type $ft-2 (func (param i32) (result i32))) + (type $ct-2 (cont $ft-2)) + + (func $handle_foo (import "foo" "handle_foo") (param (ref $ft)) (result i32)) + (func $do_foo (import "foo" "do_foo") (result i32)) + + (tag $baz (result i32)) ;; unused, but occupies defined tag entry 0 + + (func $handle_baz (param $f (ref $ft)) (result i32) + (block $on_baz (result (ref $ct-2)) + (resume $ct (on $baz $on_baz) (cont.new $ct (local.get $f))) + (return) + ) ;; on_baz + (drop) + (return (i32.const 3)) + ) + + (func $inner-baz (result i32) + (call $handle_baz (ref.func $do_foo))) + (func (export "compose-handle-foo-baz") (result i32) + (call $handle_foo (ref.func $inner-baz))) + + (func $inner-foo (result i32) + (call $handle_foo (ref.func $do_foo))) + (func (export "compose-handle-baz-foo") (result i32) + (call $handle_baz (ref.func $inner-foo))) + (elem declare func $do_foo $inner-baz $inner-foo) +) +(register "baz") +(assert_return (invoke "compose-handle-baz-foo") (i32.const 1)) +(assert_return (invoke "compose-handle-foo-baz") (i32.const 1)) + +(module $quux + (type $ft (func (result i32))) + (type $ct (cont $ft)) + + (type $ft-2 (func (param i32) (result i32))) + (type $ct-2 (cont $ft-2)) + + (func $handle_foo (import "foo" "handle_foo") (param (ref $ft)) (result i32)) + (tag $foo (import "foo" "foo") (result i32)) + + (func $do_foo (result i32) + (suspend $foo)) + + (func $my_handle_foo (param $f (ref $ft)) (result i32) + (block $on_foo (result (ref $ct-2)) + (resume $ct (on $foo $on_foo) (cont.new $ct (local.get $f))) + (return) + ) ;; on_foo + (drop) + (return (i32.const 4)) + ) + + (func $inner-my-foo (result i32) + (call $my_handle_foo (ref.func $do_foo))) + (func (export "compose-handle-foo-my-foo") (result i32) + (call $handle_foo (ref.func $inner-my-foo))) + + (func $inner-foo (result i32) + (call $handle_foo (ref.func $do_foo))) + (func (export "compose-handle-my-foo-foo") (result i32) + (call $my_handle_foo (ref.func $inner-foo))) + (elem declare func $do_foo $inner-my-foo $inner-foo) +) +(register "quux") +(assert_return (invoke "compose-handle-foo-my-foo") (i32.const 4)) +(assert_return (invoke "compose-handle-my-foo-foo") (i32.const 1)) \ No newline at end of file diff --git a/tests/wast.rs b/tests/wast.rs index 384dec76612e..7846f28971a4 100644 --- a/tests/wast.rs +++ b/tests/wast.rs @@ -193,20 +193,12 @@ fn ignore(test: &Path, strategy: Strategy) -> bool { .iter() .any(|i| test.ends_with(i)); } - - if part == "typed-continuations" { - // TODO(dhil): Tag linking is currently broken - if test.ends_with("linking_tags.wast") { - return true; - } - - // This test specifically checks that we catch a continuation being - // resumed twice, which we cannot detect in this mode. - if cfg!(feature = "unsafe_disable_continuation_linearity_check") - && test.ends_with("cont_twice.wast") - { - return true; - } + // This test specifically checks that we catch a continuation being + // resumed twice, which we cannot detect in this mode. + if cfg!(feature = "unsafe_disable_continuation_linearity_check") + && test.ends_with("cont_twice.wast") + { + return true; } } From a55095d146be31a4bfdede53ad9eabe357ac554f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hillerstr=C3=B6m?= Date: Wed, 21 Aug 2024 09:44:54 +0200 Subject: [PATCH 2/8] Fix C API --- crates/c-api/src/extern.rs | 2 ++ crates/c-api/src/types/extern.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/crates/c-api/src/extern.rs b/crates/c-api/src/extern.rs index 77aba7818d5c..74d14f1f010c 100644 --- a/crates/c-api/src/extern.rs +++ b/crates/c-api/src/extern.rs @@ -21,6 +21,7 @@ pub extern "C" fn wasm_extern_kind(e: &wasm_extern_t) -> wasm_externkind_t { Extern::Table(_) => crate::WASM_EXTERN_TABLE, Extern::Memory(_) => crate::WASM_EXTERN_MEMORY, Extern::SharedMemory(_) => todo!(), + Extern::Tag(_) => todo!(), } } @@ -141,6 +142,7 @@ impl From for wasmtime_extern_t { sharedmemory: ManuallyDrop::new(Box::new(sharedmemory)), }, }, + Extern::Tag(_) => todo!(), } } } diff --git a/crates/c-api/src/types/extern.rs b/crates/c-api/src/types/extern.rs index c85dcbb25cfd..7b6b182e76ef 100644 --- a/crates/c-api/src/types/extern.rs +++ b/crates/c-api/src/types/extern.rs @@ -25,6 +25,7 @@ impl CExternType { ExternType::Global(f) => CExternType::Global(CGlobalType::new(f)), ExternType::Memory(f) => CExternType::Memory(CMemoryType::new(f)), ExternType::Table(f) => CExternType::Table(CTableType::new(f)), + ExternType::Tag(_) => todo!(), } } } From 756d5f35d4bb16f54a4ad9075a2fa847f4150991 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hillerstr=C3=B6m?= Date: Wed, 21 Aug 2024 09:53:14 +0200 Subject: [PATCH 3/8] Fix failing test --- crates/wasmtime/src/runtime/vm/vmcontext.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 7c255c8f1bfa..b6deeab7a195 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -270,8 +270,7 @@ unsafe impl Sync for VMTagImport {} #[cfg(test)] mod test_vmtag_import { use super::VMTagImport; - use memoffset::offset_of; - use std::mem::size_of; + use core::mem::{offset_of, size_of}; use wasmtime_environ::{Module, VMOffsets}; #[test] From e7d8d3dd2402dc9b256ced667f710c7101f301f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hillerstr=C3=B6m?= Date: Wed, 21 Aug 2024 09:55:24 +0200 Subject: [PATCH 4/8] Fix failing check --- crates/wasmtime/src/runtime/vm/continuation.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/continuation.rs b/crates/wasmtime/src/runtime/vm/continuation.rs index 638b45d30289..1bc0bb34d7b8 100644 --- a/crates/wasmtime/src/runtime/vm/continuation.rs +++ b/crates/wasmtime/src/runtime/vm/continuation.rs @@ -826,7 +826,7 @@ pub mod stack_chain { #[cfg(feature = "wasmfx_baseline")] pub mod optimized { use crate::runtime::vm::{Instance, TrapReason}; - pub use wasmtime_continuations::{StackLimits, SwitchDirection}; + pub use wasmtime_continuations::{ControlEffect, StackLimits}; pub type VMContRef = super::baseline::VMContRef; @@ -857,12 +857,12 @@ pub mod optimized { _instance: &mut Instance, _contref: *mut VMContRef, _parent_stack_limits: *mut StackLimits, - ) -> Result { + ) -> Result { panic!("attempt to execute continuation::optimized::resume with `typed_continuation_baseline_implementation` toggled!") } #[inline(always)] - pub fn suspend(_instance: &mut Instance, _tag_index: u32) -> Result<(), TrapReason> { + pub fn suspend(_instance: &mut Instance, _tag_addr: *mut u8) -> Result<(), TrapReason> { panic!("attempt to execute continuation::optimized::suspend with `typed_continuation_baseline_implementation` toggled!") } } From 816af74a322e5c7e48bf0a196625d5bba24c7ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hillerstr=C3=B6m?= Date: Wed, 21 Aug 2024 10:05:59 +0200 Subject: [PATCH 5/8] Fix clippy warnings --- crates/continuations/src/lib.rs | 6 +++--- crates/fuzzing/src/oracles/dummy.rs | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/continuations/src/lib.rs b/crates/continuations/src/lib.rs index a0c2e6486f4a..65971aa3340d 100644 --- a/crates/continuations/src/lib.rs +++ b/crates/continuations/src/lib.rs @@ -259,15 +259,15 @@ pub struct ControlEffect(TaggedPointer); impl ControlEffect { pub fn suspend(ptr: *const u8) -> Self { let tptr = TaggedPointer::untagged(ptr as usize); - Self(TaggedPointer::low_tag(tptr, 0b01)) + Self(TaggedPointer::low_tag(tptr, 0b01_usize)) } pub fn return_() -> Self { - Self((0b00 as usize).into()) + Self((0b00_usize).into()) } pub fn resume() -> Self { - Self((0b11 as usize).into()) + Self((0b11_usize).into()) } fn new(raw: usize) -> Self { diff --git a/crates/fuzzing/src/oracles/dummy.rs b/crates/fuzzing/src/oracles/dummy.rs index 2530d31d27c4..7b6c8b197939 100644 --- a/crates/fuzzing/src/oracles/dummy.rs +++ b/crates/fuzzing/src/oracles/dummy.rs @@ -23,6 +23,7 @@ pub fn dummy_extern(store: &mut Store, ty: ExternType) -> Result { ExternType::Global(global_ty) => Extern::Global(dummy_global(store, global_ty)?), ExternType::Table(table_ty) => Extern::Table(dummy_table(store, table_ty)?), ExternType::Memory(mem_ty) => Extern::Memory(dummy_memory(store, mem_ty)?), + ExternType::Tag(_) => todo!(), }) } From 6137205a7d3b5e7c0e24712fd7c21be6ff31437b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hillerstr=C3=B6m?= Date: Thu, 22 Aug 2024 14:50:33 +0200 Subject: [PATCH 6/8] Address Frank's comments and change alignment of defined tags to 16. --- crates/cranelift/src/wasmfx/optimized.rs | 27 ++++++++++++--------- crates/environ/src/vmoffsets.rs | 2 +- crates/wasmtime/src/runtime/vm/vmcontext.rs | 11 +++++++-- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/crates/cranelift/src/wasmfx/optimized.rs b/crates/cranelift/src/wasmfx/optimized.rs index d10df4b38701..7f0527b574db 100644 --- a/crates/cranelift/src/wasmfx/optimized.rs +++ b/crates/cranelift/src/wasmfx/optimized.rs @@ -1379,7 +1379,7 @@ pub(crate) fn translate_resume<'a>( // Here we translate a resume instruction into several basic // blocks as follows: // - // prelude_block + // previous block // | // | // resume_block <-----------\ @@ -1393,8 +1393,10 @@ pub(crate) fn translate_resume<'a>( // | | // forward_block --/ // - // * prelude_block pushes the continuation arguments onto the - // buffer in the libcall context. + // * previous block is the current active builder block upon + // entering `translate_resume`, in this block we push the + // continuation arguments onto the buffer in the libcall + // context. // * resume_block continues a given `contref`. It jumps to // the `control_block`. // * control_block handles the control effect of resume, i.e. on @@ -1414,14 +1416,14 @@ pub(crate) fn translate_resume<'a>( // NOTE1: The dispatch block is the head of a collection of blocks // which encodes a right-leaning (almost binary) decision tree, // that is a series of nested if-then-else. The `then` branch - // contains a "leaf" node which setups the jump to a user-defined + // contains a "leaf" node which sets up the jump to a user-defined // handler block, whilst the `else` branch contains another // decision tree or the forward_block. let resume_block = builder.create_block(); let return_block = builder.create_block(); let control_block = builder.create_block(); let dispatch_block = builder.create_block(); - let forwarding_block = builder.create_block(); + let forward_block = builder.create_block(); let vmctx = env.vmctx_val(&mut builder.cursor()); @@ -1565,11 +1567,11 @@ pub(crate) fn translate_resume<'a>( tag }; - // Forwarding block: The last block in the if-then-else dispatch + // Forward block: The last block in the if-then-else dispatch // chain. Control flows to this block when the table on (resume // ...) does not have a matching mapping (on ...). { - builder.switch_to_block(forwarding_block); + builder.switch_to_block(forward_block); let parent_contref = parent_stack_chain.unwrap_continuation_or_trap( env, @@ -1635,7 +1637,7 @@ pub(crate) fn translate_resume<'a>( // Fill the preamble block. builder.switch_to_block(target_preamble_block); // Load and push arguments. - let param_types = env.tag_params(handle_tag); // TODO(dhil): Check whether this function behaves correctly for imported tags. + let param_types = env.tag_params(handle_tag); let param_types: Vec = param_types .iter() .map(|wty| crate::value_type(env.isa, *wty)) @@ -1668,8 +1670,8 @@ pub(crate) fn translate_resume<'a>( // The last tail_block unconditionally jumps to the forwarding // block. builder.switch_to_block(tail_block); - builder.ins().jump(forwarding_block, &[]); - builder.seal_block(forwarding_block); + builder.ins().jump(forward_block, &[]); + builder.seal_block(forward_block); // Return block: Jumped to by resume block if continuation // returned normally. @@ -1690,7 +1692,10 @@ pub(crate) fn translate_resume<'a>( // The continuation has returned and all `VMContObjs` to it // should have been be invalidated. We may safely deallocate - // it. + // it. NOTE(dhil): it is only safe to deallocate the stack + // object if there are no lingering references to it, + // otherwise we have to keep it alive (though it can be + // repurposed). shared::typed_continuations_drop_cont_ref(env, builder, resume_contref); return values; diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index aa978d0752b1..f2bd53c13586 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -161,7 +161,7 @@ pub trait PtrSize { /// Return the size of `VMTagDefinition`. #[inline] fn size_of_vmtag_definition(&self) -> u8 { - 8 + 16 } /// This is the size of the largest value type (i.e. a V128). diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index b6deeab7a195..644e032eedbe 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -258,7 +258,7 @@ mod test_vmglobal_import { pub struct VMTagImport { /// A pointer to the imported tag description. pub from: *mut VMTagDefinition, - /// A pointer to the `VMContext` that owns the tag description. + /// A pointer to the owning instance. pub vmctx: *mut VMContext, } @@ -663,7 +663,7 @@ mod test_vmshared_type_index { /// A WebAssembly tag defined within the instance. /// #[derive(Debug)] -#[repr(C, align(8))] +#[repr(C, align(16))] pub struct VMTagDefinition { /// Function signature's type id. pub type_index: VMSharedTypeIndex, @@ -690,6 +690,13 @@ mod test_vmtag_definition { usize::from(offsets.ptr.size_of_vmtag_definition()) ); } + + #[test] + fn check_vmtag_begins_aligned() { + let module = Module::new(); + let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); + assert_eq!(offsets.vmctx_tags_begin() % 16, 0); + } } /// The VM caller-checked "funcref" record, for caller-side signature checking. From c2c9e669e1e3fec2dc781394ac4614db89254f78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hillerstr=C3=B6m?= Date: Thu, 22 Aug 2024 17:47:25 +0200 Subject: [PATCH 7/8] 4 byte sized VMTagDefinition --- crates/environ/src/vmoffsets.rs | 3 ++- crates/wasmtime/src/runtime/vm/vmcontext.rs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index f2bd53c13586..b529ade7f864 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -161,7 +161,7 @@ pub trait PtrSize { /// Return the size of `VMTagDefinition`. #[inline] fn size_of_vmtag_definition(&self) -> u8 { - 16 + 4 } /// This is the size of the largest value type (i.e. a V128). @@ -550,6 +550,7 @@ impl From> for VMOffsets

{ = cmul(ret.num_defined_globals, ret.ptr.size_of_vmglobal_definition()), size(defined_tags) = cmul(ret.num_defined_tags, ret.ptr.size_of_vmtag_definition()), + align(16), size(defined_func_refs) = cmul( ret.num_escaped_funcs, ret.ptr.size_of_vm_func_ref(), diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 644e032eedbe..2725d47f7b0b 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -257,7 +257,7 @@ mod test_vmglobal_import { #[repr(C)] pub struct VMTagImport { /// A pointer to the imported tag description. - pub from: *mut VMTagDefinition, + pub from: *mut VMTagDefinition, // TODO(dhil): May not be heap aligned! /// A pointer to the owning instance. pub vmctx: *mut VMContext, } @@ -663,7 +663,7 @@ mod test_vmshared_type_index { /// A WebAssembly tag defined within the instance. /// #[derive(Debug)] -#[repr(C, align(16))] +#[repr(C)] pub struct VMTagDefinition { /// Function signature's type id. pub type_index: VMSharedTypeIndex, From cbb1e162f13a111528b9495afbc3816449269aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hillerstr=C3=B6m?= Date: Fri, 23 Aug 2024 11:19:52 +0200 Subject: [PATCH 8/8] Tweaks --- crates/continuations/src/lib.rs | 3 ++- crates/cranelift/src/wasmfx/optimized.rs | 1 + crates/cranelift/src/wasmfx/shared.rs | 5 +++-- crates/wasmtime/src/runtime/vm/vmcontext.rs | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/continuations/src/lib.rs b/crates/continuations/src/lib.rs index 65971aa3340d..4a2dffda7014 100644 --- a/crates/continuations/src/lib.rs +++ b/crates/continuations/src/lib.rs @@ -216,7 +216,8 @@ pub mod offsets { pub struct TaggedPointer(usize); impl TaggedPointer { - const LOW_TAG_MASK: usize = 0b11; + const LOW_TAG_BITS: usize = 2; + const LOW_TAG_MASK: usize = (1 << Self::LOW_TAG_BITS) - 1; pub fn untagged(val: usize) -> Self { Self(val) diff --git a/crates/cranelift/src/wasmfx/optimized.rs b/crates/cranelift/src/wasmfx/optimized.rs index 7f0527b574db..6a1967887d6e 100644 --- a/crates/cranelift/src/wasmfx/optimized.rs +++ b/crates/cranelift/src/wasmfx/optimized.rs @@ -1712,6 +1712,7 @@ pub(crate) fn translate_suspend<'a>( typed_continuations_store_payloads(env, builder, suspend_args); let tag_addr = shared::tag_address(env, builder, tag_index); + emit_debug_println!(env, builder, "[suspend] suspending with tag {:p}", tag_addr); call_builtin!(builder, env, tc_suspend(tag_addr)); let contref = typed_continuations_load_continuation_reference(env, builder); diff --git a/crates/cranelift/src/wasmfx/shared.rs b/crates/cranelift/src/wasmfx/shared.rs index 5ec163cb0132..312950af2a3b 100644 --- a/crates/cranelift/src/wasmfx/shared.rs +++ b/crates/cranelift/src/wasmfx/shared.rs @@ -146,8 +146,9 @@ pub(crate) fn typed_continuations_drop_cont_ref<'a>( pub struct TaggedPointer(pub ir::Value); impl TaggedPointer { - const LOW_TAG_MASK: i64 = 0b11; - const LOW_TAG_INVERSE_MASK: i64 = !0b11; + const LOW_TAG_BITS: i64 = 2; + const LOW_TAG_MASK: i64 = (1 << Self::LOW_TAG_BITS) - 1; + const LOW_TAG_INVERSE_MASK: i64 = !Self::LOW_TAG_MASK; pub fn new(val: ir::Value) -> Self { Self(val) diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 2725d47f7b0b..b7d0035eee70 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -257,7 +257,7 @@ mod test_vmglobal_import { #[repr(C)] pub struct VMTagImport { /// A pointer to the imported tag description. - pub from: *mut VMTagDefinition, // TODO(dhil): May not be heap aligned! + pub from: *mut VMTagDefinition, /// A pointer to the owning instance. pub vmctx: *mut VMContext, }