From 2ede8e45dca853d0e093c255e3b73a02e1f3714a Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 25 Nov 2020 16:10:09 -0800 Subject: [PATCH] Refactor module instantiation in the runtime. This commit refactors module instantiation in the runtime to allow for different instance allocation strategy implementations. It adds an `InstanceAllocator` trait with the current implementation put behind the `OnDemandInstanceAllocator` struct. The Wasmtime API has been updated to allow a `Config` to have an instance allocation strategy set which will determine how instances get allocated. This change is in preparation for an alternative *pooling* instance allocator that can reserve all needed host process address space in advance. This commit also makes changes to the `wasmtime_environ` crate to represent compiled modules in a way that reduces copying at instantiation time. --- cranelift/wasm/src/module_translator.rs | 2 + crates/environ/src/data_structures.rs | 2 +- crates/environ/src/module.rs | 59 +- crates/environ/src/module_environ.rs | 39 +- crates/jit/src/instantiate.rs | 157 ++--- crates/runtime/src/instance.rs | 599 +++--------------- crates/runtime/src/instance/allocator.rs | 515 +++++++++++++++ crates/runtime/src/lib.rs | 5 +- crates/runtime/src/vmcontext.rs | 4 +- crates/wasmtime/src/config.rs | 43 +- crates/wasmtime/src/func.rs | 2 +- crates/wasmtime/src/instance.rs | 62 +- crates/wasmtime/src/store.rs | 26 +- .../wasmtime/src/trampoline/create_handle.rs | 30 +- crates/wasmtime/src/trampoline/func.rs | 1 + crates/wasmtime/src/trampoline/memory.rs | 6 +- examples/interrupt.c | 1 + 17 files changed, 854 insertions(+), 699 deletions(-) create mode 100644 crates/runtime/src/instance/allocator.rs diff --git a/cranelift/wasm/src/module_translator.rs b/cranelift/wasm/src/module_translator.rs index 5fc60ccbdda5..dd77bda422be 100644 --- a/cranelift/wasm/src/module_translator.rs +++ b/cranelift/wasm/src/module_translator.rs @@ -108,6 +108,8 @@ pub fn translate_module<'data>( Payload::DataCountSection { count, range } => { validator.data_count_section(count, &range)?; + + // BUGBUG: the count here is the total segment count, not the passive segment count environ.reserve_passive_data(count)?; } diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 9e1a164675f3..474dee6ad723 100644 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -20,7 +20,7 @@ pub mod isa { } pub mod entity { - pub use cranelift_entity::{packed_option, BoxedSlice, EntityRef, PrimaryMap}; + pub use cranelift_entity::{packed_option, BoxedSlice, EntityRef, EntitySet, PrimaryMap}; } pub mod wasm { diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 8cd04647f679..977083f7c0f9 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -175,13 +175,19 @@ pub struct Module { pub table_elements: Vec, /// WebAssembly passive elements. - pub passive_elements: HashMap>, + pub passive_elements: Vec>, + + /// The map from passive element index (element segment index space) to index in `passive_elements`. + pub passive_elements_map: HashMap, /// WebAssembly passive data segments. #[serde(with = "passive_data_serde")] - pub passive_data: HashMap>, + pub passive_data: Vec>, - /// WebAssembly table initializers. + /// The map from passive data index (data segment index space) to index in `passive_data`. + pub passive_data_map: HashMap, + + /// WebAssembly function names. pub func_names: HashMap, /// Unprocessed signatures exactly as provided by `declare_signature()`. @@ -246,8 +252,10 @@ impl Module { exports: IndexMap::new(), start_func: None, table_elements: Vec::new(), - passive_elements: HashMap::new(), - passive_data: HashMap::new(), + passive_elements: Vec::new(), + passive_elements_map: HashMap::new(), + passive_data: Vec::new(), + passive_data_map: HashMap::new(), func_names: HashMap::new(), num_imported_funcs: 0, num_imported_tables: 0, @@ -266,7 +274,8 @@ impl Module { /// Get the given passive element, if it exists. pub fn get_passive_element(&self, index: ElemIndex) -> Option<&[FuncIndex]> { - self.passive_elements.get(&index).map(|es| &**es) + let index = *self.passive_elements_map.get(&index)?; + Some(self.passive_elements[index].as_ref()) } fn next_id() -> usize { @@ -376,47 +385,45 @@ impl Default for Module { } mod passive_data_serde { - use super::{Arc, DataIndex, HashMap}; - use serde::{de::MapAccess, de::Visitor, ser::SerializeMap, Deserializer, Serializer}; + use super::Arc; + use serde::{de::SeqAccess, de::Visitor, ser::SerializeSeq, Deserializer, Serializer}; use std::fmt; - pub(super) fn serialize( - data: &HashMap>, - ser: S, - ) -> Result + pub(super) fn serialize(data: &Vec>, ser: S) -> Result where S: Serializer, { - let mut map = ser.serialize_map(Some(data.len()))?; - for (k, v) in data { - map.serialize_entry(k, v.as_ref())?; + let mut seq = ser.serialize_seq(Some(data.len()))?; + for v in data { + seq.serialize_element(v.as_ref())?; } - map.end() + seq.end() } struct PassiveDataVisitor; impl<'de> Visitor<'de> for PassiveDataVisitor { - type Value = HashMap>; + type Value = Vec>; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a passive_data map") + formatter.write_str("a passive data sequence") } - fn visit_map(self, mut access: M) -> Result + + fn visit_seq(self, mut access: M) -> Result where - M: MapAccess<'de>, + M: SeqAccess<'de>, { - let mut map = HashMap::with_capacity(access.size_hint().unwrap_or(0)); - while let Some((key, value)) = access.next_entry::<_, Vec>()? { - map.insert(key, value.into()); + let mut data = Vec::with_capacity(access.size_hint().unwrap_or(0)); + while let Some(value) = access.next_element::>()? { + data.push(value.into()); } - Ok(map) + Ok(data) } } - pub(super) fn deserialize<'de, D>(de: D) -> Result>, D::Error> + pub(super) fn deserialize<'de, D>(de: D) -> Result>, D::Error> where D: Deserializer<'de>, { - de.deserialize_map(PassiveDataVisitor) + de.deserialize_seq(PassiveDataVisitor) } } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index e085751d77e7..d68e5ae67362 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -471,11 +471,13 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data elem_index: ElemIndex, segments: Box<[FuncIndex]>, ) -> WasmResult<()> { + let index = self.result.module.passive_elements.len(); + self.result.module.passive_elements.push(segments); let old = self .result .module - .passive_elements - .insert(elem_index, segments); + .passive_elements_map + .insert(elem_index, index); debug_assert!( old.is_none(), "should never get duplicate element indices, that would be a bug in `cranelift_wasm`'s \ @@ -543,17 +545,21 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data Ok(()) } - fn reserve_passive_data(&mut self, count: u32) -> WasmResult<()> { - self.result.module.passive_data.reserve(count as usize); + fn reserve_passive_data(&mut self, _count: u32) -> WasmResult<()> { + // BUGBUG: the count passed in here appears to be the *total* segment count + // There is no way to reserve for just the passive segments as they are discovered when iterating the data section entries + // Given that the total segment count might be much larger than the passive count, do not reserve Ok(()) } fn declare_passive_data(&mut self, data_index: DataIndex, data: &'data [u8]) -> WasmResult<()> { + let index = self.result.module.passive_data.len(); + self.result.module.passive_data.push(Arc::from(data)); let old = self .result .module - .passive_data - .insert(data_index, Arc::from(data)); + .passive_data_map + .insert(data_index, index); debug_assert!( old.is_none(), "a module can't have duplicate indices, this would be a cranelift-wasm bug" @@ -697,3 +703,24 @@ pub struct DataInitializer<'data> { /// The initialization data. pub data: &'data [u8], } + +/// Similar to `DataInitializer`, but owns its own copy of the data rather +/// than holding a slice of the original module. +#[derive(Serialize, Deserialize)] +pub struct OwnedDataInitializer { + /// The location where the initialization is to be performed. + pub location: DataInitializerLocation, + + /// The initialization data. + pub data: Box<[u8]>, +} + +impl OwnedDataInitializer { + /// Creates a new owned data initializer from a borrowed data initializer. + pub fn new(borrowed: DataInitializer<'_>) -> Self { + Self { + location: borrowed.location.clone(), + data: borrowed.data.into(), + } + } +} diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 3ec6f04bf1db..4974511adfc7 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -11,7 +11,6 @@ use object::File as ObjectFile; #[cfg(feature = "parallel-compilation")] use rayon::prelude::*; use serde::{Deserialize, Serialize}; -use std::any::Any; use std::sync::Arc; use thiserror::Error; use wasmtime_debug::create_gdbjit_image; @@ -19,15 +18,11 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::{DefinedFuncIndex, ModuleIndex, SignatureIndex}; use wasmtime_environ::{ - CompileError, DataInitializer, DataInitializerLocation, FunctionAddressMap, Module, - ModuleEnvironment, ModuleTranslation, StackMapInformation, TrapInformation, + CompileError, FunctionAddressMap, Module, ModuleEnvironment, ModuleTranslation, + OwnedDataInitializer, StackMapInformation, TrapInformation, }; use wasmtime_profiling::ProfilingAgent; -use wasmtime_runtime::{ - GdbJitImageRegistration, Imports, InstanceHandle, InstantiationError, RuntimeMemoryCreator, - StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, VMInterrupts, - VMSharedSignatureIndex, VMTrampoline, -}; +use wasmtime_runtime::{GdbJitImageRegistration, InstantiationError, VMFunctionBody, VMTrampoline}; /// An error condition while setting up a wasm instance, be it validation, /// compilation, or instantiation. @@ -55,7 +50,8 @@ pub enum SetupError { #[derive(Serialize, Deserialize)] pub struct CompilationArtifacts { /// Module metadata. - module: Module, + #[serde(with = "module_serde")] + module: Arc, /// ELF image with functions code. obj: Box<[u8]>, @@ -64,7 +60,8 @@ pub struct CompilationArtifacts { unwind_info: Box<[ObjectUnwindInfo]>, /// Data initiailizers. - data_initializers: Box<[OwnedDataInitializer]>, + #[serde(with = "data_initializers_serde")] + data_initializers: Arc<[OwnedDataInitializer]>, /// Descriptions of compiled functions funcs: PrimaryMap, @@ -110,7 +107,7 @@ impl CompilationArtifacts { .into_iter() .map(OwnedDataInitializer::new) .collect::>() - .into_boxed_slice(); + .into(); let obj = obj.write().map_err(|_| { SetupError::Instantiate(InstantiationError::Resource( @@ -119,7 +116,7 @@ impl CompilationArtifacts { })?; Ok(CompilationArtifacts { - module, + module: Arc::new(module), obj: obj.into_boxed_slice(), unwind_info: unwind_info.into_boxed_slice(), data_initializers, @@ -161,7 +158,6 @@ pub struct ModuleCode { /// A compiled wasm module, ready to be instantiated. pub struct CompiledModule { artifacts: CompilationArtifacts, - module: Arc, code: Arc, finished_functions: FinishedFunctions, trampolines: PrimaryMap, @@ -220,7 +216,6 @@ impl CompiledModule { let finished_functions = FinishedFunctions(finished_functions); Ok(Self { - module: Arc::new(artifacts.module.clone()), artifacts, code: Arc::new(ModuleCode { code_memory, @@ -231,62 +226,24 @@ impl CompiledModule { }) } - /// Crate an `Instance` from this `CompiledModule`. - /// - /// Note that if only one instance of this module is needed, it may be more - /// efficient to call the top-level `instantiate`, since that avoids copying - /// the data initializers. - /// - /// # Unsafety - /// - /// See `InstanceHandle::new` - pub unsafe fn instantiate( - &self, - imports: Imports<'_>, - lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, - mem_creator: Option<&dyn RuntimeMemoryCreator>, - interrupts: *const VMInterrupts, - host_state: Box, - externref_activations_table: *mut VMExternRefActivationsTable, - stack_map_registry: *mut StackMapRegistry, - ) -> Result { - InstanceHandle::new( - self.module.clone(), - &self.finished_functions.0, - imports, - mem_creator, - lookup_shared_signature, - host_state, - interrupts, - externref_activations_table, - stack_map_registry, - ) - } /// Extracts `CompilationArtifacts` from the compiled module. pub fn compilation_artifacts(&self) -> &CompilationArtifacts { &self.artifacts } - /// Returns data initializers to pass to `InstanceHandle::initialize` - pub fn data_initializers(&self) -> Vec> { - self.artifacts - .data_initializers - .iter() - .map(|init| DataInitializer { - location: init.location.clone(), - data: &*init.data, - }) - .collect() + /// Returns the data initializers from the compiled module. + pub fn data_initializers(&self) -> &Arc<[OwnedDataInitializer]> { + &self.artifacts.data_initializers } /// Return a reference-counting pointer to a module. pub fn module(&self) -> &Arc { - &self.module + &self.artifacts.module } /// Return a reference to a mutable module (if possible). pub fn module_mut(&mut self) -> Option<&mut Module> { - Arc::get_mut(&mut self.module) + Arc::get_mut(&mut self.artifacts.module) } /// Returns the map of all finished JIT functions compiled for this module @@ -350,26 +307,6 @@ impl CompiledModule { } } -/// Similar to `DataInitializer`, but owns its own copy of the data rather -/// than holding a slice of the original module. -#[derive(Clone, Serialize, Deserialize)] -pub struct OwnedDataInitializer { - /// The location where the initialization is to be performed. - location: DataInitializerLocation, - - /// The initialization data. - data: Box<[u8]>, -} - -impl OwnedDataInitializer { - fn new(borrowed: DataInitializer<'_>) -> Self { - Self { - location: borrowed.location.clone(), - data: borrowed.data.to_vec().into_boxed_slice(), - } - } -} - fn create_dbg_image( obj: Vec, code_range: (*const u8, usize), @@ -432,3 +369,69 @@ fn build_code_memory( Ok((code_memory, code_range, finished_functions, trampolines)) } + +mod module_serde { + use super::{Arc, Module}; + use serde::{de::Deserialize, ser::Serialize, Deserializer, Serializer}; + + pub(super) fn serialize(module: &Arc, ser: S) -> Result + where + S: Serializer, + { + (**module).serialize(ser) + } + + pub(super) fn deserialize<'de, D>(de: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + Ok(Arc::new(Module::deserialize(de)?)) + } +} + +mod data_initializers_serde { + use super::{Arc, OwnedDataInitializer}; + use serde::{de::SeqAccess, de::Visitor, ser::SerializeSeq, Deserializer, Serializer}; + use std::fmt; + + pub(super) fn serialize( + data: &Arc<[OwnedDataInitializer]>, + ser: S, + ) -> Result + where + S: Serializer, + { + let mut seq = ser.serialize_seq(Some(data.len()))?; + for v in data.iter() { + seq.serialize_element(v)?; + } + seq.end() + } + + struct DataInitializersVisitor; + impl<'de> Visitor<'de> for DataInitializersVisitor { + type Value = Arc<[OwnedDataInitializer]>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a data initializers") + } + + fn visit_seq(self, mut access: M) -> Result + where + M: SeqAccess<'de>, + { + let mut data = Vec::with_capacity(access.size_hint().unwrap_or(0)); + while let Some(value) = access.next_element::()? { + data.push(value.into()); + } + Ok(data.into()) + } + } + + pub(super) fn deserialize<'de, D>(de: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + de.deserialize_seq(DataInitializersVisitor) + } +} diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index a95a4b594bfd..4778c90b5449 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -4,34 +4,34 @@ use crate::export::Export; use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; -use crate::imports::Imports; -use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory, RuntimeMemoryCreator}; +use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; use crate::vmcontext::{ - VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, + VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, }; use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable}; use memoffset::offset_of; use more_asserts::assert_lt; -use std::alloc::{self, Layout}; +use std::alloc::Layout; use std::any::Any; use std::cell::RefCell; -use std::collections::HashMap; use std::convert::TryFrom; use std::ptr::NonNull; use std::sync::Arc; use std::{mem, ptr, slice}; -use thiserror::Error; -use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap}; +use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, EntitySet}; use wasmtime_environ::wasm::{ - DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, - ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, - TableElementType, TableIndex, WasmType, + DataIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, ElemIndex, EntityIndex, + FuncIndex, GlobalIndex, MemoryIndex, SignatureIndex, TableElementType, TableIndex, }; -use wasmtime_environ::{ir, DataInitializer, Module, TableElements, VMOffsets}; +use wasmtime_environ::{ir, Module, VMOffsets}; + +mod allocator; + +pub use allocator::*; /// A WebAssembly instance. /// @@ -50,17 +50,16 @@ pub(crate) struct Instance { /// WebAssembly table data. tables: BoxedSlice, - /// Passive elements in this instantiation. As `elem.drop`s happen, these - /// entries get removed. A missing entry is considered equivalent to an - /// empty slice. - passive_elements: RefCell>>, + /// Stores the dropped passive element segments in this instantiation by index. + /// If the index is present in the set, the segment has been dropped. + dropped_elements: RefCell>, - /// Passive data segments from our module. As `data.drop`s happen, entries - /// get removed. A missing entry is considered equivalent to an empty slice. - passive_data: RefCell>>, + /// Stores the dropped passive data segments in this instantiation by index. + /// If the index is present in the set, the segment has been dropped. + dropped_data: RefCell>, /// Hosts can store arbitrary per-instance information here. - host_state: Box, + host_state: Option>, /// Additional context used by compiled wasm code. This field is last, and /// represents a dynamically-sized array that extends beyond the nominal @@ -340,8 +339,8 @@ impl Instance { /// Return a reference to the custom state attached to this instance. #[inline] - pub fn host_state(&self) -> &dyn Any { - &*self.host_state + pub fn host_state(&self) -> Option<&dyn Any> { + self.host_state.as_deref() } /// Return the offset from the vmctx pointer to its containing Instance. @@ -561,11 +560,21 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init let table = self.get_table(table_index); - let passive_elements = self.passive_elements.borrow(); - let elem = passive_elements - .get(&elem_index) - .map(|e| &**e) - .unwrap_or_else(|| &[]); + let elem_index = self.module.passive_elements_map.get(&elem_index); + let elem = match elem_index { + Some(index) => { + if self + .dropped_elements + .borrow() + .contains(ElemIndex::new(*index)) + { + &[] + } else { + self.module.passive_elements[*index].as_ref() + } + } + None => &[], + }; if src .checked_add(len) @@ -577,8 +586,14 @@ impl Instance { // TODO(#983): investigate replacing this get/set loop with a `memcpy`. for (dst, src) in (dst..dst + len).zip(src..src + len) { + let elem = self + .get_caller_checked_anyfunc(elem[src as usize]) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut _ + }); + table - .set(dst, TableElement::FuncRef(elem[src as usize])) + .set(dst, TableElement::FuncRef(elem)) .expect("should never panic because we already did the bounds check above"); } @@ -589,10 +604,14 @@ impl Instance { pub(crate) fn elem_drop(&self, elem_index: ElemIndex) { // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop - let mut passive_elements = self.passive_elements.borrow_mut(); - passive_elements.remove(&elem_index); - // Note that we don't check that we actually removed an element because - // dropping a non-passive element is a no-op (not a trap). + if let Some(index) = self.module.passive_elements_map.get(&elem_index) { + self.dropped_elements + .borrow_mut() + .insert(ElemIndex::new(*index)); + } + + // Note that we don't check that we actually removed a segment because + // dropping a non-passive segment is a no-op (not a trap). } /// Do a `memory.copy` @@ -711,10 +730,17 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init let memory = self.get_memory(memory_index); - let passive_data = self.passive_data.borrow(); - let data = passive_data - .get(&data_index) - .map_or(&[][..], |data| &**data); + let data_index = self.module.passive_data_map.get(&data_index); + let data = match data_index { + Some(index) => { + if self.dropped_data.borrow().contains(DataIndex::new(*index)) { + &[] + } else { + self.module.passive_data[*index].as_ref() + } + } + None => &[], + }; if src .checked_add(len) @@ -739,8 +765,14 @@ impl Instance { /// Drop the given data segment, truncating its length to zero. pub(crate) fn data_drop(&self, data_index: DataIndex) { - let mut passive_data = self.passive_data.borrow_mut(); - passive_data.remove(&data_index); + if let Some(index) = self.module.passive_data_map.get(&data_index) { + self.dropped_data + .borrow_mut() + .insert(DataIndex::new(*index)); + } + + // Note that we don't check that we actually removed a segment because + // dropping a non-passive segment is a no-op (not a trap). } /// Get a table by index regardless of whether it is locally-defined or an @@ -790,194 +822,8 @@ pub struct InstanceHandle { } impl InstanceHandle { - /// Create a new `InstanceHandle` pointing at a new `Instance`. - /// - /// # Unsafety - /// - /// This method is not necessarily inherently unsafe to call, but in general - /// the APIs of an `Instance` are quite unsafe and have not been really - /// audited for safety that much. As a result the unsafety here on this - /// method is a low-overhead way of saying "this is an extremely unsafe type - /// to work with". - /// - /// Extreme care must be taken when working with `InstanceHandle` and it's - /// recommended to have relatively intimate knowledge of how it works - /// internally if you'd like to do so. If possible it's recommended to use - /// the `wasmtime` crate API rather than this type since that is vetted for - /// safety. - /// - /// It is your responsibility to ensure that the given raw - /// `externref_activations_table` and `stack_map_registry` outlive this - /// instance. - pub unsafe fn new( - module: Arc, - finished_functions: &PrimaryMap, - imports: Imports, - mem_creator: Option<&dyn RuntimeMemoryCreator>, - lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, - host_state: Box, - interrupts: *const VMInterrupts, - externref_activations_table: *mut VMExternRefActivationsTable, - stack_map_registry: *mut StackMapRegistry, - ) -> Result { - debug_assert!(!externref_activations_table.is_null()); - debug_assert!(!stack_map_registry.is_null()); - - let tables = create_tables(&module); - let memories = create_memories(&module, mem_creator.unwrap_or(&DefaultMemoryCreator {}))?; - - let vmctx_tables = tables - .values() - .map(Table::vmtable) - .collect::>() - .into_boxed_slice(); - - let vmctx_memories = memories - .values() - .map(|a| a.vmmemory()) - .collect::>() - .into_boxed_slice(); - - let vmctx_globals = create_globals(&module); - - let offsets = VMOffsets::new(mem::size_of::<*const u8>() as u8, &module); - - let passive_data = RefCell::new(module.passive_data.clone()); - - let handle = { - let instance = Instance { - module, - offsets, - memories, - tables, - passive_elements: Default::default(), - passive_data, - host_state, - vmctx: VMContext {}, - }; - let layout = instance.alloc_layout(); - let instance_ptr = alloc::alloc(layout) as *mut Instance; - if instance_ptr.is_null() { - alloc::handle_alloc_error(layout); - } - ptr::write(instance_ptr, instance); - InstanceHandle { - instance: instance_ptr, - } - }; - let instance = handle.instance(); - - let mut ptr = instance.signature_ids_ptr(); - for (signature, _) in handle.module().signatures.iter() { - *ptr = lookup_shared_signature(signature); - ptr = ptr.add(1); - } - - debug_assert_eq!(imports.functions.len(), handle.module().num_imported_funcs); - ptr::copy( - imports.functions.as_ptr(), - instance.imported_functions_ptr() as *mut VMFunctionImport, - imports.functions.len(), - ); - debug_assert_eq!(imports.tables.len(), handle.module().num_imported_tables); - ptr::copy( - imports.tables.as_ptr(), - instance.imported_tables_ptr() as *mut VMTableImport, - imports.tables.len(), - ); - debug_assert_eq!( - imports.memories.len(), - handle.module().num_imported_memories - ); - ptr::copy( - imports.memories.as_ptr(), - instance.imported_memories_ptr() as *mut VMMemoryImport, - imports.memories.len(), - ); - debug_assert_eq!(imports.globals.len(), handle.module().num_imported_globals); - ptr::copy( - imports.globals.as_ptr(), - instance.imported_globals_ptr() as *mut VMGlobalImport, - imports.globals.len(), - ); - ptr::copy( - vmctx_tables.values().as_slice().as_ptr(), - instance.tables_ptr() as *mut VMTableDefinition, - vmctx_tables.len(), - ); - ptr::copy( - vmctx_memories.values().as_slice().as_ptr(), - instance.memories_ptr() as *mut VMMemoryDefinition, - vmctx_memories.len(), - ); - ptr::copy( - vmctx_globals.values().as_slice().as_ptr(), - instance.globals_ptr() as *mut VMGlobalDefinition, - vmctx_globals.len(), - ); - ptr::write( - instance.builtin_functions_ptr() as *mut VMBuiltinFunctionsArray, - VMBuiltinFunctionsArray::initialized(), - ); - *instance.interrupts() = interrupts; - *instance.externref_activations_table() = externref_activations_table; - *instance.stack_map_registry() = stack_map_registry; - - for (index, sig) in instance.module.functions.iter() { - let type_index = instance.signature_id(*sig); - - let (func_ptr, vmctx) = - if let Some(def_index) = instance.module.defined_func_index(index) { - ( - NonNull::new(finished_functions[def_index] as *mut _).unwrap(), - instance.vmctx_ptr(), - ) - } else { - let import = instance.imported_function(index); - (import.body, import.vmctx) - }; - - ptr::write( - instance.anyfunc_ptr(index), - VMCallerCheckedAnyfunc { - func_ptr, - type_index, - vmctx, - }, - ); - } - - // Perform infallible initialization in this constructor, while fallible - // initialization is deferred to the `initialize` method. - initialize_passive_elements(instance); - initialize_globals(instance); - - Ok(handle) - } - - /// Finishes the instantiation process started by `Instance::new`. - /// - /// Only safe to call immediately after instantiation. - pub unsafe fn initialize( - &self, - is_bulk_memory: bool, - data_initializers: &[DataInitializer<'_>], - ) -> Result<(), InstantiationError> { - // Check initializer bounds before initializing anything. Only do this - // when bulk memory is disabled, since the bulk memory proposal changes - // instantiation such that the intermediate results of failed - // initializations are visible. - if !is_bulk_memory { - check_table_init_bounds(self.instance())?; - check_memory_init_bounds(self.instance(), data_initializers)?; - } - - // Apply fallible initializers. Note that this can "leak" state even if - // it fails. - initialize_tables(self.instance())?; - initialize_memories(self.instance(), data_initializers)?; - - Ok(()) + pub(crate) unsafe fn new(instance: *mut Instance) -> Self { + Self { instance } } /// Create a new `InstanceHandle` pointing at the instance @@ -1028,7 +874,7 @@ impl InstanceHandle { } /// Return a reference to the custom state attached to this instance. - pub fn host_state(&self) -> &dyn Any { + pub fn host_state(&self) -> Option<&dyn Any> { self.instance().host_state() } @@ -1138,305 +984,4 @@ impl InstanceHandle { instance: self.instance, } } - - /// Deallocates memory associated with this instance. - /// - /// Note that this is unsafe because there might be other handles to this - /// `InstanceHandle` elsewhere, and there's nothing preventing usage of - /// this handle after this function is called. - pub unsafe fn dealloc(&self) { - let instance = self.instance(); - let layout = instance.alloc_layout(); - ptr::drop_in_place(self.instance); - alloc::dealloc(self.instance.cast(), layout); - } -} - -fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module().table_elements { - let start = get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - let size = usize::try_from(table.size()).unwrap(); - if size < start + init.elements.len() { - return Err(InstantiationError::Link(LinkError( - "table out of bounds: elements segment does not fit".to_owned(), - ))); - } - } - - Ok(()) -} - -/// Compute the offset for a memory data initializer. -fn get_memory_init_start(init: &DataInitializer<'_>, instance: &Instance) -> usize { - let mut start = init.location.offset; - - if let Some(base) = init.location.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start -} - -/// Return a byte-slice view of a memory's data. -unsafe fn get_memory_slice<'instance>( - init: &DataInitializer<'_>, - instance: &'instance Instance, -) -> &'instance mut [u8] { - let memory = if let Some(defined_memory_index) = instance - .module - .defined_memory_index(init.location.memory_index) - { - instance.memory(defined_memory_index) - } else { - let import = instance.imported_memory(init.location.memory_index); - let foreign_instance = (&mut *(import).vmctx).instance(); - let foreign_memory = &mut *(import).from; - let foreign_index = foreign_instance.memory_index(foreign_memory); - foreign_instance.memory(foreign_index) - }; - slice::from_raw_parts_mut(memory.base, memory.current_length) -} - -fn check_memory_init_bounds( - instance: &Instance, - data_initializers: &[DataInitializer<'_>], -) -> Result<(), InstantiationError> { - for init in data_initializers { - let start = get_memory_init_start(init, instance); - unsafe { - let mem_slice = get_memory_slice(init, instance); - if mem_slice.get_mut(start..start + init.data.len()).is_none() { - return Err(InstantiationError::Link(LinkError( - "memory out of bounds: data segment does not fit".into(), - ))); - } - } - } - - Ok(()) -} - -/// Allocate memory for just the tables of the current module. -fn create_tables(module: &Module) -> BoxedSlice { - let num_imports = module.num_imported_tables; - let mut tables: PrimaryMap = - PrimaryMap::with_capacity(module.table_plans.len() - num_imports); - for table in &module.table_plans.values().as_slice()[num_imports..] { - tables.push(Table::new(table)); - } - tables.into_boxed_slice() -} - -/// Compute the offset for a table element initializer. -fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize { - let mut start = init.offset; - - if let Some(base) = init.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start -} - -/// Initialize the table memory from the provided initializers. -fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module().table_elements { - let start = get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - if start - .checked_add(init.elements.len()) - .map_or(true, |end| end > table.size() as usize) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::TableOutOfBounds, - ))); - } - - for (i, func_idx) in init.elements.iter().enumerate() { - let item = match table.element_type() { - TableElementType::Func => instance - .get_caller_checked_anyfunc(*func_idx) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc - }) - .into(), - TableElementType::Val(_) => { - assert!(*func_idx == FuncIndex::reserved_value()); - TableElement::ExternRef(None) - } - }; - table.set(u32::try_from(start + i).unwrap(), item).unwrap(); - } - } - - Ok(()) -} - -/// Initialize the `Instance::passive_elements` map by resolving the -/// `Module::passive_elements`'s `FuncIndex`s into `VMCallerCheckedAnyfunc`s for -/// this instance. -fn initialize_passive_elements(instance: &Instance) { - let mut passive_elements = instance.passive_elements.borrow_mut(); - debug_assert!( - passive_elements.is_empty(), - "should only be called once, at initialization time" - ); - - passive_elements.extend( - instance - .module - .passive_elements - .iter() - .filter(|(_, segments)| !segments.is_empty()) - .map(|(idx, segments)| { - ( - *idx, - segments - .iter() - .map(|s| { - instance.get_caller_checked_anyfunc(*s).map_or( - ptr::null_mut(), - |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut _ - }, - ) - }) - .collect(), - ) - }), - ); -} - -/// Allocate memory for just the memories of the current module. -fn create_memories( - module: &Module, - mem_creator: &dyn RuntimeMemoryCreator, -) -> Result>, InstantiationError> { - let num_imports = module.num_imported_memories; - let mut memories: PrimaryMap = - PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); - for plan in &module.memory_plans.values().as_slice()[num_imports..] { - memories.push( - mem_creator - .new_memory(plan) - .map_err(InstantiationError::Resource)?, - ); - } - Ok(memories.into_boxed_slice()) -} - -/// Initialize the table memory from the provided initializers. -fn initialize_memories( - instance: &Instance, - data_initializers: &[DataInitializer<'_>], -) -> Result<(), InstantiationError> { - for init in data_initializers { - let memory = instance.get_memory(init.location.memory_index); - - let start = get_memory_init_start(init, instance); - if start - .checked_add(init.data.len()) - .map_or(true, |end| end > memory.current_length) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::HeapOutOfBounds, - ))); - } - - unsafe { - let mem_slice = get_memory_slice(init, instance); - let end = start + init.data.len(); - let to_init = &mut mem_slice[start..end]; - to_init.copy_from_slice(init.data); - } - } - - Ok(()) -} - -/// Allocate memory for just the globals of the current module, -/// with initializers applied. -fn create_globals(module: &Module) -> BoxedSlice { - let num_imports = module.num_imported_globals; - let mut vmctx_globals = PrimaryMap::with_capacity(module.globals.len() - num_imports); - - for _ in &module.globals.values().as_slice()[num_imports..] { - vmctx_globals.push(VMGlobalDefinition::new()); - } - - vmctx_globals.into_boxed_slice() -} - -fn initialize_globals(instance: &Instance) { - let module = instance.module(); - let num_imports = module.num_imported_globals; - for (index, global) in module.globals.iter().skip(num_imports) { - let def_index = module.defined_global_index(index).unwrap(); - unsafe { - let to = instance.global_ptr(def_index); - match global.initializer { - GlobalInit::I32Const(x) => *(*to).as_i32_mut() = x, - GlobalInit::I64Const(x) => *(*to).as_i64_mut() = x, - GlobalInit::F32Const(x) => *(*to).as_f32_bits_mut() = x, - GlobalInit::F64Const(x) => *(*to).as_f64_bits_mut() = x, - GlobalInit::V128Const(x) => *(*to).as_u128_bits_mut() = x.0, - GlobalInit::GetGlobal(x) => { - let from = if let Some(def_x) = module.defined_global_index(x) { - instance.global(def_x) - } else { - *instance.imported_global(x).from - }; - *to = from; - } - GlobalInit::RefFunc(f) => { - *(*to).as_anyfunc_mut() = instance.get_caller_checked_anyfunc(f).unwrap() - as *const VMCallerCheckedAnyfunc; - } - GlobalInit::RefNullConst => match global.wasm_ty { - WasmType::FuncRef => *(*to).as_anyfunc_mut() = ptr::null(), - WasmType::ExternRef => *(*to).as_externref_mut() = None, - ty => panic!("unsupported reference type for global: {:?}", ty), - }, - GlobalInit::Import => panic!("locally-defined global initialized as import"), - } - } - } -} - -/// An link error while instantiating a module. -#[derive(Error, Debug)] -#[error("Link error: {0}")] -pub struct LinkError(pub String); - -/// An error while instantiating a module. -#[derive(Error, Debug)] -pub enum InstantiationError { - /// Insufficient resources available for execution. - #[error("Insufficient resources: {0}")] - Resource(String), - - /// A wasm link error occured. - #[error("Failed to link module")] - Link(#[from] LinkError), - - /// A trap ocurred during instantiation, after linking. - #[error("Trap occurred during instantiation")] - Trap(Trap), } diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs new file mode 100644 index 000000000000..6617490be78f --- /dev/null +++ b/crates/runtime/src/instance/allocator.rs @@ -0,0 +1,515 @@ +use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; +use crate::imports::Imports; +use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator}; +use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory}; +use crate::table::{Table, TableElement}; +use crate::traphandlers::Trap; +use crate::vmcontext::{ + VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, + VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryDefinition, VMMemoryImport, + VMSharedSignatureIndex, VMTableDefinition, VMTableImport, +}; +use std::alloc; +use std::any::Any; +use std::cell::RefCell; +use std::convert::TryFrom; +use std::ptr::{self, NonNull}; +use std::slice; +use std::sync::Arc; +use thiserror::Error; +use wasmtime_environ::entity::{ + packed_option::ReservedValue, BoxedSlice, EntityRef, EntitySet, PrimaryMap, +}; +use wasmtime_environ::wasm::{ + DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalInit, SignatureIndex, + TableElementType, WasmType, +}; +use wasmtime_environ::{ir, Module, OwnedDataInitializer, TableElements, VMOffsets}; + +/// Represents a request for a new runtime instance. +pub struct InstanceAllocationRequest<'a> { + /// The module being instantiated. + pub module: Arc, + + /// The finished (JIT) functions for the module. + pub finished_functions: &'a PrimaryMap, + + /// The imports to use for the instantiation. + pub imports: Imports<'a>, + + /// A callback for looking up shared signature indexes. + pub lookup_shared_signature: &'a dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, + + /// The host state to associate with the instance. + pub host_state: Option>, + + /// The pointer to the VM interrupts structure to use for the instance. + pub interrupts: *const VMInterrupts, + + /// The pointer to the reference activations table to use for the instance. + pub externref_activations_table: *mut VMExternRefActivationsTable, + + /// The pointer to the stack map registry to use for the instance. + pub stack_map_registry: *mut StackMapRegistry, +} + +/// An link error while instantiating a module. +#[derive(Error, Debug)] +#[error("Link error: {0}")] +pub struct LinkError(pub String); + +/// An error while instantiating a module. +#[derive(Error, Debug)] +pub enum InstantiationError { + /// Insufficient resources available for execution. + #[error("Insufficient resources: {0}")] + Resource(String), + + /// A wasm link error occured. + #[error("Failed to link module")] + Link(#[from] LinkError), + + /// A trap ocurred during instantiation, after linking. + #[error("Trap occurred during instantiation")] + Trap(Trap), +} + +/// Represents a runtime instance allocator. +/// +/// # Safety +/// +/// This trait is unsafe as it requires knowledge of Wasmtime's runtime internals to implement correctly. +pub unsafe trait InstanceAllocator: Send + Sync { + /// Allocates an instance for the given allocation request. + /// + /// # Safety + /// + /// This method is not inherently unsafe, but care must be made to ensure + /// pointers passed in the allocation request outlive the returned instance. + unsafe fn allocate( + &self, + req: InstanceAllocationRequest, + ) -> Result; + + /// Finishes the instantiation process started by an instance allocator. + /// + /// # Safety + /// + /// This method is only safe to call immediately after an instance has been allocated. + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + data_initializers: Arc<[OwnedDataInitializer]>, + ) -> Result<(), InstantiationError>; + + /// Deallocates a previously allocated instance. + /// + /// # Safety + /// + /// This function is unsafe because there are no guarantees that the given handle + /// is the only owner of the underlying instance to deallocate. + /// + /// Use extreme care when deallocating an instance so that there are no dangling instance pointers. + unsafe fn deallocate(&self, handle: &InstanceHandle); +} + +unsafe fn initialize_vmcontext( + instance: &Instance, + req: &InstanceAllocationRequest, + get_mem_def: impl Fn(DefinedMemoryIndex) -> VMMemoryDefinition, + get_table_def: impl Fn(DefinedTableIndex) -> VMTableDefinition, +) { + *instance.interrupts() = req.interrupts; + *instance.externref_activations_table() = req.externref_activations_table; + *instance.stack_map_registry() = req.stack_map_registry; + + // Initialize shared signatures + let mut ptr = instance.signature_ids_ptr(); + for (signature, _) in req.module.signatures.iter() { + *ptr = (*req.lookup_shared_signature)(signature); + ptr = ptr.add(1); + } + + // Initialize the built-in functions + ptr::write( + instance.builtin_functions_ptr() as *mut VMBuiltinFunctionsArray, + VMBuiltinFunctionsArray::initialized(), + ); + + // Initialize the imports + debug_assert_eq!(req.imports.functions.len(), req.module.num_imported_funcs); + ptr::copy( + req.imports.functions.as_ptr(), + instance.imported_functions_ptr() as *mut VMFunctionImport, + req.imports.functions.len(), + ); + debug_assert_eq!(req.imports.tables.len(), req.module.num_imported_tables); + ptr::copy( + req.imports.tables.as_ptr(), + instance.imported_tables_ptr() as *mut VMTableImport, + req.imports.tables.len(), + ); + debug_assert_eq!(req.imports.memories.len(), req.module.num_imported_memories); + ptr::copy( + req.imports.memories.as_ptr(), + instance.imported_memories_ptr() as *mut VMMemoryImport, + req.imports.memories.len(), + ); + debug_assert_eq!(req.imports.globals.len(), req.module.num_imported_globals); + ptr::copy( + req.imports.globals.as_ptr(), + instance.imported_globals_ptr() as *mut VMGlobalImport, + req.imports.globals.len(), + ); + + // Initialize the defined functions + for (index, sig) in req.module.functions.iter() { + let type_index = instance.signature_id(*sig); + + let (func_ptr, vmctx) = if let Some(def_index) = req.module.defined_func_index(index) { + ( + NonNull::new(req.finished_functions[def_index] as *mut _).unwrap(), + instance.vmctx_ptr(), + ) + } else { + let import = instance.imported_function(index); + (import.body, import.vmctx) + }; + + ptr::write( + instance.anyfunc_ptr(index), + VMCallerCheckedAnyfunc { + func_ptr, + type_index, + vmctx, + }, + ); + } + + // Initialize the defined tables + let mut ptr = instance.tables_ptr(); + for i in 0..req.module.table_plans.len() - req.module.num_imported_tables { + ptr::write(ptr, get_table_def(DefinedTableIndex::new(i))); + ptr = ptr.add(1); + } + + // Initialize the defined memories + let mut ptr = instance.memories_ptr(); + for i in 0..req.module.memory_plans.len() - req.module.num_imported_memories { + ptr::write(ptr, get_mem_def(DefinedMemoryIndex::new(i))); + ptr = ptr.add(1); + } + + // Initialize the defined globals + initialize_vmcontext_globals(instance); +} + +unsafe fn initialize_vmcontext_globals(instance: &Instance) { + let module = instance.module(); + let num_imports = module.num_imported_globals; + for (index, global) in module.globals.iter().skip(num_imports) { + let def_index = module.defined_global_index(index).unwrap(); + let to = instance.global_ptr(def_index); + + // Initialize the global before writing to it + ptr::write(to, VMGlobalDefinition::new()); + + match global.initializer { + GlobalInit::I32Const(x) => *(*to).as_i32_mut() = x, + GlobalInit::I64Const(x) => *(*to).as_i64_mut() = x, + GlobalInit::F32Const(x) => *(*to).as_f32_bits_mut() = x, + GlobalInit::F64Const(x) => *(*to).as_f64_bits_mut() = x, + GlobalInit::V128Const(x) => *(*to).as_u128_bits_mut() = x.0, + GlobalInit::GetGlobal(x) => { + let from = if let Some(def_x) = module.defined_global_index(x) { + instance.global(def_x) + } else { + *instance.imported_global(x).from + }; + *to = from; + } + GlobalInit::RefFunc(f) => { + *(*to).as_anyfunc_mut() = instance.get_caller_checked_anyfunc(f).unwrap() + as *const VMCallerCheckedAnyfunc; + } + GlobalInit::RefNullConst => match global.wasm_ty { + WasmType::FuncRef => *(*to).as_anyfunc_mut() = ptr::null(), + WasmType::ExternRef => *(*to).as_externref_mut() = None, + ty => panic!("unsupported reference type for global: {:?}", ty), + }, + GlobalInit::Import => panic!("locally-defined global initialized as import"), + } + } +} + +/// Represents the on-demand instance allocator. +#[derive(Clone)] +pub struct OnDemandInstanceAllocator { + mem_creator: Option>, +} + +impl OnDemandInstanceAllocator { + /// Creates a new on-demand instance allocator. + pub fn new(mem_creator: Option>) -> Self { + Self { mem_creator } + } + + fn create_tables(module: &Module) -> BoxedSlice { + let num_imports = module.num_imported_tables; + let mut tables: PrimaryMap = + PrimaryMap::with_capacity(module.table_plans.len() - num_imports); + for table in &module.table_plans.values().as_slice()[num_imports..] { + tables.push(Table::new(table)); + } + tables.into_boxed_slice() + } + + fn create_memories( + &self, + module: &Module, + ) -> Result>, InstantiationError> + { + let creator = self + .mem_creator + .as_deref() + .unwrap_or_else(|| &DefaultMemoryCreator); + let num_imports = module.num_imported_memories; + let mut memories: PrimaryMap = + PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); + for plan in &module.memory_plans.values().as_slice()[num_imports..] { + memories.push( + creator + .new_memory(plan) + .map_err(InstantiationError::Resource)?, + ); + } + Ok(memories.into_boxed_slice()) + } + + fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module().table_elements { + let start = Self::get_table_init_start(init, instance); + let table = instance.get_table(init.table_index); + + let size = usize::try_from(table.size()).unwrap(); + if size < start + init.elements.len() { + return Err(InstantiationError::Link(LinkError( + "table out of bounds: elements segment does not fit".to_owned(), + ))); + } + } + + Ok(()) + } + + fn get_memory_init_start(init: &OwnedDataInitializer, instance: &Instance) -> usize { + let mut start = init.location.offset; + + if let Some(base) = init.location.base { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + start += usize::try_from(val).unwrap(); + } + + start + } + + unsafe fn get_memory_slice<'instance>( + init: &OwnedDataInitializer, + instance: &'instance Instance, + ) -> &'instance mut [u8] { + let memory = if let Some(defined_memory_index) = instance + .module + .defined_memory_index(init.location.memory_index) + { + instance.memory(defined_memory_index) + } else { + let import = instance.imported_memory(init.location.memory_index); + let foreign_instance = (&mut *(import).vmctx).instance(); + let foreign_memory = &mut *(import).from; + let foreign_index = foreign_instance.memory_index(foreign_memory); + foreign_instance.memory(foreign_index) + }; + slice::from_raw_parts_mut(memory.base, memory.current_length) + } + + fn check_memory_init_bounds( + instance: &Instance, + data_initializers: &[OwnedDataInitializer], + ) -> Result<(), InstantiationError> { + for init in data_initializers { + let start = Self::get_memory_init_start(init, instance); + unsafe { + let mem_slice = Self::get_memory_slice(init, instance); + if mem_slice.get_mut(start..start + init.data.len()).is_none() { + return Err(InstantiationError::Link(LinkError( + "memory out of bounds: data segment does not fit".into(), + ))); + } + } + } + + Ok(()) + } + + fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize { + let mut start = init.offset; + + if let Some(base) = init.base { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + start += usize::try_from(val).unwrap(); + } + + start + } + + fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module().table_elements { + let start = Self::get_table_init_start(init, instance); + let table = instance.get_table(init.table_index); + + if start + .checked_add(init.elements.len()) + .map_or(true, |end| end > table.size() as usize) + { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::TableOutOfBounds, + ))); + } + + for (i, func_idx) in init.elements.iter().enumerate() { + let item = match table.element_type() { + TableElementType::Func => instance + .get_caller_checked_anyfunc(*func_idx) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc + }) + .into(), + TableElementType::Val(_) => { + assert!(*func_idx == FuncIndex::reserved_value()); + TableElement::ExternRef(None) + } + }; + table.set(u32::try_from(start + i).unwrap(), item).unwrap(); + } + } + + Ok(()) + } + + /// Initialize the table memory from the provided initializers. + fn initialize_memories( + instance: &Instance, + data_initializers: &[OwnedDataInitializer], + ) -> Result<(), InstantiationError> { + for init in data_initializers { + let memory = instance.get_memory(init.location.memory_index); + + let start = Self::get_memory_init_start(init, instance); + if start + .checked_add(init.data.len()) + .map_or(true, |end| end > memory.current_length) + { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::HeapOutOfBounds, + ))); + } + + unsafe { + let mem_slice = Self::get_memory_slice(init, instance); + let end = start + init.data.len(); + let to_init = &mut mem_slice[start..end]; + to_init.copy_from_slice(&init.data); + } + } + + Ok(()) + } +} + +unsafe impl InstanceAllocator for OnDemandInstanceAllocator { + unsafe fn allocate( + &self, + mut req: InstanceAllocationRequest, + ) -> Result { + debug_assert!(!req.externref_activations_table.is_null()); + debug_assert!(!req.stack_map_registry.is_null()); + + let memories = self.create_memories(&req.module)?; + let tables = Self::create_tables(&req.module); + + let handle = { + let instance = Instance { + module: req.module.clone(), + offsets: VMOffsets::new(std::mem::size_of::<*const u8>() as u8, &req.module), + memories, + tables, + dropped_elements: RefCell::new(EntitySet::with_capacity( + req.module.passive_elements.len(), + )), + dropped_data: RefCell::new(EntitySet::with_capacity(req.module.passive_data.len())), + host_state: req.host_state.take(), + vmctx: VMContext {}, + }; + let layout = instance.alloc_layout(); + let instance_ptr = alloc::alloc(layout) as *mut Instance; + if instance_ptr.is_null() { + alloc::handle_alloc_error(layout); + } + ptr::write(instance_ptr, instance); + InstanceHandle::new(instance_ptr) + }; + + let instance = handle.instance(); + initialize_vmcontext( + instance, + &req, + &|index| instance.memories[index].vmmemory(), + &|index| instance.tables[index].vmtable(), + ); + + Ok(handle) + } + + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + data_initializers: Arc<[OwnedDataInitializer]>, + ) -> Result<(), InstantiationError> { + // Check initializer bounds before initializing anything. Only do this + // when bulk memory is disabled, since the bulk memory proposal changes + // instantiation such that the intermediate results of failed + // initializations are visible. + if !is_bulk_memory { + Self::check_table_init_bounds(handle.instance())?; + Self::check_memory_init_bounds(handle.instance(), data_initializers.as_ref())?; + } + + // Apply fallible initializers. Note that this can "leak" state even if + // it fails. + Self::initialize_tables(handle.instance())?; + Self::initialize_memories(handle.instance(), data_initializers.as_ref())?; + + Ok(()) + } + + unsafe fn deallocate(&self, handle: &InstanceHandle) { + let instance = handle.instance(); + let layout = instance.alloc_layout(); + ptr::drop_in_place(instance as *const Instance as *mut Instance); + alloc::dealloc(instance as *const Instance as *mut _, layout); + } +} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 97e3934ef0c8..3bafaddd42ce 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -37,7 +37,10 @@ pub mod libcalls; pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; -pub use crate::instance::{InstanceHandle, InstantiationError, LinkError}; +pub use crate::instance::{ + InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstantiationError, LinkError, + OnDemandInstanceAllocator, +}; pub use crate::jit_int::GdbJitImageRegistration; pub use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; pub use crate::mmap::Mmap; diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index 23d368802abe..8232b66df4ac 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -723,12 +723,12 @@ impl VMContext { /// This is unsafe because it doesn't work on just any `VMContext`, it must /// be a `VMContext` allocated as part of an `Instance`. #[inline] - pub unsafe fn host_state(&self) -> &dyn Any { + pub unsafe fn host_state(&self) -> Option<&dyn Any> { self.instance().host_state() } } -/// +/// Trampoline function pointer type. pub type VMTrampoline = unsafe extern "C" fn( *mut VMContext, // callee vmctx *mut VMContext, // caller vmctx diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 0cb776ffdc16..a277bcafae08 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -14,6 +14,25 @@ use wasmtime_environ::settings::{self, Configurable, SetError}; use wasmtime_environ::{isa, isa::TargetIsa, Tunables}; use wasmtime_jit::{native, CompilationStrategy, Compiler}; use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; +use wasmtime_runtime::{InstanceAllocator, RuntimeMemoryCreator}; + +/// Represents the module instance allocation strategy to use. +#[derive(Clone)] +pub enum InstanceAllocationStrategy { + /// The on-demand instance allocation strategy. + /// + /// Resources related to a module instance are allocated at instantiation time and + /// immediately deallocated when the `Store` referencing the instance is dropped. + /// + /// This is the default allocation strategy for Wasmtime. + OnDemand, +} + +impl Default for InstanceAllocationStrategy { + fn default() -> Self { + Self::OnDemand + } +} /// Global configuration options used to create an [`Engine`](crate::Engine) /// and customize its behavior. @@ -29,7 +48,8 @@ pub struct Config { #[cfg(feature = "cache")] pub(crate) cache_config: CacheConfig, pub(crate) profiler: Arc, - pub(crate) memory_creator: Option, + pub(crate) mem_creator: Option>, + pub(crate) instance_allocator: Option>, pub(crate) max_wasm_stack: usize, pub(crate) features: WasmFeatures, } @@ -69,7 +89,8 @@ impl Config { #[cfg(feature = "cache")] cache_config: CacheConfig::new_cache_disabled(), profiler: Arc::new(NullProfilerAgent), - memory_creator: None, + mem_creator: None, + instance_allocator: None, max_wasm_stack: 1 << 20, features: WasmFeatures { reference_types: true, @@ -430,9 +451,23 @@ impl Config { Ok(self) } - /// Sets a custom memory creator + /// Sets a custom memory creator. + /// + /// Custom memory creators are used when creating host `Memory` objects or when + /// creating instance linear memories for the default instance allocation strategy. pub fn with_host_memory(&mut self, mem_creator: Arc) -> &mut Self { - self.memory_creator = Some(MemoryCreatorProxy { mem_creator }); + self.mem_creator = Some(Arc::new(MemoryCreatorProxy(mem_creator))); + self + } + + /// Sets the instance allocation strategy to use. + pub fn with_instance_allocation_strategy( + &mut self, + strategy: InstanceAllocationStrategy, + ) -> &mut Self { + self.instance_allocator = match strategy { + InstanceAllocationStrategy::OnDemand => None, + }; self } diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index b57936d67d98..6c2fb33397f9 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -1593,7 +1593,7 @@ macro_rules! impl_into_func { $( $args: WasmTy, )* R: WasmRet, { - let state = (*vmctx).host_state(); + let state = (*vmctx).host_state().unwrap(); // Double-check ourselves in debug mode, but we control // the `Any` here so an unsafe downcast should also // work. diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index b375b4884889..a4b62c05c1c6 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -6,15 +6,16 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::{EntityIndex, FuncIndex, GlobalIndex, MemoryIndex, TableIndex}; use wasmtime_jit::CompiledModule; use wasmtime_runtime::{ - Imports, InstantiationError, StackMapRegistry, VMContext, VMExternRefActivationsTable, - VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport, + Imports, InstanceAllocationRequest, InstantiationError, OnDemandInstanceAllocator, + StackMapRegistry, VMContext, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, + VMGlobalImport, VMMemoryImport, VMTableImport, }; -fn instantiate( +fn instantiate_module( store: &Store, compiled_module: &CompiledModule, all_modules: &[CompiledModule], - imports: &mut ImportsBuilder<'_>, + imports: &ImportsBuilder, ) -> Result { let env_module = compiled_module.module(); @@ -56,7 +57,7 @@ fn instantiate( EntityIndex::Instance(_) => unimplemented!(), } } - instantiate(store, compiled_module, all_modules, &mut builder)?; + instantiate_module(store, compiled_module, all_modules, &builder)?; } // Register the module just before instantiation to ensure we have a @@ -65,16 +66,27 @@ fn instantiate( store.register_module(compiled_module); let config = store.engine().config(); + + let default_allocator = OnDemandInstanceAllocator::new(config.mem_creator.clone()); + + let allocator = config + .instance_allocator + .as_deref() + .unwrap_or_else(|| &default_allocator); + let instance = unsafe { - let instance = compiled_module.instantiate( - imports.imports(), - &store.lookup_shared_signature(compiled_module.module()), - config.memory_creator.as_ref().map(|a| a as _), - store.interrupts(), - Box::new(()), - store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - store.stack_map_registry() as *const StackMapRegistry as *mut _, - )?; + let handle = allocator.allocate(InstanceAllocationRequest { + module: compiled_module.module().clone(), + finished_functions: compiled_module.finished_functions(), + imports: imports.imports(), + lookup_shared_signature: &store.lookup_shared_signature(compiled_module.module()), + host_state: None, + interrupts: store.interrupts(), + externref_activations_table: store.externref_activations_table() + as *const VMExternRefActivationsTable + as *mut _, + stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, + })?; // After we've created the `InstanceHandle` we still need to run // initialization to set up data/elements/etc. We do this after adding @@ -83,11 +95,13 @@ fn instantiate( // initializers may have run which placed elements into other instance's // tables. This means that from this point on, regardless of whether // initialization is successful, we need to keep the instance alive. - let instance = store.add_instance(instance); - instance + let instance = store.add_instance(handle); + + allocator .initialize( + &instance.handle, config.features.bulk_memory, - &compiled_module.data_initializers(), + compiled_module.data_initializers().clone(), ) .map_err(|e| -> Error { match e { @@ -113,7 +127,7 @@ fn instantiate( }; let vmctx_ptr = instance.handle.vmctx_ptr(); unsafe { - super::func::invoke_wasm_and_catch_traps(vmctx_ptr, store, || { + crate::func::invoke_wasm_and_catch_traps(vmctx_ptr, store, || { mem::transmute::< *const VMFunctionBody, unsafe extern "C" fn(*mut VMContext, *mut VMContext), @@ -145,7 +159,7 @@ fn instantiate( /// call any code or execute anything! #[derive(Clone)] pub struct Instance { - pub(crate) handle: StoreInstanceHandle, + handle: StoreInstanceHandle, module: Module, } @@ -203,9 +217,9 @@ impl Instance { /// [inst]: https://webassembly.github.io/spec/core/exec/modules.html#exec-instantiation /// [issue]: https://github.com/bytecodealliance/wasmtime/issues/727 /// [`ExternType`]: crate::ExternType - pub fn new(store: &Store, module: &Module, imports: &[Extern]) -> Result { + pub fn new(store: &Store, module: &Module, imports: &[Extern]) -> Result { if !Engine::same(store.engine(), module.engine()) { - bail!("cross-`Engine` instantiation is not currently supported"); + bail!("cross-engine instantiation is not currently supported"); } let mut builder = ImportsBuilder::new(module.compiled_module().module(), store); @@ -223,11 +237,11 @@ impl Instance { } } builder.validate_all_imports_provided()?; - let handle = instantiate( + let handle = instantiate_module( store, module.compiled_module(), &module.compiled, - &mut builder, + &builder, )?; Ok(Instance { @@ -413,7 +427,7 @@ impl<'a> ImportsBuilder<'a> { Ok(()) } - fn imports(&self) -> Imports<'_> { + fn imports(&self) -> Imports { Imports { tables: self.tables.values().as_slice(), globals: self.globals.values().as_slice(), diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 5317631cc4f7..257b5c956909 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -13,8 +13,8 @@ use std::sync::Arc; use wasmtime_environ::wasm; use wasmtime_jit::{CompiledModule, ModuleCode}; use wasmtime_runtime::{ - InstanceHandle, RuntimeMemoryCreator, SignalHandler, StackMapRegistry, TrapInfo, VMExternRef, - VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, + InstanceHandle, OnDemandInstanceAllocator, SignalHandler, StackMapRegistry, TrapInfo, + VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, }; /// A `Store` is a collection of WebAssembly instances and host-defined items. @@ -122,15 +122,6 @@ impl Store { &self.inner.engine } - /// Returns an optional reference to a ['RuntimeMemoryCreator'] - pub(crate) fn memory_creator(&self) -> Option<&dyn RuntimeMemoryCreator> { - self.engine() - .config() - .memory_creator - .as_ref() - .map(|x| x as _) - } - pub(crate) fn signatures(&self) -> &RefCell { &self.inner.signatures } @@ -416,9 +407,18 @@ impl fmt::Debug for Store { impl Drop for StoreInner { fn drop(&mut self) { - for instance in self.instances.get_mut().iter() { + let config = self.engine.config(); + + let default_allocator = OnDemandInstanceAllocator::new(config.mem_creator.clone()); + + let allocator = config + .instance_allocator + .as_deref() + .unwrap_or_else(|| &default_allocator); + + for instance in self.instances.borrow().iter() { unsafe { - instance.dealloc(); + allocator.deallocate(instance); } } } diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs index 874431861cca..9d5b72e49edf 100644 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ b/crates/wasmtime/src/trampoline/create_handle.rs @@ -9,8 +9,8 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::Module; use wasmtime_runtime::{ - Imports, InstanceHandle, StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, - VMFunctionImport, + Imports, InstanceAllocationRequest, InstanceAllocator, OnDemandInstanceAllocator, + StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, }; pub(crate) fn create_handle( @@ -23,20 +23,24 @@ pub(crate) fn create_handle( let mut imports = Imports::default(); imports.functions = func_imports; let module = Arc::new(module); - let module2 = module.clone(); unsafe { - let handle = InstanceHandle::new( - module, - &finished_functions, + // Use the on-demand allocator when creating handles associated with host objects + let allocator = OnDemandInstanceAllocator::new(store.engine().config().mem_creator.clone()); + + let handle = allocator.allocate(InstanceAllocationRequest { + module: module.clone(), + finished_functions: &finished_functions, imports, - store.memory_creator(), - &store.lookup_shared_signature(&module2), - state, - store.interrupts(), - store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - store.stack_map_registry() as *const StackMapRegistry as *mut _, - )?; + lookup_shared_signature: &store.lookup_shared_signature(&module), + host_state: Some(state), + interrupts: store.interrupts(), + externref_activations_table: store.externref_activations_table() + as *const VMExternRefActivationsTable + as *mut _, + stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, + })?; + Ok(store.add_instance(handle)) } } diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 0050f7333046..53b641ab561c 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -72,6 +72,7 @@ unsafe extern "C" fn stub_fn( let instance = InstanceHandle::from_vmctx(vmctx); let state = &instance .host_state() + .unwrap() .downcast_ref::() .expect("state"); (state.func)(caller_vmctx, values_vec) diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs index 974be5b63b18..97382e4d31f6 100644 --- a/crates/wasmtime/src/trampoline/memory.rs +++ b/crates/wasmtime/src/trampoline/memory.rs @@ -54,9 +54,7 @@ impl RuntimeLinearMemory for LinearMemoryProxy { } #[derive(Clone)] -pub(crate) struct MemoryCreatorProxy { - pub(crate) mem_creator: Arc, -} +pub(crate) struct MemoryCreatorProxy(pub Arc); impl RuntimeMemoryCreator for MemoryCreatorProxy { fn new_memory(&self, plan: &MemoryPlan) -> Result, String> { @@ -65,7 +63,7 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy { MemoryStyle::Static { bound } => Some(bound as u64 * WASM_PAGE_SIZE as u64), MemoryStyle::Dynamic => None, }; - self.mem_creator + self.0 .new_memory(ty, reserved_size_in_bytes, plan.offset_guard_size) .map(|mem| Box::new(LinearMemoryProxy { mem }) as Box) } diff --git a/examples/interrupt.c b/examples/interrupt.c index d8ccd9a8f422..5af56e1c900c 100644 --- a/examples/interrupt.c +++ b/examples/interrupt.c @@ -42,6 +42,7 @@ static void* helper(void *_handle) { printf("Sending an interrupt\n"); wasmtime_interrupt_handle_interrupt(handle); wasmtime_interrupt_handle_delete(handle); + return 0; } static void spawn_interrupt(wasmtime_interrupt_handle_t *handle) {