From 28a8416aae693c0aee69b6990c810bb414e4f981 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 25 Nov 2020 16:10:09 -0800 Subject: [PATCH] Refactor module instantiation in the runtime. This commit refactors module instantiation in the runtime to allow for different instance allocation strategy implementations. It adds an `InstanceAllocator` trait with the current implementation put behind the `OnDemandInstanceAllocator` struct. The Wasmtime API has been updated to allow a `Config` to have an instance allocation strategy set which will determine how instances get allocated. This change is in preparation for an alternative *pooling* instance allocator that can reserve all needed host process address space in advance. This commit also makes changes to the `wasmtime_environ` crate to represent compiled modules in a way that reduces copying at instantiation time. --- cranelift/wasm/src/module_translator.rs | 2 + crates/environ/src/data_structures.rs | 2 +- crates/environ/src/module.rs | 53 +- crates/environ/src/module_environ.rs | 39 +- crates/jit/src/instantiate.rs | 134 ++-- crates/runtime/src/instance.rs | 594 ++---------------- crates/runtime/src/instance/allocator.rs | 536 ++++++++++++++++ crates/runtime/src/lib.rs | 5 +- crates/runtime/src/vmcontext.rs | 2 +- crates/wasmtime/src/config.rs | 52 +- crates/wasmtime/src/instance.rs | 37 +- crates/wasmtime/src/store.rs | 18 +- .../wasmtime/src/trampoline/create_handle.rs | 35 +- crates/wasmtime/src/trampoline/memory.rs | 6 +- 14 files changed, 829 insertions(+), 686 deletions(-) create mode 100644 crates/runtime/src/instance/allocator.rs diff --git a/cranelift/wasm/src/module_translator.rs b/cranelift/wasm/src/module_translator.rs index ed71d7ce1ca2..819834f20e0b 100644 --- a/cranelift/wasm/src/module_translator.rs +++ b/cranelift/wasm/src/module_translator.rs @@ -101,6 +101,8 @@ pub fn translate_module<'data>( Payload::DataCountSection { count, range } => { validator.data_count_section(count, &range)?; + + // NOTE: the count here is the total segment count, not the passive segment count environ.reserve_passive_data(count)?; } diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 2fc9e9ecd05d..07f2aedaeca9 100644 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -20,7 +20,7 @@ pub mod isa { } pub mod entity { - pub use cranelift_entity::{packed_option, BoxedSlice, EntityRef, PrimaryMap}; + pub use cranelift_entity::{packed_option, BoxedSlice, EntityRef, EntitySet, PrimaryMap}; } pub mod wasm { diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index b25682acf215..957feb55f0a7 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -158,13 +158,19 @@ pub struct Module { pub table_elements: Vec, /// WebAssembly passive elements. - pub passive_elements: HashMap>, + pub passive_elements: Vec>, + + /// The map from passive element index (element segment index space) to index in `passive_elements`. + pub passive_elements_map: HashMap, /// WebAssembly passive data segments. #[serde(with = "passive_data_serde")] - pub passive_data: HashMap>, + pub passive_data: Vec>, - /// WebAssembly table initializers. + /// The map from passive data index (data segment index space) to index in `passive_data`. + pub passive_data_map: HashMap, + + /// WebAssembly function names. pub func_names: HashMap, /// Types declared in the wasm module. @@ -272,7 +278,8 @@ impl Module { /// Get the given passive element, if it exists. pub fn get_passive_element(&self, index: ElemIndex) -> Option<&[FuncIndex]> { - self.passive_elements.get(&index).map(|es| &**es) + let index = *self.passive_elements_map.get(&index)?; + Some(self.passive_elements[index].as_ref()) } /// Convert a `DefinedFuncIndex` into a `FuncIndex`. @@ -419,47 +426,45 @@ pub struct InstanceSignature { } mod passive_data_serde { - use super::{Arc, DataIndex, HashMap}; - use serde::{de::MapAccess, de::Visitor, ser::SerializeMap, Deserializer, Serializer}; + use super::Arc; + use serde::{de::SeqAccess, de::Visitor, ser::SerializeSeq, Deserializer, Serializer}; use std::fmt; - pub(super) fn serialize( - data: &HashMap>, - ser: S, - ) -> Result + pub(super) fn serialize(data: &Vec>, ser: S) -> Result where S: Serializer, { - let mut map = ser.serialize_map(Some(data.len()))?; - for (k, v) in data { - map.serialize_entry(k, v.as_ref())?; + let mut seq = ser.serialize_seq(Some(data.len()))?; + for v in data { + seq.serialize_element(v.as_ref())?; } - map.end() + seq.end() } struct PassiveDataVisitor; impl<'de> Visitor<'de> for PassiveDataVisitor { - type Value = HashMap>; + type Value = Vec>; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a passive_data map") + formatter.write_str("a passive data sequence") } - fn visit_map(self, mut access: M) -> Result + + fn visit_seq(self, mut access: M) -> Result where - M: MapAccess<'de>, + M: SeqAccess<'de>, { - let mut map = HashMap::with_capacity(access.size_hint().unwrap_or(0)); - while let Some((key, value)) = access.next_entry::<_, Vec>()? { - map.insert(key, value.into()); + let mut data = Vec::with_capacity(access.size_hint().unwrap_or(0)); + while let Some(value) = access.next_element::>()? { + data.push(value.into()); } - Ok(map) + Ok(data) } } - pub(super) fn deserialize<'de, D>(de: D) -> Result>, D::Error> + pub(super) fn deserialize<'de, D>(de: D) -> Result>, D::Error> where D: Deserializer<'de>, { - de.deserialize_map(PassiveDataVisitor) + de.deserialize_seq(PassiveDataVisitor) } } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 9d1f89cb1f82..800455358032 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -710,11 +710,13 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data elem_index: ElemIndex, segments: Box<[FuncIndex]>, ) -> WasmResult<()> { + let index = self.result.module.passive_elements.len(); + self.result.module.passive_elements.push(segments); let old = self .result .module - .passive_elements - .insert(elem_index, segments); + .passive_elements_map + .insert(elem_index, index); debug_assert!( old.is_none(), "should never get duplicate element indices, that would be a bug in `cranelift_wasm`'s \ @@ -782,17 +784,21 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data Ok(()) } - fn reserve_passive_data(&mut self, count: u32) -> WasmResult<()> { - self.result.module.passive_data.reserve(count as usize); + fn reserve_passive_data(&mut self, _count: u32) -> WasmResult<()> { + // Note: the count passed in here is the *total* segment count + // There is no way to reserve for just the passive segments as they are discovered when iterating the data section entries + // Given that the total segment count might be much larger than the passive count, do not reserve Ok(()) } fn declare_passive_data(&mut self, data_index: DataIndex, data: &'data [u8]) -> WasmResult<()> { + let index = self.result.module.passive_data.len(); + self.result.module.passive_data.push(Arc::from(data)); let old = self .result .module - .passive_data - .insert(data_index, Arc::from(data)); + .passive_data_map + .insert(data_index, index); debug_assert!( old.is_none(), "a module can't have duplicate indices, this would be a cranelift-wasm bug" @@ -1088,3 +1094,24 @@ pub struct DataInitializer<'data> { /// The initialization data. pub data: &'data [u8], } + +/// Similar to `DataInitializer`, but owns its own copy of the data rather +/// than holding a slice of the original module. +#[derive(Serialize, Deserialize)] +pub struct OwnedDataInitializer { + /// The location where the initialization is to be performed. + pub location: DataInitializerLocation, + + /// The initialization data. + pub data: Box<[u8]>, +} + +impl OwnedDataInitializer { + /// Creates a new owned data initializer from a borrowed data initializer. + pub fn new(borrowed: DataInitializer<'_>) -> Self { + Self { + location: borrowed.location.clone(), + data: borrowed.data.into(), + } + } +} diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 655432e00f47..a1647cde5a9b 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -11,7 +11,6 @@ use object::File as ObjectFile; #[cfg(feature = "parallel-compilation")] use rayon::prelude::*; use serde::{Deserialize, Serialize}; -use std::any::Any; use std::ops::Range; use std::sync::Arc; use thiserror::Error; @@ -22,16 +21,11 @@ use wasmtime_environ::wasm::{ DefinedFuncIndex, InstanceTypeIndex, ModuleTypeIndex, SignatureIndex, WasmFuncType, }; use wasmtime_environ::{ - CompileError, DataInitializer, DataInitializerLocation, DebugInfoData, FunctionAddressMap, - InstanceSignature, Module, ModuleEnvironment, ModuleSignature, ModuleTranslation, - StackMapInformation, TrapInformation, + CompileError, DebugInfoData, FunctionAddressMap, InstanceSignature, Module, ModuleEnvironment, + ModuleSignature, ModuleTranslation, OwnedDataInitializer, StackMapInformation, TrapInformation, }; use wasmtime_profiling::ProfilingAgent; -use wasmtime_runtime::{ - GdbJitImageRegistration, Imports, InstanceHandle, InstantiationError, RuntimeMemoryCreator, - StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, VMInterrupts, - VMSharedSignatureIndex, VMTrampoline, -}; +use wasmtime_runtime::{GdbJitImageRegistration, InstantiationError, VMFunctionBody, VMTrampoline}; /// An error condition while setting up a wasm instance, be it validation, /// compilation, or instantiation. @@ -59,7 +53,8 @@ pub enum SetupError { #[derive(Serialize, Deserialize)] pub struct CompilationArtifacts { /// Module metadata. - module: Module, + #[serde(with = "arc_serde")] + module: Arc, /// ELF image with functions code. obj: Box<[u8]>, @@ -68,7 +63,8 @@ pub struct CompilationArtifacts { unwind_info: Box<[ObjectUnwindInfo]>, /// Data initiailizers. - data_initializers: Box<[OwnedDataInitializer]>, + #[serde(with = "arc_slice_serde")] + data_initializers: Arc<[OwnedDataInitializer]>, /// Descriptions of compiled functions funcs: PrimaryMap, @@ -134,7 +130,7 @@ impl CompilationArtifacts { .into_iter() .map(OwnedDataInitializer::new) .collect::>() - .into_boxed_slice(); + .into(); let obj = obj.write().map_err(|_| { SetupError::Instantiate(InstantiationError::Resource( @@ -143,7 +139,7 @@ impl CompilationArtifacts { })?; Ok(CompilationArtifacts { - module, + module: Arc::new(module), obj: obj.into_boxed_slice(), unwind_info: unwind_info.into_boxed_slice(), data_initializers, @@ -208,7 +204,6 @@ pub struct ModuleCode { /// A compiled wasm module, ready to be instantiated. pub struct CompiledModule { artifacts: CompilationArtifacts, - module: Arc, code: Arc, finished_functions: FinishedFunctions, trampolines: PrimaryMap, @@ -267,7 +262,6 @@ impl CompiledModule { let finished_functions = FinishedFunctions(finished_functions); Ok(Arc::new(Self { - module: Arc::new(artifacts.module.clone()), artifacts, code: Arc::new(ModuleCode { code_memory, @@ -278,62 +272,24 @@ impl CompiledModule { })) } - /// Crate an `Instance` from this `CompiledModule`. - /// - /// Note that if only one instance of this module is needed, it may be more - /// efficient to call the top-level `instantiate`, since that avoids copying - /// the data initializers. - /// - /// # Unsafety - /// - /// See `InstanceHandle::new` - pub unsafe fn instantiate( - &self, - imports: Imports<'_>, - lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, - mem_creator: Option<&dyn RuntimeMemoryCreator>, - interrupts: *const VMInterrupts, - host_state: Box, - externref_activations_table: *mut VMExternRefActivationsTable, - stack_map_registry: *mut StackMapRegistry, - ) -> Result { - InstanceHandle::new( - self.module.clone(), - &self.finished_functions.0, - imports, - mem_creator, - lookup_shared_signature, - host_state, - interrupts, - externref_activations_table, - stack_map_registry, - ) - } /// Extracts `CompilationArtifacts` from the compiled module. pub fn compilation_artifacts(&self) -> &CompilationArtifacts { &self.artifacts } - /// Returns data initializers to pass to `InstanceHandle::initialize` - pub fn data_initializers(&self) -> Vec> { - self.artifacts - .data_initializers - .iter() - .map(|init| DataInitializer { - location: init.location.clone(), - data: &*init.data, - }) - .collect() + /// Returns the data initializers from the compiled module. + pub fn data_initializers(&self) -> &Arc<[OwnedDataInitializer]> { + &self.artifacts.data_initializers } /// Return a reference-counting pointer to a module. pub fn module(&self) -> &Arc { - &self.module + &self.artifacts.module } /// Return a reference to a mutable module (if possible). pub fn module_mut(&mut self) -> Option<&mut Module> { - Arc::get_mut(&mut self.module) + Arc::get_mut(&mut self.artifacts.module) } /// Returns the map of all finished JIT functions compiled for this module @@ -470,26 +426,6 @@ impl SymbolizeContext { } } -/// Similar to `DataInitializer`, but owns its own copy of the data rather -/// than holding a slice of the original module. -#[derive(Clone, Serialize, Deserialize)] -pub struct OwnedDataInitializer { - /// The location where the initialization is to be performed. - location: DataInitializerLocation, - - /// The initialization data. - data: Box<[u8]>, -} - -impl OwnedDataInitializer { - fn new(borrowed: DataInitializer<'_>) -> Self { - Self { - location: borrowed.location.clone(), - data: borrowed.data.to_vec().into_boxed_slice(), - } - } -} - fn create_dbg_image( obj: Vec, code_range: (*const u8, usize), @@ -586,3 +522,45 @@ impl From> for DebugInfo { } } } + +mod arc_serde { + use super::Arc; + use serde::{de::Deserialize, ser::Serialize, Deserializer, Serializer}; + + pub(super) fn serialize(arc: &Arc, ser: S) -> Result + where + S: Serializer, + T: Serialize, + { + (**arc).serialize(ser) + } + + pub(super) fn deserialize<'de, D, T>(de: D) -> Result, D::Error> + where + D: Deserializer<'de>, + T: Deserialize<'de>, + { + Ok(Arc::new(T::deserialize(de)?)) + } +} + +mod arc_slice_serde { + use super::Arc; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub(super) fn serialize(arc: &Arc<[T]>, ser: S) -> Result + where + S: Serializer, + T: Serialize, + { + (**arc).serialize(ser) + } + + pub(super) fn deserialize<'de, D, T>(de: D) -> Result, D::Error> + where + D: Deserializer<'de>, + T: Deserialize<'de>, + { + Ok(Vec::::deserialize(de)?.into()) + } +} diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 1df1cfb64065..cdb892842f1a 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -4,12 +4,11 @@ use crate::export::Export; use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; -use crate::imports::Imports; -use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory, RuntimeMemoryCreator}; +use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; use crate::vmcontext::{ - VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, + VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, }; @@ -17,23 +16,24 @@ use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable}; use indexmap::IndexMap; use memoffset::offset_of; use more_asserts::assert_lt; -use std::alloc::{self, Layout}; +use std::alloc::Layout; use std::any::Any; use std::cell::RefCell; -use std::collections::HashMap; use std::convert::TryFrom; use std::ptr::NonNull; use std::rc::Rc; use std::sync::Arc; use std::{mem, ptr, slice}; -use thiserror::Error; -use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap}; +use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, EntitySet}; use wasmtime_environ::wasm::{ - DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, - ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, - TableElementType, TableIndex, WasmType, + DataIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, ElemIndex, EntityIndex, + FuncIndex, GlobalIndex, MemoryIndex, TableElementType, TableIndex, }; -use wasmtime_environ::{ir, DataInitializer, Module, ModuleType, TableElements, VMOffsets}; +use wasmtime_environ::{ir, Module, VMOffsets}; + +mod allocator; + +pub use allocator::*; /// Runtime representation of an instance value, which erases all `Instance` /// information since instances are just a collection of values. @@ -56,14 +56,13 @@ pub(crate) struct Instance { /// WebAssembly table data. tables: BoxedSlice, - /// Passive elements in this instantiation. As `elem.drop`s happen, these - /// entries get removed. A missing entry is considered equivalent to an - /// empty slice. - passive_elements: RefCell>>, + /// Stores the dropped passive element segments in this instantiation by index. + /// If the index is present in the set, the segment has been dropped. + dropped_elements: RefCell>, - /// Passive data segments from our module. As `data.drop`s happen, entries - /// get removed. A missing entry is considered equivalent to an empty slice. - passive_data: RefCell>>, + /// Stores the dropped passive data segments in this instantiation by index. + /// If the index is present in the set, the segment has been dropped. + dropped_data: RefCell>, /// Hosts can store arbitrary per-instance information here. host_state: Box, @@ -551,11 +550,21 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init let table = self.get_table(table_index); - let passive_elements = self.passive_elements.borrow(); - let elem = passive_elements - .get(&elem_index) - .map(|e| &**e) - .unwrap_or_else(|| &[]); + let elem_index = self.module.passive_elements_map.get(&elem_index); + let elem = match elem_index { + Some(index) => { + if self + .dropped_elements + .borrow() + .contains(ElemIndex::new(*index)) + { + &[] + } else { + self.module.passive_elements[*index].as_ref() + } + } + None => &[], + }; if src .checked_add(len) @@ -567,8 +576,14 @@ impl Instance { // TODO(#983): investigate replacing this get/set loop with a `memcpy`. for (dst, src) in (dst..dst + len).zip(src..src + len) { + let elem = self + .get_caller_checked_anyfunc(elem[src as usize]) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut _ + }); + table - .set(dst, TableElement::FuncRef(elem[src as usize])) + .set(dst, TableElement::FuncRef(elem)) .expect("should never panic because we already did the bounds check above"); } @@ -579,10 +594,14 @@ impl Instance { pub(crate) fn elem_drop(&self, elem_index: ElemIndex) { // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop - let mut passive_elements = self.passive_elements.borrow_mut(); - passive_elements.remove(&elem_index); - // Note that we don't check that we actually removed an element because - // dropping a non-passive element is a no-op (not a trap). + if let Some(index) = self.module.passive_elements_map.get(&elem_index) { + self.dropped_elements + .borrow_mut() + .insert(ElemIndex::new(*index)); + } + + // Note that we don't check that we actually removed a segment because + // dropping a non-passive segment is a no-op (not a trap). } /// Do a `memory.copy` @@ -701,10 +720,17 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init let memory = self.get_memory(memory_index); - let passive_data = self.passive_data.borrow(); - let data = passive_data - .get(&data_index) - .map_or(&[][..], |data| &**data); + let data_index = self.module.passive_data_map.get(&data_index); + let data = match data_index { + Some(index) => { + if self.dropped_data.borrow().contains(DataIndex::new(*index)) { + &[] + } else { + self.module.passive_data[*index].as_ref() + } + } + None => &[], + }; if src .checked_add(len) @@ -729,8 +755,14 @@ impl Instance { /// Drop the given data segment, truncating its length to zero. pub(crate) fn data_drop(&self, data_index: DataIndex) { - let mut passive_data = self.passive_data.borrow_mut(); - passive_data.remove(&data_index); + if let Some(index) = self.module.passive_data_map.get(&data_index) { + self.dropped_data + .borrow_mut() + .insert(DataIndex::new(*index)); + } + + // Note that we don't check that we actually removed a segment because + // dropping a non-passive segment is a no-op (not a trap). } /// Get a table by index regardless of whether it is locally-defined or an @@ -780,197 +812,8 @@ pub struct InstanceHandle { } impl InstanceHandle { - /// Create a new `InstanceHandle` pointing at a new `Instance`. - /// - /// # Unsafety - /// - /// This method is not necessarily inherently unsafe to call, but in general - /// the APIs of an `Instance` are quite unsafe and have not been really - /// audited for safety that much. As a result the unsafety here on this - /// method is a low-overhead way of saying "this is an extremely unsafe type - /// to work with". - /// - /// Extreme care must be taken when working with `InstanceHandle` and it's - /// recommended to have relatively intimate knowledge of how it works - /// internally if you'd like to do so. If possible it's recommended to use - /// the `wasmtime` crate API rather than this type since that is vetted for - /// safety. - /// - /// It is your responsibility to ensure that the given raw - /// `externref_activations_table` and `stack_map_registry` outlive this - /// instance. - pub unsafe fn new( - module: Arc, - finished_functions: &PrimaryMap, - imports: Imports, - mem_creator: Option<&dyn RuntimeMemoryCreator>, - lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, - host_state: Box, - interrupts: *const VMInterrupts, - externref_activations_table: *mut VMExternRefActivationsTable, - stack_map_registry: *mut StackMapRegistry, - ) -> Result { - debug_assert!(!externref_activations_table.is_null()); - debug_assert!(!stack_map_registry.is_null()); - - let tables = create_tables(&module); - let memories = create_memories(&module, mem_creator.unwrap_or(&DefaultMemoryCreator {}))?; - - let vmctx_tables = tables - .values() - .map(Table::vmtable) - .collect::>() - .into_boxed_slice(); - - let vmctx_memories = memories - .values() - .map(|a| a.vmmemory()) - .collect::>() - .into_boxed_slice(); - - let vmctx_globals = create_globals(&module); - - let offsets = VMOffsets::new(mem::size_of::<*const u8>() as u8, &module); - - let passive_data = RefCell::new(module.passive_data.clone()); - - let handle = { - let instance = Instance { - module, - offsets, - memories, - tables, - passive_elements: Default::default(), - passive_data, - host_state, - vmctx: VMContext {}, - }; - let layout = instance.alloc_layout(); - let instance_ptr = alloc::alloc(layout) as *mut Instance; - if instance_ptr.is_null() { - alloc::handle_alloc_error(layout); - } - ptr::write(instance_ptr, instance); - InstanceHandle { - instance: instance_ptr, - } - }; - let instance = handle.instance(); - - let mut ptr = instance.signature_ids_ptr(); - for sig in handle.module().types.values() { - *ptr = match sig { - ModuleType::Function(sig) => lookup_shared_signature(*sig), - _ => VMSharedSignatureIndex::new(u32::max_value()), - }; - ptr = ptr.add(1); - } - - debug_assert_eq!(imports.functions.len(), handle.module().num_imported_funcs); - ptr::copy( - imports.functions.as_ptr(), - instance.imported_functions_ptr() as *mut VMFunctionImport, - imports.functions.len(), - ); - debug_assert_eq!(imports.tables.len(), handle.module().num_imported_tables); - ptr::copy( - imports.tables.as_ptr(), - instance.imported_tables_ptr() as *mut VMTableImport, - imports.tables.len(), - ); - debug_assert_eq!( - imports.memories.len(), - handle.module().num_imported_memories - ); - ptr::copy( - imports.memories.as_ptr(), - instance.imported_memories_ptr() as *mut VMMemoryImport, - imports.memories.len(), - ); - debug_assert_eq!(imports.globals.len(), handle.module().num_imported_globals); - ptr::copy( - imports.globals.as_ptr(), - instance.imported_globals_ptr() as *mut VMGlobalImport, - imports.globals.len(), - ); - ptr::copy( - vmctx_tables.values().as_slice().as_ptr(), - instance.tables_ptr() as *mut VMTableDefinition, - vmctx_tables.len(), - ); - ptr::copy( - vmctx_memories.values().as_slice().as_ptr(), - instance.memories_ptr() as *mut VMMemoryDefinition, - vmctx_memories.len(), - ); - ptr::copy( - vmctx_globals.values().as_slice().as_ptr(), - instance.globals_ptr() as *mut VMGlobalDefinition, - vmctx_globals.len(), - ); - ptr::write( - instance.builtin_functions_ptr() as *mut VMBuiltinFunctionsArray, - VMBuiltinFunctionsArray::initialized(), - ); - *instance.interrupts() = interrupts; - *instance.externref_activations_table() = externref_activations_table; - *instance.stack_map_registry() = stack_map_registry; - - for (index, sig) in instance.module.functions.iter() { - let type_index = lookup_shared_signature(*sig); - - let (func_ptr, vmctx) = - if let Some(def_index) = instance.module.defined_func_index(index) { - ( - NonNull::new(finished_functions[def_index] as *mut _).unwrap(), - instance.vmctx_ptr(), - ) - } else { - let import = instance.imported_function(index); - (import.body, import.vmctx) - }; - - ptr::write( - instance.anyfunc_ptr(index), - VMCallerCheckedAnyfunc { - func_ptr, - type_index, - vmctx, - }, - ); - } - - // Perform infallible initialization in this constructor, while fallible - // initialization is deferred to the `initialize` method. - initialize_passive_elements(instance); - initialize_globals(instance); - - Ok(handle) - } - - /// Finishes the instantiation process started by `Instance::new`. - /// - /// Only safe to call immediately after instantiation. - pub unsafe fn initialize( - &self, - is_bulk_memory: bool, - data_initializers: &[DataInitializer<'_>], - ) -> Result<(), InstantiationError> { - // Check initializer bounds before initializing anything. Only do this - // when bulk memory is disabled, since the bulk memory proposal changes - // instantiation such that the intermediate results of failed - // initializations are visible. - if !is_bulk_memory { - check_table_init_bounds(self.instance())?; - check_memory_init_bounds(self.instance(), data_initializers)?; - } - - // Apply fallible initializers. Note that this can "leak" state even if - // it fails. - initialize_tables(self.instance())?; - initialize_memories(self.instance(), data_initializers)?; - - Ok(()) + pub(crate) unsafe fn new(instance: *mut Instance) -> Self { + Self { instance } } /// Create a new `InstanceHandle` pointing at the instance @@ -1126,305 +969,4 @@ impl InstanceHandle { instance: self.instance, } } - - /// Deallocates memory associated with this instance. - /// - /// Note that this is unsafe because there might be other handles to this - /// `InstanceHandle` elsewhere, and there's nothing preventing usage of - /// this handle after this function is called. - pub unsafe fn dealloc(&self) { - let instance = self.instance(); - let layout = instance.alloc_layout(); - ptr::drop_in_place(self.instance); - alloc::dealloc(self.instance.cast(), layout); - } -} - -fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module().table_elements { - let start = get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - let size = usize::try_from(table.size()).unwrap(); - if size < start + init.elements.len() { - return Err(InstantiationError::Link(LinkError( - "table out of bounds: elements segment does not fit".to_owned(), - ))); - } - } - - Ok(()) -} - -/// Compute the offset for a memory data initializer. -fn get_memory_init_start(init: &DataInitializer<'_>, instance: &Instance) -> usize { - let mut start = init.location.offset; - - if let Some(base) = init.location.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start -} - -/// Return a byte-slice view of a memory's data. -unsafe fn get_memory_slice<'instance>( - init: &DataInitializer<'_>, - instance: &'instance Instance, -) -> &'instance mut [u8] { - let memory = if let Some(defined_memory_index) = instance - .module - .defined_memory_index(init.location.memory_index) - { - instance.memory(defined_memory_index) - } else { - let import = instance.imported_memory(init.location.memory_index); - let foreign_instance = (&mut *(import).vmctx).instance(); - let foreign_memory = &mut *(import).from; - let foreign_index = foreign_instance.memory_index(foreign_memory); - foreign_instance.memory(foreign_index) - }; - slice::from_raw_parts_mut(memory.base, memory.current_length) -} - -fn check_memory_init_bounds( - instance: &Instance, - data_initializers: &[DataInitializer<'_>], -) -> Result<(), InstantiationError> { - for init in data_initializers { - let start = get_memory_init_start(init, instance); - unsafe { - let mem_slice = get_memory_slice(init, instance); - if mem_slice.get_mut(start..start + init.data.len()).is_none() { - return Err(InstantiationError::Link(LinkError( - "memory out of bounds: data segment does not fit".into(), - ))); - } - } - } - - Ok(()) -} - -/// Allocate memory for just the tables of the current module. -fn create_tables(module: &Module) -> BoxedSlice { - let num_imports = module.num_imported_tables; - let mut tables: PrimaryMap = - PrimaryMap::with_capacity(module.table_plans.len() - num_imports); - for table in &module.table_plans.values().as_slice()[num_imports..] { - tables.push(Table::new(table)); - } - tables.into_boxed_slice() -} - -/// Compute the offset for a table element initializer. -fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize { - let mut start = init.offset; - - if let Some(base) = init.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start -} - -/// Initialize the table memory from the provided initializers. -fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module().table_elements { - let start = get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - if start - .checked_add(init.elements.len()) - .map_or(true, |end| end > table.size() as usize) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::TableOutOfBounds, - ))); - } - - for (i, func_idx) in init.elements.iter().enumerate() { - let item = match table.element_type() { - TableElementType::Func => instance - .get_caller_checked_anyfunc(*func_idx) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc - }) - .into(), - TableElementType::Val(_) => { - assert!(*func_idx == FuncIndex::reserved_value()); - TableElement::ExternRef(None) - } - }; - table.set(u32::try_from(start + i).unwrap(), item).unwrap(); - } - } - - Ok(()) -} - -/// Initialize the `Instance::passive_elements` map by resolving the -/// `Module::passive_elements`'s `FuncIndex`s into `VMCallerCheckedAnyfunc`s for -/// this instance. -fn initialize_passive_elements(instance: &Instance) { - let mut passive_elements = instance.passive_elements.borrow_mut(); - debug_assert!( - passive_elements.is_empty(), - "should only be called once, at initialization time" - ); - - passive_elements.extend( - instance - .module - .passive_elements - .iter() - .filter(|(_, segments)| !segments.is_empty()) - .map(|(idx, segments)| { - ( - *idx, - segments - .iter() - .map(|s| { - instance.get_caller_checked_anyfunc(*s).map_or( - ptr::null_mut(), - |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut _ - }, - ) - }) - .collect(), - ) - }), - ); -} - -/// Allocate memory for just the memories of the current module. -fn create_memories( - module: &Module, - mem_creator: &dyn RuntimeMemoryCreator, -) -> Result>, InstantiationError> { - let num_imports = module.num_imported_memories; - let mut memories: PrimaryMap = - PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); - for plan in &module.memory_plans.values().as_slice()[num_imports..] { - memories.push( - mem_creator - .new_memory(plan) - .map_err(InstantiationError::Resource)?, - ); - } - Ok(memories.into_boxed_slice()) -} - -/// Initialize the table memory from the provided initializers. -fn initialize_memories( - instance: &Instance, - data_initializers: &[DataInitializer<'_>], -) -> Result<(), InstantiationError> { - for init in data_initializers { - let memory = instance.get_memory(init.location.memory_index); - - let start = get_memory_init_start(init, instance); - if start - .checked_add(init.data.len()) - .map_or(true, |end| end > memory.current_length) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::HeapOutOfBounds, - ))); - } - - unsafe { - let mem_slice = get_memory_slice(init, instance); - let end = start + init.data.len(); - let to_init = &mut mem_slice[start..end]; - to_init.copy_from_slice(init.data); - } - } - - Ok(()) -} - -/// Allocate memory for just the globals of the current module, -/// with initializers applied. -fn create_globals(module: &Module) -> BoxedSlice { - let num_imports = module.num_imported_globals; - let mut vmctx_globals = PrimaryMap::with_capacity(module.globals.len() - num_imports); - - for _ in &module.globals.values().as_slice()[num_imports..] { - vmctx_globals.push(VMGlobalDefinition::new()); - } - - vmctx_globals.into_boxed_slice() -} - -fn initialize_globals(instance: &Instance) { - let module = instance.module(); - let num_imports = module.num_imported_globals; - for (index, global) in module.globals.iter().skip(num_imports) { - let def_index = module.defined_global_index(index).unwrap(); - unsafe { - let to = instance.global_ptr(def_index); - match global.initializer { - GlobalInit::I32Const(x) => *(*to).as_i32_mut() = x, - GlobalInit::I64Const(x) => *(*to).as_i64_mut() = x, - GlobalInit::F32Const(x) => *(*to).as_f32_bits_mut() = x, - GlobalInit::F64Const(x) => *(*to).as_f64_bits_mut() = x, - GlobalInit::V128Const(x) => *(*to).as_u128_bits_mut() = x.0, - GlobalInit::GetGlobal(x) => { - let from = if let Some(def_x) = module.defined_global_index(x) { - instance.global(def_x) - } else { - *instance.imported_global(x).from - }; - *to = from; - } - GlobalInit::RefFunc(f) => { - *(*to).as_anyfunc_mut() = instance.get_caller_checked_anyfunc(f).unwrap() - as *const VMCallerCheckedAnyfunc; - } - GlobalInit::RefNullConst => match global.wasm_ty { - WasmType::FuncRef => *(*to).as_anyfunc_mut() = ptr::null(), - WasmType::ExternRef => *(*to).as_externref_mut() = None, - ty => panic!("unsupported reference type for global: {:?}", ty), - }, - GlobalInit::Import => panic!("locally-defined global initialized as import"), - } - } - } -} - -/// An link error while instantiating a module. -#[derive(Error, Debug)] -#[error("Link error: {0}")] -pub struct LinkError(pub String); - -/// An error while instantiating a module. -#[derive(Error, Debug)] -pub enum InstantiationError { - /// Insufficient resources available for execution. - #[error("Insufficient resources: {0}")] - Resource(String), - - /// A wasm link error occured. - #[error("Failed to link module")] - Link(#[from] LinkError), - - /// A trap ocurred during instantiation, after linking. - #[error("Trap occurred during instantiation")] - Trap(Trap), } diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs new file mode 100644 index 000000000000..395b7c120fd0 --- /dev/null +++ b/crates/runtime/src/instance/allocator.rs @@ -0,0 +1,536 @@ +use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; +use crate::imports::Imports; +use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator}; +use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory}; +use crate::table::{Table, TableElement}; +use crate::traphandlers::Trap; +use crate::vmcontext::{ + VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, + VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryDefinition, VMMemoryImport, + VMSharedSignatureIndex, VMTableDefinition, VMTableImport, +}; +use std::alloc; +use std::any::Any; +use std::cell::RefCell; +use std::convert::TryFrom; +use std::ptr::{self, NonNull}; +use std::slice; +use std::sync::Arc; +use thiserror::Error; +use wasmtime_environ::entity::{ + packed_option::ReservedValue, BoxedSlice, EntityRef, EntitySet, PrimaryMap, +}; +use wasmtime_environ::wasm::{ + DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalInit, SignatureIndex, + TableElementType, WasmType, +}; +use wasmtime_environ::{ir, Module, ModuleType, OwnedDataInitializer, TableElements, VMOffsets}; + +/// Represents a request for a new runtime instance. +pub struct InstanceAllocationRequest<'a> { + /// The module being instantiated. + pub module: Arc, + + /// The finished (JIT) functions for the module. + pub finished_functions: &'a PrimaryMap, + + /// The imports to use for the instantiation. + pub imports: Imports<'a>, + + /// A callback for looking up shared signature indexes. + pub lookup_shared_signature: &'a dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, + + /// The host state to associate with the instance. + pub host_state: Box, + + /// The pointer to the VM interrupts structure to use for the instance. + pub interrupts: *const VMInterrupts, + + /// The pointer to the reference activations table to use for the instance. + pub externref_activations_table: *mut VMExternRefActivationsTable, + + /// The pointer to the stack map registry to use for the instance. + pub stack_map_registry: *mut StackMapRegistry, +} + +/// An link error while instantiating a module. +#[derive(Error, Debug)] +#[error("Link error: {0}")] +pub struct LinkError(pub String); + +/// An error while instantiating a module. +#[derive(Error, Debug)] +pub enum InstantiationError { + /// Insufficient resources available for execution. + #[error("Insufficient resources: {0}")] + Resource(String), + + /// A wasm link error occured. + #[error("Failed to link module")] + Link(#[from] LinkError), + + /// A trap ocurred during instantiation, after linking. + #[error("Trap occurred during instantiation")] + Trap(Trap), +} + +/// Represents a runtime instance allocator. +/// +/// # Safety +/// +/// This trait is unsafe as it requires knowledge of Wasmtime's runtime internals to implement correctly. +pub unsafe trait InstanceAllocator: Send + Sync { + /// Allocates an instance for the given allocation request. + /// + /// # Safety + /// + /// This method is not inherently unsafe, but care must be made to ensure + /// pointers passed in the allocation request outlive the returned instance. + unsafe fn allocate( + &self, + req: InstanceAllocationRequest, + ) -> Result; + + /// Finishes the instantiation process started by an instance allocator. + /// + /// # Safety + /// + /// This method is only safe to call immediately after an instance has been allocated. + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + data_initializers: &Arc<[OwnedDataInitializer]>, + ) -> Result<(), InstantiationError>; + + /// Deallocates a previously allocated instance. + /// + /// # Safety + /// + /// This function is unsafe because there are no guarantees that the given handle + /// is the only owner of the underlying instance to deallocate. + /// + /// Use extreme care when deallocating an instance so that there are no dangling instance pointers. + unsafe fn deallocate(&self, handle: &InstanceHandle); +} + +unsafe fn initialize_vmcontext( + instance: &Instance, + functions: &[VMFunctionImport], + tables: &[VMTableImport], + memories: &[VMMemoryImport], + globals: &[VMGlobalImport], + finished_functions: &PrimaryMap, + lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, + interrupts: *const VMInterrupts, + externref_activations_table: *mut VMExternRefActivationsTable, + stack_map_registry: *mut StackMapRegistry, + get_mem_def: impl Fn(DefinedMemoryIndex) -> VMMemoryDefinition, + get_table_def: impl Fn(DefinedTableIndex) -> VMTableDefinition, +) { + let module = &instance.module; + + *instance.interrupts() = interrupts; + *instance.externref_activations_table() = externref_activations_table; + *instance.stack_map_registry() = stack_map_registry; + + // Initialize shared signatures + let mut ptr = instance.signature_ids_ptr(); + for sig in module.types.values() { + *ptr = match sig { + ModuleType::Function(sig) => lookup_shared_signature(*sig), + _ => VMSharedSignatureIndex::new(u32::max_value()), + }; + ptr = ptr.add(1); + } + + // Initialize the built-in functions + ptr::write( + instance.builtin_functions_ptr() as *mut VMBuiltinFunctionsArray, + VMBuiltinFunctionsArray::initialized(), + ); + + // Initialize the imports + debug_assert_eq!(functions.len(), module.num_imported_funcs); + ptr::copy( + functions.as_ptr(), + instance.imported_functions_ptr() as *mut VMFunctionImport, + functions.len(), + ); + debug_assert_eq!(tables.len(), module.num_imported_tables); + ptr::copy( + tables.as_ptr(), + instance.imported_tables_ptr() as *mut VMTableImport, + tables.len(), + ); + debug_assert_eq!(memories.len(), module.num_imported_memories); + ptr::copy( + memories.as_ptr(), + instance.imported_memories_ptr() as *mut VMMemoryImport, + memories.len(), + ); + debug_assert_eq!(globals.len(), module.num_imported_globals); + ptr::copy( + globals.as_ptr(), + instance.imported_globals_ptr() as *mut VMGlobalImport, + globals.len(), + ); + + // Initialize the defined functions + for (index, sig) in instance.module.functions.iter() { + let type_index = lookup_shared_signature(*sig); + + let (func_ptr, vmctx) = if let Some(def_index) = instance.module.defined_func_index(index) { + ( + NonNull::new(finished_functions[def_index] as *mut _).unwrap(), + instance.vmctx_ptr(), + ) + } else { + let import = instance.imported_function(index); + (import.body, import.vmctx) + }; + + ptr::write( + instance.anyfunc_ptr(index), + VMCallerCheckedAnyfunc { + func_ptr, + type_index, + vmctx, + }, + ); + } + + // Initialize the defined tables + let mut ptr = instance.tables_ptr(); + for i in 0..module.table_plans.len() - module.num_imported_tables { + ptr::write(ptr, get_table_def(DefinedTableIndex::new(i))); + ptr = ptr.add(1); + } + + // Initialize the defined memories + let mut ptr = instance.memories_ptr(); + for i in 0..module.memory_plans.len() - module.num_imported_memories { + ptr::write(ptr, get_mem_def(DefinedMemoryIndex::new(i))); + ptr = ptr.add(1); + } + + // Initialize the defined globals + initialize_vmcontext_globals(instance); +} + +unsafe fn initialize_vmcontext_globals(instance: &Instance) { + let module = &instance.module; + let num_imports = module.num_imported_globals; + for (index, global) in module.globals.iter().skip(num_imports) { + let def_index = module.defined_global_index(index).unwrap(); + let to = instance.global_ptr(def_index); + + // Initialize the global before writing to it + ptr::write(to, VMGlobalDefinition::new()); + + match global.initializer { + GlobalInit::I32Const(x) => *(*to).as_i32_mut() = x, + GlobalInit::I64Const(x) => *(*to).as_i64_mut() = x, + GlobalInit::F32Const(x) => *(*to).as_f32_bits_mut() = x, + GlobalInit::F64Const(x) => *(*to).as_f64_bits_mut() = x, + GlobalInit::V128Const(x) => *(*to).as_u128_bits_mut() = x.0, + GlobalInit::GetGlobal(x) => { + let from = if let Some(def_x) = module.defined_global_index(x) { + instance.global(def_x) + } else { + *instance.imported_global(x).from + }; + *to = from; + } + GlobalInit::RefFunc(f) => { + *(*to).as_anyfunc_mut() = instance.get_caller_checked_anyfunc(f).unwrap() + as *const VMCallerCheckedAnyfunc; + } + GlobalInit::RefNullConst => match global.wasm_ty { + WasmType::FuncRef => *(*to).as_anyfunc_mut() = ptr::null(), + WasmType::ExternRef => *(*to).as_externref_mut() = None, + ty => panic!("unsupported reference type for global: {:?}", ty), + }, + GlobalInit::Import => panic!("locally-defined global initialized as import"), + } + } +} + +/// Represents the on-demand instance allocator. +#[derive(Clone)] +pub struct OnDemandInstanceAllocator { + mem_creator: Option>, +} + +impl OnDemandInstanceAllocator { + /// Creates a new on-demand instance allocator. + pub fn new(mem_creator: Option>) -> Self { + Self { mem_creator } + } + + fn create_tables(module: &Module) -> BoxedSlice { + let num_imports = module.num_imported_tables; + let mut tables: PrimaryMap = + PrimaryMap::with_capacity(module.table_plans.len() - num_imports); + for table in &module.table_plans.values().as_slice()[num_imports..] { + tables.push(Table::new(table)); + } + tables.into_boxed_slice() + } + + fn create_memories( + &self, + module: &Module, + ) -> Result>, InstantiationError> + { + let creator = self + .mem_creator + .as_deref() + .unwrap_or_else(|| &DefaultMemoryCreator); + let num_imports = module.num_imported_memories; + let mut memories: PrimaryMap = + PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); + for plan in &module.memory_plans.values().as_slice()[num_imports..] { + memories.push( + creator + .new_memory(plan) + .map_err(InstantiationError::Resource)?, + ); + } + Ok(memories.into_boxed_slice()) + } + + fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module.table_elements { + let start = Self::get_table_init_start(init, instance); + let table = instance.get_table(init.table_index); + + let size = usize::try_from(table.size()).unwrap(); + if size < start + init.elements.len() { + return Err(InstantiationError::Link(LinkError( + "table out of bounds: elements segment does not fit".to_owned(), + ))); + } + } + + Ok(()) + } + + fn get_memory_init_start(init: &OwnedDataInitializer, instance: &Instance) -> usize { + let mut start = init.location.offset; + + if let Some(base) = init.location.base { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + start += usize::try_from(val).unwrap(); + } + + start + } + + unsafe fn get_memory_slice<'instance>( + init: &OwnedDataInitializer, + instance: &'instance Instance, + ) -> &'instance mut [u8] { + let memory = if let Some(defined_memory_index) = instance + .module + .defined_memory_index(init.location.memory_index) + { + instance.memory(defined_memory_index) + } else { + let import = instance.imported_memory(init.location.memory_index); + let foreign_instance = (&mut *(import).vmctx).instance(); + let foreign_memory = &mut *(import).from; + let foreign_index = foreign_instance.memory_index(foreign_memory); + foreign_instance.memory(foreign_index) + }; + slice::from_raw_parts_mut(memory.base, memory.current_length) + } + + fn check_memory_init_bounds( + instance: &Instance, + data_initializers: &[OwnedDataInitializer], + ) -> Result<(), InstantiationError> { + for init in data_initializers { + let start = Self::get_memory_init_start(init, instance); + unsafe { + let mem_slice = Self::get_memory_slice(init, instance); + if mem_slice.get_mut(start..start + init.data.len()).is_none() { + return Err(InstantiationError::Link(LinkError( + "memory out of bounds: data segment does not fit".into(), + ))); + } + } + } + + Ok(()) + } + + fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize { + let mut start = init.offset; + + if let Some(base) = init.base { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + start += usize::try_from(val).unwrap(); + } + + start + } + + fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module.table_elements { + let start = Self::get_table_init_start(init, instance); + let table = instance.get_table(init.table_index); + + if start + .checked_add(init.elements.len()) + .map_or(true, |end| end > table.size() as usize) + { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::TableOutOfBounds, + ))); + } + + for (i, func_idx) in init.elements.iter().enumerate() { + let item = match table.element_type() { + TableElementType::Func => instance + .get_caller_checked_anyfunc(*func_idx) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc + }) + .into(), + TableElementType::Val(_) => { + assert!(*func_idx == FuncIndex::reserved_value()); + TableElement::ExternRef(None) + } + }; + table.set(u32::try_from(start + i).unwrap(), item).unwrap(); + } + } + + Ok(()) + } + + /// Initialize the table memory from the provided initializers. + fn initialize_memories( + instance: &Instance, + data_initializers: &[OwnedDataInitializer], + ) -> Result<(), InstantiationError> { + for init in data_initializers { + let memory = instance.get_memory(init.location.memory_index); + + let start = Self::get_memory_init_start(init, instance); + if start + .checked_add(init.data.len()) + .map_or(true, |end| end > memory.current_length) + { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::HeapOutOfBounds, + ))); + } + + unsafe { + let mem_slice = Self::get_memory_slice(init, instance); + let end = start + init.data.len(); + let to_init = &mut mem_slice[start..end]; + to_init.copy_from_slice(&init.data); + } + } + + Ok(()) + } +} + +unsafe impl InstanceAllocator for OnDemandInstanceAllocator { + unsafe fn allocate( + &self, + req: InstanceAllocationRequest, + ) -> Result { + debug_assert!(!req.externref_activations_table.is_null()); + debug_assert!(!req.stack_map_registry.is_null()); + + let memories = self.create_memories(&req.module)?; + let tables = Self::create_tables(&req.module); + + let handle = { + let instance = Instance { + module: req.module.clone(), + offsets: VMOffsets::new(std::mem::size_of::<*const u8>() as u8, &req.module), + memories, + tables, + dropped_elements: RefCell::new(EntitySet::with_capacity( + req.module.passive_elements.len(), + )), + dropped_data: RefCell::new(EntitySet::with_capacity(req.module.passive_data.len())), + host_state: req.host_state, + vmctx: VMContext {}, + }; + let layout = instance.alloc_layout(); + let instance_ptr = alloc::alloc(layout) as *mut Instance; + if instance_ptr.is_null() { + alloc::handle_alloc_error(layout); + } + ptr::write(instance_ptr, instance); + InstanceHandle::new(instance_ptr) + }; + + let instance = handle.instance(); + initialize_vmcontext( + instance, + req.imports.functions, + req.imports.tables, + req.imports.memories, + req.imports.globals, + req.finished_functions, + req.lookup_shared_signature, + req.interrupts, + req.externref_activations_table, + req.stack_map_registry, + &|index| instance.memories[index].vmmemory(), + &|index| instance.tables[index].vmtable(), + ); + + Ok(handle) + } + + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + data_initializers: &Arc<[OwnedDataInitializer]>, + ) -> Result<(), InstantiationError> { + // Check initializer bounds before initializing anything. Only do this + // when bulk memory is disabled, since the bulk memory proposal changes + // instantiation such that the intermediate results of failed + // initializations are visible. + if !is_bulk_memory { + Self::check_table_init_bounds(handle.instance())?; + Self::check_memory_init_bounds(handle.instance(), data_initializers.as_ref())?; + } + + // Apply fallible initializers. Note that this can "leak" state even if + // it fails. + Self::initialize_tables(handle.instance())?; + Self::initialize_memories(handle.instance(), data_initializers.as_ref())?; + + Ok(()) + } + + unsafe fn deallocate(&self, handle: &InstanceHandle) { + let instance = handle.instance(); + let layout = instance.alloc_layout(); + ptr::drop_in_place(instance as *const Instance as *mut Instance); + alloc::dealloc(instance as *const Instance as *mut _, layout); + } +} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 00539c36b4c5..eff31fdeead5 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -37,7 +37,10 @@ pub mod libcalls; pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; -pub use crate::instance::{InstanceHandle, InstantiationError, LinkError, RuntimeInstance}; +pub use crate::instance::{ + InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstantiationError, LinkError, + OnDemandInstanceAllocator, RuntimeInstance, +}; pub use crate::jit_int::GdbJitImageRegistration; pub use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; pub use crate::mmap::Mmap; diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index c20a42b5ed4a..f4dffeee2c3b 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -750,7 +750,7 @@ impl VMContext { } } -/// +/// Trampoline function pointer type. pub type VMTrampoline = unsafe extern "C" fn( *mut VMContext, // callee vmctx *mut VMContext, // caller vmctx diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 6ddaa022c4d7..60adfb3005f7 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -14,6 +14,25 @@ use wasmtime_environ::settings::{self, Configurable, SetError}; use wasmtime_environ::{isa, isa::TargetIsa, Tunables}; use wasmtime_jit::{native, CompilationStrategy, Compiler}; use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; +use wasmtime_runtime::{InstanceAllocator, OnDemandInstanceAllocator}; + +/// Represents the module instance allocation strategy to use. +#[derive(Clone)] +pub enum InstanceAllocationStrategy { + /// The on-demand instance allocation strategy. + /// + /// Resources related to a module instance are allocated at instantiation time and + /// immediately deallocated when the `Store` referencing the instance is dropped. + /// + /// This is the default allocation strategy for Wasmtime. + OnDemand, +} + +impl Default for InstanceAllocationStrategy { + fn default() -> Self { + Self::OnDemand + } +} /// Global configuration options used to create an [`Engine`](crate::Engine) /// and customize its behavior. @@ -29,7 +48,10 @@ pub struct Config { #[cfg(feature = "cache")] pub(crate) cache_config: CacheConfig, pub(crate) profiler: Arc, - pub(crate) memory_creator: Option, + pub(crate) instance_allocator: Option>, + // The default instance allocator is used for instantiating host objects + // and for module instatiation when `instance_allocator` is None + pub(crate) default_instance_allocator: OnDemandInstanceAllocator, pub(crate) max_wasm_stack: usize, pub(crate) features: WasmFeatures, pub(crate) wasm_backtrace_details_env_used: bool, @@ -73,7 +95,8 @@ impl Config { #[cfg(feature = "cache")] cache_config: CacheConfig::new_cache_disabled(), profiler: Arc::new(NullProfilerAgent), - memory_creator: None, + instance_allocator: None, + default_instance_allocator: OnDemandInstanceAllocator::new(None), max_wasm_stack: 1 << 20, wasm_backtrace_details_env_used: false, features: WasmFeatures { @@ -504,9 +527,24 @@ impl Config { Ok(self) } - /// Sets a custom memory creator + /// Sets a custom memory creator. + /// + /// Custom memory creators are used when creating host `Memory` objects or when + /// creating instance linear memories for the on-demand instance allocation strategy. pub fn with_host_memory(&mut self, mem_creator: Arc) -> &mut Self { - self.memory_creator = Some(MemoryCreatorProxy { mem_creator }); + self.default_instance_allocator = + OnDemandInstanceAllocator::new(Some(Arc::new(MemoryCreatorProxy(mem_creator)))); + self + } + + /// Sets the instance allocation strategy to use. + pub fn with_instance_allocation_strategy( + &mut self, + strategy: InstanceAllocationStrategy, + ) -> &mut Self { + self.instance_allocator = match strategy { + InstanceAllocationStrategy::OnDemand => None, + }; self } @@ -728,6 +766,12 @@ impl Config { let isa = self.target_isa(); Compiler::new(isa, self.strategy, self.tunables.clone(), self.features) } + + pub(crate) fn instance_allocator(&self) -> &dyn InstanceAllocator { + self.instance_allocator + .as_deref() + .unwrap_or(&self.default_instance_allocator) + } } fn round_up_to_pages(val: u64) -> u64 { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 2ddaee2a6834..87936fd535ee 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -12,9 +12,9 @@ use wasmtime_environ::wasm::{ }; use wasmtime_environ::Initializer; use wasmtime_runtime::{ - Imports, InstantiationError, RuntimeInstance, StackMapRegistry, VMContext, - VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, - VMTableImport, + Imports, InstanceAllocationRequest, InstantiationError, RuntimeInstance, StackMapRegistry, + VMContext, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, + VMMemoryImport, VMTableImport, }; /// An instantiated WebAssembly module. @@ -492,18 +492,26 @@ impl<'a> Instantiator<'a> { // compiled JIT code within the `Store`. self.store.register_module(&self.cur.module); - let config = self.store.engine().config(); unsafe { - let instance = compiled_module.instantiate( - self.cur.build(), - &self.store.lookup_shared_signature(self.cur.module.types()), - config.memory_creator.as_ref().map(|a| a as _), - self.store.interrupts(), - Box::new(()), - self.store.externref_activations_table() as *const VMExternRefActivationsTable + let config = self.store.engine().config(); + + let allocator = config.instance_allocator(); + + let instance = allocator.allocate(InstanceAllocationRequest { + module: compiled_module.module().clone(), + finished_functions: compiled_module.finished_functions(), + imports: self.cur.build(), + lookup_shared_signature: &self + .store + .lookup_shared_signature(self.cur.module.types()), + host_state: Box::new(()), + interrupts: self.store.interrupts(), + externref_activations_table: self.store.externref_activations_table() + as *const VMExternRefActivationsTable as *mut _, - self.store.stack_map_registry() as *const StackMapRegistry as *mut _, - )?; + stack_map_registry: self.store.stack_map_registry() as *const StackMapRegistry + as *mut _, + })?; // After we've created the `InstanceHandle` we still need to run // initialization to set up data/elements/etc. We do this after adding @@ -513,8 +521,9 @@ impl<'a> Instantiator<'a> { // tables. This means that from this point on, regardless of whether // initialization is successful, we need to keep the instance alive. let instance = self.store.add_instance(instance); - instance + allocator .initialize( + &instance.handle, config.features.bulk_memory, &compiled_module.data_initializers(), ) diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index c66aed23a317..c5dde7dad40e 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -18,8 +18,8 @@ use std::task::{Context, Poll}; use wasmtime_environ::wasm; use wasmtime_jit::{CompiledModule, ModuleCode, TypeTables}; use wasmtime_runtime::{ - InstanceHandle, RuntimeMemoryCreator, SignalHandler, StackMapRegistry, TrapInfo, VMContext, - VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, + InstanceHandle, SignalHandler, StackMapRegistry, TrapInfo, VMContext, VMExternRef, + VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, }; /// A `Store` is a collection of WebAssembly instances and host-defined items. @@ -254,15 +254,6 @@ impl Store { &self.inner.engine } - /// Returns an optional reference to a ['RuntimeMemoryCreator'] - pub(crate) fn memory_creator(&self) -> Option<&dyn RuntimeMemoryCreator> { - self.engine() - .config() - .memory_creator - .as_ref() - .map(|x| x as _) - } - pub(crate) fn signatures(&self) -> &RefCell { &self.inner.signatures } @@ -964,9 +955,10 @@ impl fmt::Debug for Store { impl Drop for StoreInner { fn drop(&mut self) { - for instance in self.instances.get_mut().iter() { + let allocator = self.engine.config().instance_allocator(); + for instance in self.instances.borrow().iter() { unsafe { - instance.dealloc(); + allocator.deallocate(instance); } } } diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs index f597987e1b7b..ff088b9aa17d 100644 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ b/crates/wasmtime/src/trampoline/create_handle.rs @@ -9,15 +9,15 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::Module; use wasmtime_runtime::{ - Imports, InstanceHandle, StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, - VMFunctionImport, VMSharedSignatureIndex, + Imports, InstanceAllocationRequest, InstanceAllocator, StackMapRegistry, + VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMSharedSignatureIndex, }; pub(crate) fn create_handle( module: Module, store: &Store, finished_functions: PrimaryMap, - state: Box, + host_state: Box, func_imports: &[VMFunctionImport], shared_signature_id: Option, ) -> Result { @@ -26,17 +26,24 @@ pub(crate) fn create_handle( let module = Arc::new(module); unsafe { - let handle = InstanceHandle::new( - module, - &finished_functions, - imports, - store.memory_creator(), - &|_| shared_signature_id.unwrap(), - state, - store.interrupts(), - store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - store.stack_map_registry() as *const StackMapRegistry as *mut _, - )?; + // Use the default allocator when creating handles associated with host objects + let handle = store + .engine() + .config() + .default_instance_allocator + .allocate(InstanceAllocationRequest { + module: module.clone(), + finished_functions: &finished_functions, + imports, + lookup_shared_signature: &|_| shared_signature_id.unwrap(), + host_state, + interrupts: store.interrupts(), + externref_activations_table: store.externref_activations_table() + as *const VMExternRefActivationsTable + as *mut _, + stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, + })?; + Ok(store.add_instance(handle)) } } diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs index 0bfb2bfff9b8..af2b9547920b 100644 --- a/crates/wasmtime/src/trampoline/memory.rs +++ b/crates/wasmtime/src/trampoline/memory.rs @@ -54,9 +54,7 @@ impl RuntimeLinearMemory for LinearMemoryProxy { } #[derive(Clone)] -pub(crate) struct MemoryCreatorProxy { - pub(crate) mem_creator: Arc, -} +pub(crate) struct MemoryCreatorProxy(pub Arc); impl RuntimeMemoryCreator for MemoryCreatorProxy { fn new_memory(&self, plan: &MemoryPlan) -> Result, String> { @@ -65,7 +63,7 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy { MemoryStyle::Static { bound } => Some(bound as u64 * WASM_PAGE_SIZE as u64), MemoryStyle::Dynamic => None, }; - self.mem_creator + self.0 .new_memory(ty, reserved_size_in_bytes, plan.offset_guard_size) .map(|mem| Box::new(LinearMemoryProxy { mem }) as Box) }