diff --git a/winch/codegen/src/abi/mod.rs b/winch/codegen/src/abi/mod.rs index c2fa325c1f50..c205f7ea54c4 100644 --- a/winch/codegen/src/abi/mod.rs +++ b/winch/codegen/src/abi/mod.rs @@ -212,21 +212,6 @@ impl ABIOperand { _ => unreachable!(), } } - - /// Get the register associated to this [`ABIOperand`]. - pub fn get_reg(&self) -> Option { - match *self { - ABIOperand::Reg { reg, .. } => Some(reg), - _ => None, - } - } - - /// Get the type associated to this [`ABIOperand`]. - pub fn ty(&self) -> WasmValType { - match *self { - ABIOperand::Reg { ty, .. } | ABIOperand::Stack { ty, .. } => ty, - } - } } /// Information about the [`ABIOperand`] information used in [`ABISig`]. diff --git a/winch/codegen/src/codegen/bounds.rs b/winch/codegen/src/codegen/bounds.rs index 1904c8b68128..37ebd1088327 100644 --- a/winch/codegen/src/codegen/bounds.rs +++ b/winch/codegen/src/codegen/bounds.rs @@ -4,7 +4,7 @@ use super::env::HeapData; use crate::{ abi::{scratch, vmctx}, - codegen::CodeGenContext, + codegen::{CodeGenContext, Emission}, isa::reg::{writable, Reg}, masm::{IntCmpKind, MacroAssembler, OperandSize, RegImm, TrapCode}, stack::TypedReg, @@ -82,7 +82,7 @@ impl Index { /// Loads the bounds of the dynamic heap. pub(crate) fn load_dynamic_heap_bounds( - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, heap: &HeapData, ptr_size: OperandSize, @@ -149,7 +149,7 @@ pub(crate) fn ensure_index_and_offset( /// criteria is in bounds. pub(crate) fn load_heap_addr_checked( masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ptr_size: OperandSize, heap: &HeapData, enable_spectre_mitigation: bool, diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rs index ee3b4415cd5c..2cbb06bf96b5 100644 --- a/winch/codegen/src/codegen/call.rs +++ b/winch/codegen/src/codegen/call.rs @@ -58,7 +58,7 @@ use crate::{ abi::{scratch, vmctx, ABIOperand, ABISig, RetArea}, - codegen::{BuiltinFunction, BuiltinType, Callee, CodeGenContext}, + codegen::{BuiltinFunction, BuiltinType, Callee, CodeGenContext, Emission}, masm::{ CalleeKind, ContextArgs, MacroAssembler, MemMoveDirection, OperandSize, SPOffset, VMContextLoc, @@ -85,7 +85,7 @@ impl FnCall { pub fn emit( env: &mut FuncEnv, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, callee: Callee, ) { let (kind, callee_context) = Self::lower(env, context.vmoffsets, &callee, context, masm); @@ -129,7 +129,7 @@ impl FnCall { env: &mut FuncEnv, vmoffsets: &VMOffsets, callee: &Callee, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) -> (CalleeKind, ContextArgs) { let ptr = vmoffsets.ptr.size(); @@ -177,7 +177,7 @@ impl FnCall { fn lower_import( index: FuncIndex, sig: &ABISig, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, vmoffsets: &VMOffsets

, ) -> (CalleeKind, ContextArgs) { @@ -204,7 +204,7 @@ impl FnCall { fn lower_funcref( sig: &ABISig, ptr: impl PtrSize, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) -> (CalleeKind, ContextArgs) { // Pop the funcref pointer to a register and allocate a register to hold the @@ -275,7 +275,7 @@ impl FnCall { sig: &ABISig, callee_context: &ContextArgs, ret_area: Option<&RetArea>, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) { let arg_count = sig.params.len_without_retptr(); @@ -337,7 +337,7 @@ impl FnCall { reserved_space: u32, ret_area: Option, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) { // Free any registers holding any function references. match callee_kind { diff --git a/winch/codegen/src/codegen/context.rs b/winch/codegen/src/codegen/context.rs index 8cb315e696a6..99b9dc765667 100644 --- a/winch/codegen/src/codegen/context.rs +++ b/winch/codegen/src/codegen/context.rs @@ -3,6 +3,7 @@ use wasmtime_environ::{VMOffsets, WasmHeapType, WasmValType}; use super::ControlStackFrame; use crate::{ abi::{scratch, vmctx, ABIOperand, ABIResults, RetArea}, + codegen::{CodeGenPhase, Emission, Prologue}, frame::Frame, isa::reg::RegClass, masm::{MacroAssembler, OperandSize, RegImm, SPOffset, ShiftKind, StackSlot}, @@ -26,25 +27,78 @@ use crate::{ /// generation process. The code generation context should /// be generally used as the single entry point to access /// the compound functionality provided by its elements. -pub(crate) struct CodeGenContext<'a> { +pub(crate) struct CodeGenContext<'a, P: CodeGenPhase> { /// The register allocator. pub regalloc: RegAlloc, /// The value stack. pub stack: Stack, /// The current function's frame. - pub frame: Frame, + pub frame: Frame

, /// Reachability state. pub reachable: bool, /// A reference to the VMOffsets. pub vmoffsets: &'a VMOffsets, } -impl<'a> CodeGenContext<'a> { +impl<'a> CodeGenContext<'a, Emission> { + /// Prepares arguments for emitting an i32 shift operation. + pub fn i32_shift(&mut self, masm: &mut M, kind: ShiftKind) + where + M: MacroAssembler, + { + let top = self.stack.peek().expect("value at stack top"); + + if top.is_i32_const() { + let val = self + .stack + .pop_i32_const() + .expect("i32 const value at stack top"); + let typed_reg = self.pop_to_reg(masm, None); + masm.shift_ir( + writable!(typed_reg.reg), + val as u64, + typed_reg.reg, + kind, + OperandSize::S32, + ); + self.stack.push(typed_reg.into()); + } else { + masm.shift(self, kind, OperandSize::S32); + } + } + + /// Prepares arguments for emitting an i64 binary operation. + pub fn i64_shift(&mut self, masm: &mut M, kind: ShiftKind) + where + M: MacroAssembler, + { + let top = self.stack.peek().expect("value at stack top"); + if top.is_i64_const() { + let val = self + .stack + .pop_i64_const() + .expect("i64 const value at stack top"); + let typed_reg = self.pop_to_reg(masm, None); + masm.shift_ir( + writable!(typed_reg.reg), + val as u64, + typed_reg.reg, + kind, + OperandSize::S64, + ); + self.stack.push(typed_reg.into()); + } else { + masm.shift(self, kind, OperandSize::S64); + }; + } +} + +impl<'a> CodeGenContext<'a, Prologue> { /// Create a new code generation context. pub fn new( regalloc: RegAlloc, stack: Stack, - frame: Frame, + frame: Frame, vmoffsets: &'a VMOffsets, ) -> Self { Self { @@ -56,6 +110,19 @@ impl<'a> CodeGenContext<'a> { } } + /// Prepares the frame for the [`Emission`] code generation phase. + pub fn for_emission(self) -> CodeGenContext<'a, Emission> { + CodeGenContext { + regalloc: self.regalloc, + stack: self.stack, + reachable: self.reachable, + vmoffsets: self.vmoffsets, + frame: self.frame.for_emission(), + } + } +} + +impl<'a> CodeGenContext<'a, Emission> { /// Request a specific register to the register allocator, /// spilling if not available. pub fn reg(&mut self, named: Reg, masm: &mut M) -> Reg { @@ -325,57 +392,6 @@ impl<'a> CodeGenContext<'a> { }; } - /// Prepares arguments for emitting an i32 shift operation. - pub fn i32_shift(&mut self, masm: &mut M, kind: ShiftKind) - where - M: MacroAssembler, - { - let top = self.stack.peek().expect("value at stack top"); - - if top.is_i32_const() { - let val = self - .stack - .pop_i32_const() - .expect("i32 const value at stack top"); - let typed_reg = self.pop_to_reg(masm, None); - masm.shift_ir( - writable!(typed_reg.reg), - val as u64, - typed_reg.reg, - kind, - OperandSize::S32, - ); - self.stack.push(typed_reg.into()); - } else { - masm.shift(self, kind, OperandSize::S32); - } - } - - /// Prepares arguments for emitting an i64 binary operation. - pub fn i64_shift(&mut self, masm: &mut M, kind: ShiftKind) - where - M: MacroAssembler, - { - let top = self.stack.peek().expect("value at stack top"); - if top.is_i64_const() { - let val = self - .stack - .pop_i64_const() - .expect("i64 const value at stack top"); - let typed_reg = self.pop_to_reg(masm, None); - masm.shift_ir( - writable!(typed_reg.reg), - val as u64, - typed_reg.reg, - kind, - OperandSize::S64, - ); - self.stack.push(typed_reg.into()); - } else { - masm.shift(self, kind, OperandSize::S64); - }; - } - /// Prepares arguments for emitting a convert operation. pub fn convert_op(&mut self, masm: &mut M, dst_ty: WasmValType, mut emit: F) where @@ -511,7 +527,7 @@ impl<'a> CodeGenContext<'a> { mut calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { let area = results .on_stack() @@ -558,7 +574,7 @@ impl<'a> CodeGenContext<'a> { where M: MacroAssembler, { - let addr = masm.local_address(&self.frame.vmctx_slot); + let addr = masm.local_address(&self.frame.vmctx_slot()); masm.load_ptr(addr, writable!(vmctx!(M))); } @@ -570,7 +586,7 @@ impl<'a> CodeGenContext<'a> { fn spill_impl( stack: &mut Stack, regalloc: &mut RegAlloc, - frame: &Frame, + frame: &Frame, masm: &mut M, ) { stack.inner_mut().iter_mut().for_each(|v| match v { diff --git a/winch/codegen/src/codegen/control.rs b/winch/codegen/src/codegen/control.rs index d9fc7fc7ee8e..d9bab98880bd 100644 --- a/winch/codegen/src/codegen/control.rs +++ b/winch/codegen/src/codegen/control.rs @@ -9,6 +9,7 @@ use super::{CodeGenContext, OperandSize, Reg, TypedReg}; use crate::{ abi::{ABIOperand, ABIResults, ABISig, RetArea, ABI}, + codegen::Emission, masm::{IntCmpKind, MacroAssembler, MemMoveDirection, RegImm, SPOffset}, reg::writable, stack::Val, @@ -248,7 +249,7 @@ impl ControlStackFrame { pub fn r#if( sig: BlockSig, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) -> Self { let mut control = Self::If { cont: masm.get_label(), @@ -266,7 +267,7 @@ impl ControlStackFrame { pub fn block( sig: BlockSig, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) -> Self { let mut control = Self::Block { sig, @@ -283,7 +284,7 @@ impl ControlStackFrame { pub fn r#loop( sig: BlockSig, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) -> Self { let mut control = Self::Loop { stack_state: Default::default(), @@ -295,7 +296,7 @@ impl ControlStackFrame { control } - fn init(&mut self, masm: &mut M, context: &mut CodeGenContext) { + fn init(&mut self, masm: &mut M, context: &mut CodeGenContext) { self.calculate_stack_state(context, masm); // If the block has stack results, immediately resolve the return area // base. @@ -336,7 +337,7 @@ impl ControlStackFrame { /// Calculates the [StackState] of the block. fn calculate_stack_state( &mut self, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) { use ControlStackFrame::*; @@ -388,7 +389,7 @@ impl ControlStackFrame { pub fn ensure_stack_state( &mut self, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) { let state = self.stack_state(); // This assumes that at jump sites, the machine stack pointer will be @@ -416,7 +417,7 @@ impl ControlStackFrame { } } - fn emit(&mut self, masm: &mut M, context: &mut CodeGenContext) { + fn emit(&mut self, masm: &mut M, context: &mut CodeGenContext) { use ControlStackFrame::*; // Do not perform any emissions if we are in an unreachable state. @@ -455,7 +456,11 @@ impl ControlStackFrame { /// Handles the else branch if the current control stack frame is /// [`ControlStackFrame::If`]. - pub fn emit_else(&mut self, masm: &mut M, context: &mut CodeGenContext) { + pub fn emit_else( + &mut self, + masm: &mut M, + context: &mut CodeGenContext, + ) { debug_assert!(self.is_if()); let state = self.stack_state(); @@ -467,7 +472,11 @@ impl ControlStackFrame { /// Binds the else branch label and converts `self` to /// [`ControlStackFrame::Else`]. - pub fn bind_else(&mut self, masm: &mut M, context: &mut CodeGenContext) { + pub fn bind_else( + &mut self, + masm: &mut M, + context: &mut CodeGenContext, + ) { use ControlStackFrame::*; match self { If { @@ -510,7 +519,11 @@ impl ControlStackFrame { } /// Handles the end of a control stack frame. - pub fn emit_end(&mut self, masm: &mut M, context: &mut CodeGenContext) { + pub fn emit_end( + &mut self, + masm: &mut M, + context: &mut CodeGenContext, + ) { use ControlStackFrame::*; match self { If { stack_state, .. } | Else { stack_state, .. } | Block { stack_state, .. } => { @@ -527,7 +540,11 @@ impl ControlStackFrame { /// Binds the exit label of the current control stack frame and pushes the /// ABI results to the value stack. - pub fn bind_end(&mut self, masm: &mut M, context: &mut CodeGenContext) { + pub fn bind_end( + &mut self, + masm: &mut M, + context: &mut CodeGenContext, + ) { self.push_abi_results(context, masm); self.bind_exit_label(masm); } @@ -627,12 +644,12 @@ impl ControlStackFrame { /// updated. pub fn pop_abi_results( &mut self, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { Self::pop_abi_results_impl(self.results::(), context, masm, calculate_ret_area) } @@ -648,12 +665,12 @@ impl ControlStackFrame { /// results should be interpreted. pub fn pop_abi_results_impl( results: &mut ABIResults, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, mut calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { let mut iter = results.operands().iter().rev().peekable(); @@ -690,7 +707,7 @@ impl ControlStackFrame { /// Convenience wrapper around [CodeGenContext::push_abi_results] using the /// results of the current frame. - fn push_abi_results(&mut self, context: &mut CodeGenContext, masm: &mut M) + fn push_abi_results(&mut self, context: &mut CodeGenContext, masm: &mut M) where M: MacroAssembler, { @@ -705,12 +722,12 @@ impl ControlStackFrame { /// taken. pub fn top_abi_results( &mut self, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { Self::top_abi_results_impl::(self.results::(), context, masm, calculate_ret_area) } @@ -720,12 +737,12 @@ impl ControlStackFrame { /// needed. fn top_abi_results_impl( results: &mut ABIResults, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, mut calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { let mut area = None; Self::pop_abi_results_impl::(results, context, masm, |r, context, masm| { @@ -764,7 +781,7 @@ impl ControlStackFrame { fn adjust_stack_results( ret_area: RetArea, results: &ABIResults, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) where M: MacroAssembler, @@ -889,7 +906,7 @@ impl ControlStackFrame { /// Ensures that there is enough space for return values on the stack. /// This function is called at the end of all blocks and when branching from /// within blocks. - fn ensure_ret_area(ret_area: &RetArea, context: &mut CodeGenContext, masm: &mut M) + fn ensure_ret_area(ret_area: &RetArea, context: &mut CodeGenContext, masm: &mut M) where M: MacroAssembler, { @@ -907,7 +924,7 @@ impl ControlStackFrame { fn maybe_load_retptr( ret_area: Option<&RetArea>, results: &ABIResults, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) -> Option where diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index a3721867b2c5..63f442c23444 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -13,6 +13,7 @@ use cranelift_codegen::{ ir::{RelSourceLoc, SourceLoc}, }; use smallvec::SmallVec; +use std::marker::PhantomData; use wasmparser::{ BinaryReader, FuncValidator, MemArg, Operator, ValidatorResources, VisitOperator, VisitSimdOperator, @@ -37,6 +38,9 @@ pub(crate) mod bounds; use bounds::{Bounds, ImmOffset, Index}; +mod phase; +pub(crate) use phase::*; + /// Holds metadata about the source code location and the machine code emission. /// The fields of this struct are opaque and are not interpreted in any way. /// They serve as a mapping between source code and machine code. @@ -50,15 +54,16 @@ pub(crate) struct SourceLocation { } /// The code generation abstraction. -pub(crate) struct CodeGen<'a, 'translation: 'a, 'data: 'translation, M> +pub(crate) struct CodeGen<'a, 'translation: 'a, 'data: 'translation, M, P> where M: MacroAssembler, + P: CodeGenPhase, { /// The ABI-specific representation of the function signature, excluding results. pub sig: ABISig, /// The code generation context. - pub context: CodeGenContext<'a>, + pub context: CodeGenContext<'a, P>, /// A reference to the function compilation environment. pub env: FuncEnv<'a, 'translation, 'data, M::Ptr>, @@ -83,19 +88,20 @@ where /// Local counter to track fuel consumption. pub fuel_consumed: i64, + phase: PhantomData

, } -impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> +impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Prologue> where M: MacroAssembler, { pub fn new( tunables: &'a Tunables, masm: &'a mut M, - context: CodeGenContext<'a>, + context: CodeGenContext<'a, Prologue>, env: FuncEnv<'a, 'translation, 'data, M::Ptr>, sig: ABISig, - ) -> Self { + ) -> CodeGen<'a, 'translation, 'data, M, Prologue> { Self { sig, context, @@ -107,32 +113,11 @@ where found_unsupported_instruction: None, // Empty functions should consume at least 1 fuel unit. fuel_consumed: 1, + phase: PhantomData, } } - /// Emit the function body to machine code. - pub fn emit( - &mut self, - body: &mut BinaryReader<'a>, - validator: &mut FuncValidator, - ) -> Result<()> { - self.emit_start() - .and_then(|_| self.emit_body(body, validator)) - .and_then(|_| self.emit_end())?; - - Ok(()) - } - - /// Derives a [RelSourceLoc] from a [SourceLoc]. - pub fn source_loc_from(&mut self, loc: SourceLoc) -> RelSourceLoc { - if self.source_location.base.is_none() && !loc.is_default() { - self.source_location.base = Some(loc); - } - - RelSourceLoc::from_base_offset(self.source_location.base.unwrap_or_default(), loc) - } - - fn emit_start(&mut self) -> Result<()> { + pub fn emit_prologue(mut self) -> Result> { let vmctx = self .sig .params() @@ -153,32 +138,95 @@ where ); self.masm.reserve_stack(self.context.frame.locals_size); + self.spill_register_arguments(); + + let defined_locals_range = &self.context.frame.defined_locals_range; + self.masm.zero_mem_range(defined_locals_range.as_range()); + + // Save the results base parameter register into its slot. + self.sig.params.has_retptr().then(|| { + match self.sig.params.unwrap_results_area_operand() { + ABIOperand::Reg { ty, reg, .. } => { + let results_base_slot = self.context.frame.results_base_slot.as_ref().unwrap(); + debug_assert!(results_base_slot.addressed_from_sp()); + let addr = self.masm.local_address(results_base_slot); + self.masm.store((*reg).into(), addr, (*ty).into()); + } + // The result base parameter is a stack parameter, addressed + // from FP. + _ => {} + } + }); self.masm.end_source_loc(); - if self.tunables.consume_fuel { - self.emit_fuel_check(); + Ok(CodeGen { + sig: self.sig, + context: self.context.for_emission(), + masm: self.masm, + env: self.env, + tunables: self.tunables, + source_location: self.source_location, + control_frames: self.control_frames, + found_unsupported_instruction: self.found_unsupported_instruction, + fuel_consumed: self.fuel_consumed, + phase: PhantomData, + }) + } + + fn spill_register_arguments(&mut self) { + use WasmValType::*; + for (operand, slot) in self + .sig + .params_without_retptr() + .iter() + .zip(self.context.frame.locals()) + { + match (operand, slot) { + (ABIOperand::Reg { ty, reg, .. }, slot) => { + let addr = self.masm.local_address(slot); + match &ty { + I32 | I64 | F32 | F64 | V128 => { + self.masm.store((*reg).into(), addr, (*ty).into()) + } + Ref(rt) => match rt.heap_type { + WasmHeapType::Func | WasmHeapType::Extern => { + self.masm.store_ptr((*reg).into(), addr) + } + ht => unimplemented!("Support for WasmHeapType: {ht}"), + }, + } + } + // Skip non-register arguments + _ => {} + } } + } +} - // Once we have emitted the epilogue and reserved stack space for the locals, we push the - // base control flow block. - self.control_frames.push(ControlStackFrame::block( - BlockSig::from_sig(self.sig.clone()), - self.masm, - &mut self.context, - )); +impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> +where + M: MacroAssembler, +{ + /// Emit the function body to machine code. + pub fn emit( + &mut self, + body: &mut BinaryReader<'a>, + validator: &mut FuncValidator, + ) -> Result<()> { + self.emit_body(body, validator) + .and_then(|_| self.emit_end())?; - // Set the return area of the results *after* initializing the block. In - // the function body block case, we'll treat the results as any other - // case, addressed from the stack pointer, and when ending the function - // the return area will be set to the return pointer. - if self.sig.params.has_retptr() { - self.sig - .results - .set_ret_area(RetArea::slot(self.context.frame.results_base_slot.unwrap())); + Ok(()) + } + + /// Derives a [RelSourceLoc] from a [SourceLoc]. + pub fn source_loc_from(&mut self, loc: SourceLoc) -> RelSourceLoc { + if self.source_location.base.is_none() && !loc.is_default() { + self.source_location.base = Some(loc); } - Ok(()) + RelSourceLoc::from_base_offset(self.source_location.base.unwrap_or_default(), loc) } /// The following two helpers, handle else or end instructions when the @@ -224,24 +272,27 @@ where body: &mut BinaryReader<'a>, validator: &mut FuncValidator, ) -> Result<()> { - self.spill_register_arguments(); - let defined_locals_range = &self.context.frame.defined_locals_range; - self.masm.zero_mem_range(defined_locals_range.as_range()); + if self.tunables.consume_fuel { + self.emit_fuel_check(); + } - // Save the results base parameter register into its slot. - self.sig.params.has_retptr().then(|| { - match self.sig.params.unwrap_results_area_operand() { - ABIOperand::Reg { ty, reg, .. } => { - let results_base_slot = self.context.frame.results_base_slot.as_ref().unwrap(); - debug_assert!(results_base_slot.addressed_from_sp()); - let addr = self.masm.local_address(results_base_slot); - self.masm.store((*reg).into(), addr, (*ty).into()); - } - // The result base parameter is a stack parameter, addressed - // from FP. - _ => {} - } - }); + // Once we have emitted the epilogue and reserved stack space for the locals, we push the + // base control flow block. + self.control_frames.push(ControlStackFrame::block( + BlockSig::from_sig(self.sig.clone()), + self.masm, + &mut self.context, + )); + + // Set the return area of the results *after* initializing the block. In + // the function body block case, we'll treat the results as any other + // case, addressed from the stack pointer, and when ending the function + // the return area will be set to the return pointer. + if self.sig.params.has_retptr() { + self.sig + .results + .set_ret_area(RetArea::slot(self.context.frame.results_base_slot.unwrap())); + } while !body.eof() { let offset = body.original_position(); @@ -307,7 +358,7 @@ where } impl<'a, 'translation, 'data, M: MacroAssembler> VisitorHooks - for CodeGen<'a, 'translation, 'data, M> + for CodeGen<'a, 'translation, 'data, M, Emission> { fn visit(&self, op: &Operator) -> bool { self.context.reachable || visit_op_when_unreachable(op) @@ -431,35 +482,6 @@ where Ok(()) } - fn spill_register_arguments(&mut self) { - use WasmValType::*; - self.sig - // Skip the results base param if any; [Self::emit_body], - // will handle spilling the results base param if it's in a register. - .params_without_retptr() - .iter() - .enumerate() - .filter(|(_, a)| a.is_reg()) - .for_each(|(index, arg)| { - let ty = arg.ty(); - let local = self.context.frame.get_frame_local(index); - let addr = self.masm.local_address(local); - let src = arg - .get_reg() - .expect("arg should be associated to a register"); - - match &ty { - I32 | I64 | F32 | F64 | V128 => self.masm.store(src.into(), addr, ty.into()), - Ref(rt) => match rt.heap_type { - WasmHeapType::Func | WasmHeapType::Extern => { - self.masm.store_ptr(src.into(), addr) - } - ht => unimplemented!("Support for WasmHeapType: {ht}"), - }, - } - }); - } - /// Pops the value at the stack top and assigns it to the local at /// the given index, returning the typed register holding the /// source value. @@ -633,8 +655,12 @@ where // * index + offset + access_size overflows // OR // * index + offset + access_size > bound - let bounds = - bounds::load_dynamic_heap_bounds(&mut self.context, self.masm, &heap, ptr_size); + let bounds = bounds::load_dynamic_heap_bounds::<_>( + &mut self.context, + self.masm, + &heap, + ptr_size, + ); let index_reg = index.as_typed_reg().reg; // Allocate a temporary register to hold @@ -978,6 +1004,24 @@ where self.context.free_reg(fuel_var); } + /// Emits a series of instructions that load the `fuel_consumed` field from + /// `VMRuntimeLimits`. + fn emit_load_fuel_consumed(&mut self, fuel_var: Reg) { + let limits_offset = self.env.vmoffsets.ptr.vmctx_runtime_limits(); + let fuel_offset = self.env.vmoffsets.ptr.vmruntime_limits_fuel_consumed(); + self.masm.load_ptr( + self.masm.address_at_vmctx(u32::from(limits_offset)), + writable!(fuel_var), + ); + + self.masm.load( + self.masm.address_at_reg(fuel_var, u32::from(fuel_offset)), + writable!(fuel_var), + // Fuel is an i64. + OperandSize::S64, + ); + } + /// Increments the fuel consumed in `VMRuntimeLimits` by flushing /// `self.fuel_consumed` to memory. fn emit_fuel_increment(&mut self) { @@ -1022,24 +1066,6 @@ where self.context.free_reg(limits_var); } - /// Emits a series of instructions that load the `fuel_consumed` field from - /// `VMRuntimeLimits`. - fn emit_load_fuel_consumed(&mut self, fuel_var: Reg) { - let limits_offset = self.env.vmoffsets.ptr.vmctx_runtime_limits(); - let fuel_offset = self.env.vmoffsets.ptr.vmruntime_limits_fuel_consumed(); - self.masm.load_ptr( - self.masm.address_at_vmctx(u32::from(limits_offset)), - writable!(fuel_var), - ); - - self.masm.load( - self.masm.address_at_reg(fuel_var, u32::from(fuel_offset)), - writable!(fuel_var), - // Fuel is an i64. - OperandSize::S64, - ); - } - /// Hook to handle fuel before visiting an operator. fn fuel_before_visit_op(&mut self, op: &Operator) { if !self.context.reachable { diff --git a/winch/codegen/src/codegen/phase.rs b/winch/codegen/src/codegen/phase.rs new file mode 100644 index 000000000000..de2d2e0206fa --- /dev/null +++ b/winch/codegen/src/codegen/phase.rs @@ -0,0 +1,24 @@ +//! Type-based states to represent code generation phases. +//! These states help enforce code generation invariants at compile time. +//! +//! Currently two phases are defined for code generation: +//! +//! * Prologue: responsible of setting up the function's frame. +//! * Emission: emission of Wasm code to machine code. + +/// A code generation phase. +pub trait CodeGenPhase {} + +/// The prologue phase. +/// +/// Its main responsibility is to setup the function's frame, by creating the +/// well known local slots. In this phase, writes to such slots is allowed. +/// After this phase, the frame is considered immutable. +pub struct Prologue; +/// The code emission phase. +/// +/// Its main responsibility is to emit Wasm code to machine code. +pub struct Emission; + +impl CodeGenPhase for Prologue {} +impl CodeGenPhase for Emission {} diff --git a/winch/codegen/src/frame/mod.rs b/winch/codegen/src/frame/mod.rs index 1ebdace8daf3..c813141d7184 100644 --- a/winch/codegen/src/frame/mod.rs +++ b/winch/codegen/src/frame/mod.rs @@ -1,18 +1,27 @@ use crate::{ abi::{align_to, ABIOperand, ABISig, LocalSlot, ABI}, + codegen::{CodeGenPhase, Emission, Prologue}, masm::MacroAssembler, }; use anyhow::Result; use smallvec::SmallVec; +use std::marker::PhantomData; use std::ops::Range; use wasmparser::{BinaryReader, FuncValidator, ValidatorResources}; use wasmtime_environ::{TypeConvert, WasmValType}; +/// WebAssembly locals. // TODO: // SpiderMonkey's implementation uses 16; // (ref: https://searchfox.org/mozilla-central/source/js/src/wasm/WasmBCFrame.h#585) // during instrumentation we should measure to verify if this is a good default. -pub(crate) type Locals = SmallVec<[LocalSlot; 16]>; +pub(crate) type WasmLocals = SmallVec<[LocalSlot; 16]>; +/// Special local slots used by the compiler. +// Winch's ABI uses two extra parameters to store the callee and caller +// VMContext pointers. +// These arguments are spilled and treated as frame locals, but not +// WebAssembly locals. +pub(crate) type SpecialLocals = [LocalSlot; 2]; /// Function defined locals start and end in the frame. pub(crate) struct DefinedLocalsRange(Range); @@ -28,7 +37,7 @@ impl DefinedLocalsRange { #[derive(Default)] pub(crate) struct DefinedLocals { /// The defined locals for a function. - pub defined_locals: Locals, + pub defined_locals: WasmLocals, /// The size of the defined locals. pub stack_size: u32, } @@ -43,7 +52,7 @@ impl DefinedLocals { let mut next_stack: u32 = 0; // The first 32 bits of a Wasm binary function describe the number of locals. let local_count = reader.read_var_u32()?; - let mut slots: Locals = Default::default(); + let mut slots: WasmLocals = Default::default(); for _ in 0..local_count { let position = reader.original_position(); @@ -67,7 +76,7 @@ impl DefinedLocals { } /// Frame handler abstraction. -pub(crate) struct Frame { +pub(crate) struct Frame { /// The size of the entire local area; the arguments plus the function defined locals. pub locals_size: u32, @@ -78,23 +87,24 @@ pub(crate) struct Frame { /// /// Locals get calculated when allocating a frame and are readonly /// through the function compilation lifetime. - locals: Locals, - - /// The offset to the slot containing the `VMContext`. - pub vmctx_slot: LocalSlot, + wasm_locals: WasmLocals, + /// Special locals used by the internal ABI. See [`SpecialLocals`]. + special_locals: SpecialLocals, /// The slot holding the address of the results area. pub results_base_slot: Option, + marker: PhantomData

, } -impl Frame { +impl Frame { /// Allocate a new [`Frame`]. - pub fn new(sig: &ABISig, defined_locals: &DefinedLocals) -> Result { - let (mut locals, defined_locals_start) = Self::compute_arg_slots::(sig)?; + pub fn new(sig: &ABISig, defined_locals: &DefinedLocals) -> Result> { + let (special_locals, mut wasm_locals, defined_locals_start) = + Self::compute_arg_slots::(sig)?; // The defined locals have a zero-based offset by default // so we need to add the defined locals start to the offset. - locals.extend( + wasm_locals.extend( defined_locals .defined_locals .iter() @@ -138,66 +148,37 @@ impl Frame { (None, defined_locals_end) }; - let vmctx_slot = *locals.get(0).expect("LocalSlot for VMContext"); Ok(Self { - locals, + wasm_locals, + special_locals, locals_size, - vmctx_slot, defined_locals_range: DefinedLocalsRange( defined_locals_start..(defined_locals_start + defined_locals.stack_size), ), results_base_slot, + marker: PhantomData, }) } - // Winch's ABI uses two extra parameters to store the callee and caller - // VMContext pointers. - // These arguments are spilled and treated as frame locals, but not - // WebAssembly locals. - const WASM_LOCALS_OFFSET: usize = 2; - - /// Get the [LocalSlot] for a WebAssembly local. - /// This method assumes that the index is bound to u32::MAX, representing - /// the index space for WebAssembly locals. - /// - /// # Panics - /// This method panics if the index is not associated to a valid WebAssembly - /// local. - pub fn get_wasm_local(&self, index: u32) -> &LocalSlot { - let local_index = Self::WASM_LOCALS_OFFSET + index as usize; - self.locals - .get(local_index) - .unwrap_or_else(|| panic!(" Expected WebAssembly local at slot: {index}")) - } - - /// Get the [LocalSlot] for a frame local. - /// This method doesn't make any asumptions about the local index passed in, - /// and simply delegates the [LocalSlot] retrieval to the underlying locals - /// vector. - /// - /// # Panics - /// This method panics if the index is not associated to a valid WebAssembly - /// local. - pub fn get_frame_local(&self, index: usize) -> &LocalSlot { - self.locals - .get(index) - .unwrap_or_else(|| panic!(" Expected Frame local at slot: {index}")) + /// Returns an iterator over all the [`LocalSlot`]s in the frame, including + /// the [`SpecialLocals`]. + pub fn locals(&self) -> impl Iterator { + self.special_locals.iter().chain(self.wasm_locals.iter()) } - /// Returns the address of the local at the given index. - /// - /// # Panics - /// This function panics if the index is not associated to a local. - pub fn get_local_address( - &self, - index: u32, - masm: &mut M, - ) -> (WasmValType, M::Address) { - let slot = self.get_wasm_local(index); - (slot.ty, masm.local_address(&slot)) + /// Prepares the frame for the [`Emission`] code generation phase. + pub fn for_emission(self) -> Frame { + Frame { + wasm_locals: self.wasm_locals, + special_locals: self.special_locals, + locals_size: self.locals_size, + defined_locals_range: self.defined_locals_range, + results_base_slot: self.results_base_slot, + marker: PhantomData, + } } - fn compute_arg_slots(sig: &ABISig) -> Result<(Locals, u32)> { + fn compute_arg_slots(sig: &ABISig) -> Result<(SpecialLocals, WasmLocals, u32)> { // Go over the function ABI-signature and // calculate the stack slots. // @@ -231,13 +212,24 @@ impl Frame { // Skip the results base param; if present, the [Frame] will create // a dedicated slot for it. - let slots: Locals = sig - .params_without_retptr() - .into_iter() + let mut params_iter = sig.params_without_retptr().into_iter(); + + // Handle special local slots. + let callee_vmctx = params_iter + .next() + .map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset)) + .expect("Slot for VMContext"); + + let caller_vmctx = params_iter + .next() + .map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset)) + .expect("Slot for VMContext"); + + let slots: WasmLocals = params_iter .map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset)) .collect(); - Ok((slots, next_stack)) + Ok(([callee_vmctx, caller_vmctx], slots, next_stack)) } fn abi_arg_slot(arg: &ABIOperand, next_stack: &mut u32, arg_base_offset: u32) -> LocalSlot { @@ -256,3 +248,47 @@ impl Frame { } } } + +impl Frame { + /// Get the [`LocalSlot`] for a WebAssembly local. + /// This method assumes that the index is bound to u32::MAX, representing + /// the index space for WebAssembly locals. + /// + /// # Panics + /// This method panics if the index is not associated to a valid WebAssembly + /// local. + pub fn get_wasm_local(&self, index: u32) -> &LocalSlot { + self.wasm_locals + .get(index as usize) + .unwrap_or_else(|| panic!(" Expected WebAssembly local at slot: {index}")) + } + + /// Get the [`LocalSlot`] for a special local. + /// + /// # Panics + /// This method panics if the index is not associated to a valid special + /// local. + pub fn get_special_local(&self, index: usize) -> &LocalSlot { + self.special_locals + .get(index) + .unwrap_or_else(|| panic!(" Expected special local at slot: {index}")) + } + + /// Get the special [`LocalSlot`] for the `VMContext`. + pub fn vmctx_slot(&self) -> &LocalSlot { + self.get_special_local(0) + } + + /// Returns the address of the local at the given index. + /// + /// # Panics + /// This function panics if the index is not associated to a local. + pub fn get_local_address( + &self, + index: u32, + masm: &mut M, + ) -> (WasmValType, M::Address) { + let slot = self.get_wasm_local(index); + (slot.ty, masm.local_address(&slot)) + } +} diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs index 02226387baa3..9be8a60f772a 100644 --- a/winch/codegen/src/isa/aarch64/masm.rs +++ b/winch/codegen/src/isa/aarch64/masm.rs @@ -1,7 +1,7 @@ use super::{abi::Aarch64ABI, address::Address, asm::Assembler, regs}; use crate::{ abi::local::LocalSlot, - codegen::{ptr_type_from_ptr_size, CodeGenContext, FuncEnv}, + codegen::{ptr_type_from_ptr_size, CodeGenContext, Emission, FuncEnv}, isa::reg::{writable, Reg, WritableReg}, masm::{ CalleeKind, DivKind, ExtendKind, FloatCmpKind, Imm as I, IntCmpKind, @@ -350,11 +350,11 @@ impl Masm for MacroAssembler { self.asm.fabs_rr(dst.to_reg(), dst, size); } - fn float_round, &mut CodeGenContext, &mut Self)>( + fn float_round, &mut CodeGenContext, &mut Self)>( &mut self, mode: RoundingMode, _env: &mut FuncEnv, - context: &mut CodeGenContext, + context: &mut CodeGenContext, size: OperandSize, _fallback: F, ) { @@ -433,7 +433,12 @@ impl Masm for MacroAssembler { self.asm.shift_ir(imm, lhs, dst, kind, size) } - fn shift(&mut self, context: &mut CodeGenContext, kind: ShiftKind, size: OperandSize) { + fn shift( + &mut self, + context: &mut CodeGenContext, + kind: ShiftKind, + size: OperandSize, + ) { let src = context.pop_to_reg(self, None); let dst = context.pop_to_reg(self, None); @@ -444,11 +449,11 @@ impl Masm for MacroAssembler { context.stack.push(dst.into()); } - fn div(&mut self, _context: &mut CodeGenContext, _kind: DivKind, _size: OperandSize) { + fn div(&mut self, _context: &mut CodeGenContext, _kind: DivKind, _size: OperandSize) { todo!() } - fn rem(&mut self, _context: &mut CodeGenContext, _kind: RemKind, _size: OperandSize) { + fn rem(&mut self, _context: &mut CodeGenContext, _kind: RemKind, _size: OperandSize) { todo!() } @@ -456,7 +461,7 @@ impl Masm for MacroAssembler { self.asm.load_constant(0, reg); } - fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize) { + fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize) { let src = context.pop_to_reg(self, None); let tmp = regs::float_scratch(); self.asm.mov_to_fpu(src.into(), writable!(tmp), size); @@ -700,7 +705,7 @@ impl Masm for MacroAssembler { todo!() } - fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind) { + fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind) { let _ = (context, kind); todo!() } diff --git a/winch/codegen/src/isa/aarch64/mod.rs b/winch/codegen/src/isa/aarch64/mod.rs index 17c49577a6fb..e54ffa0c9d4e 100644 --- a/winch/codegen/src/isa/aarch64/mod.rs +++ b/winch/codegen/src/isa/aarch64/mod.rs @@ -123,11 +123,12 @@ impl TargetIsa for Aarch64 { ); let regalloc = RegAlloc::from(gpr, fpr); let codegen_context = CodeGenContext::new(regalloc, stack, frame, &vmoffsets); - let mut codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); + let codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); - codegen.emit(&mut body, validator)?; - let names = codegen.env.take_name_map(); - let base = codegen.source_location.base; + let mut body_codegen = codegen.emit_prologue()?; + body_codegen.emit(&mut body, validator)?; + let names = body_codegen.env.take_name_map(); + let base = body_codegen.source_location.base; Ok(CompiledFunction::new( masm.finalize(base), names, diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs index 4d72b79e6965..9937d165e9b9 100644 --- a/winch/codegen/src/isa/x64/masm.rs +++ b/winch/codegen/src/isa/x64/masm.rs @@ -12,7 +12,7 @@ use crate::masm::{ }; use crate::{ abi::{self, align_to, calculate_frame_adjustment, LocalSlot}, - codegen::{ptr_type_from_ptr_size, CodeGenContext, FuncEnv}, + codegen::{ptr_type_from_ptr_size, CodeGenContext, Emission, FuncEnv}, stack::{TypedReg, Val}, }; use crate::{ @@ -474,11 +474,11 @@ impl Masm for MacroAssembler { self.asm.xmm_and_rr(scratch_xmm, dst, size); } - fn float_round, &mut CodeGenContext, &mut Self)>( + fn float_round, &mut CodeGenContext, &mut Self)>( &mut self, mode: RoundingMode, env: &mut FuncEnv, - context: &mut CodeGenContext, + context: &mut CodeGenContext, size: OperandSize, mut fallback: F, ) { @@ -565,7 +565,12 @@ impl Masm for MacroAssembler { self.asm.shift_ir(imm as u8, dst, kind, size) } - fn shift(&mut self, context: &mut CodeGenContext, kind: ShiftKind, size: OperandSize) { + fn shift( + &mut self, + context: &mut CodeGenContext, + kind: ShiftKind, + size: OperandSize, + ) { // Number of bits to shift must be in the CL register. let src = context.pop_to_reg(self, Some(regs::rcx())); let dst = context.pop_to_reg(self, None); @@ -577,7 +582,7 @@ impl Masm for MacroAssembler { context.stack.push(dst.into()); } - fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize) { + fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize) { // Allocate rdx:rax. let rdx = context.reg(regs::rdx(), self); let rax = context.reg(regs::rax(), self); @@ -599,7 +604,7 @@ impl Masm for MacroAssembler { context.stack.push(rax.into()); } - fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize) { + fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize) { // Allocate rdx:rax. let rdx = context.reg(regs::rdx(), self); let rax = context.reg(regs::rax(), self); @@ -787,7 +792,7 @@ impl Masm for MacroAssembler { self.asm.jmp(target); } - fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize) { + fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize) { let src = context.pop_to_reg(self, None); if self.flags.has_popcnt() && self.flags.has_sse42() { self.asm.popcnt(src.into(), size); @@ -1028,7 +1033,7 @@ impl Masm for MacroAssembler { self.asm.sbb_rr(rhs_hi, dst_hi, OperandSize::S64); } - fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind) { + fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind) { // Reserve rax/rdx since they're required by the `mul_wide` instruction // being used here. let rax = context.reg(regs::rax(), self); diff --git a/winch/codegen/src/isa/x64/mod.rs b/winch/codegen/src/isa/x64/mod.rs index ae038f6d7776..de0029d08d00 100644 --- a/winch/codegen/src/isa/x64/mod.rs +++ b/winch/codegen/src/isa/x64/mod.rs @@ -134,12 +134,14 @@ impl TargetIsa for X64 { let regalloc = RegAlloc::from(gpr, fpr); let codegen_context = CodeGenContext::new(regalloc, stack, frame, &vmoffsets); - let mut codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); + let codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); - codegen.emit(&mut body, validator)?; - let base = codegen.source_location.base; + let mut body_codegen = codegen.emit_prologue()?; - let names = codegen.env.take_name_map(); + body_codegen.emit(&mut body, validator)?; + let base = body_codegen.source_location.base; + + let names = body_codegen.env.take_name_map(); Ok(CompiledFunction::new( masm.finalize(base), names, diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs index 95de6e789225..1bd5a7a59b4e 100644 --- a/winch/codegen/src/masm.rs +++ b/winch/codegen/src/masm.rs @@ -1,5 +1,5 @@ use crate::abi::{self, align_to, scratch, LocalSlot}; -use crate::codegen::{CodeGenContext, FuncEnv}; +use crate::codegen::{CodeGenContext, Emission, FuncEnv}; use crate::isa::reg::{writable, Reg, WritableReg}; use cranelift_codegen::{ binemit::CodeOffset, @@ -744,11 +744,11 @@ pub(crate) trait MacroAssembler { fn float_neg(&mut self, dst: WritableReg, size: OperandSize); /// Perform a floating point floor operation. - fn float_round, &mut CodeGenContext, &mut Self)>( + fn float_round, &mut CodeGenContext, &mut Self)>( &mut self, mode: RoundingMode, env: &mut FuncEnv, - context: &mut CodeGenContext, + context: &mut CodeGenContext, size: OperandSize, fallback: F, ); @@ -781,7 +781,7 @@ pub(crate) trait MacroAssembler { /// caller from having to deal with the architecture specific constraints /// we give this function access to the code generation context, allowing /// each implementation to decide the lowering path. - fn shift(&mut self, context: &mut CodeGenContext, kind: ShiftKind, size: OperandSize); + fn shift(&mut self, context: &mut CodeGenContext, kind: ShiftKind, size: OperandSize); /// Perform division operation. /// Division is special in that some architectures have specific @@ -794,10 +794,10 @@ pub(crate) trait MacroAssembler { /// unconstrained binary operation, the caller can decide to use /// the `CodeGenContext::i32_binop` or `CodeGenContext::i64_binop` /// functions. - fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize); + fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize); /// Calculate remainder. - fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize); + fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize); /// Compares `src1` against `src2` for the side effect of setting processor /// flags. @@ -852,7 +852,7 @@ pub(crate) trait MacroAssembler { /// Count the number of 1 bits in src and put the result in dst. In x64, /// this will emit multiple instructions if the `has_popcnt` flag is false. - fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize); + fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize); /// Converts an i64 to an i32 by discarding the high 32 bits. fn wrap(&mut self, dst: WritableReg, src: Reg); @@ -1053,5 +1053,5 @@ pub(crate) trait MacroAssembler { /// /// Note that some platforms require special handling of registers in this /// instruction (e.g. x64) so full access to `CodeGenContext` is provided. - fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind); + fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind); } diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs index c38cd41ac1ed..1fae5644330e 100644 --- a/winch/codegen/src/visitor.rs +++ b/winch/codegen/src/visitor.rs @@ -5,7 +5,7 @@ //! machine code emitter. use crate::abi::RetArea; -use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, FnCall}; +use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, Emission, FnCall}; use crate::masm::{ DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, MulWideKind, OperandSize, RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, @@ -253,7 +253,7 @@ macro_rules! def_unsupported { (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; } -impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M> +impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> where M: MacroAssembler, { @@ -2212,7 +2212,8 @@ where wasmparser::for_each_visit_operator!(def_unsupported); } -impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> for CodeGen<'a, 'translation, 'data, M> +impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> + for CodeGen<'a, 'translation, 'data, M, Emission> where M: MacroAssembler, { @@ -2231,7 +2232,7 @@ where wasmparser::for_each_visit_simd_operator!(def_unsupported); } -impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> +impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> where M: MacroAssembler, {