From 7ef8f2e271aa8f5bfefca16b06757808ee6e0a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C3=BAl=20Cabrera?= Date: Wed, 4 Dec 2024 14:12:09 -0500 Subject: [PATCH] winch: Improve frame handling (#9708) This commit addresses issues identified while working on issue #8091. It improves the frame handling in Winch to prevent subtle bugs and enhance the robustness of the code generation process. Previously, there was no clear mechanism to verify when the frame was fully set up and safe to access the local slots allocated for register arguments, including the special slots used for the `VMContext`. As a result, it was possible to inadvertently read from uninitialized memory if calls were made before the frame was properly set up and sealed. This commit introduces two main changes with the objective to help reduce the risk of introducing bugs related to the above: * A `CodeGenPhase` trait, used via the type state pattern to clearly gate the operations allowed during each phase of the code generation process. * Improve the semantics of locals, by clearly separating the notion of Wasm locals and special locals used by the compiler. This specialization allows a more accurate representation of the semantics of Wasm locals and their index space. --- winch/codegen/src/abi/mod.rs | 15 -- winch/codegen/src/codegen/bounds.rs | 6 +- winch/codegen/src/codegen/call.rs | 14 +- winch/codegen/src/codegen/context.rs | 132 ++++++++------ winch/codegen/src/codegen/control.rs | 63 ++++--- winch/codegen/src/codegen/mod.rs | 252 ++++++++++++++------------ winch/codegen/src/codegen/phase.rs | 24 +++ winch/codegen/src/frame/mod.rs | 162 ++++++++++------- winch/codegen/src/isa/aarch64/masm.rs | 21 ++- winch/codegen/src/isa/aarch64/mod.rs | 9 +- winch/codegen/src/isa/x64/masm.rs | 21 ++- winch/codegen/src/isa/x64/mod.rs | 10 +- winch/codegen/src/masm.rs | 16 +- winch/codegen/src/visitor.rs | 9 +- 14 files changed, 436 insertions(+), 318 deletions(-) create mode 100644 winch/codegen/src/codegen/phase.rs diff --git a/winch/codegen/src/abi/mod.rs b/winch/codegen/src/abi/mod.rs index c2fa325c1f50..c205f7ea54c4 100644 --- a/winch/codegen/src/abi/mod.rs +++ b/winch/codegen/src/abi/mod.rs @@ -212,21 +212,6 @@ impl ABIOperand { _ => unreachable!(), } } - - /// Get the register associated to this [`ABIOperand`]. - pub fn get_reg(&self) -> Option { - match *self { - ABIOperand::Reg { reg, .. } => Some(reg), - _ => None, - } - } - - /// Get the type associated to this [`ABIOperand`]. - pub fn ty(&self) -> WasmValType { - match *self { - ABIOperand::Reg { ty, .. } | ABIOperand::Stack { ty, .. } => ty, - } - } } /// Information about the [`ABIOperand`] information used in [`ABISig`]. diff --git a/winch/codegen/src/codegen/bounds.rs b/winch/codegen/src/codegen/bounds.rs index 1904c8b68128..37ebd1088327 100644 --- a/winch/codegen/src/codegen/bounds.rs +++ b/winch/codegen/src/codegen/bounds.rs @@ -4,7 +4,7 @@ use super::env::HeapData; use crate::{ abi::{scratch, vmctx}, - codegen::CodeGenContext, + codegen::{CodeGenContext, Emission}, isa::reg::{writable, Reg}, masm::{IntCmpKind, MacroAssembler, OperandSize, RegImm, TrapCode}, stack::TypedReg, @@ -82,7 +82,7 @@ impl Index { /// Loads the bounds of the dynamic heap. pub(crate) fn load_dynamic_heap_bounds( - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, heap: &HeapData, ptr_size: OperandSize, @@ -149,7 +149,7 @@ pub(crate) fn ensure_index_and_offset( /// criteria is in bounds. pub(crate) fn load_heap_addr_checked( masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ptr_size: OperandSize, heap: &HeapData, enable_spectre_mitigation: bool, diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rs index ee3b4415cd5c..2cbb06bf96b5 100644 --- a/winch/codegen/src/codegen/call.rs +++ b/winch/codegen/src/codegen/call.rs @@ -58,7 +58,7 @@ use crate::{ abi::{scratch, vmctx, ABIOperand, ABISig, RetArea}, - codegen::{BuiltinFunction, BuiltinType, Callee, CodeGenContext}, + codegen::{BuiltinFunction, BuiltinType, Callee, CodeGenContext, Emission}, masm::{ CalleeKind, ContextArgs, MacroAssembler, MemMoveDirection, OperandSize, SPOffset, VMContextLoc, @@ -85,7 +85,7 @@ impl FnCall { pub fn emit( env: &mut FuncEnv, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, callee: Callee, ) { let (kind, callee_context) = Self::lower(env, context.vmoffsets, &callee, context, masm); @@ -129,7 +129,7 @@ impl FnCall { env: &mut FuncEnv, vmoffsets: &VMOffsets, callee: &Callee, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) -> (CalleeKind, ContextArgs) { let ptr = vmoffsets.ptr.size(); @@ -177,7 +177,7 @@ impl FnCall { fn lower_import( index: FuncIndex, sig: &ABISig, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, vmoffsets: &VMOffsets

, ) -> (CalleeKind, ContextArgs) { @@ -204,7 +204,7 @@ impl FnCall { fn lower_funcref( sig: &ABISig, ptr: impl PtrSize, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) -> (CalleeKind, ContextArgs) { // Pop the funcref pointer to a register and allocate a register to hold the @@ -275,7 +275,7 @@ impl FnCall { sig: &ABISig, callee_context: &ContextArgs, ret_area: Option<&RetArea>, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) { let arg_count = sig.params.len_without_retptr(); @@ -337,7 +337,7 @@ impl FnCall { reserved_space: u32, ret_area: Option, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) { // Free any registers holding any function references. match callee_kind { diff --git a/winch/codegen/src/codegen/context.rs b/winch/codegen/src/codegen/context.rs index 8cb315e696a6..99b9dc765667 100644 --- a/winch/codegen/src/codegen/context.rs +++ b/winch/codegen/src/codegen/context.rs @@ -3,6 +3,7 @@ use wasmtime_environ::{VMOffsets, WasmHeapType, WasmValType}; use super::ControlStackFrame; use crate::{ abi::{scratch, vmctx, ABIOperand, ABIResults, RetArea}, + codegen::{CodeGenPhase, Emission, Prologue}, frame::Frame, isa::reg::RegClass, masm::{MacroAssembler, OperandSize, RegImm, SPOffset, ShiftKind, StackSlot}, @@ -26,25 +27,78 @@ use crate::{ /// generation process. The code generation context should /// be generally used as the single entry point to access /// the compound functionality provided by its elements. -pub(crate) struct CodeGenContext<'a> { +pub(crate) struct CodeGenContext<'a, P: CodeGenPhase> { /// The register allocator. pub regalloc: RegAlloc, /// The value stack. pub stack: Stack, /// The current function's frame. - pub frame: Frame, + pub frame: Frame

, /// Reachability state. pub reachable: bool, /// A reference to the VMOffsets. pub vmoffsets: &'a VMOffsets, } -impl<'a> CodeGenContext<'a> { +impl<'a> CodeGenContext<'a, Emission> { + /// Prepares arguments for emitting an i32 shift operation. + pub fn i32_shift(&mut self, masm: &mut M, kind: ShiftKind) + where + M: MacroAssembler, + { + let top = self.stack.peek().expect("value at stack top"); + + if top.is_i32_const() { + let val = self + .stack + .pop_i32_const() + .expect("i32 const value at stack top"); + let typed_reg = self.pop_to_reg(masm, None); + masm.shift_ir( + writable!(typed_reg.reg), + val as u64, + typed_reg.reg, + kind, + OperandSize::S32, + ); + self.stack.push(typed_reg.into()); + } else { + masm.shift(self, kind, OperandSize::S32); + } + } + + /// Prepares arguments for emitting an i64 binary operation. + pub fn i64_shift(&mut self, masm: &mut M, kind: ShiftKind) + where + M: MacroAssembler, + { + let top = self.stack.peek().expect("value at stack top"); + if top.is_i64_const() { + let val = self + .stack + .pop_i64_const() + .expect("i64 const value at stack top"); + let typed_reg = self.pop_to_reg(masm, None); + masm.shift_ir( + writable!(typed_reg.reg), + val as u64, + typed_reg.reg, + kind, + OperandSize::S64, + ); + self.stack.push(typed_reg.into()); + } else { + masm.shift(self, kind, OperandSize::S64); + }; + } +} + +impl<'a> CodeGenContext<'a, Prologue> { /// Create a new code generation context. pub fn new( regalloc: RegAlloc, stack: Stack, - frame: Frame, + frame: Frame, vmoffsets: &'a VMOffsets, ) -> Self { Self { @@ -56,6 +110,19 @@ impl<'a> CodeGenContext<'a> { } } + /// Prepares the frame for the [`Emission`] code generation phase. + pub fn for_emission(self) -> CodeGenContext<'a, Emission> { + CodeGenContext { + regalloc: self.regalloc, + stack: self.stack, + reachable: self.reachable, + vmoffsets: self.vmoffsets, + frame: self.frame.for_emission(), + } + } +} + +impl<'a> CodeGenContext<'a, Emission> { /// Request a specific register to the register allocator, /// spilling if not available. pub fn reg(&mut self, named: Reg, masm: &mut M) -> Reg { @@ -325,57 +392,6 @@ impl<'a> CodeGenContext<'a> { }; } - /// Prepares arguments for emitting an i32 shift operation. - pub fn i32_shift(&mut self, masm: &mut M, kind: ShiftKind) - where - M: MacroAssembler, - { - let top = self.stack.peek().expect("value at stack top"); - - if top.is_i32_const() { - let val = self - .stack - .pop_i32_const() - .expect("i32 const value at stack top"); - let typed_reg = self.pop_to_reg(masm, None); - masm.shift_ir( - writable!(typed_reg.reg), - val as u64, - typed_reg.reg, - kind, - OperandSize::S32, - ); - self.stack.push(typed_reg.into()); - } else { - masm.shift(self, kind, OperandSize::S32); - } - } - - /// Prepares arguments for emitting an i64 binary operation. - pub fn i64_shift(&mut self, masm: &mut M, kind: ShiftKind) - where - M: MacroAssembler, - { - let top = self.stack.peek().expect("value at stack top"); - if top.is_i64_const() { - let val = self - .stack - .pop_i64_const() - .expect("i64 const value at stack top"); - let typed_reg = self.pop_to_reg(masm, None); - masm.shift_ir( - writable!(typed_reg.reg), - val as u64, - typed_reg.reg, - kind, - OperandSize::S64, - ); - self.stack.push(typed_reg.into()); - } else { - masm.shift(self, kind, OperandSize::S64); - }; - } - /// Prepares arguments for emitting a convert operation. pub fn convert_op(&mut self, masm: &mut M, dst_ty: WasmValType, mut emit: F) where @@ -511,7 +527,7 @@ impl<'a> CodeGenContext<'a> { mut calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { let area = results .on_stack() @@ -558,7 +574,7 @@ impl<'a> CodeGenContext<'a> { where M: MacroAssembler, { - let addr = masm.local_address(&self.frame.vmctx_slot); + let addr = masm.local_address(&self.frame.vmctx_slot()); masm.load_ptr(addr, writable!(vmctx!(M))); } @@ -570,7 +586,7 @@ impl<'a> CodeGenContext<'a> { fn spill_impl( stack: &mut Stack, regalloc: &mut RegAlloc, - frame: &Frame, + frame: &Frame, masm: &mut M, ) { stack.inner_mut().iter_mut().for_each(|v| match v { diff --git a/winch/codegen/src/codegen/control.rs b/winch/codegen/src/codegen/control.rs index d9fc7fc7ee8e..d9bab98880bd 100644 --- a/winch/codegen/src/codegen/control.rs +++ b/winch/codegen/src/codegen/control.rs @@ -9,6 +9,7 @@ use super::{CodeGenContext, OperandSize, Reg, TypedReg}; use crate::{ abi::{ABIOperand, ABIResults, ABISig, RetArea, ABI}, + codegen::Emission, masm::{IntCmpKind, MacroAssembler, MemMoveDirection, RegImm, SPOffset}, reg::writable, stack::Val, @@ -248,7 +249,7 @@ impl ControlStackFrame { pub fn r#if( sig: BlockSig, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) -> Self { let mut control = Self::If { cont: masm.get_label(), @@ -266,7 +267,7 @@ impl ControlStackFrame { pub fn block( sig: BlockSig, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) -> Self { let mut control = Self::Block { sig, @@ -283,7 +284,7 @@ impl ControlStackFrame { pub fn r#loop( sig: BlockSig, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) -> Self { let mut control = Self::Loop { stack_state: Default::default(), @@ -295,7 +296,7 @@ impl ControlStackFrame { control } - fn init(&mut self, masm: &mut M, context: &mut CodeGenContext) { + fn init(&mut self, masm: &mut M, context: &mut CodeGenContext) { self.calculate_stack_state(context, masm); // If the block has stack results, immediately resolve the return area // base. @@ -336,7 +337,7 @@ impl ControlStackFrame { /// Calculates the [StackState] of the block. fn calculate_stack_state( &mut self, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) { use ControlStackFrame::*; @@ -388,7 +389,7 @@ impl ControlStackFrame { pub fn ensure_stack_state( &mut self, masm: &mut M, - context: &mut CodeGenContext, + context: &mut CodeGenContext, ) { let state = self.stack_state(); // This assumes that at jump sites, the machine stack pointer will be @@ -416,7 +417,7 @@ impl ControlStackFrame { } } - fn emit(&mut self, masm: &mut M, context: &mut CodeGenContext) { + fn emit(&mut self, masm: &mut M, context: &mut CodeGenContext) { use ControlStackFrame::*; // Do not perform any emissions if we are in an unreachable state. @@ -455,7 +456,11 @@ impl ControlStackFrame { /// Handles the else branch if the current control stack frame is /// [`ControlStackFrame::If`]. - pub fn emit_else(&mut self, masm: &mut M, context: &mut CodeGenContext) { + pub fn emit_else( + &mut self, + masm: &mut M, + context: &mut CodeGenContext, + ) { debug_assert!(self.is_if()); let state = self.stack_state(); @@ -467,7 +472,11 @@ impl ControlStackFrame { /// Binds the else branch label and converts `self` to /// [`ControlStackFrame::Else`]. - pub fn bind_else(&mut self, masm: &mut M, context: &mut CodeGenContext) { + pub fn bind_else( + &mut self, + masm: &mut M, + context: &mut CodeGenContext, + ) { use ControlStackFrame::*; match self { If { @@ -510,7 +519,11 @@ impl ControlStackFrame { } /// Handles the end of a control stack frame. - pub fn emit_end(&mut self, masm: &mut M, context: &mut CodeGenContext) { + pub fn emit_end( + &mut self, + masm: &mut M, + context: &mut CodeGenContext, + ) { use ControlStackFrame::*; match self { If { stack_state, .. } | Else { stack_state, .. } | Block { stack_state, .. } => { @@ -527,7 +540,11 @@ impl ControlStackFrame { /// Binds the exit label of the current control stack frame and pushes the /// ABI results to the value stack. - pub fn bind_end(&mut self, masm: &mut M, context: &mut CodeGenContext) { + pub fn bind_end( + &mut self, + masm: &mut M, + context: &mut CodeGenContext, + ) { self.push_abi_results(context, masm); self.bind_exit_label(masm); } @@ -627,12 +644,12 @@ impl ControlStackFrame { /// updated. pub fn pop_abi_results( &mut self, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { Self::pop_abi_results_impl(self.results::(), context, masm, calculate_ret_area) } @@ -648,12 +665,12 @@ impl ControlStackFrame { /// results should be interpreted. pub fn pop_abi_results_impl( results: &mut ABIResults, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, mut calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { let mut iter = results.operands().iter().rev().peekable(); @@ -690,7 +707,7 @@ impl ControlStackFrame { /// Convenience wrapper around [CodeGenContext::push_abi_results] using the /// results of the current frame. - fn push_abi_results(&mut self, context: &mut CodeGenContext, masm: &mut M) + fn push_abi_results(&mut self, context: &mut CodeGenContext, masm: &mut M) where M: MacroAssembler, { @@ -705,12 +722,12 @@ impl ControlStackFrame { /// taken. pub fn top_abi_results( &mut self, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { Self::top_abi_results_impl::(self.results::(), context, masm, calculate_ret_area) } @@ -720,12 +737,12 @@ impl ControlStackFrame { /// needed. fn top_abi_results_impl( results: &mut ABIResults, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, mut calculate_ret_area: F, ) where M: MacroAssembler, - F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, + F: FnMut(&ABIResults, &mut CodeGenContext, &mut M) -> Option, { let mut area = None; Self::pop_abi_results_impl::(results, context, masm, |r, context, masm| { @@ -764,7 +781,7 @@ impl ControlStackFrame { fn adjust_stack_results( ret_area: RetArea, results: &ABIResults, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) where M: MacroAssembler, @@ -889,7 +906,7 @@ impl ControlStackFrame { /// Ensures that there is enough space for return values on the stack. /// This function is called at the end of all blocks and when branching from /// within blocks. - fn ensure_ret_area(ret_area: &RetArea, context: &mut CodeGenContext, masm: &mut M) + fn ensure_ret_area(ret_area: &RetArea, context: &mut CodeGenContext, masm: &mut M) where M: MacroAssembler, { @@ -907,7 +924,7 @@ impl ControlStackFrame { fn maybe_load_retptr( ret_area: Option<&RetArea>, results: &ABIResults, - context: &mut CodeGenContext, + context: &mut CodeGenContext, masm: &mut M, ) -> Option where diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index a3721867b2c5..63f442c23444 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -13,6 +13,7 @@ use cranelift_codegen::{ ir::{RelSourceLoc, SourceLoc}, }; use smallvec::SmallVec; +use std::marker::PhantomData; use wasmparser::{ BinaryReader, FuncValidator, MemArg, Operator, ValidatorResources, VisitOperator, VisitSimdOperator, @@ -37,6 +38,9 @@ pub(crate) mod bounds; use bounds::{Bounds, ImmOffset, Index}; +mod phase; +pub(crate) use phase::*; + /// Holds metadata about the source code location and the machine code emission. /// The fields of this struct are opaque and are not interpreted in any way. /// They serve as a mapping between source code and machine code. @@ -50,15 +54,16 @@ pub(crate) struct SourceLocation { } /// The code generation abstraction. -pub(crate) struct CodeGen<'a, 'translation: 'a, 'data: 'translation, M> +pub(crate) struct CodeGen<'a, 'translation: 'a, 'data: 'translation, M, P> where M: MacroAssembler, + P: CodeGenPhase, { /// The ABI-specific representation of the function signature, excluding results. pub sig: ABISig, /// The code generation context. - pub context: CodeGenContext<'a>, + pub context: CodeGenContext<'a, P>, /// A reference to the function compilation environment. pub env: FuncEnv<'a, 'translation, 'data, M::Ptr>, @@ -83,19 +88,20 @@ where /// Local counter to track fuel consumption. pub fuel_consumed: i64, + phase: PhantomData

, } -impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> +impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Prologue> where M: MacroAssembler, { pub fn new( tunables: &'a Tunables, masm: &'a mut M, - context: CodeGenContext<'a>, + context: CodeGenContext<'a, Prologue>, env: FuncEnv<'a, 'translation, 'data, M::Ptr>, sig: ABISig, - ) -> Self { + ) -> CodeGen<'a, 'translation, 'data, M, Prologue> { Self { sig, context, @@ -107,32 +113,11 @@ where found_unsupported_instruction: None, // Empty functions should consume at least 1 fuel unit. fuel_consumed: 1, + phase: PhantomData, } } - /// Emit the function body to machine code. - pub fn emit( - &mut self, - body: &mut BinaryReader<'a>, - validator: &mut FuncValidator, - ) -> Result<()> { - self.emit_start() - .and_then(|_| self.emit_body(body, validator)) - .and_then(|_| self.emit_end())?; - - Ok(()) - } - - /// Derives a [RelSourceLoc] from a [SourceLoc]. - pub fn source_loc_from(&mut self, loc: SourceLoc) -> RelSourceLoc { - if self.source_location.base.is_none() && !loc.is_default() { - self.source_location.base = Some(loc); - } - - RelSourceLoc::from_base_offset(self.source_location.base.unwrap_or_default(), loc) - } - - fn emit_start(&mut self) -> Result<()> { + pub fn emit_prologue(mut self) -> Result> { let vmctx = self .sig .params() @@ -153,32 +138,95 @@ where ); self.masm.reserve_stack(self.context.frame.locals_size); + self.spill_register_arguments(); + + let defined_locals_range = &self.context.frame.defined_locals_range; + self.masm.zero_mem_range(defined_locals_range.as_range()); + + // Save the results base parameter register into its slot. + self.sig.params.has_retptr().then(|| { + match self.sig.params.unwrap_results_area_operand() { + ABIOperand::Reg { ty, reg, .. } => { + let results_base_slot = self.context.frame.results_base_slot.as_ref().unwrap(); + debug_assert!(results_base_slot.addressed_from_sp()); + let addr = self.masm.local_address(results_base_slot); + self.masm.store((*reg).into(), addr, (*ty).into()); + } + // The result base parameter is a stack parameter, addressed + // from FP. + _ => {} + } + }); self.masm.end_source_loc(); - if self.tunables.consume_fuel { - self.emit_fuel_check(); + Ok(CodeGen { + sig: self.sig, + context: self.context.for_emission(), + masm: self.masm, + env: self.env, + tunables: self.tunables, + source_location: self.source_location, + control_frames: self.control_frames, + found_unsupported_instruction: self.found_unsupported_instruction, + fuel_consumed: self.fuel_consumed, + phase: PhantomData, + }) + } + + fn spill_register_arguments(&mut self) { + use WasmValType::*; + for (operand, slot) in self + .sig + .params_without_retptr() + .iter() + .zip(self.context.frame.locals()) + { + match (operand, slot) { + (ABIOperand::Reg { ty, reg, .. }, slot) => { + let addr = self.masm.local_address(slot); + match &ty { + I32 | I64 | F32 | F64 | V128 => { + self.masm.store((*reg).into(), addr, (*ty).into()) + } + Ref(rt) => match rt.heap_type { + WasmHeapType::Func | WasmHeapType::Extern => { + self.masm.store_ptr((*reg).into(), addr) + } + ht => unimplemented!("Support for WasmHeapType: {ht}"), + }, + } + } + // Skip non-register arguments + _ => {} + } } + } +} - // Once we have emitted the epilogue and reserved stack space for the locals, we push the - // base control flow block. - self.control_frames.push(ControlStackFrame::block( - BlockSig::from_sig(self.sig.clone()), - self.masm, - &mut self.context, - )); +impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> +where + M: MacroAssembler, +{ + /// Emit the function body to machine code. + pub fn emit( + &mut self, + body: &mut BinaryReader<'a>, + validator: &mut FuncValidator, + ) -> Result<()> { + self.emit_body(body, validator) + .and_then(|_| self.emit_end())?; - // Set the return area of the results *after* initializing the block. In - // the function body block case, we'll treat the results as any other - // case, addressed from the stack pointer, and when ending the function - // the return area will be set to the return pointer. - if self.sig.params.has_retptr() { - self.sig - .results - .set_ret_area(RetArea::slot(self.context.frame.results_base_slot.unwrap())); + Ok(()) + } + + /// Derives a [RelSourceLoc] from a [SourceLoc]. + pub fn source_loc_from(&mut self, loc: SourceLoc) -> RelSourceLoc { + if self.source_location.base.is_none() && !loc.is_default() { + self.source_location.base = Some(loc); } - Ok(()) + RelSourceLoc::from_base_offset(self.source_location.base.unwrap_or_default(), loc) } /// The following two helpers, handle else or end instructions when the @@ -224,24 +272,27 @@ where body: &mut BinaryReader<'a>, validator: &mut FuncValidator, ) -> Result<()> { - self.spill_register_arguments(); - let defined_locals_range = &self.context.frame.defined_locals_range; - self.masm.zero_mem_range(defined_locals_range.as_range()); + if self.tunables.consume_fuel { + self.emit_fuel_check(); + } - // Save the results base parameter register into its slot. - self.sig.params.has_retptr().then(|| { - match self.sig.params.unwrap_results_area_operand() { - ABIOperand::Reg { ty, reg, .. } => { - let results_base_slot = self.context.frame.results_base_slot.as_ref().unwrap(); - debug_assert!(results_base_slot.addressed_from_sp()); - let addr = self.masm.local_address(results_base_slot); - self.masm.store((*reg).into(), addr, (*ty).into()); - } - // The result base parameter is a stack parameter, addressed - // from FP. - _ => {} - } - }); + // Once we have emitted the epilogue and reserved stack space for the locals, we push the + // base control flow block. + self.control_frames.push(ControlStackFrame::block( + BlockSig::from_sig(self.sig.clone()), + self.masm, + &mut self.context, + )); + + // Set the return area of the results *after* initializing the block. In + // the function body block case, we'll treat the results as any other + // case, addressed from the stack pointer, and when ending the function + // the return area will be set to the return pointer. + if self.sig.params.has_retptr() { + self.sig + .results + .set_ret_area(RetArea::slot(self.context.frame.results_base_slot.unwrap())); + } while !body.eof() { let offset = body.original_position(); @@ -307,7 +358,7 @@ where } impl<'a, 'translation, 'data, M: MacroAssembler> VisitorHooks - for CodeGen<'a, 'translation, 'data, M> + for CodeGen<'a, 'translation, 'data, M, Emission> { fn visit(&self, op: &Operator) -> bool { self.context.reachable || visit_op_when_unreachable(op) @@ -431,35 +482,6 @@ where Ok(()) } - fn spill_register_arguments(&mut self) { - use WasmValType::*; - self.sig - // Skip the results base param if any; [Self::emit_body], - // will handle spilling the results base param if it's in a register. - .params_without_retptr() - .iter() - .enumerate() - .filter(|(_, a)| a.is_reg()) - .for_each(|(index, arg)| { - let ty = arg.ty(); - let local = self.context.frame.get_frame_local(index); - let addr = self.masm.local_address(local); - let src = arg - .get_reg() - .expect("arg should be associated to a register"); - - match &ty { - I32 | I64 | F32 | F64 | V128 => self.masm.store(src.into(), addr, ty.into()), - Ref(rt) => match rt.heap_type { - WasmHeapType::Func | WasmHeapType::Extern => { - self.masm.store_ptr(src.into(), addr) - } - ht => unimplemented!("Support for WasmHeapType: {ht}"), - }, - } - }); - } - /// Pops the value at the stack top and assigns it to the local at /// the given index, returning the typed register holding the /// source value. @@ -633,8 +655,12 @@ where // * index + offset + access_size overflows // OR // * index + offset + access_size > bound - let bounds = - bounds::load_dynamic_heap_bounds(&mut self.context, self.masm, &heap, ptr_size); + let bounds = bounds::load_dynamic_heap_bounds::<_>( + &mut self.context, + self.masm, + &heap, + ptr_size, + ); let index_reg = index.as_typed_reg().reg; // Allocate a temporary register to hold @@ -978,6 +1004,24 @@ where self.context.free_reg(fuel_var); } + /// Emits a series of instructions that load the `fuel_consumed` field from + /// `VMRuntimeLimits`. + fn emit_load_fuel_consumed(&mut self, fuel_var: Reg) { + let limits_offset = self.env.vmoffsets.ptr.vmctx_runtime_limits(); + let fuel_offset = self.env.vmoffsets.ptr.vmruntime_limits_fuel_consumed(); + self.masm.load_ptr( + self.masm.address_at_vmctx(u32::from(limits_offset)), + writable!(fuel_var), + ); + + self.masm.load( + self.masm.address_at_reg(fuel_var, u32::from(fuel_offset)), + writable!(fuel_var), + // Fuel is an i64. + OperandSize::S64, + ); + } + /// Increments the fuel consumed in `VMRuntimeLimits` by flushing /// `self.fuel_consumed` to memory. fn emit_fuel_increment(&mut self) { @@ -1022,24 +1066,6 @@ where self.context.free_reg(limits_var); } - /// Emits a series of instructions that load the `fuel_consumed` field from - /// `VMRuntimeLimits`. - fn emit_load_fuel_consumed(&mut self, fuel_var: Reg) { - let limits_offset = self.env.vmoffsets.ptr.vmctx_runtime_limits(); - let fuel_offset = self.env.vmoffsets.ptr.vmruntime_limits_fuel_consumed(); - self.masm.load_ptr( - self.masm.address_at_vmctx(u32::from(limits_offset)), - writable!(fuel_var), - ); - - self.masm.load( - self.masm.address_at_reg(fuel_var, u32::from(fuel_offset)), - writable!(fuel_var), - // Fuel is an i64. - OperandSize::S64, - ); - } - /// Hook to handle fuel before visiting an operator. fn fuel_before_visit_op(&mut self, op: &Operator) { if !self.context.reachable { diff --git a/winch/codegen/src/codegen/phase.rs b/winch/codegen/src/codegen/phase.rs new file mode 100644 index 000000000000..de2d2e0206fa --- /dev/null +++ b/winch/codegen/src/codegen/phase.rs @@ -0,0 +1,24 @@ +//! Type-based states to represent code generation phases. +//! These states help enforce code generation invariants at compile time. +//! +//! Currently two phases are defined for code generation: +//! +//! * Prologue: responsible of setting up the function's frame. +//! * Emission: emission of Wasm code to machine code. + +/// A code generation phase. +pub trait CodeGenPhase {} + +/// The prologue phase. +/// +/// Its main responsibility is to setup the function's frame, by creating the +/// well known local slots. In this phase, writes to such slots is allowed. +/// After this phase, the frame is considered immutable. +pub struct Prologue; +/// The code emission phase. +/// +/// Its main responsibility is to emit Wasm code to machine code. +pub struct Emission; + +impl CodeGenPhase for Prologue {} +impl CodeGenPhase for Emission {} diff --git a/winch/codegen/src/frame/mod.rs b/winch/codegen/src/frame/mod.rs index 1ebdace8daf3..c813141d7184 100644 --- a/winch/codegen/src/frame/mod.rs +++ b/winch/codegen/src/frame/mod.rs @@ -1,18 +1,27 @@ use crate::{ abi::{align_to, ABIOperand, ABISig, LocalSlot, ABI}, + codegen::{CodeGenPhase, Emission, Prologue}, masm::MacroAssembler, }; use anyhow::Result; use smallvec::SmallVec; +use std::marker::PhantomData; use std::ops::Range; use wasmparser::{BinaryReader, FuncValidator, ValidatorResources}; use wasmtime_environ::{TypeConvert, WasmValType}; +/// WebAssembly locals. // TODO: // SpiderMonkey's implementation uses 16; // (ref: https://searchfox.org/mozilla-central/source/js/src/wasm/WasmBCFrame.h#585) // during instrumentation we should measure to verify if this is a good default. -pub(crate) type Locals = SmallVec<[LocalSlot; 16]>; +pub(crate) type WasmLocals = SmallVec<[LocalSlot; 16]>; +/// Special local slots used by the compiler. +// Winch's ABI uses two extra parameters to store the callee and caller +// VMContext pointers. +// These arguments are spilled and treated as frame locals, but not +// WebAssembly locals. +pub(crate) type SpecialLocals = [LocalSlot; 2]; /// Function defined locals start and end in the frame. pub(crate) struct DefinedLocalsRange(Range); @@ -28,7 +37,7 @@ impl DefinedLocalsRange { #[derive(Default)] pub(crate) struct DefinedLocals { /// The defined locals for a function. - pub defined_locals: Locals, + pub defined_locals: WasmLocals, /// The size of the defined locals. pub stack_size: u32, } @@ -43,7 +52,7 @@ impl DefinedLocals { let mut next_stack: u32 = 0; // The first 32 bits of a Wasm binary function describe the number of locals. let local_count = reader.read_var_u32()?; - let mut slots: Locals = Default::default(); + let mut slots: WasmLocals = Default::default(); for _ in 0..local_count { let position = reader.original_position(); @@ -67,7 +76,7 @@ impl DefinedLocals { } /// Frame handler abstraction. -pub(crate) struct Frame { +pub(crate) struct Frame { /// The size of the entire local area; the arguments plus the function defined locals. pub locals_size: u32, @@ -78,23 +87,24 @@ pub(crate) struct Frame { /// /// Locals get calculated when allocating a frame and are readonly /// through the function compilation lifetime. - locals: Locals, - - /// The offset to the slot containing the `VMContext`. - pub vmctx_slot: LocalSlot, + wasm_locals: WasmLocals, + /// Special locals used by the internal ABI. See [`SpecialLocals`]. + special_locals: SpecialLocals, /// The slot holding the address of the results area. pub results_base_slot: Option, + marker: PhantomData

, } -impl Frame { +impl Frame { /// Allocate a new [`Frame`]. - pub fn new(sig: &ABISig, defined_locals: &DefinedLocals) -> Result { - let (mut locals, defined_locals_start) = Self::compute_arg_slots::(sig)?; + pub fn new(sig: &ABISig, defined_locals: &DefinedLocals) -> Result> { + let (special_locals, mut wasm_locals, defined_locals_start) = + Self::compute_arg_slots::(sig)?; // The defined locals have a zero-based offset by default // so we need to add the defined locals start to the offset. - locals.extend( + wasm_locals.extend( defined_locals .defined_locals .iter() @@ -138,66 +148,37 @@ impl Frame { (None, defined_locals_end) }; - let vmctx_slot = *locals.get(0).expect("LocalSlot for VMContext"); Ok(Self { - locals, + wasm_locals, + special_locals, locals_size, - vmctx_slot, defined_locals_range: DefinedLocalsRange( defined_locals_start..(defined_locals_start + defined_locals.stack_size), ), results_base_slot, + marker: PhantomData, }) } - // Winch's ABI uses two extra parameters to store the callee and caller - // VMContext pointers. - // These arguments are spilled and treated as frame locals, but not - // WebAssembly locals. - const WASM_LOCALS_OFFSET: usize = 2; - - /// Get the [LocalSlot] for a WebAssembly local. - /// This method assumes that the index is bound to u32::MAX, representing - /// the index space for WebAssembly locals. - /// - /// # Panics - /// This method panics if the index is not associated to a valid WebAssembly - /// local. - pub fn get_wasm_local(&self, index: u32) -> &LocalSlot { - let local_index = Self::WASM_LOCALS_OFFSET + index as usize; - self.locals - .get(local_index) - .unwrap_or_else(|| panic!(" Expected WebAssembly local at slot: {index}")) - } - - /// Get the [LocalSlot] for a frame local. - /// This method doesn't make any asumptions about the local index passed in, - /// and simply delegates the [LocalSlot] retrieval to the underlying locals - /// vector. - /// - /// # Panics - /// This method panics if the index is not associated to a valid WebAssembly - /// local. - pub fn get_frame_local(&self, index: usize) -> &LocalSlot { - self.locals - .get(index) - .unwrap_or_else(|| panic!(" Expected Frame local at slot: {index}")) + /// Returns an iterator over all the [`LocalSlot`]s in the frame, including + /// the [`SpecialLocals`]. + pub fn locals(&self) -> impl Iterator { + self.special_locals.iter().chain(self.wasm_locals.iter()) } - /// Returns the address of the local at the given index. - /// - /// # Panics - /// This function panics if the index is not associated to a local. - pub fn get_local_address( - &self, - index: u32, - masm: &mut M, - ) -> (WasmValType, M::Address) { - let slot = self.get_wasm_local(index); - (slot.ty, masm.local_address(&slot)) + /// Prepares the frame for the [`Emission`] code generation phase. + pub fn for_emission(self) -> Frame { + Frame { + wasm_locals: self.wasm_locals, + special_locals: self.special_locals, + locals_size: self.locals_size, + defined_locals_range: self.defined_locals_range, + results_base_slot: self.results_base_slot, + marker: PhantomData, + } } - fn compute_arg_slots(sig: &ABISig) -> Result<(Locals, u32)> { + fn compute_arg_slots(sig: &ABISig) -> Result<(SpecialLocals, WasmLocals, u32)> { // Go over the function ABI-signature and // calculate the stack slots. // @@ -231,13 +212,24 @@ impl Frame { // Skip the results base param; if present, the [Frame] will create // a dedicated slot for it. - let slots: Locals = sig - .params_without_retptr() - .into_iter() + let mut params_iter = sig.params_without_retptr().into_iter(); + + // Handle special local slots. + let callee_vmctx = params_iter + .next() + .map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset)) + .expect("Slot for VMContext"); + + let caller_vmctx = params_iter + .next() + .map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset)) + .expect("Slot for VMContext"); + + let slots: WasmLocals = params_iter .map(|arg| Self::abi_arg_slot(&arg, &mut next_stack, arg_base_offset)) .collect(); - Ok((slots, next_stack)) + Ok(([callee_vmctx, caller_vmctx], slots, next_stack)) } fn abi_arg_slot(arg: &ABIOperand, next_stack: &mut u32, arg_base_offset: u32) -> LocalSlot { @@ -256,3 +248,47 @@ impl Frame { } } } + +impl Frame { + /// Get the [`LocalSlot`] for a WebAssembly local. + /// This method assumes that the index is bound to u32::MAX, representing + /// the index space for WebAssembly locals. + /// + /// # Panics + /// This method panics if the index is not associated to a valid WebAssembly + /// local. + pub fn get_wasm_local(&self, index: u32) -> &LocalSlot { + self.wasm_locals + .get(index as usize) + .unwrap_or_else(|| panic!(" Expected WebAssembly local at slot: {index}")) + } + + /// Get the [`LocalSlot`] for a special local. + /// + /// # Panics + /// This method panics if the index is not associated to a valid special + /// local. + pub fn get_special_local(&self, index: usize) -> &LocalSlot { + self.special_locals + .get(index) + .unwrap_or_else(|| panic!(" Expected special local at slot: {index}")) + } + + /// Get the special [`LocalSlot`] for the `VMContext`. + pub fn vmctx_slot(&self) -> &LocalSlot { + self.get_special_local(0) + } + + /// Returns the address of the local at the given index. + /// + /// # Panics + /// This function panics if the index is not associated to a local. + pub fn get_local_address( + &self, + index: u32, + masm: &mut M, + ) -> (WasmValType, M::Address) { + let slot = self.get_wasm_local(index); + (slot.ty, masm.local_address(&slot)) + } +} diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs index 02226387baa3..9be8a60f772a 100644 --- a/winch/codegen/src/isa/aarch64/masm.rs +++ b/winch/codegen/src/isa/aarch64/masm.rs @@ -1,7 +1,7 @@ use super::{abi::Aarch64ABI, address::Address, asm::Assembler, regs}; use crate::{ abi::local::LocalSlot, - codegen::{ptr_type_from_ptr_size, CodeGenContext, FuncEnv}, + codegen::{ptr_type_from_ptr_size, CodeGenContext, Emission, FuncEnv}, isa::reg::{writable, Reg, WritableReg}, masm::{ CalleeKind, DivKind, ExtendKind, FloatCmpKind, Imm as I, IntCmpKind, @@ -350,11 +350,11 @@ impl Masm for MacroAssembler { self.asm.fabs_rr(dst.to_reg(), dst, size); } - fn float_round, &mut CodeGenContext, &mut Self)>( + fn float_round, &mut CodeGenContext, &mut Self)>( &mut self, mode: RoundingMode, _env: &mut FuncEnv, - context: &mut CodeGenContext, + context: &mut CodeGenContext, size: OperandSize, _fallback: F, ) { @@ -433,7 +433,12 @@ impl Masm for MacroAssembler { self.asm.shift_ir(imm, lhs, dst, kind, size) } - fn shift(&mut self, context: &mut CodeGenContext, kind: ShiftKind, size: OperandSize) { + fn shift( + &mut self, + context: &mut CodeGenContext, + kind: ShiftKind, + size: OperandSize, + ) { let src = context.pop_to_reg(self, None); let dst = context.pop_to_reg(self, None); @@ -444,11 +449,11 @@ impl Masm for MacroAssembler { context.stack.push(dst.into()); } - fn div(&mut self, _context: &mut CodeGenContext, _kind: DivKind, _size: OperandSize) { + fn div(&mut self, _context: &mut CodeGenContext, _kind: DivKind, _size: OperandSize) { todo!() } - fn rem(&mut self, _context: &mut CodeGenContext, _kind: RemKind, _size: OperandSize) { + fn rem(&mut self, _context: &mut CodeGenContext, _kind: RemKind, _size: OperandSize) { todo!() } @@ -456,7 +461,7 @@ impl Masm for MacroAssembler { self.asm.load_constant(0, reg); } - fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize) { + fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize) { let src = context.pop_to_reg(self, None); let tmp = regs::float_scratch(); self.asm.mov_to_fpu(src.into(), writable!(tmp), size); @@ -700,7 +705,7 @@ impl Masm for MacroAssembler { todo!() } - fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind) { + fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind) { let _ = (context, kind); todo!() } diff --git a/winch/codegen/src/isa/aarch64/mod.rs b/winch/codegen/src/isa/aarch64/mod.rs index 17c49577a6fb..e54ffa0c9d4e 100644 --- a/winch/codegen/src/isa/aarch64/mod.rs +++ b/winch/codegen/src/isa/aarch64/mod.rs @@ -123,11 +123,12 @@ impl TargetIsa for Aarch64 { ); let regalloc = RegAlloc::from(gpr, fpr); let codegen_context = CodeGenContext::new(regalloc, stack, frame, &vmoffsets); - let mut codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); + let codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); - codegen.emit(&mut body, validator)?; - let names = codegen.env.take_name_map(); - let base = codegen.source_location.base; + let mut body_codegen = codegen.emit_prologue()?; + body_codegen.emit(&mut body, validator)?; + let names = body_codegen.env.take_name_map(); + let base = body_codegen.source_location.base; Ok(CompiledFunction::new( masm.finalize(base), names, diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs index 4d72b79e6965..9937d165e9b9 100644 --- a/winch/codegen/src/isa/x64/masm.rs +++ b/winch/codegen/src/isa/x64/masm.rs @@ -12,7 +12,7 @@ use crate::masm::{ }; use crate::{ abi::{self, align_to, calculate_frame_adjustment, LocalSlot}, - codegen::{ptr_type_from_ptr_size, CodeGenContext, FuncEnv}, + codegen::{ptr_type_from_ptr_size, CodeGenContext, Emission, FuncEnv}, stack::{TypedReg, Val}, }; use crate::{ @@ -474,11 +474,11 @@ impl Masm for MacroAssembler { self.asm.xmm_and_rr(scratch_xmm, dst, size); } - fn float_round, &mut CodeGenContext, &mut Self)>( + fn float_round, &mut CodeGenContext, &mut Self)>( &mut self, mode: RoundingMode, env: &mut FuncEnv, - context: &mut CodeGenContext, + context: &mut CodeGenContext, size: OperandSize, mut fallback: F, ) { @@ -565,7 +565,12 @@ impl Masm for MacroAssembler { self.asm.shift_ir(imm as u8, dst, kind, size) } - fn shift(&mut self, context: &mut CodeGenContext, kind: ShiftKind, size: OperandSize) { + fn shift( + &mut self, + context: &mut CodeGenContext, + kind: ShiftKind, + size: OperandSize, + ) { // Number of bits to shift must be in the CL register. let src = context.pop_to_reg(self, Some(regs::rcx())); let dst = context.pop_to_reg(self, None); @@ -577,7 +582,7 @@ impl Masm for MacroAssembler { context.stack.push(dst.into()); } - fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize) { + fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize) { // Allocate rdx:rax. let rdx = context.reg(regs::rdx(), self); let rax = context.reg(regs::rax(), self); @@ -599,7 +604,7 @@ impl Masm for MacroAssembler { context.stack.push(rax.into()); } - fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize) { + fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize) { // Allocate rdx:rax. let rdx = context.reg(regs::rdx(), self); let rax = context.reg(regs::rax(), self); @@ -787,7 +792,7 @@ impl Masm for MacroAssembler { self.asm.jmp(target); } - fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize) { + fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize) { let src = context.pop_to_reg(self, None); if self.flags.has_popcnt() && self.flags.has_sse42() { self.asm.popcnt(src.into(), size); @@ -1028,7 +1033,7 @@ impl Masm for MacroAssembler { self.asm.sbb_rr(rhs_hi, dst_hi, OperandSize::S64); } - fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind) { + fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind) { // Reserve rax/rdx since they're required by the `mul_wide` instruction // being used here. let rax = context.reg(regs::rax(), self); diff --git a/winch/codegen/src/isa/x64/mod.rs b/winch/codegen/src/isa/x64/mod.rs index ae038f6d7776..de0029d08d00 100644 --- a/winch/codegen/src/isa/x64/mod.rs +++ b/winch/codegen/src/isa/x64/mod.rs @@ -134,12 +134,14 @@ impl TargetIsa for X64 { let regalloc = RegAlloc::from(gpr, fpr); let codegen_context = CodeGenContext::new(regalloc, stack, frame, &vmoffsets); - let mut codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); + let codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); - codegen.emit(&mut body, validator)?; - let base = codegen.source_location.base; + let mut body_codegen = codegen.emit_prologue()?; - let names = codegen.env.take_name_map(); + body_codegen.emit(&mut body, validator)?; + let base = body_codegen.source_location.base; + + let names = body_codegen.env.take_name_map(); Ok(CompiledFunction::new( masm.finalize(base), names, diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs index 95de6e789225..1bd5a7a59b4e 100644 --- a/winch/codegen/src/masm.rs +++ b/winch/codegen/src/masm.rs @@ -1,5 +1,5 @@ use crate::abi::{self, align_to, scratch, LocalSlot}; -use crate::codegen::{CodeGenContext, FuncEnv}; +use crate::codegen::{CodeGenContext, Emission, FuncEnv}; use crate::isa::reg::{writable, Reg, WritableReg}; use cranelift_codegen::{ binemit::CodeOffset, @@ -744,11 +744,11 @@ pub(crate) trait MacroAssembler { fn float_neg(&mut self, dst: WritableReg, size: OperandSize); /// Perform a floating point floor operation. - fn float_round, &mut CodeGenContext, &mut Self)>( + fn float_round, &mut CodeGenContext, &mut Self)>( &mut self, mode: RoundingMode, env: &mut FuncEnv, - context: &mut CodeGenContext, + context: &mut CodeGenContext, size: OperandSize, fallback: F, ); @@ -781,7 +781,7 @@ pub(crate) trait MacroAssembler { /// caller from having to deal with the architecture specific constraints /// we give this function access to the code generation context, allowing /// each implementation to decide the lowering path. - fn shift(&mut self, context: &mut CodeGenContext, kind: ShiftKind, size: OperandSize); + fn shift(&mut self, context: &mut CodeGenContext, kind: ShiftKind, size: OperandSize); /// Perform division operation. /// Division is special in that some architectures have specific @@ -794,10 +794,10 @@ pub(crate) trait MacroAssembler { /// unconstrained binary operation, the caller can decide to use /// the `CodeGenContext::i32_binop` or `CodeGenContext::i64_binop` /// functions. - fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize); + fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize); /// Calculate remainder. - fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize); + fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize); /// Compares `src1` against `src2` for the side effect of setting processor /// flags. @@ -852,7 +852,7 @@ pub(crate) trait MacroAssembler { /// Count the number of 1 bits in src and put the result in dst. In x64, /// this will emit multiple instructions if the `has_popcnt` flag is false. - fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize); + fn popcnt(&mut self, context: &mut CodeGenContext, size: OperandSize); /// Converts an i64 to an i32 by discarding the high 32 bits. fn wrap(&mut self, dst: WritableReg, src: Reg); @@ -1053,5 +1053,5 @@ pub(crate) trait MacroAssembler { /// /// Note that some platforms require special handling of registers in this /// instruction (e.g. x64) so full access to `CodeGenContext` is provided. - fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind); + fn mul_wide(&mut self, context: &mut CodeGenContext, kind: MulWideKind); } diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs index c38cd41ac1ed..1fae5644330e 100644 --- a/winch/codegen/src/visitor.rs +++ b/winch/codegen/src/visitor.rs @@ -5,7 +5,7 @@ //! machine code emitter. use crate::abi::RetArea; -use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, FnCall}; +use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, Emission, FnCall}; use crate::masm::{ DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, MulWideKind, OperandSize, RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, @@ -253,7 +253,7 @@ macro_rules! def_unsupported { (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; } -impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M> +impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> where M: MacroAssembler, { @@ -2212,7 +2212,8 @@ where wasmparser::for_each_visit_operator!(def_unsupported); } -impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> for CodeGen<'a, 'translation, 'data, M> +impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> + for CodeGen<'a, 'translation, 'data, M, Emission> where M: MacroAssembler, { @@ -2231,7 +2232,7 @@ where wasmparser::for_each_visit_simd_operator!(def_unsupported); } -impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> +impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> where M: MacroAssembler, {