From 4674f5c74d5a1a7a6c60bbe90ea6b93581bc4a99 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Mon, 30 Mar 2020 19:48:02 -0700 Subject: [PATCH] Refactor unwind generation in Cranelift. This commit makes the following changes to unwind information generation in Cranelift: * Remove frame layout change implementation in favor of processing the prologue and epilogue instructions when unwind information is requested. This also means this work is no longer performed for Windows, which didn't utilize it. It also helps simplify the prologue and epilogue generation code. * Remove the unwind sink implementation that required each unwind information to be represented in final form. For FDEs, this meant writing a complete frame table per function, which wastes 20 bytes or so for each function with duplicate CIEs. This also enables Cranelift users to collect the unwind information and write it as a single frame table. * For System V calling convention, the unwind information is no longer stored in code memory (it's only a requirement for Windows ABI to do so). This allows for more compact code memory for modules with a lot of functions. * Deletes some duplicate code relating to frame table generation. Users can now simply use gimli to create a frame table from each function's unwind information. Fixes #1181. --- Cargo.lock | 1 + cranelift/codegen/src/binemit/mod.rs | 30 - cranelift/codegen/src/context.rs | 20 +- cranelift/codegen/src/ir/framelayout.rs | 74 -- cranelift/codegen/src/ir/function.rs | 23 +- cranelift/codegen/src/ir/mod.rs | 2 - cranelift/codegen/src/isa/mod.rs | 33 +- cranelift/codegen/src/isa/unwind.rs | 16 + cranelift/codegen/src/isa/unwind/systemv.rs | 122 ++++ cranelift/codegen/src/isa/x86/abi.rs | 296 ++------ cranelift/codegen/src/isa/x86/fde.rs | 451 ------------ cranelift/codegen/src/isa/x86/mod.rs | 26 +- cranelift/codegen/src/isa/x86/unwind.rs | 539 +-------------- .../codegen/src/isa/x86/unwind/systemv.rs | 489 ++++++++++++++ .../codegen/src/isa/x86/unwind/windows.rs | 522 ++++++++++++++ .../filetests/isa/x86/systemv_x64_unwind.clif | 197 ++++++ .../isa/x86/windows_fastcall_x64_unwind.clif | 267 +++----- .../isa/x86/windows_systemv_x64_fde.clif | 54 -- cranelift/filetests/src/lib.rs | 2 - cranelift/filetests/src/test_fde.rs | 415 ------------ cranelift/filetests/src/test_unwind.rs | 639 ++++++++++++++---- crates/api/src/trampoline/func.rs | 9 +- crates/debug/src/frame.rs | 140 ---- crates/debug/src/lib.rs | 50 +- crates/debug/src/transform/expression.rs | 3 +- crates/environ/src/cache.rs | 5 - crates/environ/src/cache/tests.rs | 1 - crates/environ/src/compilation.rs | 133 +--- crates/environ/src/cranelift.rs | 57 +- crates/environ/src/data_structures.rs | 5 +- crates/environ/src/frame_layout.rs | 21 - crates/environ/src/lib.rs | 7 +- crates/environ/src/lightbeam.rs | 2 +- crates/jit/Cargo.toml | 1 + crates/jit/src/code_memory.rs | 146 ++-- crates/jit/src/compiler.rs | 77 +-- crates/jit/src/function_table.rs | 224 ------ crates/jit/src/lib.rs | 2 +- crates/jit/src/unwind.rs | 11 + crates/jit/src/unwind/systemv.rs | 140 ++++ crates/jit/src/unwind/winx64.rs | 91 +++ src/obj.rs | 57 +- 42 files changed, 2463 insertions(+), 2937 deletions(-) delete mode 100644 cranelift/codegen/src/ir/framelayout.rs create mode 100644 cranelift/codegen/src/isa/unwind.rs create mode 100644 cranelift/codegen/src/isa/unwind/systemv.rs delete mode 100644 cranelift/codegen/src/isa/x86/fde.rs create mode 100644 cranelift/codegen/src/isa/x86/unwind/systemv.rs create mode 100644 cranelift/codegen/src/isa/x86/unwind/windows.rs create mode 100644 cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif delete mode 100644 cranelift/filetests/filetests/isa/x86/windows_systemv_x64_fde.clif delete mode 100644 cranelift/filetests/src/test_fde.rs delete mode 100644 crates/debug/src/frame.rs delete mode 100644 crates/environ/src/frame_layout.rs delete mode 100644 crates/jit/src/function_table.rs create mode 100644 crates/jit/src/unwind.rs create mode 100644 crates/jit/src/unwind/systemv.rs create mode 100644 crates/jit/src/unwind/winx64.rs diff --git a/Cargo.lock b/Cargo.lock index 5859533f3c31..10bd289642c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2298,6 +2298,7 @@ dependencies = [ "cranelift-frontend", "cranelift-native", "cranelift-wasm", + "gimli", "log", "more-asserts", "region", diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index 347e3bbadd5d..7219b70622f6 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -164,36 +164,6 @@ pub trait CodeSink { fn add_stackmap(&mut self, _: &[Value], _: &Function, _: &dyn TargetIsa); } -/// Type of the frame unwind information. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum FrameUnwindKind { - /// Windows fastcall unwinding (as in .pdata). - Fastcall, - /// FDE entry for libunwind (similar to .eh_frame format). - Libunwind, -} - -/// Offset in frame unwind information buffer. -pub type FrameUnwindOffset = usize; - -/// Sink for frame unwind information. -pub trait FrameUnwindSink { - /// Get the current position. - fn len(&self) -> FrameUnwindOffset; - - /// Add bytes to the code section. - fn bytes(&mut self, _: &[u8]); - - /// Reserves bytes in the buffer. - fn reserve(&mut self, _len: usize) {} - - /// Add a relocation entry. - fn reloc(&mut self, _: Reloc, _: FrameUnwindOffset); - - /// Specified offset to main structure. - fn set_entry_offset(&mut self, _: FrameUnwindOffset); -} - /// Report a bad encoding error. #[cold] pub fn bad_encoding(func: &Function, inst: Inst) -> ! { diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index ca70293c05fb..f67282cd5324 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -10,8 +10,8 @@ //! single ISA instance. use crate::binemit::{ - relax_branches, shrink_instructions, CodeInfo, FrameUnwindKind, FrameUnwindSink, - MemoryCodeSink, RelocSink, StackmapSink, TrapSink, + relax_branches, shrink_instructions, CodeInfo, MemoryCodeSink, RelocSink, StackmapSink, + TrapSink, }; use crate::dce::do_dce; use crate::dominator_tree::DominatorTree; @@ -195,19 +195,15 @@ impl Context { sink.info } - /// Emit unwind information. + /// Creates unwind information for the function. /// - /// Requires that the function layout be calculated (see `relax_branches`). - /// - /// Only some calling conventions (e.g. Windows fastcall) will have unwind information. - /// This is a no-op if the function has no unwind information. - pub fn emit_unwind_info( + /// Returns `None` if the function has no unwind information. + #[cfg(feature = "unwind")] + pub fn create_unwind_info( &self, isa: &dyn TargetIsa, - kind: FrameUnwindKind, - sink: &mut dyn FrameUnwindSink, - ) { - isa.emit_unwind_info(&self.func, kind, sink); + ) -> Option { + isa.create_unwind_info(&self.func) } /// Run the verifier on the function. diff --git a/cranelift/codegen/src/ir/framelayout.rs b/cranelift/codegen/src/ir/framelayout.rs deleted file mode 100644 index f4cb5bb909f3..000000000000 --- a/cranelift/codegen/src/ir/framelayout.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Frame layout item changes. - -use crate::ir::entities::Inst; -use crate::isa::RegUnit; -use std::boxed::Box; - -use crate::HashMap; - -#[cfg(feature = "enable-serde")] -use serde::{Deserialize, Serialize}; - -/// Change in the frame layout information. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] -pub enum FrameLayoutChange { - /// Base CallFrameAddress (CFA) pointer moved to different register/offset. - CallFrameAddressAt { - /// CFA register. - reg: RegUnit, - /// CFA offset. - offset: isize, - }, - /// Register saved at. - RegAt { - /// Saved register. - reg: RegUnit, - /// Offset in the frame (offset from CFA). - cfa_offset: isize, - }, - /// Return address saved at. - ReturnAddressAt { - /// Offset in the frame (offset from CFA). - cfa_offset: isize, - }, - /// The entire frame layout must be preserved somewhere to be restored at a corresponding - /// `Restore` change. - /// - /// This likely maps to the DWARF call frame instruction `.cfa_remember_state`. - Preserve, - /// Restore the entire frame layout from a corresponding prior `Preserve` frame change. - /// - /// This likely maps to the DWARF call frame instruction `.cfa_restore_state`. - Restore, -} - -/// Set of frame layout changes. -pub type FrameLayoutChanges = Box<[FrameLayoutChange]>; - -/// Frame items layout for (prologue/epilogue) instructions. -#[derive(Debug, Clone)] -pub struct FrameLayout { - /// Initial frame layout. - pub initial: FrameLayoutChanges, - - /// Instruction frame layout (changes). Because the map will not be dense, - /// a HashMap is used instead of a SecondaryMap. - pub instructions: HashMap, -} - -impl FrameLayout { - /// Create instance of FrameLayout. - pub fn new() -> Self { - Self { - initial: vec![].into_boxed_slice(), - instructions: HashMap::new(), - } - } - - /// Clear the structure. - pub fn clear(&mut self) { - self.initial = vec![].into_boxed_slice(); - self.instructions.clear(); - } -} diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 1e72d2bc48c8..b9568e09b9e7 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -10,13 +10,14 @@ use crate::ir::{ Block, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, Inst, JumpTable, JumpTableData, Opcode, SigRef, StackSlot, StackSlotData, Table, TableData, }; -use crate::ir::{BlockOffsets, FrameLayout, InstEncodings, SourceLocs, StackSlots, ValueLocations}; +use crate::ir::{BlockOffsets, InstEncodings, SourceLocs, StackSlots, ValueLocations}; use crate::ir::{DataFlowGraph, ExternalName, Layout, Signature}; use crate::ir::{JumpTableOffsets, JumpTables}; use crate::isa::{CallConv, EncInfo, Encoding, Legalize, TargetIsa}; use crate::regalloc::{EntryRegDiversions, RegDiversions}; use crate::value_label::ValueLabelsRanges; use crate::write::write_function; +use alloc::vec::Vec; use core::fmt; /// A function. @@ -87,15 +88,13 @@ pub struct Function { /// Instruction that marks the end (inclusive) of the function's prologue. /// - /// This is used for some calling conventions to track the end of unwind information. + /// This is used for some ABIs to generate unwind information. pub prologue_end: Option, - /// Frame layout for the instructions. + /// The instructions that mark the start (inclusive) of an epilogue in the function. /// - /// The stack unwinding requires to have information about which registers and where they - /// are saved in the frame. This information is created during the prologue and epilogue - /// passes. - pub frame_layout: Option, + /// This is used for some ABIs to generate unwind information. + pub epilogues_start: Vec, } impl Function { @@ -119,7 +118,7 @@ impl Function { jt_offsets: SecondaryMap::new(), srclocs: SecondaryMap::new(), prologue_end: None, - frame_layout: None, + epilogues_start: Vec::new(), } } @@ -140,7 +139,7 @@ impl Function { self.jt_offsets.clear(); self.srclocs.clear(); self.prologue_end = None; - self.frame_layout = None; + self.epilogues_start.clear(); } /// Create a new empty, anonymous function with a Fast calling convention. @@ -250,12 +249,6 @@ impl Function { /// Starts collection of debug information. pub fn collect_debug_info(&mut self) { self.dfg.collect_debug_info(); - self.collect_frame_layout_info(); - } - - /// Starts collection of frame layout information. - pub fn collect_frame_layout_info(&mut self) { - self.frame_layout = Some(FrameLayout::new()); } /// Changes the destination of a jump or branch instruction. diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index 3c222ca9f525..7f3c36b7be0f 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -6,7 +6,6 @@ pub mod dfg; pub mod entities; mod extfunc; mod extname; -mod framelayout; pub mod function; mod globalvalue; mod heap; @@ -40,7 +39,6 @@ pub use crate::ir::extfunc::{ AbiParam, ArgumentExtension, ArgumentPurpose, ExtFuncData, Signature, }; pub use crate::ir::extname::ExternalName; -pub use crate::ir::framelayout::{FrameLayout, FrameLayoutChange, FrameLayoutChanges}; pub use crate::ir::function::{DisplayFunctionAnnotations, Function}; pub use crate::ir::globalvalue::GlobalValueData; pub use crate::ir::heap::{HeapData, HeapStyle}; diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index eaa1e599b41d..97120f369926 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -74,18 +74,15 @@ mod riscv; #[cfg(feature = "x86")] mod x86; -#[cfg(all(feature = "x86", feature = "unwind"))] -/// Expose the register-mapping functionality necessary for exception handling, debug, etc. -pub mod fde { - pub use super::x86::map_reg; -} - #[cfg(feature = "arm32")] mod arm32; #[cfg(feature = "arm64")] mod arm64; +#[cfg(feature = "unwind")] +pub mod unwind; + mod call_conv; mod constraints; mod enc_tables; @@ -385,16 +382,22 @@ pub trait TargetIsa: fmt::Display + Send + Sync { /// IntCC condition for Unsigned Subtraction Overflow (Borrow/Carry). fn unsigned_sub_overflow_condition(&self) -> ir::condcodes::IntCC; - /// Emit unwind information for the given function. + /// Creates unwind information for the function. /// - /// Only some calling conventions (e.g. Windows fastcall) will have unwind information. - fn emit_unwind_info( - &self, - _func: &ir::Function, - _kind: binemit::FrameUnwindKind, - _sink: &mut dyn binemit::FrameUnwindSink, - ) { - // No-op by default + /// Returns `None` if there is no unwind information for the function. + #[cfg(feature = "unwind")] + fn create_unwind_info(&self, _func: &ir::Function) -> Option { + // By default, an ISA has no unwind information + None + } + + /// Creates a new System V Common Information Entry for the ISA. + /// + /// Returns `None` if the ISA does not support System V unwind information. + #[cfg(feature = "unwind")] + fn create_systemv_cie(&self) -> Option { + // By default, an ISA cannot create a System V CIE + None } } diff --git a/cranelift/codegen/src/isa/unwind.rs b/cranelift/codegen/src/isa/unwind.rs new file mode 100644 index 000000000000..14f1c14e8cad --- /dev/null +++ b/cranelift/codegen/src/isa/unwind.rs @@ -0,0 +1,16 @@ +//! Represents information relating to function unwinding. +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +pub mod systemv; + +/// Represents unwind information for a single function. +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub enum UnwindInfo { + /// Windows x64 ABI unwind information. + #[cfg(feature = "x86")] + WindowsX64(super::x86::unwind::windows::UnwindInfo), + /// System V ABI unwind information. + SystemV(systemv::UnwindInfo), +} diff --git a/cranelift/codegen/src/isa/unwind/systemv.rs b/cranelift/codegen/src/isa/unwind/systemv.rs new file mode 100644 index 000000000000..0742943a7a2a --- /dev/null +++ b/cranelift/codegen/src/isa/unwind/systemv.rs @@ -0,0 +1,122 @@ +//! System V ABI unwind information. + +use alloc::vec::Vec; +use gimli::write::{Address, FrameDescriptionEntry}; + +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +type Register = u16; +type Expression = Vec; + +/// Module for System V x86-64 ABI utility functions. +#[cfg(feature = "x86")] +pub mod x86 { + pub use crate::isa::x86::unwind::systemv::map_reg; +} + +// This mirrors gimli's CallFrameInstruction, but is serializable +// TODO: if gimli ever adds serialization support, remove this type +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub(crate) enum CallFrameInstruction { + Cfa(Register, i32), + CfaRegister(Register), + CfaOffset(i32), + CfaExpression(Expression), + Restore(Register), + Undefined(Register), + SameValue(Register), + Offset(Register, i32), + ValOffset(Register, i32), + Register(Register, Register), + Expression(Register, Expression), + ValExpression(Register, Expression), + RememberState, + RestoreState, + ArgsSize(u32), +} + +impl From for CallFrameInstruction { + fn from(cfi: gimli::write::CallFrameInstruction) -> Self { + use gimli::write::CallFrameInstruction; + + match cfi { + CallFrameInstruction::Cfa(reg, offset) => Self::Cfa(reg.0, offset), + CallFrameInstruction::CfaRegister(reg) => Self::CfaRegister(reg.0), + CallFrameInstruction::CfaOffset(offset) => Self::CfaOffset(offset), + CallFrameInstruction::CfaExpression(expr) => Self::CfaExpression(expr.0), + CallFrameInstruction::Restore(reg) => Self::Restore(reg.0), + CallFrameInstruction::Undefined(reg) => Self::Undefined(reg.0), + CallFrameInstruction::SameValue(reg) => Self::SameValue(reg.0), + CallFrameInstruction::Offset(reg, offset) => Self::Offset(reg.0, offset), + CallFrameInstruction::ValOffset(reg, offset) => Self::ValOffset(reg.0, offset), + CallFrameInstruction::Register(reg1, reg2) => Self::Register(reg1.0, reg2.0), + CallFrameInstruction::Expression(reg, expr) => Self::Expression(reg.0, expr.0), + CallFrameInstruction::ValExpression(reg, expr) => Self::ValExpression(reg.0, expr.0), + CallFrameInstruction::RememberState => Self::RememberState, + CallFrameInstruction::RestoreState => Self::RestoreState, + CallFrameInstruction::ArgsSize(size) => Self::ArgsSize(size), + } + } +} + +impl Into for CallFrameInstruction { + fn into(self) -> gimli::write::CallFrameInstruction { + use gimli::{ + write::{CallFrameInstruction, Expression}, + Register, + }; + + match self { + Self::Cfa(reg, offset) => CallFrameInstruction::Cfa(Register(reg), offset), + Self::CfaRegister(reg) => CallFrameInstruction::CfaRegister(Register(reg)), + Self::CfaOffset(offset) => CallFrameInstruction::CfaOffset(offset), + Self::CfaExpression(expr) => CallFrameInstruction::CfaExpression(Expression(expr)), + Self::Restore(reg) => CallFrameInstruction::Restore(Register(reg)), + Self::Undefined(reg) => CallFrameInstruction::Undefined(Register(reg)), + Self::SameValue(reg) => CallFrameInstruction::SameValue(Register(reg)), + Self::Offset(reg, offset) => CallFrameInstruction::Offset(Register(reg), offset), + Self::ValOffset(reg, offset) => CallFrameInstruction::ValOffset(Register(reg), offset), + Self::Register(reg1, reg2) => { + CallFrameInstruction::Register(Register(reg1), Register(reg2)) + } + Self::Expression(reg, expr) => { + CallFrameInstruction::Expression(Register(reg), Expression(expr)) + } + Self::ValExpression(reg, expr) => { + CallFrameInstruction::ValExpression(Register(reg), Expression(expr)) + } + Self::RememberState => CallFrameInstruction::RememberState, + Self::RestoreState => CallFrameInstruction::RestoreState, + Self::ArgsSize(size) => CallFrameInstruction::ArgsSize(size), + } + } +} + +/// Represents unwind information for a single System V ABI function. +/// +/// This representation is not ISA specific. +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct UnwindInfo { + instructions: Vec<(u32, CallFrameInstruction)>, + len: u32, +} + +impl UnwindInfo { + pub(crate) fn new(instructions: Vec<(u32, CallFrameInstruction)>, len: u32) -> Self { + Self { instructions, len } + } + + /// Converts the unwind information into a `FrameDescriptionEntry`. + pub fn to_fde(&self, address: Address) -> gimli::write::FrameDescriptionEntry { + let mut fde = FrameDescriptionEntry::new(address, self.len); + + for (offset, inst) in &self.instructions { + fde.add_instruction(*offset, inst.clone().into()); + } + + fde + } +} diff --git a/cranelift/codegen/src/isa/x86/abi.rs b/cranelift/codegen/src/isa/x86/abi.rs index db67457a6c76..f5dc99ab76ee 100644 --- a/cranelift/codegen/src/isa/x86/abi.rs +++ b/cranelift/codegen/src/isa/x86/abi.rs @@ -1,22 +1,16 @@ //! x86 ABI implementation. use super::super::settings as shared_settings; -#[cfg(feature = "unwind")] -use super::fde::emit_fde; use super::registers::{FPR, GPR, RU}; use super::settings as isa_settings; -#[cfg(feature = "unwind")] -use super::unwind::UnwindInfo; use crate::abi::{legalize_args, ArgAction, ArgAssigner, ValueConversion}; -#[cfg(feature = "unwind")] -use crate::binemit::{FrameUnwindKind, FrameUnwindSink}; use crate::cursor::{Cursor, CursorPosition, EncCursor}; use crate::ir; use crate::ir::immediates::Imm64; use crate::ir::stackslot::{StackOffset, StackSize}; use crate::ir::{ - get_probestack_funcref, AbiParam, ArgumentExtension, ArgumentLoc, ArgumentPurpose, - FrameLayoutChange, InstBuilder, ValueLoc, + get_probestack_funcref, AbiParam, ArgumentExtension, ArgumentLoc, ArgumentPurpose, InstBuilder, + ValueLoc, }; use crate::isa::{CallConv, RegClass, RegUnit, TargetIsa}; use crate::regalloc::RegisterSet; @@ -24,7 +18,6 @@ use crate::result::CodegenResult; use crate::stack_layout::layout_stack; use alloc::borrow::Cow; use core::i32; -use std::boxed::Box; use target_lexicon::{PointerWidth, Triple}; /// Argument registers for x86-64 @@ -476,32 +469,6 @@ fn baldrdash_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> Ok(()) } -/// CFAState is cranelift's model of the call frame layout at any particular point in a function. -/// It describes the call frame's layout in terms of a call frame address, where it is with respect -/// to the start of the call frame, and the where the top of the stack is with respect to it. -/// -/// Changes in this layout are used to derive appropriate `ir::FrameLayoutChange` to record for -/// relevant instructions. -#[derive(Clone)] -struct CFAState { - /// The register from which we can derive the call frame address. On x86_64, this is typically - /// `rbp`, but at function entry and exit may be `rsp` while the call frame is being - /// established. - cf_ptr_reg: RegUnit, - /// Given that `cf_ptr_reg` is a register containing a pointer to some memory, `cf_ptr_offset` - /// is the offset from that pointer to the address of the start of this function's call frame. - /// - /// For a concrete x86_64 example, we will start this at 8 - the call frame begins immediately - /// before the return address. This will typically then be set to 16, after pushing `rbp` to - /// preserve the parent call frame. It is very unlikely the offset should be anything other - /// than one or two pointer widths. - cf_ptr_offset: isize, - /// The offset between the start of the call frame and the current stack pointer. This is - /// primarily useful to point to where on the stack preserved registers are, but is maintained - /// through the whole function for consistency. - current_depth: isize, -} - /// Implementation of the fastcall-based Win64 calling convention described at [1] /// [1] https://docs.microsoft.com/en-us/cpp/build/x64-calling-convention fn fastcall_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> { @@ -553,19 +520,11 @@ fn fastcall_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> C // Set up the cursor and insert the prologue let entry_block = func.layout.entry_block().expect("missing entry block"); let mut pos = EncCursor::new(func, isa).at_first_insertion_point(entry_block); - let prologue_cfa_state = - insert_common_prologue(&mut pos, local_stack_size, reg_type, &csrs, isa); + insert_common_prologue(&mut pos, local_stack_size, reg_type, &csrs, isa); // Reset the cursor and insert the epilogue let mut pos = pos.at_position(CursorPosition::Nowhere); - insert_common_epilogues( - &mut pos, - local_stack_size, - reg_type, - &csrs, - isa, - prologue_cfa_state, - ); + insert_common_epilogues(&mut pos, local_stack_size, reg_type, &csrs); Ok(()) } @@ -614,19 +573,11 @@ fn system_v_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> C // Set up the cursor and insert the prologue let entry_block = func.layout.entry_block().expect("missing entry block"); let mut pos = EncCursor::new(func, isa).at_first_insertion_point(entry_block); - let prologue_cfa_state = - insert_common_prologue(&mut pos, local_stack_size, reg_type, &csrs, isa); + insert_common_prologue(&mut pos, local_stack_size, reg_type, &csrs, isa); // Reset the cursor and insert the epilogue let mut pos = pos.at_position(CursorPosition::Nowhere); - insert_common_epilogues( - &mut pos, - local_stack_size, - reg_type, - &csrs, - isa, - prologue_cfa_state, - ); + insert_common_epilogues(&mut pos, local_stack_size, reg_type, &csrs); Ok(()) } @@ -639,8 +590,7 @@ fn insert_common_prologue( reg_type: ir::types::Type, csrs: &RegisterSet, isa: &dyn TargetIsa, -) -> Option { - let word_size = isa.pointer_bytes() as isize; +) { if stack_size > 0 { // Check if there is a special stack limit parameter. If so insert stack check. if let Some(stack_limit_arg) = pos.func.special_param(ArgumentPurpose::StackLimit) { @@ -649,110 +599,36 @@ fn insert_common_prologue( // Also, the size of a return address, implicitly pushed by a x86 `call` instruction, // also should be accounted for. // TODO: Check if the function body actually contains a `call` instruction. - let total_stack_size = (csrs.iter(GPR).len() + 1 + 1) as i64 * word_size as i64; + let total_stack_size = + (csrs.iter(GPR).len() + 1 + 1) as i64 * (isa.pointer_bytes() as isize) as i64; insert_stack_check(pos, total_stack_size, stack_limit_arg); } } - let mut cfa_state = if let Some(ref mut frame_layout) = pos.func.frame_layout { - let cfa_state = CFAState { - cf_ptr_reg: RU::rsp as RegUnit, - cf_ptr_offset: word_size, - current_depth: -word_size, - }; - - frame_layout.initial = vec![ - FrameLayoutChange::CallFrameAddressAt { - reg: cfa_state.cf_ptr_reg, - offset: cfa_state.cf_ptr_offset, - }, - FrameLayoutChange::ReturnAddressAt { - cfa_offset: cfa_state.current_depth, - }, - ] - .into_boxed_slice(); - - Some(cfa_state) - } else { - None - }; - // Append param to entry block let block = pos.current_block().expect("missing block under cursor"); let fp = pos.func.dfg.append_block_param(block, reg_type); pos.func.locations[fp] = ir::ValueLoc::Reg(RU::rbp as RegUnit); - let push_fp_inst = pos.ins().x86_push(fp); - - if let Some(ref mut frame_layout) = pos.func.frame_layout { - let cfa_state = cfa_state - .as_mut() - .expect("cfa state exists when recording frame layout"); - cfa_state.current_depth -= word_size; - cfa_state.cf_ptr_offset += word_size; - frame_layout.instructions.insert( - push_fp_inst, - vec![ - FrameLayoutChange::CallFrameAddressAt { - reg: cfa_state.cf_ptr_reg, - offset: cfa_state.cf_ptr_offset, - }, - FrameLayoutChange::RegAt { - reg: RU::rbp as RegUnit, - cfa_offset: cfa_state.current_depth, - }, - ] - .into_boxed_slice(), - ); - } + pos.ins().x86_push(fp); let mov_sp_inst = pos .ins() .copy_special(RU::rsp as RegUnit, RU::rbp as RegUnit); - if let Some(ref mut frame_layout) = pos.func.frame_layout { - let mut cfa_state = cfa_state - .as_mut() - .expect("cfa state exists when recording frame layout"); - cfa_state.cf_ptr_reg = RU::rbp as RegUnit; - frame_layout.instructions.insert( - mov_sp_inst, - vec![FrameLayoutChange::CallFrameAddressAt { - reg: cfa_state.cf_ptr_reg, - offset: cfa_state.cf_ptr_offset, - }] - .into_boxed_slice(), - ); - } - + let mut last_csr_push = None; for reg in csrs.iter(GPR) { // Append param to entry block let csr_arg = pos.func.dfg.append_block_param(block, reg_type); // Assign it a location pos.func.locations[csr_arg] = ir::ValueLoc::Reg(reg); - - // Remember it so we can push it momentarily - let reg_push_inst = pos.ins().x86_push(csr_arg); - - if let Some(ref mut frame_layout) = pos.func.frame_layout { - let mut cfa_state = cfa_state - .as_mut() - .expect("cfa state exists when recording frame layout"); - cfa_state.current_depth -= word_size; - frame_layout.instructions.insert( - reg_push_inst, - vec![FrameLayoutChange::RegAt { - reg, - cfa_offset: cfa_state.current_depth, - }] - .into_boxed_slice(), - ); - } + last_csr_push = Some(pos.ins().x86_push(csr_arg)); } // Allocate stack frame storage. + let mut adjust_sp_inst = None; if stack_size > 0 { if isa.flags().enable_probestack() && stack_size > (1 << isa.flags().probestack_size_log2()) { @@ -788,15 +664,15 @@ fn insert_common_prologue( if !isa.flags().probestack_func_adjusts_sp() { let result = pos.func.dfg.inst_results(call)[0]; pos.func.locations[result] = rax_val; - pos.func.prologue_end = Some(pos.ins().adjust_sp_down(result)); + adjust_sp_inst = Some(pos.ins().adjust_sp_down(result)); } } else { // Simply decrement the stack pointer. - pos.func.prologue_end = Some(pos.ins().adjust_sp_down_imm(Imm64::new(stack_size))); + adjust_sp_inst = Some(pos.ins().adjust_sp_down_imm(Imm64::new(stack_size))); } } - cfa_state + pos.func.prologue_end = Some(adjust_sp_inst.or(last_csr_push).unwrap_or(mov_sp_inst)); } /// Insert a check that generates a trap if the stack pointer goes @@ -828,24 +704,12 @@ fn insert_common_epilogues( stack_size: i64, reg_type: ir::types::Type, csrs: &RegisterSet, - isa: &dyn TargetIsa, - cfa_state: Option, ) { while let Some(block) = pos.next_block() { pos.goto_last_inst(block); if let Some(inst) = pos.current_inst() { if pos.func.dfg[inst].opcode().is_return() { - let is_last = pos.func.layout.last_block() == Some(block); - insert_common_epilogue( - inst, - stack_size, - pos, - reg_type, - csrs, - isa, - is_last, - cfa_state.clone(), - ); + insert_common_epilogue(inst, stack_size, pos, reg_type, csrs); } } } @@ -859,113 +723,49 @@ fn insert_common_epilogue( pos: &mut EncCursor, reg_type: ir::types::Type, csrs: &RegisterSet, - isa: &dyn TargetIsa, - is_last: bool, - mut cfa_state: Option, ) { - let word_size = isa.pointer_bytes() as isize; - if stack_size > 0 { - pos.ins().adjust_sp_up_imm(Imm64::new(stack_size)); - } + let mut sp_adjust_inst = None; - // Pop all the callee-saved registers, stepping backward each time to - // preserve the correct order. - let fp_ret = pos.ins().x86_pop(reg_type); - let fp_pop_inst = pos.built_inst(); - - if let Some(ref mut cfa_state) = cfa_state.as_mut() { - // Account for CFA state in the reverse of `insert_common_prologue`. - cfa_state.current_depth += word_size; - cfa_state.cf_ptr_offset -= word_size; - // And now that we're going to overwrite `rbp`, `rsp` is the only way to get to the call frame. - // We don't apply a frame layout change *yet* because we check that at return the depth is - // exactly one `word_size`. - cfa_state.cf_ptr_reg = RU::rsp as RegUnit; + if stack_size > 0 { + sp_adjust_inst = Some(pos.ins().adjust_sp_up_imm(Imm64::new(stack_size))); } - pos.prev_inst(); - - pos.func.locations[fp_ret] = ir::ValueLoc::Reg(RU::rbp as RegUnit); - pos.func.dfg.append_inst_arg(inst, fp_ret); + // Insert the pop of the frame pointer + let fp_pop = pos.ins().x86_pop(reg_type); + let fp_pop_inst = pos.prev_inst().unwrap(); + pos.func.locations[fp_pop] = ir::ValueLoc::Reg(RU::rbp as RegUnit); + pos.func.dfg.append_inst_arg(inst, fp_pop); + // Insert the CSR pops + let mut first_csr_pop_inst = None; for reg in csrs.iter(GPR) { - let csr_ret = pos.ins().x86_pop(reg_type); - if let Some(ref mut cfa_state) = cfa_state.as_mut() { - // Note: don't bother recording a frame layout change because the popped value is - // still correct in memory, and won't be overwritten until we've returned where the - // current frame's layout would no longer matter. Only adjust `current_depth` for a - // consistency check later. - cfa_state.current_depth += word_size; - } - pos.prev_inst(); - - pos.func.locations[csr_ret] = ir::ValueLoc::Reg(reg); - pos.func.dfg.append_inst_arg(inst, csr_ret); + let csr_pop = pos.ins().x86_pop(reg_type); + first_csr_pop_inst = Some(pos.prev_inst().unwrap()); + pos.func.locations[csr_pop] = ir::ValueLoc::Reg(reg); + pos.func.dfg.append_inst_arg(inst, csr_pop); } - if let Some(ref mut frame_layout) = pos.func.frame_layout { - let cfa_state = cfa_state - .as_mut() - .expect("cfa state exists when recording frame layout"); - // Validity checks - if we accounted correctly, CFA state at a return will match CFA state - // at the entry of a function. - // - // Current_depth starts assuming a return address is pushed, and cf_ptr_offset is one - // pointer below current_depth. - assert_eq!(cfa_state.current_depth, -word_size); - assert_eq!(cfa_state.cf_ptr_offset, word_size); - - // Inserting preserve CFA state operation after FP pop instructions. - let new_cfa = FrameLayoutChange::CallFrameAddressAt { - reg: cfa_state.cf_ptr_reg, - offset: cfa_state.cf_ptr_offset, - }; - let new_cfa = if is_last { - vec![new_cfa] - } else { - vec![FrameLayoutChange::Preserve, new_cfa] - }; - - frame_layout - .instructions - .entry(fp_pop_inst) - .and_modify(|insts| { - *insts = insts - .iter() - .cloned() - .chain(new_cfa.clone().into_iter()) - .collect::>(); - }) - .or_insert_with(|| new_cfa.into_boxed_slice()); - - if !is_last { - // Inserting restore CFA state operation after each return. - frame_layout - .instructions - .insert(inst, vec![FrameLayoutChange::Restore].into_boxed_slice()); - } - } + pos.func + .epilogues_start + .push(sp_adjust_inst.or(first_csr_pop_inst).unwrap_or(fp_pop_inst)); } #[cfg(feature = "unwind")] -pub fn emit_unwind_info( +pub fn create_unwind_info( func: &ir::Function, isa: &dyn TargetIsa, - kind: FrameUnwindKind, - sink: &mut dyn FrameUnwindSink, -) { - match kind { - FrameUnwindKind::Fastcall => { - // Assumption: RBP is being used as the frame pointer - // In the future, Windows fastcall codegen should usually omit the frame pointer - if let Some(info) = UnwindInfo::try_from_func(func, isa, Some(RU::rbp.into())) { - info.emit(sink); - } - } - FrameUnwindKind::Libunwind => { - if func.frame_layout.is_some() { - emit_fde(func, isa, sink); - } - } +) -> Option { + use crate::isa::unwind::UnwindInfo; + + // Assumption: RBP is being used as the frame pointer for both calling conventions + // In the future, we should be omitting frame pointer as an optimization, so this will change + match func.signature.call_conv { + CallConv::Fast | CallConv::Cold | CallConv::SystemV => Some(UnwindInfo::SystemV( + super::unwind::systemv::create_unwind_info(func, isa, Some(RU::rbp.into()))?, + )), + CallConv::WindowsFastcall => Some(UnwindInfo::WindowsX64( + super::unwind::windows::create_unwind_info(func, isa, Some(RU::rbp.into()))?, + )), + _ => None, } } diff --git a/cranelift/codegen/src/isa/x86/fde.rs b/cranelift/codegen/src/isa/x86/fde.rs deleted file mode 100644 index 17dee88cb3f0..000000000000 --- a/cranelift/codegen/src/isa/x86/fde.rs +++ /dev/null @@ -1,451 +0,0 @@ -//! Support for FDE data generation. - -use crate::binemit::{FrameUnwindOffset, FrameUnwindSink, Reloc}; -use crate::ir::{FrameLayoutChange, Function}; -use crate::isa::{CallConv, RegUnit, TargetIsa}; -use alloc::vec::Vec; -use core::convert::TryInto; -use gimli::write::{ - Address, CallFrameInstruction, CommonInformationEntry, EhFrame, EndianVec, - FrameDescriptionEntry, FrameTable, Result, Writer, -}; -use gimli::{Encoding, Format, LittleEndian, Register, X86_64}; -use thiserror::Error; - -pub type FDERelocEntry = (FrameUnwindOffset, Reloc); - -const FUNCTION_ENTRY_ADDRESS: Address = Address::Symbol { - symbol: 0, - addend: 0, -}; - -#[derive(Clone)] -struct FDEWriter { - vec: EndianVec, - relocs: Vec, -} - -impl FDEWriter { - fn new() -> Self { - Self { - vec: EndianVec::new(LittleEndian), - relocs: Vec::new(), - } - } - fn into_vec_and_relocs(self) -> (Vec, Vec) { - (self.vec.into_vec(), self.relocs) - } -} - -impl Writer for FDEWriter { - type Endian = LittleEndian; - fn endian(&self) -> Self::Endian { - LittleEndian - } - fn len(&self) -> usize { - self.vec.len() - } - fn write(&mut self, bytes: &[u8]) -> Result<()> { - self.vec.write(bytes) - } - fn write_at(&mut self, offset: usize, bytes: &[u8]) -> Result<()> { - self.vec.write_at(offset, bytes) - } - fn write_address(&mut self, address: Address, size: u8) -> Result<()> { - match address { - Address::Constant(_) => self.vec.write_address(address, size), - Address::Symbol { .. } => { - assert_eq!(address, FUNCTION_ENTRY_ADDRESS); - let rt = match size { - 4 => Reloc::Abs4, - 8 => Reloc::Abs8, - _ => { - panic!("Unexpected address size at FDEWriter::write_address"); - } - }; - self.relocs.push((self.vec.len().try_into().unwrap(), rt)); - self.vec.write_udata(0, size) - } - } - } -} - -fn return_address_reg(isa: &dyn TargetIsa) -> Register { - assert!(isa.name() == "x86" && isa.pointer_bits() == 64); - X86_64::RA -} - -/// Map Cranelift registers to their corresponding Gimli registers. -pub fn map_reg( - isa: &dyn TargetIsa, - reg: RegUnit, -) -> core::result::Result { - if isa.name() != "x86" || isa.pointer_bits() != 64 { - return Err(RegisterMappingError::UnsupportedArchitecture); - } - - // Mapping from https://github.com/bytecodealliance/cranelift/pull/902 by @iximeow - const X86_GP_REG_MAP: [gimli::Register; 16] = [ - X86_64::RAX, - X86_64::RCX, - X86_64::RDX, - X86_64::RBX, - X86_64::RSP, - X86_64::RBP, - X86_64::RSI, - X86_64::RDI, - X86_64::R8, - X86_64::R9, - X86_64::R10, - X86_64::R11, - X86_64::R12, - X86_64::R13, - X86_64::R14, - X86_64::R15, - ]; - const X86_XMM_REG_MAP: [gimli::Register; 16] = [ - X86_64::XMM0, - X86_64::XMM1, - X86_64::XMM2, - X86_64::XMM3, - X86_64::XMM4, - X86_64::XMM5, - X86_64::XMM6, - X86_64::XMM7, - X86_64::XMM8, - X86_64::XMM9, - X86_64::XMM10, - X86_64::XMM11, - X86_64::XMM12, - X86_64::XMM13, - X86_64::XMM14, - X86_64::XMM15, - ]; - - let reg_info = isa.register_info(); - let bank = reg_info - .bank_containing_regunit(reg) - .ok_or_else(|| RegisterMappingError::MissingBank)?; - match bank.name { - "IntRegs" => { - // x86 GP registers have a weird mapping to DWARF registers, so we use a - // lookup table. - Ok(X86_GP_REG_MAP[(reg - bank.first_unit) as usize]) - } - "FloatRegs" => Ok(X86_XMM_REG_MAP[(reg - bank.first_unit) as usize]), - _ => Err(RegisterMappingError::UnsupportedRegisterBank(bank.name)), - } -} - -#[derive(Error, Debug)] -pub enum RegisterMappingError { - #[error("unable to find bank for register info")] - MissingBank, - #[error("register mapping is currently only implemented for x86_64")] - UnsupportedArchitecture, - #[error("unsupported register bank: {0}")] - UnsupportedRegisterBank(&'static str), -} - -fn to_cfi( - isa: &dyn TargetIsa, - change: &FrameLayoutChange, - cfa_def_reg: &mut Register, - cfa_def_offset: &mut i32, -) -> Option { - Some(match change { - FrameLayoutChange::CallFrameAddressAt { reg, offset } => { - let mapped = map_reg(isa, *reg).expect("a register mapping from cranelift to gimli"); - let offset = (*offset) as i32; - if mapped != *cfa_def_reg && offset != *cfa_def_offset { - *cfa_def_reg = mapped; - *cfa_def_offset = offset; - CallFrameInstruction::Cfa(mapped, offset) - } else if offset != *cfa_def_offset { - *cfa_def_offset = offset; - CallFrameInstruction::CfaOffset(offset) - } else if mapped != *cfa_def_reg { - *cfa_def_reg = mapped; - CallFrameInstruction::CfaRegister(mapped) - } else { - return None; - } - } - FrameLayoutChange::RegAt { reg, cfa_offset } => { - assert!(cfa_offset % -8 == 0); - let cfa_offset = *cfa_offset as i32; - let mapped = map_reg(isa, *reg).expect("a register mapping from cranelift to gimli"); - CallFrameInstruction::Offset(mapped, cfa_offset) - } - FrameLayoutChange::ReturnAddressAt { cfa_offset } => { - assert!(cfa_offset % -8 == 0); - let cfa_offset = *cfa_offset as i32; - CallFrameInstruction::Offset(X86_64::RA, cfa_offset) - } - FrameLayoutChange::Preserve => CallFrameInstruction::RememberState, - FrameLayoutChange::Restore => CallFrameInstruction::RestoreState, - }) -} - -/// Creates FDE structure from FrameLayout. -pub fn emit_fde(func: &Function, isa: &dyn TargetIsa, sink: &mut dyn FrameUnwindSink) { - assert!(isa.name() == "x86"); - - // Expecting function with System V prologue - assert!( - func.signature.call_conv == CallConv::Fast - || func.signature.call_conv == CallConv::Cold - || func.signature.call_conv == CallConv::SystemV - ); - - assert!(func.frame_layout.is_some(), "expected func.frame_layout"); - let frame_layout = func.frame_layout.as_ref().unwrap(); - - let mut blocks = func.layout.blocks().collect::>(); - blocks.sort_by_key(|block| func.offsets[*block]); // Ensure inst offsets always increase - - let encinfo = isa.encoding_info(); - let mut last_offset = 0; - let mut changes = Vec::new(); - for block in blocks { - for (offset, inst, size) in func.inst_offsets(block, &encinfo) { - let address_offset = (offset + size) as usize; - assert!(last_offset <= address_offset); - if let Some(cmds) = frame_layout.instructions.get(&inst) { - for cmd in cmds.iter() { - changes.push((address_offset, *cmd)); - } - } - last_offset = address_offset; - } - } - - let len = last_offset as u32; - - let word_size = isa.pointer_bytes() as i32; - - let encoding = Encoding { - format: Format::Dwarf32, - version: 1, - address_size: word_size as u8, - }; - let mut frames = FrameTable::default(); - - let mut cfa_def_reg = return_address_reg(isa); - let mut cfa_def_offset = 0i32; - - let mut cie = CommonInformationEntry::new( - encoding, - /* code_alignment_factor = */ 1, - /* data_alignment_factor = */ -word_size as i8, - return_address_reg(isa), - ); - for ch in frame_layout.initial.iter() { - if let Some(cfi) = to_cfi(isa, ch, &mut cfa_def_reg, &mut cfa_def_offset) { - cie.add_instruction(cfi); - } - } - - let cie_id = frames.add_cie(cie); - - let mut fde = FrameDescriptionEntry::new(FUNCTION_ENTRY_ADDRESS, len); - - for (addr, ch) in changes.iter() { - if let Some(cfi) = to_cfi(isa, ch, &mut cfa_def_reg, &mut cfa_def_offset) { - fde.add_instruction((*addr) as u32, cfi); - } - } - - frames.add_fde(cie_id, fde); - - let mut eh_frame = EhFrame::from(FDEWriter::new()); - frames.write_eh_frame(&mut eh_frame).unwrap(); - - let (bytes, relocs) = eh_frame.clone().into_vec_and_relocs(); - - let unwind_start = sink.len(); - sink.bytes(&bytes); - - for (off, r) in relocs { - sink.reloc(r, off + unwind_start); - } - - let cie_len = u32::from_le_bytes(bytes.as_slice()[..4].try_into().unwrap()); - let fde_offset = cie_len as usize + 4; - sink.set_entry_offset(unwind_start + fde_offset); - - // Need 0 marker for GCC unwind to end FDE "list". - sink.bytes(&[0, 0, 0, 0]); -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::binemit::{FrameUnwindOffset, Reloc}; - use crate::cursor::{Cursor, FuncCursor}; - use crate::ir::{ - types, AbiParam, ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind, - TrapCode, - }; - use crate::isa::{lookup, CallConv}; - use crate::settings::{builder, Flags}; - use crate::Context; - use std::str::FromStr; - use target_lexicon::triple; - - struct SimpleUnwindSink(pub Vec, pub usize, pub Vec<(Reloc, usize)>); - impl FrameUnwindSink for SimpleUnwindSink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() - } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); - } - fn reloc(&mut self, r: Reloc, off: FrameUnwindOffset) { - self.2.push((r, off)); - } - fn set_entry_offset(&mut self, off: FrameUnwindOffset) { - self.1 = off; - } - } - - #[test] - fn test_simple_func() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function( - CallConv::SystemV, - Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), - )); - context.func.collect_frame_layout_info(); - - context.compile(&*isa).expect("expected compilation"); - - let mut sink = SimpleUnwindSink(Vec::new(), 0, Vec::new()); - emit_fde(&context.func, &*isa, &mut sink); - - assert_eq!( - sink.0, - vec![ - 20, 0, 0, 0, // CIE len - 0, 0, 0, 0, // CIE marker - 1, // version - 0, // augmentation string - 1, // code aligment = 1 - 120, // data alignment = -8 - 16, // RA = r16 - 0x0c, 0x07, 0x08, // DW_CFA_def_cfa r7, 8 - 0x90, 0x01, // DW_CFA_offset r16, -8 * 1 - 0, 0, 0, 0, 0, 0, // padding - 36, 0, 0, 0, // FDE len - 28, 0, 0, 0, // CIE offset - 0, 0, 0, 0, 0, 0, 0, 0, // addr reloc - 16, 0, 0, 0, 0, 0, 0, 0, // function length - 0x42, // DW_CFA_advance_loc 2 - 0x0e, 0x10, // DW_CFA_def_cfa_offset 16 - 0x86, 0x02, // DW_CFA_offset r6, -8 * 2 - 0x43, // DW_CFA_advance_loc 3 - 0x0d, 0x06, // DW_CFA_def_cfa_register - 0x4a, // DW_CFA_advance_loc 10 - 0x0c, 0x07, 0x08, // DW_CFA_def_cfa r7, 8 - 0, 0, 0, 0, // padding - 0, 0, 0, 0, // End of FDEs - ] - ); - assert_eq!(sink.1, 24); - assert_eq!(sink.2.len(), 1); - } - - fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); - - let block0 = func.dfg.make_block(); - let mut pos = FuncCursor::new(&mut func); - pos.insert_block(block0); - pos.ins().return_(&[]); - - if let Some(stack_slot) = stack_slot { - func.stack_slots.push(stack_slot); - } - - func - } - - #[test] - fn test_multi_return_func() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_multi_return_function(CallConv::SystemV)); - context.func.collect_frame_layout_info(); - - context.compile(&*isa).expect("expected compilation"); - - let mut sink = SimpleUnwindSink(Vec::new(), 0, Vec::new()); - emit_fde(&context.func, &*isa, &mut sink); - - assert_eq!( - sink.0, - vec![ - 20, 0, 0, 0, // CIE len - 0, 0, 0, 0, // CIE marker - 1, // version - 0, // augmentation string - 1, // code aligment = 1 - 120, // data alignment = -8 - 16, // RA = r16 - 0x0c, 0x07, 0x08, // DW_CFA_def_cfa r7, 8 - 0x90, 0x01, // DW_CFA_offset r16, -8 * 1 - 0, 0, 0, 0, 0, 0, // padding - 36, 0, 0, 0, // FDE len - 28, 0, 0, 0, // CIE offset - 0, 0, 0, 0, 0, 0, 0, 0, // addr reloc - 15, 0, 0, 0, 0, 0, 0, 0, // function length - 0x42, // DW_CFA_advance_loc 2 - 0x0e, 0x10, // DW_CFA_def_cfa_offset 16 - 0x86, 0x02, // DW_CFA_offset r6, -8 * 2 - 0x43, // DW_CFA_advance_loc 3 - 0x0d, 0x06, // DW_CFA_def_cfa_register - 0x47, // DW_CFA_advance_loc 10 - 0x0a, // DW_CFA_preserve_state - 0x0c, 0x07, 0x08, // DW_CFA_def_cfa r7, 8 - 0x41, // DW_CFA_advance_loc 1 - 0x0b, // DW_CFA_restore_state - // NOTE: no additional CFA directives -- DW_CFA_restore_state - // is done before trap and it is last instruction in the function. - 0, // padding - 0, 0, 0, 0, // End of FDEs - ] - ); - assert_eq!(sink.1, 24); - assert_eq!(sink.2.len(), 1); - } - - fn create_multi_return_function(call_conv: CallConv) -> Function { - let mut sig = Signature::new(call_conv); - sig.params.push(AbiParam::new(types::I32)); - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); - - let block0 = func.dfg.make_block(); - let v0 = func.dfg.append_block_param(block0, types::I32); - let block1 = func.dfg.make_block(); - let block2 = func.dfg.make_block(); - - let mut pos = FuncCursor::new(&mut func); - pos.insert_block(block0); - pos.ins().brnz(v0, block2, &[]); - pos.ins().jump(block1, &[]); - - pos.insert_block(block1); - pos.ins().return_(&[]); - - pos.insert_block(block2); - pos.ins().trap(TrapCode::User(0)); - - func - } -} diff --git a/cranelift/codegen/src/isa/x86/mod.rs b/cranelift/codegen/src/isa/x86/mod.rs index f7c49cd21b7e..b832e7defffc 100644 --- a/cranelift/codegen/src/isa/x86/mod.rs +++ b/cranelift/codegen/src/isa/x86/mod.rs @@ -3,22 +3,15 @@ mod abi; mod binemit; mod enc_tables; -#[cfg(feature = "unwind")] -mod fde; mod registers; pub mod settings; #[cfg(feature = "unwind")] -mod unwind; - -#[cfg(feature = "unwind")] -pub use fde::map_reg; +pub mod unwind; use super::super::settings as shared_settings; #[cfg(feature = "testing_hooks")] use crate::binemit::CodeSink; use crate::binemit::{emit_function, MemoryCodeSink}; -#[cfg(feature = "unwind")] -use crate::binemit::{FrameUnwindKind, FrameUnwindSink}; use crate::ir; use crate::isa::enc_tables::{self as shared_enc_tables, lookup_enclist, Encodings}; use crate::isa::Builder as IsaBuilder; @@ -161,17 +154,14 @@ impl TargetIsa for Isa { ir::condcodes::IntCC::UnsignedLessThan } - /// Emit unwind information for the given function. - /// - /// Only some calling conventions (e.g. Windows fastcall) will have unwind information. #[cfg(feature = "unwind")] - fn emit_unwind_info( - &self, - func: &ir::Function, - kind: FrameUnwindKind, - sink: &mut dyn FrameUnwindSink, - ) { - abi::emit_unwind_info(func, self, kind, sink); + fn create_unwind_info(&self, func: &ir::Function) -> Option { + abi::create_unwind_info(func, self) + } + + #[cfg(feature = "unwind")] + fn create_systemv_cie(&self) -> Option { + Some(unwind::systemv::create_cie()) } } diff --git a/cranelift/codegen/src/isa/x86/unwind.rs b/cranelift/codegen/src/isa/x86/unwind.rs index de653b89a68f..5f4488d40e23 100644 --- a/cranelift/codegen/src/isa/x86/unwind.rs +++ b/cranelift/codegen/src/isa/x86/unwind.rs @@ -1,537 +1,4 @@ -//! Unwind information for x64 Windows. +//! Module for x86 unwind generation for supported ABIs. -use super::registers::{GPR, RU}; -use crate::binemit::FrameUnwindSink; -use crate::ir::{Function, InstructionData, Opcode}; -use crate::isa::{CallConv, RegUnit, TargetIsa}; -use alloc::vec::Vec; -use byteorder::{ByteOrder, LittleEndian}; - -/// Maximum (inclusive) size of a "small" stack allocation -const SMALL_ALLOC_MAX_SIZE: u32 = 128; -/// Maximum (inclusive) size of a "large" stack allocation that can represented in 16-bits -const LARGE_ALLOC_16BIT_MAX_SIZE: u32 = 524280; - -fn write_u8(sink: &mut dyn FrameUnwindSink, v: u8) { - sink.bytes(&[v]); -} - -fn write_u16(sink: &mut dyn FrameUnwindSink, v: u16) { - let mut buf = [0; 2]; - T::write_u16(&mut buf, v); - sink.bytes(&buf); -} - -fn write_u32(sink: &mut dyn FrameUnwindSink, v: u32) { - let mut buf = [0; 4]; - T::write_u32(&mut buf, v); - sink.bytes(&buf); -} - -/// The supported unwind codes for the x64 Windows ABI. -/// -/// See: https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 -/// Only what is needed to describe the prologues generated by the Cranelift x86 ISA are represented here. -/// Note: the Cranelift x86 ISA RU enum matches the Windows unwind GPR encoding values. -#[derive(Debug, PartialEq, Eq)] -enum UnwindCode { - PushRegister { offset: u8, reg: RegUnit }, - StackAlloc { offset: u8, size: u32 }, - SetFramePointer { offset: u8, sp_offset: u8 }, -} - -impl UnwindCode { - fn emit(&self, sink: &mut dyn FrameUnwindSink) { - enum UnwindOperation { - PushNonvolatileRegister, - LargeStackAlloc, - SmallStackAlloc, - SetFramePointer, - } - - match self { - Self::PushRegister { offset, reg } => { - write_u8(sink, *offset); - write_u8( - sink, - ((GPR.index_of(*reg) as u8) << 4) - | (UnwindOperation::PushNonvolatileRegister as u8), - ); - } - Self::StackAlloc { offset, size } => { - // Stack allocations on Windows must be a multiple of 8 and be at least 1 slot - assert!(*size >= 8); - assert!((*size % 8) == 0); - - write_u8(sink, *offset); - if *size <= SMALL_ALLOC_MAX_SIZE { - write_u8( - sink, - ((((*size - 8) / 8) as u8) << 4) | UnwindOperation::SmallStackAlloc as u8, - ); - } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { - write_u8(sink, UnwindOperation::LargeStackAlloc as u8); - write_u16::(sink, (*size / 8) as u16); - } else { - write_u8(sink, (1 << 4) | (UnwindOperation::LargeStackAlloc as u8)); - write_u32::(sink, *size); - } - } - Self::SetFramePointer { offset, sp_offset } => { - write_u8(sink, *offset); - write_u8( - sink, - (*sp_offset << 4) | (UnwindOperation::SetFramePointer as u8), - ); - } - }; - } - - fn node_count(&self) -> usize { - match self { - Self::StackAlloc { size, .. } => { - if *size <= SMALL_ALLOC_MAX_SIZE { - 1 - } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { - 2 - } else { - 3 - } - } - _ => 1, - } - } -} - -/// Represents Windows x64 unwind information. -/// -/// For information about Windows x64 unwind info, see: -/// https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 -#[derive(Debug, PartialEq, Eq)] -pub struct UnwindInfo { - flags: u8, - prologue_size: u8, - frame_register: Option, - frame_register_offset: u8, - unwind_codes: Vec, -} - -impl UnwindInfo { - pub fn try_from_func( - func: &Function, - isa: &dyn TargetIsa, - frame_register: Option, - ) -> Option { - // Only Windows fastcall is supported for unwind information - if func.signature.call_conv != CallConv::WindowsFastcall || func.prologue_end.is_none() { - return None; - } - - let prologue_end = func.prologue_end.unwrap(); - let entry_block = func.layout.blocks().nth(0).expect("missing entry block"); - - // Stores the stack size when SP is not adjusted via an immediate value - let mut stack_size = None; - let mut prologue_size = 0; - let mut unwind_codes = Vec::new(); - let mut found_end = false; - - for (offset, inst, size) in func.inst_offsets(entry_block, &isa.encoding_info()) { - // x64 ABI prologues cannot exceed 255 bytes in length - if (offset + size) > 255 { - panic!("function prologues cannot exceed 255 bytes in size for Windows x64"); - } - - prologue_size += size; - - let unwind_offset = (offset + size) as u8; - - match func.dfg[inst] { - InstructionData::Unary { opcode, arg } => { - match opcode { - Opcode::X86Push => { - unwind_codes.push(UnwindCode::PushRegister { - offset: unwind_offset, - reg: func.locations[arg].unwrap_reg(), - }); - } - Opcode::AdjustSpDown => { - // This is used when calling a stack check function - // We need to track the assignment to RAX which has the size of the stack - unwind_codes.push(UnwindCode::StackAlloc { - offset: unwind_offset, - size: stack_size - .expect("expected a previous stack size instruction"), - }); - } - _ => {} - } - } - InstructionData::CopySpecial { src, dst, .. } => { - if let Some(frame_register) = frame_register { - if src == (RU::rsp as RegUnit) && dst == frame_register { - unwind_codes.push(UnwindCode::SetFramePointer { - offset: unwind_offset, - sp_offset: 0, - }); - } - } - } - InstructionData::UnaryImm { opcode, imm } => { - match opcode { - Opcode::Iconst => { - let imm: i64 = imm.into(); - assert!(imm <= core::u32::MAX as i64); - assert!(stack_size.is_none()); - - // This instruction should only appear in a prologue to pass an - // argument of the stack size to a stack check function. - // Record the stack size so we know what it is when we encounter the adjustment - // instruction (which will adjust via the register assigned to this instruction). - stack_size = Some(imm as u32); - } - Opcode::AdjustSpDownImm => { - let imm: i64 = imm.into(); - assert!(imm <= core::u32::MAX as i64); - - unwind_codes.push(UnwindCode::StackAlloc { - offset: unwind_offset, - size: imm as u32, - }); - } - _ => {} - } - } - _ => {} - }; - - if inst == prologue_end { - found_end = true; - break; - } - } - - if !found_end { - return None; - } - - Some(Self { - flags: 0, // this assumes cranelift functions have no SEH handlers - prologue_size: prologue_size as u8, - frame_register, - frame_register_offset: 0, - unwind_codes, - }) - } - - pub fn size(&self) -> usize { - let node_count = self.node_count(); - - // Calculation of the size requires no SEH handler or chained info - assert!(self.flags == 0); - - // Size of fixed part of UNWIND_INFO is 4 bytes - // Then comes the UNWIND_CODE nodes (2 bytes each) - // Then comes 2 bytes of padding for the unwind codes if necessary - // Next would come the SEH data, but we assert above that the function doesn't have SEH data - - 4 + (node_count * 2) + if (node_count & 1) == 1 { 2 } else { 0 } - } - - pub fn node_count(&self) -> usize { - self.unwind_codes - .iter() - .fold(0, |nodes, c| nodes + c.node_count()) - } - - pub fn emit(&self, sink: &mut dyn FrameUnwindSink) { - const UNWIND_INFO_VERSION: u8 = 1; - - let size = self.size(); - let offset = sink.len(); - - // Ensure the memory is 32-bit aligned - assert_eq!(offset % 4, 0); - - sink.reserve(offset + size); - - let node_count = self.node_count(); - assert!(node_count <= 256); - - write_u8(sink, (self.flags << 3) | UNWIND_INFO_VERSION); - write_u8(sink, self.prologue_size); - write_u8(sink, node_count as u8); - - if let Some(reg) = self.frame_register { - write_u8( - sink, - (self.frame_register_offset << 4) | GPR.index_of(reg) as u8, - ); - } else { - write_u8(sink, 0); - } - - // Unwind codes are written in reverse order (prologue offset descending) - for code in self.unwind_codes.iter().rev() { - code.emit(sink); - } - - // To keep a 32-bit alignment, emit 2 bytes of padding if there's an odd number of 16-bit nodes - if (node_count & 1) == 1 { - write_u16::(sink, 0); - } - - // Ensure the correct number of bytes was emitted - assert_eq!(sink.len() - offset, size); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::binemit::{FrameUnwindOffset, Reloc}; - use crate::cursor::{Cursor, FuncCursor}; - use crate::ir::{ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind}; - use crate::isa::{lookup, CallConv}; - use crate::settings::{builder, Flags}; - use crate::Context; - use std::str::FromStr; - use target_lexicon::triple; - - struct SimpleUnwindSink(pub Vec); - impl FrameUnwindSink for SimpleUnwindSink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() - } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); - } - fn reloc(&mut self, _: Reloc, _: FrameUnwindOffset) {} - fn set_entry_offset(&mut self, _: FrameUnwindOffset) {} - } - - #[test] - fn test_wrong_calling_convention() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function(CallConv::SystemV, None)); - - context.compile(&*isa).expect("expected compilation"); - - assert_eq!(UnwindInfo::try_from_func(&context.func, &*isa, None), None); - } - - #[test] - fn test_small_alloc() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function( - CallConv::WindowsFastcall, - Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), - )); - - context.compile(&*isa).expect("expected compilation"); - - let unwind = UnwindInfo::try_from_func(&context.func, &*isa, Some(RU::rbp.into())) - .expect("expected unwind info"); - - assert_eq!( - unwind, - UnwindInfo { - flags: 0, - prologue_size: 9, - frame_register: Some(RU::rbp.into()), - frame_register_offset: 0, - unwind_codes: vec![ - UnwindCode::PushRegister { - offset: 2, - reg: RU::rbp.into() - }, - UnwindCode::SetFramePointer { - offset: 5, - sp_offset: 0 - }, - UnwindCode::StackAlloc { - offset: 9, - size: 64 + 32 - } - ] - } - ); - - assert_eq!(unwind.size(), 12); - - let mut sink = SimpleUnwindSink(Vec::new()); - unwind.emit(&mut sink); - - assert_eq!( - sink.0, - [ - 0x01, // Version and flags (version 1, no flags) - 0x09, // Prologue size - 0x03, // Unwind code count (1 for stack alloc, 1 for save frame reg, 1 for push reg) - 0x05, // Frame register + offset (RBP with 0 offset) - 0x09, // Prolog offset - 0xB2, // Operation 2 (small stack alloc), size = 0xB slots (e.g. (0xB * 8) + 8 = 96 (64 + 32) bytes) - 0x05, // Prolog offset - 0x03, // Operation 3 (save frame register), stack pointer offset = 0 - 0x02, // Prolog offset - 0x50, // Operation 0 (save nonvolatile register), reg = 5 (RBP) - 0x00, // Padding byte - 0x00, // Padding byte - ] - ); - } - - #[test] - fn test_medium_alloc() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function( - CallConv::WindowsFastcall, - Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 10000)), - )); - - context.compile(&*isa).expect("expected compilation"); - - let unwind = UnwindInfo::try_from_func(&context.func, &*isa, Some(RU::rbp.into())) - .expect("expected unwind info"); - - assert_eq!( - unwind, - UnwindInfo { - flags: 0, - prologue_size: 27, - frame_register: Some(RU::rbp.into()), - frame_register_offset: 0, - unwind_codes: vec![ - UnwindCode::PushRegister { - offset: 2, - reg: RU::rbp.into() - }, - UnwindCode::SetFramePointer { - offset: 5, - sp_offset: 0 - }, - UnwindCode::StackAlloc { - offset: 27, - size: 10000 + 32 - } - ] - } - ); - - assert_eq!(unwind.size(), 12); - - let mut sink = SimpleUnwindSink(Vec::new()); - unwind.emit(&mut sink); - - assert_eq!( - sink.0, - [ - 0x01, // Version and flags (version 1, no flags) - 0x1B, // Prologue size - 0x04, // Unwind code count (2 for stack alloc, 1 for save frame reg, 1 for push reg) - 0x05, // Frame register + offset (RBP with 0 offset) - 0x1B, // Prolog offset - 0x01, // Operation 1 (large stack alloc), size is scaled 16-bits (info = 0) - 0xE6, // Low size byte - 0x04, // High size byte (e.g. 0x04E6 * 8 = 100032 (10000 + 32) bytes) - 0x05, // Prolog offset - 0x03, // Operation 3 (save frame register), stack pointer offset = 0 - 0x02, // Prolog offset - 0x50, // Operation 0 (push nonvolatile register), reg = 5 (RBP) - ] - ); - } - - #[test] - fn test_large_alloc() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function( - CallConv::WindowsFastcall, - Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 1000000)), - )); - - context.compile(&*isa).expect("expected compilation"); - - let unwind = UnwindInfo::try_from_func(&context.func, &*isa, Some(RU::rbp.into())) - .expect("expected unwind info"); - - assert_eq!( - unwind, - UnwindInfo { - flags: 0, - prologue_size: 27, - frame_register: Some(RU::rbp.into()), - frame_register_offset: 0, - unwind_codes: vec![ - UnwindCode::PushRegister { - offset: 2, - reg: RU::rbp.into() - }, - UnwindCode::SetFramePointer { - offset: 5, - sp_offset: 0 - }, - UnwindCode::StackAlloc { - offset: 27, - size: 1000000 + 32 - } - ] - } - ); - - assert_eq!(unwind.size(), 16); - - let mut sink = SimpleUnwindSink(Vec::new()); - unwind.emit(&mut sink); - - assert_eq!( - sink.0, - [ - 0x01, // Version and flags (version 1, no flags) - 0x1B, // Prologue size - 0x05, // Unwind code count (3 for stack alloc, 1 for save frame reg, 1 for push reg) - 0x05, // Frame register + offset (RBP with 0 offset) - 0x1B, // Prolog offset - 0x11, // Operation 1 (large stack alloc), size is unscaled 32-bits (info = 1) - 0x60, // Byte 1 of size - 0x42, // Byte 2 of size - 0x0F, // Byte 3 of size - 0x00, // Byte 4 of size (size is 0xF4260 = 1000032 (1000000 + 32) bytes) - 0x05, // Prolog offset - 0x03, // Operation 3 (save frame register), stack pointer offset = 0 - 0x02, // Prolog offset - 0x50, // Operation 0 (push nonvolatile register), reg = 5 (RBP) - 0x00, // Padding byte - 0x00, // Padding byte - ] - ); - } - - fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); - - let block0 = func.dfg.make_block(); - let mut pos = FuncCursor::new(&mut func); - pos.insert_block(block0); - pos.ins().return_(&[]); - - if let Some(stack_slot) = stack_slot { - func.stack_slots.push(stack_slot); - } - - func - } -} +pub mod systemv; +pub mod windows; diff --git a/cranelift/codegen/src/isa/x86/unwind/systemv.rs b/cranelift/codegen/src/isa/x86/unwind/systemv.rs new file mode 100644 index 000000000000..347e14151017 --- /dev/null +++ b/cranelift/codegen/src/isa/x86/unwind/systemv.rs @@ -0,0 +1,489 @@ +//! Unwind information for System V ABI (x86-64). + +use crate::ir::{Function, Inst, InstructionData, Opcode, Value}; +use crate::isa::{ + unwind::systemv::{CallFrameInstruction, UnwindInfo}, + x86::registers::RU, + CallConv, RegUnit, TargetIsa, +}; +use alloc::vec::Vec; +use gimli::{write::CommonInformationEntry, Encoding, Format, Register, X86_64}; +use thiserror::Error; + +/// Creates a new x86-64 common information entry (CIE). +pub fn create_cie() -> CommonInformationEntry { + use gimli::write::CallFrameInstruction; + + let mut entry = CommonInformationEntry::new( + Encoding { + address_size: 8, + format: Format::Dwarf32, + version: 1, + }, + 1, // Code alignment factor + -8, // Data alignment factor + X86_64::RA, + ); + + // Every frame will start with the call frame address (CFA) at RSP+8 + // It is +8 to account for the push of the return address by the call instruction + entry.add_instruction(CallFrameInstruction::Cfa(X86_64::RSP, 8)); + + // Every frame will start with the return address at RSP (CFA-8 = RSP+8-8 = RSP) + entry.add_instruction(CallFrameInstruction::Offset(X86_64::RA, -8)); + + entry +} + +/// Map Cranelift registers to their corresponding Gimli registers. +pub fn map_reg(isa: &dyn TargetIsa, reg: RegUnit) -> Result { + if isa.name() != "x86" || isa.pointer_bits() != 64 { + return Err(RegisterMappingError::UnsupportedArchitecture); + } + + // Mapping from https://github.com/bytecodealliance/cranelift/pull/902 by @iximeow + const X86_GP_REG_MAP: [gimli::Register; 16] = [ + X86_64::RAX, + X86_64::RCX, + X86_64::RDX, + X86_64::RBX, + X86_64::RSP, + X86_64::RBP, + X86_64::RSI, + X86_64::RDI, + X86_64::R8, + X86_64::R9, + X86_64::R10, + X86_64::R11, + X86_64::R12, + X86_64::R13, + X86_64::R14, + X86_64::R15, + ]; + const X86_XMM_REG_MAP: [gimli::Register; 16] = [ + X86_64::XMM0, + X86_64::XMM1, + X86_64::XMM2, + X86_64::XMM3, + X86_64::XMM4, + X86_64::XMM5, + X86_64::XMM6, + X86_64::XMM7, + X86_64::XMM8, + X86_64::XMM9, + X86_64::XMM10, + X86_64::XMM11, + X86_64::XMM12, + X86_64::XMM13, + X86_64::XMM14, + X86_64::XMM15, + ]; + + let reg_info = isa.register_info(); + let bank = reg_info + .bank_containing_regunit(reg) + .ok_or_else(|| RegisterMappingError::MissingBank)?; + match bank.name { + "IntRegs" => { + // x86 GP registers have a weird mapping to DWARF registers, so we use a + // lookup table. + Ok(X86_GP_REG_MAP[(reg - bank.first_unit) as usize]) + } + "FloatRegs" => Ok(X86_XMM_REG_MAP[(reg - bank.first_unit) as usize]), + _ => Err(RegisterMappingError::UnsupportedRegisterBank(bank.name)), + } +} + +#[derive(Error, Debug)] +pub enum RegisterMappingError { + #[error("unable to find bank for register info")] + MissingBank, + #[error("register mapping is currently only implemented for x86_64")] + UnsupportedArchitecture, + #[error("unsupported register bank: {0}")] + UnsupportedRegisterBank(&'static str), +} + +struct InstructionBuilder<'a> { + func: &'a Function, + isa: &'a dyn TargetIsa, + cfa_offset: i32, + frame_register: Option, + instructions: Vec<(u32, CallFrameInstruction)>, + stack_size: Option, + epilogue_pop_offsets: Vec, +} + +impl<'a> InstructionBuilder<'a> { + fn new(func: &'a Function, isa: &'a dyn TargetIsa, frame_register: Option) -> Self { + Self { + func, + isa, + cfa_offset: 8, // CFA offset starts at 8 to account to return address on stack + frame_register, + instructions: Vec::new(), + stack_size: None, + epilogue_pop_offsets: Vec::new(), + } + } + + fn push_reg(&mut self, offset: u32, arg: Value) { + self.cfa_offset += 8; + + let reg = self.func.locations[arg].unwrap_reg(); + + // Update the CFA if this is the save of the frame pointer register or if a frame pointer isn't being used + // When using a frame pointer, we only need to update the CFA to account for the push of the frame pointer itself + if match self.frame_register { + Some(fp) => reg == fp, + None => true, + } { + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + // Pushes in the prologue are register saves, so record an offset of the save + self.instructions.push(( + offset, + CallFrameInstruction::Offset( + map_reg(self.isa, reg) + .expect("a register mapping from cranelift to gimli") + .0, + -self.cfa_offset, + ), + )); + } + + fn adjust_sp_down(&mut self, offset: u32) { + // Don't adjust the CFA if we're using a frame pointer + if self.frame_register.is_some() { + return; + } + + self.cfa_offset += self + .stack_size + .expect("expected a previous stack size instruction"); + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + fn adjust_sp_down_imm(&mut self, offset: u32, imm: i64) { + assert!(imm <= core::u32::MAX as i64); + + // Don't adjust the CFA if we're using a frame pointer + if self.frame_register.is_some() { + return; + } + + self.cfa_offset += imm as i32; + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + fn adjust_sp_up_imm(&mut self, offset: u32, imm: i64) { + assert!(imm <= core::u32::MAX as i64); + + // Don't adjust the CFA if we're using a frame pointer + if self.frame_register.is_some() { + return; + } + + self.cfa_offset -= imm as i32; + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + fn move_reg(&mut self, offset: u32, src: RegUnit, dst: RegUnit) { + if let Some(fp) = self.frame_register { + // Check for change in CFA register (RSP is always the starting CFA) + if src == (RU::rsp as RegUnit) && dst == fp { + self.instructions.push(( + offset, + CallFrameInstruction::CfaRegister( + map_reg(self.isa, dst) + .expect("a register mapping from cranelift to gimli") + .0, + ), + )); + } + } + } + + fn prologue_imm_const(&mut self, imm: i64) { + assert!(imm <= core::u32::MAX as i64); + assert!(self.stack_size.is_none()); + + // This instruction should only appear in a prologue to pass an + // argument of the stack size to a stack check function. + // Record the stack size so we know what it is when we encounter the adjustment + // instruction (which will adjust via the register assigned to this instruction). + self.stack_size = Some(imm as i32); + } + + fn ret(&mut self, inst: Inst) { + let args = self.func.dfg.inst_args(inst); + + for (i, arg) in args.iter().rev().enumerate() { + // Only walk back the args for the pop instructions encountered + if i >= self.epilogue_pop_offsets.len() { + break; + } + + self.cfa_offset -= 8; + let reg = self.func.locations[*arg].unwrap_reg(); + + // Update the CFA if this is the restore of the frame pointer register or if a frame pointer isn't being used + match self.frame_register { + Some(fp) => { + if reg == fp { + self.instructions.push(( + self.epilogue_pop_offsets[i], + CallFrameInstruction::Cfa( + map_reg(self.isa, RU::rsp as RegUnit) + .expect("a register mapping from cranelift to gimli") + .0, + self.cfa_offset, + ), + )); + } + } + None => { + self.instructions.push(( + self.epilogue_pop_offsets[i], + CallFrameInstruction::CfaOffset(self.cfa_offset), + )); + + // Pops in the epilogue are register restores, so record a "same value" for the register + // This isn't necessary when using a frame pointer as the CFA doesn't change for CSR restores + self.instructions.push(( + self.epilogue_pop_offsets[i], + CallFrameInstruction::SameValue( + map_reg(self.isa, reg) + .expect("a register mapping from cranelift to gimli") + .0, + ), + )); + } + }; + } + + self.epilogue_pop_offsets.clear(); + } + + fn insert_pop_offset(&mut self, offset: u32) { + self.epilogue_pop_offsets.push(offset); + } + + fn remember_state(&mut self, offset: u32) { + self.instructions + .push((offset, CallFrameInstruction::RememberState)); + } + + fn restore_state(&mut self, offset: u32) { + self.instructions + .push((offset, CallFrameInstruction::RestoreState)); + } + + fn is_prologue_end(&self, inst: Inst) -> bool { + self.func.prologue_end == Some(inst) + } + + fn is_epilogue_start(&self, inst: Inst) -> bool { + self.func.epilogues_start.contains(&inst) + } +} + +pub(crate) fn create_unwind_info( + func: &Function, + isa: &dyn TargetIsa, + frame_register: Option, +) -> Option { + // Only System V-like calling conventions are supported + match func.signature.call_conv { + CallConv::Fast | CallConv::Cold | CallConv::SystemV => {} + _ => return None, + } + + if func.prologue_end.is_none() || isa.name() != "x86" || isa.pointer_bits() != 64 { + return None; + } + + let mut builder = InstructionBuilder::new(func, isa, frame_register); + let mut in_prologue = true; + let mut in_epilogue = false; + let mut len = 0; + + let mut blocks = func.layout.blocks().collect::>(); + blocks.sort_by_key(|b| func.offsets[*b]); + + for (i, block) in blocks.iter().enumerate() { + for (offset, inst, size) in func.inst_offsets(*block, &isa.encoding_info()) { + let offset = offset + size; + assert!(len <= offset); + len = offset; + + let is_last_block = i == blocks.len() - 1; + + if in_prologue { + // Check for prologue end (inclusive) + in_prologue = !builder.is_prologue_end(inst); + } else if !in_epilogue && builder.is_epilogue_start(inst) { + // Now in an epilogue, emit a remember state instruction if not last block + in_epilogue = true; + + if !is_last_block { + builder.remember_state(offset); + } + } else if !in_epilogue { + // Ignore normal instructions + continue; + } + + match builder.func.dfg[inst] { + InstructionData::Unary { opcode, arg } => match opcode { + Opcode::X86Push => { + builder.push_reg(offset, arg); + } + Opcode::AdjustSpDown => { + builder.adjust_sp_down(offset); + } + _ => {} + }, + InstructionData::CopySpecial { src, dst, .. } => { + builder.move_reg(offset, src, dst); + } + InstructionData::NullAry { opcode } => match opcode { + Opcode::X86Pop => { + builder.insert_pop_offset(offset); + } + _ => {} + }, + InstructionData::UnaryImm { opcode, imm } => match opcode { + Opcode::Iconst => { + builder.prologue_imm_const(imm.into()); + } + Opcode::AdjustSpDownImm => { + builder.adjust_sp_down_imm(offset, imm.into()); + } + Opcode::AdjustSpUpImm => { + builder.adjust_sp_up_imm(offset, imm.into()); + } + _ => {} + }, + InstructionData::MultiAry { opcode, .. } => match opcode { + Opcode::Return => { + builder.ret(inst); + + if !is_last_block { + builder.restore_state(offset); + } + + in_epilogue = false; + } + _ => {} + }, + _ => {} + }; + } + } + + Some(UnwindInfo::new(builder.instructions, len)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cursor::{Cursor, FuncCursor}; + use crate::ir::{ + types, AbiParam, ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind, + }; + use crate::isa::{lookup, CallConv}; + use crate::settings::{builder, Flags}; + use crate::Context; + use gimli::write::Address; + use std::str::FromStr; + use target_lexicon::triple; + + #[test] + fn test_simple_func() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::SystemV, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let fde = match isa.create_unwind_info(&context.func) { + Some(crate::isa::unwind::UnwindInfo::SystemV(info)) => { + info.to_fde(Address::Constant(1234)) + } + _ => panic!("expected unwind information"), + }; + + assert_eq!(format!("{:?}", fde), "FrameDescriptionEntry { address: Constant(1234), length: 16, lsda: None, instructions: [(2, CfaOffset(16)), (2, Offset(Register(6), -16)), (5, CfaRegister(Register(6))), (15, Cfa(Register(7), 8))] }"); + } + + fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { + let mut func = + Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + + let block0 = func.dfg.make_block(); + let mut pos = FuncCursor::new(&mut func); + pos.insert_block(block0); + pos.ins().return_(&[]); + + if let Some(stack_slot) = stack_slot { + func.stack_slots.push(stack_slot); + } + + func + } + + #[test] + fn test_multi_return_func() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_multi_return_function(CallConv::SystemV)); + + context.compile(&*isa).expect("expected compilation"); + + let fde = match isa.create_unwind_info(&context.func) { + Some(crate::isa::unwind::UnwindInfo::SystemV(info)) => { + info.to_fde(Address::Constant(4321)) + } + _ => panic!("expected unwind information"), + }; + + assert_eq!(format!("{:?}", fde), "FrameDescriptionEntry { address: Constant(4321), length: 16, lsda: None, instructions: [(2, CfaOffset(16)), (2, Offset(Register(6), -16)), (5, CfaRegister(Register(6))), (12, RememberState), (12, Cfa(Register(7), 8)), (13, RestoreState), (15, Cfa(Register(7), 0))] }"); + } + + fn create_multi_return_function(call_conv: CallConv) -> Function { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(types::I32)); + let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + + let block0 = func.dfg.make_block(); + let v0 = func.dfg.append_block_param(block0, types::I32); + let block1 = func.dfg.make_block(); + let block2 = func.dfg.make_block(); + + let mut pos = FuncCursor::new(&mut func); + pos.insert_block(block0); + pos.ins().brnz(v0, block2, &[]); + pos.ins().jump(block1, &[]); + + pos.insert_block(block1); + pos.ins().return_(&[]); + + pos.insert_block(block2); + pos.ins().return_(&[]); + + func + } +} diff --git a/cranelift/codegen/src/isa/x86/unwind/windows.rs b/cranelift/codegen/src/isa/x86/unwind/windows.rs new file mode 100644 index 000000000000..4b3333a106d9 --- /dev/null +++ b/cranelift/codegen/src/isa/x86/unwind/windows.rs @@ -0,0 +1,522 @@ +//! Unwind information for Windows x64 ABI. + +use crate::ir::{Function, InstructionData, Opcode}; +use crate::isa::x86::registers::{GPR, RU}; +use crate::isa::{CallConv, RegUnit, TargetIsa}; +use alloc::vec::Vec; +use byteorder::{ByteOrder, LittleEndian}; + +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +/// Maximum (inclusive) size of a "small" stack allocation +const SMALL_ALLOC_MAX_SIZE: u32 = 128; +/// Maximum (inclusive) size of a "large" stack allocation that can represented in 16-bits +const LARGE_ALLOC_16BIT_MAX_SIZE: u32 = 524280; + +struct Writer<'a> { + buf: &'a mut [u8], + offset: usize, +} + +impl<'a> Writer<'a> { + pub fn new(buf: &'a mut [u8]) -> Self { + Self { buf, offset: 0 } + } + + fn write_u8(&mut self, v: u8) { + self.buf[self.offset] = v; + self.offset += 1; + } + + fn write_u16(&mut self, v: u16) { + T::write_u16(&mut self.buf[self.offset..(self.offset + 2)], v); + self.offset += 2; + } + + fn write_u32(&mut self, v: u32) { + T::write_u32(&mut self.buf[self.offset..(self.offset + 4)], v); + self.offset += 4; + } +} + +/// The supported unwind codes for the x64 Windows ABI. +/// +/// See: https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 +/// Only what is needed to describe the prologues generated by the Cranelift x86 ISA are represented here. +/// Note: the Cranelift x86 ISA RU enum matches the Windows unwind GPR encoding values. +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +enum UnwindCode { + PushRegister { offset: u8, reg: u8 }, + StackAlloc { offset: u8, size: u32 }, + SetFramePointer { offset: u8, sp_offset: u8 }, +} + +impl UnwindCode { + fn emit(&self, writer: &mut Writer) { + enum UnwindOperation { + PushNonvolatileRegister, + LargeStackAlloc, + SmallStackAlloc, + SetFramePointer, + } + + match self { + Self::PushRegister { offset, reg } => { + writer.write_u8(*offset); + writer.write_u8((*reg << 4) | (UnwindOperation::PushNonvolatileRegister as u8)); + } + Self::StackAlloc { offset, size } => { + // Stack allocations on Windows must be a multiple of 8 and be at least 1 slot + assert!(*size >= 8); + assert!((*size % 8) == 0); + + writer.write_u8(*offset); + if *size <= SMALL_ALLOC_MAX_SIZE { + writer.write_u8( + ((((*size - 8) / 8) as u8) << 4) | UnwindOperation::SmallStackAlloc as u8, + ); + } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { + writer.write_u8(UnwindOperation::LargeStackAlloc as u8); + writer.write_u16::((*size / 8) as u16); + } else { + writer.write_u8((1 << 4) | (UnwindOperation::LargeStackAlloc as u8)); + writer.write_u32::(*size); + } + } + Self::SetFramePointer { offset, sp_offset } => { + writer.write_u8(*offset); + writer.write_u8((*sp_offset << 4) | (UnwindOperation::SetFramePointer as u8)); + } + }; + } + + fn node_count(&self) -> usize { + match self { + Self::StackAlloc { size, .. } => { + if *size <= SMALL_ALLOC_MAX_SIZE { + 1 + } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { + 2 + } else { + 3 + } + } + _ => 1, + } + } +} + +pub(crate) fn create_unwind_info( + func: &Function, + isa: &dyn TargetIsa, + frame_register: Option, +) -> Option { + // Only Windows fastcall is supported for unwind information + if func.signature.call_conv != CallConv::WindowsFastcall || func.prologue_end.is_none() { + return None; + } + + let prologue_end = func.prologue_end.unwrap(); + let entry_block = func.layout.entry_block().expect("missing entry block"); + + // Stores the stack size when SP is not adjusted via an immediate value + let mut stack_size = None; + let mut prologue_size = 0; + let mut unwind_codes = Vec::new(); + let mut found_end = false; + + for (offset, inst, size) in func.inst_offsets(entry_block, &isa.encoding_info()) { + // x64 ABI prologues cannot exceed 255 bytes in length + if (offset + size) > 255 { + panic!("function prologues cannot exceed 255 bytes in size for Windows x64"); + } + + prologue_size += size; + + let unwind_offset = (offset + size) as u8; + + match func.dfg[inst] { + InstructionData::Unary { opcode, arg } => { + match opcode { + Opcode::X86Push => { + unwind_codes.push(UnwindCode::PushRegister { + offset: unwind_offset, + reg: GPR.index_of(func.locations[arg].unwrap_reg()) as u8, + }); + } + Opcode::AdjustSpDown => { + // This is used when calling a stack check function + // We need to track the assignment to RAX which has the size of the stack + unwind_codes.push(UnwindCode::StackAlloc { + offset: unwind_offset, + size: stack_size.expect("expected a previous stack size instruction"), + }); + } + _ => {} + } + } + InstructionData::CopySpecial { src, dst, .. } => { + if let Some(frame_register) = frame_register { + if src == (RU::rsp as RegUnit) && dst == frame_register { + unwind_codes.push(UnwindCode::SetFramePointer { + offset: unwind_offset, + sp_offset: 0, + }); + } + } + } + InstructionData::UnaryImm { opcode, imm } => { + match opcode { + Opcode::Iconst => { + let imm: i64 = imm.into(); + assert!(imm <= core::u32::MAX as i64); + assert!(stack_size.is_none()); + + // This instruction should only appear in a prologue to pass an + // argument of the stack size to a stack check function. + // Record the stack size so we know what it is when we encounter the adjustment + // instruction (which will adjust via the register assigned to this instruction). + stack_size = Some(imm as u32); + } + Opcode::AdjustSpDownImm => { + let imm: i64 = imm.into(); + assert!(imm <= core::u32::MAX as i64); + + unwind_codes.push(UnwindCode::StackAlloc { + offset: unwind_offset, + size: imm as u32, + }); + } + _ => {} + } + } + _ => {} + }; + + if inst == prologue_end { + found_end = true; + break; + } + } + + assert!(found_end); + + Some(UnwindInfo { + flags: 0, // this assumes cranelift functions have no SEH handlers + prologue_size: prologue_size as u8, + frame_register: frame_register.map(|r| GPR.index_of(r) as u8), + frame_register_offset: 0, + unwind_codes, + }) +} + +/// Represents Windows x64 unwind information. +/// +/// For information about Windows x64 unwind info, see: +/// https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct UnwindInfo { + flags: u8, + prologue_size: u8, + frame_register: Option, + frame_register_offset: u8, + unwind_codes: Vec, +} + +impl UnwindInfo { + /// Gets the emit size of the unwind information, in bytes. + pub fn emit_size(&self) -> usize { + let node_count = self.node_count(); + + // Calculation of the size requires no SEH handler or chained info + assert!(self.flags == 0); + + // Size of fixed part of UNWIND_INFO is 4 bytes + // Then comes the UNWIND_CODE nodes (2 bytes each) + // Then comes 2 bytes of padding for the unwind codes if necessary + // Next would come the SEH data, but we assert above that the function doesn't have SEH data + + 4 + (node_count * 2) + if (node_count & 1) == 1 { 2 } else { 0 } + } + + /// Emits the unwind information into the given mutable byte slice. + /// + /// This function will panic if the slice is not at least `emit_size` in length. + pub fn emit(&self, buf: &mut [u8]) { + const UNWIND_INFO_VERSION: u8 = 1; + + let node_count = self.node_count(); + assert!(node_count <= 256); + + let mut writer = Writer::new(buf); + + writer.write_u8((self.flags << 3) | UNWIND_INFO_VERSION); + writer.write_u8(self.prologue_size); + writer.write_u8(node_count as u8); + + if let Some(reg) = self.frame_register { + writer.write_u8((self.frame_register_offset << 4) | reg); + } else { + writer.write_u8(0); + } + + // Unwind codes are written in reverse order (prologue offset descending) + for code in self.unwind_codes.iter().rev() { + code.emit(&mut writer); + } + + // To keep a 32-bit alignment, emit 2 bytes of padding if there's an odd number of 16-bit nodes + if (node_count & 1) == 1 { + writer.write_u16::(0); + } + + // Ensure the correct number of bytes was emitted + assert_eq!(writer.offset, self.emit_size()); + } + + fn node_count(&self) -> usize { + self.unwind_codes + .iter() + .fold(0, |nodes, c| nodes + c.node_count()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cursor::{Cursor, FuncCursor}; + use crate::ir::{ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind}; + use crate::isa::{lookup, CallConv}; + use crate::settings::{builder, Flags}; + use crate::Context; + use std::str::FromStr; + use target_lexicon::triple; + + #[test] + fn test_wrong_calling_convention() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function(CallConv::SystemV, None)); + + context.compile(&*isa).expect("expected compilation"); + + assert_eq!(create_unwind_info(&context.func, &*isa, None), None); + } + + #[test] + fn test_small_alloc() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::WindowsFastcall, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, Some(RU::rbp.into())) + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + flags: 0, + prologue_size: 9, + frame_register: Some(GPR.index_of(RU::rbp.into()) as u8), + frame_register_offset: 0, + unwind_codes: vec![ + UnwindCode::PushRegister { + offset: 2, + reg: GPR.index_of(RU::rbp.into()) as u8 + }, + UnwindCode::SetFramePointer { + offset: 5, + sp_offset: 0 + }, + UnwindCode::StackAlloc { + offset: 9, + size: 64 + 32 + } + ] + } + ); + + assert_eq!(unwind.emit_size(), 12); + + let mut buf = [0u8; 12]; + unwind.emit(&mut buf); + + assert_eq!( + buf, + [ + 0x01, // Version and flags (version 1, no flags) + 0x09, // Prologue size + 0x03, // Unwind code count (1 for stack alloc, 1 for save frame reg, 1 for push reg) + 0x05, // Frame register + offset (RBP with 0 offset) + 0x09, // Prolog offset + 0xB2, // Operation 2 (small stack alloc), size = 0xB slots (e.g. (0xB * 8) + 8 = 96 (64 + 32) bytes) + 0x05, // Prolog offset + 0x03, // Operation 3 (save frame register), stack pointer offset = 0 + 0x02, // Prolog offset + 0x50, // Operation 0 (save nonvolatile register), reg = 5 (RBP) + 0x00, // Padding byte + 0x00, // Padding byte + ] + ); + } + + #[test] + fn test_medium_alloc() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::WindowsFastcall, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 10000)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, Some(RU::rbp.into())) + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + flags: 0, + prologue_size: 27, + frame_register: Some(GPR.index_of(RU::rbp.into()) as u8), + frame_register_offset: 0, + unwind_codes: vec![ + UnwindCode::PushRegister { + offset: 2, + reg: GPR.index_of(RU::rbp.into()) as u8 + }, + UnwindCode::SetFramePointer { + offset: 5, + sp_offset: 0 + }, + UnwindCode::StackAlloc { + offset: 27, + size: 10000 + 32 + } + ] + } + ); + + assert_eq!(unwind.emit_size(), 12); + + let mut buf = [0u8; 12]; + unwind.emit(&mut buf); + + assert_eq!( + buf, + [ + 0x01, // Version and flags (version 1, no flags) + 0x1B, // Prologue size + 0x04, // Unwind code count (2 for stack alloc, 1 for save frame reg, 1 for push reg) + 0x05, // Frame register + offset (RBP with 0 offset) + 0x1B, // Prolog offset + 0x01, // Operation 1 (large stack alloc), size is scaled 16-bits (info = 0) + 0xE6, // Low size byte + 0x04, // High size byte (e.g. 0x04E6 * 8 = 100032 (10000 + 32) bytes) + 0x05, // Prolog offset + 0x03, // Operation 3 (save frame register), stack pointer offset = 0 + 0x02, // Prolog offset + 0x50, // Operation 0 (push nonvolatile register), reg = 5 (RBP) + ] + ); + } + + #[test] + fn test_large_alloc() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::WindowsFastcall, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 1000000)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, Some(RU::rbp.into())) + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + flags: 0, + prologue_size: 27, + frame_register: Some(GPR.index_of(RU::rbp.into()) as u8), + frame_register_offset: 0, + unwind_codes: vec![ + UnwindCode::PushRegister { + offset: 2, + reg: GPR.index_of(RU::rbp.into()) as u8 + }, + UnwindCode::SetFramePointer { + offset: 5, + sp_offset: 0 + }, + UnwindCode::StackAlloc { + offset: 27, + size: 1000000 + 32 + } + ] + } + ); + + assert_eq!(unwind.emit_size(), 16); + + let mut buf = [0u8; 16]; + unwind.emit(&mut buf); + + assert_eq!( + buf, + [ + 0x01, // Version and flags (version 1, no flags) + 0x1B, // Prologue size + 0x05, // Unwind code count (3 for stack alloc, 1 for save frame reg, 1 for push reg) + 0x05, // Frame register + offset (RBP with 0 offset) + 0x1B, // Prolog offset + 0x11, // Operation 1 (large stack alloc), size is unscaled 32-bits (info = 1) + 0x60, // Byte 1 of size + 0x42, // Byte 2 of size + 0x0F, // Byte 3 of size + 0x00, // Byte 4 of size (size is 0xF4260 = 1000032 (1000000 + 32) bytes) + 0x05, // Prolog offset + 0x03, // Operation 3 (save frame register), stack pointer offset = 0 + 0x02, // Prolog offset + 0x50, // Operation 0 (push nonvolatile register), reg = 5 (RBP) + 0x00, // Padding byte + 0x00, // Padding byte + ] + ); + } + + fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { + let mut func = + Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + + let block0 = func.dfg.make_block(); + let mut pos = FuncCursor::new(&mut func); + pos.insert_block(block0); + pos.ins().return_(&[]); + + if let Some(stack_slot) = stack_slot { + func.stack_slots.push(stack_slot); + } + + func + } +} diff --git a/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif b/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif new file mode 100644 index 000000000000..7ff0ea861578 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif @@ -0,0 +1,197 @@ +test unwind +set opt_level=speed_and_size +set is_pic +target x86_64 haswell + +; check the unwind information with a function with no args +function %no_args() system_v { +block0: + return +} +; sameln: 0x00000000: CIE +; nextln: length: 0x00000014 +; nextln: version: 0x01 +; nextln: code_align: 1 +; nextln: data_align: -8 +; nextln: ra_register: 0x10 +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_offset (r16, 1) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: Instructions: Init State: +; nextln: +; nextln: +; nextln: 0x00000018: FDE +; nextln: length: 0x00000024 +; nextln: CIE_pointer: 0x00000000 +; nextln: start_addr: 0x0000000000000000 +; nextln: range_size: 0x0000000000000006 (end_addr = 0x0000000000000006) +; nextln: Instructions: +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa_offset (16) +; nextln: DW_CFA_offset (r6, 2) +; nextln: DW_CFA_advance_loc (3) +; nextln: DW_CFA_def_cfa_register (r6) +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop + +; check a function with medium-sized stack alloc +function %medium_stack() system_v { + ss0 = explicit_slot 100000 +block0: + return +} +; sameln: 0x00000000: CIE +; nextln: length: 0x00000014 +; nextln: version: 0x01 +; nextln: code_align: 1 +; nextln: data_align: -8 +; nextln: ra_register: 0x10 +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_offset (r16, 1) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: Instructions: Init State: +; nextln: +; nextln: +; nextln: 0x00000018: FDE +; nextln: length: 0x00000024 +; nextln: CIE_pointer: 0x00000000 +; nextln: start_addr: 0x0000000000000000 +; nextln: range_size: 0x000000000000001a (end_addr = 0x000000000000001a) +; nextln: Instructions: +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa_offset (16) +; nextln: DW_CFA_offset (r6, 2) +; nextln: DW_CFA_advance_loc (3) +; nextln: DW_CFA_def_cfa_register (r6) +; nextln: DW_CFA_advance_loc (21) +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop + +; check a function with large-sized stack alloc +function %large_stack() system_v { + ss0 = explicit_slot 524288 +block0: + return +} +; sameln: 0x00000000: CIE +; nextln: length: 0x00000014 +; nextln: version: 0x01 +; nextln: code_align: 1 +; nextln: data_align: -8 +; nextln: ra_register: 0x10 +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_offset (r16, 1) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: Instructions: Init State: +; nextln: +; nextln: +; nextln: 0x00000018: FDE +; nextln: length: 0x00000024 +; nextln: CIE_pointer: 0x00000000 +; nextln: start_addr: 0x0000000000000000 +; nextln: range_size: 0x000000000000001a (end_addr = 0x000000000000001a) +; nextln: Instructions: +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa_offset (16) +; nextln: DW_CFA_offset (r6, 2) +; nextln: DW_CFA_advance_loc (3) +; nextln: DW_CFA_def_cfa_register (r6) +; nextln: DW_CFA_advance_loc (21) +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: + +; check a function that has CSRs +function %lots_of_registers(i64, i64) system_v { +block0(v0: i64, v1: i64): + v2 = load.i32 v0+0 + v3 = load.i32 v0+8 + v4 = load.i32 v0+16 + v5 = load.i32 v0+24 + v6 = load.i32 v0+32 + v7 = load.i32 v0+40 + v8 = load.i32 v0+48 + v9 = load.i32 v0+56 + v10 = load.i32 v0+64 + v11 = load.i32 v0+72 + v12 = load.i32 v0+80 + v13 = load.i32 v0+88 + v14 = load.i32 v0+96 + store.i32 v2, v1+0 + store.i32 v3, v1+8 + store.i32 v4, v1+16 + store.i32 v5, v1+24 + store.i32 v6, v1+32 + store.i32 v7, v1+40 + store.i32 v8, v1+48 + store.i32 v9, v1+56 + store.i32 v10, v1+64 + store.i32 v11, v1+72 + store.i32 v12, v1+80 + store.i32 v13, v1+88 + store.i32 v14, v1+96 + return +} +; sameln: 0x00000000: CIE +; nextln: length: 0x00000014 +; nextln: version: 0x01 +; nextln: code_align: 1 +; nextln: data_align: -8 +; nextln: ra_register: 0x10 +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_offset (r16, 1) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: Instructions: Init State: +; nextln: +; nextln: +; nextln: 0x00000018: FDE +; nextln: length: 0x00000034 +; nextln: CIE_pointer: 0x00000000 +; nextln: start_addr: 0x0000000000000000 +; nextln: range_size: 0x0000000000000074 (end_addr = 0x0000000000000074) +; nextln: Instructions: +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa_offset (16) +; nextln: DW_CFA_offset (r6, 2) +; nextln: DW_CFA_advance_loc (3) +; nextln: DW_CFA_def_cfa_register (r6) +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_offset (r3, 3) +; nextln: DW_CFA_advance_loc (2) +; nextln: DW_CFA_offset (r12, 4) +; nextln: DW_CFA_advance_loc (2) +; nextln: DW_CFA_offset (r13, 5) +; nextln: DW_CFA_advance_loc (2) +; nextln: DW_CFA_offset (r14, 6) +; nextln: DW_CFA_advance_loc (2) +; nextln: DW_CFA_offset (r15, 7) +; nextln: DW_CFA_advance_loc (102) +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop diff --git a/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif b/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif index b146f0ac7696..da9acaf56e0f 100644 --- a/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif +++ b/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif @@ -3,46 +3,29 @@ set opt_level=speed_and_size set is_pic target x86_64 haswell -; check that there is no unwind information for a system_v function -function %not_fastcall() system_v { -block0: - return -} -; sameln: No unwind information. - ; check the unwind information with a function with no args function %no_args() windows_fastcall { block0: return } -; sameln: UnwindInfo { -; nextln: version: 1, -; nextln: flags: 0, -; nextln: prologue_size: 8, -; nextln: unwind_code_count_raw: 3, -; nextln: frame_register: 5, -; nextln: frame_register_offset: 0, -; nextln: unwind_codes: [ -; nextln: UnwindCode { -; nextln: offset: 8, -; nextln: op: SmallStackAlloc, -; nextln: info: 3, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 4, -; nextln: op: SetFramePointer, -; nextln: info: 0, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 1, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 5, -; nextln: value: None, -; nextln: }, -; nextln: ], -; nextln: } +; sameln: version: 1 +; nextln: flags: 0 +; nextln: prologue size: 8 +; nextln: frame register: 5 +; nextln: frame register offset: 0 +; nextln: unwind codes: 3 +; nextln: +; nextln: offset: 1 +; nextln: op: PushNonvolatileRegister +; nextln: info: 5 +; nextln: +; nextln: offset: 4 +; nextln: op: SetFramePointer +; nextln: info: 0 +; nextln: +; nextln: offset: 8 +; nextln: op: SmallStackAlloc +; nextln: info: 3 ; check a function with medium-sized stack alloc function %medium_stack() windows_fastcall { @@ -50,36 +33,25 @@ function %medium_stack() windows_fastcall { block0: return } -; sameln: UnwindInfo { -; nextln: version: 1, -; nextln: flags: 0, -; nextln: prologue_size: 17, -; nextln: unwind_code_count_raw: 4, -; nextln: frame_register: 5, -; nextln: frame_register_offset: 0, -; nextln: unwind_codes: [ -; nextln: UnwindCode { -; nextln: offset: 17, -; nextln: op: LargeStackAlloc, -; nextln: info: 0, -; nextln: value: U16( -; nextln: 12504, -; nextln: ), -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 4, -; nextln: op: SetFramePointer, -; nextln: info: 0, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 1, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 5, -; nextln: value: None, -; nextln: }, -; nextln: ], -; nextln: } +; sameln: version: 1 +; nextln: flags: 0 +; nextln: prologue size: 17 +; nextln: frame register: 5 +; nextln: frame register offset: 0 +; nextln: unwind codes: 3 +; nextln: +; nextln: offset: 1 +; nextln: op: PushNonvolatileRegister +; nextln: info: 5 +; nextln: +; nextln: offset: 4 +; nextln: op: SetFramePointer +; nextln: info: 0 +; nextln: +; nextln: offset: 17 +; nextln: op: LargeStackAlloc +; nextln: info: 0 +; nextln: value: 12504 (u16) ; check a function with large-sized stack alloc function %large_stack() windows_fastcall { @@ -87,36 +59,25 @@ function %large_stack() windows_fastcall { block0: return } -; sameln: UnwindInfo { -; nextln: version: 1, -; nextln: flags: 0, -; nextln: prologue_size: 17, -; nextln: unwind_code_count_raw: 5, -; nextln: frame_register: 5, -; nextln: frame_register_offset: 0, -; nextln: unwind_codes: [ -; nextln: UnwindCode { -; nextln: offset: 17, -; nextln: op: LargeStackAlloc, -; nextln: info: 1, -; nextln: value: U32( -; nextln: 524320, -; nextln: ), -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 4, -; nextln: op: SetFramePointer, -; nextln: info: 0, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 1, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 5, -; nextln: value: None, -; nextln: }, -; nextln: ], -; nextln: } +; sameln: version: 1 +; nextln: flags: 0 +; nextln: prologue size: 17 +; nextln: frame register: 5 +; nextln: frame register offset: 0 +; nextln: unwind codes: 3 +; nextln: +; nextln: offset: 1 +; nextln: op: PushNonvolatileRegister +; nextln: info: 5 +; nextln: +; nextln: offset: 4 +; nextln: op: SetFramePointer +; nextln: info: 0 +; nextln: +; nextln: offset: 17 +; nextln: op: LargeStackAlloc +; nextln: info: 1 +; nextln: value: 524320 (u32) ; check a function that has CSRs function %lots_of_registers(i64, i64) windows_fastcall { @@ -149,73 +110,49 @@ block0(v0: i64, v1: i64): store.i32 v14, v1+96 return } -; sameln: UnwindInfo { -; nextln: version: 1, -; nextln: flags: 0, -; nextln: prologue_size: 19, -; nextln: unwind_code_count_raw: 10, -; nextln: frame_register: 5, -; nextln: frame_register_offset: 0, -; nextln: unwind_codes: [ -; nextln: UnwindCode { -; nextln: offset: 19, -; nextln: op: SmallStackAlloc, -; nextln: info: 3, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 15, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 15, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 13, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 14, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 11, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 13, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 9, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 12, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 7, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 7, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 6, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 6, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 5, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 3, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 4, -; nextln: op: SetFramePointer, -; nextln: info: 0, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 1, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 5, -; nextln: value: None, -; nextln: }, -; nextln: ], -; nextln: } +; sameln: version: 1 +; nextln: flags: 0 +; nextln: prologue size: 19 +; nextln: frame register: 5 +; nextln: frame register offset: 0 +; nextln: unwind codes: 10 +; nextln: +; nextln: offset: 1 +; nextln: op: PushNonvolatileRegister +; nextln: info: 5 +; nextln: +; nextln: offset: 4 +; nextln: op: SetFramePointer +; nextln: info: 0 +; nextln: +; nextln: offset: 5 +; nextln: op: PushNonvolatileRegister +; nextln: info: 3 +; nextln: +; nextln: offset: 6 +; nextln: op: PushNonvolatileRegister +; nextln: info: 6 +; nextln: +; nextln: offset: 7 +; nextln: op: PushNonvolatileRegister +; nextln: info: 7 +; nextln: +; nextln: offset: 9 +; nextln: op: PushNonvolatileRegister +; nextln: info: 12 +; nextln: +; nextln: offset: 11 +; nextln: op: PushNonvolatileRegister +; nextln: info: 13 +; nextln: +; nextln: offset: 13 +; nextln: op: PushNonvolatileRegister +; nextln: info: 14 +; nextln: +; nextln: offset: 15 +; nextln: op: PushNonvolatileRegister +; nextln: info: 15 +; nextln: +; nextln: offset: 19 +; nextln: op: SmallStackAlloc +; nextln: info: 3 \ No newline at end of file diff --git a/cranelift/filetests/filetests/isa/x86/windows_systemv_x64_fde.clif b/cranelift/filetests/filetests/isa/x86/windows_systemv_x64_fde.clif deleted file mode 100644 index 31b75b6c16d9..000000000000 --- a/cranelift/filetests/filetests/isa/x86/windows_systemv_x64_fde.clif +++ /dev/null @@ -1,54 +0,0 @@ -test fde -set opt_level=speed_and_size -set is_pic -target x86_64 haswell - -; check that there is no libunwind information for a windows_fastcall function -function %not_fastcall() windows_fastcall { -block0: - return -} -; sameln: No unwind information. - -; check the libunwind information with a function with no args -function %no_args() system_v { -block0: - return -} -; sameln: 0x00000000: CIE -; nextln: length: 0x00000014 -; nextln: version: 0x01 -; nextln: code_align: 1 -; nextln: data_align: -8 -; nextln: ra_register: 0x10 -; nextln: DW_CFA_def_cfa (r7, 8) -; nextln: DW_CFA_offset (r16, 1) -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: Instructions: Init State: -; nextln: -; nextln: -; nextln: 0x00000018: FDE -; nextln: length: 0x00000024 -; nextln: CIE_pointer: 0x00000000 -; nextln: start_addr: 0x0000000000000000 -; nextln: range_size: 0x0000000000000006 (end_addr = 0x0000000000000006) -; nextln: Instructions: -; nextln: DW_CFA_advance_loc (1) -; nextln: DW_CFA_def_cfa_offset (16) -; nextln: DW_CFA_offset (r6, 2) -; nextln: DW_CFA_advance_loc (3) -; nextln: DW_CFA_def_cfa_register (r6) -; nextln: DW_CFA_advance_loc (1) -; nextln: DW_CFA_def_cfa (r7, 8) -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: -; nextln: Entry: 24 -; nextln: Relocs: [(Abs8, 32)] diff --git a/cranelift/filetests/src/lib.rs b/cranelift/filetests/src/lib.rs index 0d3b12e45890..ded3acd16c51 100644 --- a/cranelift/filetests/src/lib.rs +++ b/cranelift/filetests/src/lib.rs @@ -42,7 +42,6 @@ mod test_cat; mod test_compile; mod test_dce; mod test_domtree; -mod test_fde; mod test_legalizer; mod test_licm; mod test_postopt; @@ -138,7 +137,6 @@ fn new_subtest(parsed: &TestCommand) -> subtest::SubtestResult test_preopt::subtest(parsed), "safepoint" => test_safepoint::subtest(parsed), "unwind" => test_unwind::subtest(parsed), - "fde" => test_fde::subtest(parsed), _ => Err(format!("unknown test command '{}'", parsed.command)), } } diff --git a/cranelift/filetests/src/test_fde.rs b/cranelift/filetests/src/test_fde.rs deleted file mode 100644 index 3e3747fdde2c..000000000000 --- a/cranelift/filetests/src/test_fde.rs +++ /dev/null @@ -1,415 +0,0 @@ -//! Test command for verifying the unwind emitted for each function. -//! -//! The `unwind` test command runs each function through the full code generator pipeline. -#![cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] - -use crate::subtest::{run_filecheck, Context, SubTest, SubtestResult}; -use cranelift_codegen; -use cranelift_codegen::binemit::{FrameUnwindKind, FrameUnwindOffset, FrameUnwindSink, Reloc}; -use cranelift_codegen::ir; -use cranelift_reader::TestCommand; -use std::borrow::Cow; -use std::fmt::Write; - -struct TestUnwind; - -pub fn subtest(parsed: &TestCommand) -> SubtestResult> { - assert_eq!(parsed.command, "fde"); - if !parsed.options.is_empty() { - Err(format!("No options allowed on {}", parsed)) - } else { - Ok(Box::new(TestUnwind)) - } -} - -impl SubTest for TestUnwind { - fn name(&self) -> &'static str { - "fde" - } - - fn is_mutating(&self) -> bool { - false - } - - fn needs_isa(&self) -> bool { - true - } - - fn run(&self, func: Cow, context: &Context) -> SubtestResult<()> { - let isa = context.isa.expect("unwind needs an ISA"); - - if func.signature.call_conv != cranelift_codegen::isa::CallConv::SystemV { - return run_filecheck(&"No unwind information.", context); - } - - let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned()); - comp_ctx.func.collect_frame_layout_info(); - - comp_ctx.compile(isa).expect("failed to compile function"); - - struct SimpleUnwindSink(pub Vec, pub usize, pub Vec<(Reloc, usize)>); - impl FrameUnwindSink for SimpleUnwindSink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() - } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); - } - fn reloc(&mut self, r: Reloc, off: FrameUnwindOffset) { - self.2.push((r, off)); - } - fn set_entry_offset(&mut self, off: FrameUnwindOffset) { - self.1 = off; - } - } - - let mut sink = SimpleUnwindSink(Vec::new(), 0, Vec::new()); - comp_ctx.emit_unwind_info(isa, FrameUnwindKind::Libunwind, &mut sink); - - let mut text = String::new(); - if sink.0.is_empty() { - writeln!(text, "No unwind information.").unwrap(); - } else { - print_unwind_info(&mut text, &sink.0, isa.pointer_bytes()); - writeln!(text, "Entry: {}", sink.1).unwrap(); - writeln!(text, "Relocs: {:?}", sink.2).unwrap(); - } - - run_filecheck(&text, context) - } -} - -fn register_name<'a>(register: gimli::Register) -> std::borrow::Cow<'a, str> { - Cow::Owned(format!("r{}", register.0)) -} - -fn print_unwind_info(text: &mut String, mem: &[u8], address_size: u8) { - let mut eh_frame = gimli::EhFrame::new(mem, gimli::LittleEndian); - eh_frame.set_address_size(address_size); - let bases = gimli::BaseAddresses::default(); - dwarfdump::dump_eh_frame(text, &eh_frame, &bases, ®ister_name).unwrap(); -} - -mod dwarfdump { - // Copied from https://github.com/gimli-rs/gimli/blob/1e49ffc9af4ec64a1b7316924d73c933dd7157c5/examples/dwarfdump.rs - use gimli::UnwindSection; - use std::borrow::Cow; - use std::collections::HashMap; - use std::fmt::{self, Debug, Write}; - use std::result; - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) enum Error { - GimliError(gimli::Error), - IoError, - } - - impl fmt::Display for Error { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> { - Debug::fmt(self, f) - } - } - - impl From for Error { - fn from(err: gimli::Error) -> Self { - Self::GimliError(err) - } - } - - impl From for Error { - fn from(_: fmt::Error) -> Self { - Self::IoError - } - } - - pub(super) type Result = result::Result; - - pub(super) trait Reader: gimli::Reader + Send + Sync {} - - impl<'input, Endian> Reader for gimli::EndianSlice<'input, Endian> where - Endian: gimli::Endianity + Send + Sync - { - } - - pub(super) fn dump_eh_frame( - w: &mut W, - eh_frame: &gimli::EhFrame, - bases: &gimli::BaseAddresses, - register_name: &dyn Fn(gimli::Register) -> Cow<'static, str>, - ) -> Result<()> { - let mut cies = HashMap::new(); - - let mut entries = eh_frame.entries(bases); - loop { - match entries.next()? { - None => return Ok(()), - Some(gimli::CieOrFde::Cie(cie)) => { - writeln!(w, "{:#010x}: CIE", cie.offset())?; - writeln!(w, " length: {:#010x}", cie.entry_len())?; - // TODO: CIE_id - writeln!(w, " version: {:#04x}", cie.version())?; - // TODO: augmentation - writeln!(w, " code_align: {}", cie.code_alignment_factor())?; - writeln!(w, " data_align: {}", cie.data_alignment_factor())?; - writeln!(w, " ra_register: {:#x}", cie.return_address_register().0)?; - if let Some(encoding) = cie.lsda_encoding() { - writeln!(w, " lsda_encoding: {:#02x}", encoding.0)?; - } - if let Some((encoding, personality)) = cie.personality_with_encoding() { - write!(w, " personality: {:#02x} ", encoding.0)?; - dump_pointer(w, personality)?; - writeln!(w)?; - } - if let Some(encoding) = cie.fde_address_encoding() { - writeln!(w, " fde_encoding: {:#02x}", encoding.0)?; - } - dump_cfi_instructions( - w, - cie.instructions(eh_frame, bases), - true, - register_name, - )?; - writeln!(w)?; - } - Some(gimli::CieOrFde::Fde(partial)) => { - let mut offset = None; - let fde = partial.parse(|_, bases, o| { - offset = Some(o); - cies.entry(o) - .or_insert_with(|| eh_frame.cie_from_offset(bases, o)) - .clone() - })?; - - writeln!(w)?; - writeln!(w, "{:#010x}: FDE", fde.offset())?; - writeln!(w, " length: {:#010x}", fde.entry_len())?; - writeln!(w, " CIE_pointer: {:#010x}", offset.unwrap().0)?; - // TODO: symbolicate the start address like the canonical dwarfdump does. - writeln!(w, " start_addr: {:#018x}", fde.initial_address())?; - writeln!( - w, - " range_size: {:#018x} (end_addr = {:#018x})", - fde.len(), - fde.initial_address() + fde.len() - )?; - if let Some(lsda) = fde.lsda() { - write!(w, " lsda: ")?; - dump_pointer(w, lsda)?; - writeln!(w)?; - } - dump_cfi_instructions( - w, - fde.instructions(eh_frame, bases), - false, - register_name, - )?; - writeln!(w)?; - } - } - } - } - - fn dump_pointer(w: &mut W, p: gimli::Pointer) -> Result<()> { - match p { - gimli::Pointer::Direct(p) => { - write!(w, "{:#018x}", p)?; - } - gimli::Pointer::Indirect(p) => { - write!(w, "({:#018x})", p)?; - } - } - Ok(()) - } - - #[allow(clippy::unneeded_field_pattern)] - fn dump_cfi_instructions( - w: &mut W, - mut insns: gimli::CallFrameInstructionIter, - is_initial: bool, - register_name: &dyn Fn(gimli::Register) -> Cow<'static, str>, - ) -> Result<()> { - use gimli::CallFrameInstruction::*; - - // TODO: we need to actually evaluate these instructions as we iterate them - // so we can print the initialized state for CIEs, and each unwind row's - // registers for FDEs. - // - // TODO: We should print DWARF expressions for the CFI instructions that - // embed DWARF expressions within themselves. - - if !is_initial { - writeln!(w, " Instructions:")?; - } - - loop { - match insns.next() { - Err(e) => { - writeln!(w, "Failed to decode CFI instruction: {}", e)?; - return Ok(()); - } - Ok(None) => { - if is_initial { - writeln!(w, " Instructions: Init State:")?; - } - return Ok(()); - } - Ok(Some(op)) => match op { - SetLoc { address } => { - writeln!(w, " DW_CFA_set_loc ({:#x})", address)?; - } - AdvanceLoc { delta } => { - writeln!(w, " DW_CFA_advance_loc ({})", delta)?; - } - DefCfa { register, offset } => { - writeln!( - w, - " DW_CFA_def_cfa ({}, {})", - register_name(register), - offset - )?; - } - DefCfaSf { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_def_cfa_sf ({}, {})", - register_name(register), - factored_offset - )?; - } - DefCfaRegister { register } => { - writeln!( - w, - " DW_CFA_def_cfa_register ({})", - register_name(register) - )?; - } - DefCfaOffset { offset } => { - writeln!(w, " DW_CFA_def_cfa_offset ({})", offset)?; - } - DefCfaOffsetSf { factored_offset } => { - writeln!( - w, - " DW_CFA_def_cfa_offset_sf ({})", - factored_offset - )?; - } - DefCfaExpression { expression: _ } => { - writeln!(w, " DW_CFA_def_cfa_expression (...)")?; - } - Undefined { register } => { - writeln!( - w, - " DW_CFA_undefined ({})", - register_name(register) - )?; - } - SameValue { register } => { - writeln!( - w, - " DW_CFA_same_value ({})", - register_name(register) - )?; - } - Offset { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_offset ({}, {})", - register_name(register), - factored_offset - )?; - } - OffsetExtendedSf { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_offset_extended_sf ({}, {})", - register_name(register), - factored_offset - )?; - } - ValOffset { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_val_offset ({}, {})", - register_name(register), - factored_offset - )?; - } - ValOffsetSf { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_val_offset_sf ({}, {})", - register_name(register), - factored_offset - )?; - } - Register { - dest_register, - src_register, - } => { - writeln!( - w, - " DW_CFA_register ({}, {})", - register_name(dest_register), - register_name(src_register) - )?; - } - Expression { - register, - expression: _, - } => { - writeln!( - w, - " DW_CFA_expression ({}, ...)", - register_name(register) - )?; - } - ValExpression { - register, - expression: _, - } => { - writeln!( - w, - " DW_CFA_val_expression ({}, ...)", - register_name(register) - )?; - } - Restore { register } => { - writeln!( - w, - " DW_CFA_restore ({})", - register_name(register) - )?; - } - RememberState => { - writeln!(w, " DW_CFA_remember_state")?; - } - RestoreState => { - writeln!(w, " DW_CFA_restore_state")?; - } - ArgsSize { size } => { - writeln!(w, " DW_CFA_GNU_args_size ({})", size)?; - } - Nop => { - writeln!(w, " DW_CFA_nop")?; - } - }, - } - } - } -} diff --git a/cranelift/filetests/src/test_unwind.rs b/cranelift/filetests/src/test_unwind.rs index 3db1cbf8299e..b54c242f5f94 100644 --- a/cranelift/filetests/src/test_unwind.rs +++ b/cranelift/filetests/src/test_unwind.rs @@ -4,13 +4,13 @@ #![cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] use crate::subtest::{run_filecheck, Context, SubTest, SubtestResult}; -use byteorder::{ByteOrder, LittleEndian}; -use cranelift_codegen; -use cranelift_codegen::binemit::{FrameUnwindKind, FrameUnwindOffset, FrameUnwindSink, Reloc}; -use cranelift_codegen::ir; +use cranelift_codegen::{self, ir, isa::unwind::UnwindInfo}; use cranelift_reader::TestCommand; +use gimli::{ + write::{Address, EhFrame, EndianVec, FrameTable}, + LittleEndian, +}; use std::borrow::Cow; -use std::fmt::Write; struct TestUnwind; @@ -42,173 +42,526 @@ impl SubTest for TestUnwind { comp_ctx.compile(isa).expect("failed to compile function"); - struct Sink(Vec); - impl FrameUnwindSink for Sink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() + let mut text = String::new(); + match comp_ctx.create_unwind_info(isa) { + Some(UnwindInfo::WindowsX64(info)) => { + let mut mem = vec![0; info.emit_size()]; + info.emit(&mut mem); + windowsx64::dump(&mut text, &mem); } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); + Some(UnwindInfo::SystemV(info)) => { + let mut table = FrameTable::default(); + let cie = isa + .create_systemv_cie() + .expect("the ISA should support a System V CIE"); + + let cie_id = table.add_cie(cie); + table.add_fde(cie_id, info.to_fde(Address::Constant(0))); + + let mut eh_frame = EhFrame(EndianVec::new(LittleEndian)); + table.write_eh_frame(&mut eh_frame).unwrap(); + systemv::dump(&mut text, &eh_frame.0.into_vec(), isa.pointer_bytes()) } - fn reloc(&mut self, _: Reloc, _: FrameUnwindOffset) { - unimplemented!(); + None => {} + } + + run_filecheck(&text, context) + } +} + +mod windowsx64 { + use byteorder::{ByteOrder, LittleEndian}; + use std::fmt::Write; + + pub fn dump(text: &mut W, mem: &[u8]) { + let info = UnwindInfo::from_slice(mem); + + writeln!(text, " version: {}", info.version).unwrap(); + writeln!(text, " flags: {}", info.flags).unwrap(); + writeln!(text, " prologue size: {}", info.prologue_size).unwrap(); + writeln!(text, " frame register: {}", info.frame_register).unwrap(); + writeln!( + text, + "frame register offset: {}", + info.frame_register_offset + ) + .unwrap(); + writeln!(text, " unwind codes: {}", info.unwind_codes.len()).unwrap(); + + for code in info.unwind_codes.iter().rev() { + writeln!(text).unwrap(); + writeln!(text, " offset: {}", code.offset).unwrap(); + writeln!(text, " op: {:?}", code.op).unwrap(); + writeln!(text, " info: {}", code.info).unwrap(); + match code.value { + UnwindValue::None => {} + UnwindValue::U16(v) => { + writeln!(text, " value: {} (u16)", v).unwrap() + } + UnwindValue::U32(v) => { + writeln!(text, " value: {} (u32)", v).unwrap() + } + }; + } + } + + #[derive(Debug)] + struct UnwindInfo { + version: u8, + flags: u8, + prologue_size: u8, + unwind_code_count_raw: u8, + frame_register: u8, + frame_register_offset: u8, + unwind_codes: Vec, + } + + impl UnwindInfo { + fn from_slice(mem: &[u8]) -> Self { + let version_and_flags = mem[0]; + let prologue_size = mem[1]; + let unwind_code_count_raw = mem[2]; + let frame_register_and_offset = mem[3]; + let mut unwind_codes = Vec::new(); + + let mut i = 0; + while i < unwind_code_count_raw { + let code = UnwindCode::from_slice(&mem[(4 + (i * 2) as usize)..]); + + i += match &code.value { + UnwindValue::None => 1, + UnwindValue::U16(_) => 2, + UnwindValue::U32(_) => 3, + }; + + unwind_codes.push(code); } - fn set_entry_offset(&mut self, _: FrameUnwindOffset) { - unimplemented!(); + + Self { + version: version_and_flags & 0x3, + flags: (version_and_flags & 0xF8) >> 3, + prologue_size, + unwind_code_count_raw, + frame_register: frame_register_and_offset & 0xF, + frame_register_offset: (frame_register_and_offset & 0xF0) >> 4, + unwind_codes, } } + } - let mut sink = Sink(Vec::new()); - comp_ctx.emit_unwind_info(isa, FrameUnwindKind::Fastcall, &mut sink); + #[derive(Debug)] + struct UnwindCode { + offset: u8, + op: UnwindOperation, + info: u8, + value: UnwindValue, + } - let mut text = String::new(); - if sink.0.is_empty() { - writeln!(text, "No unwind information.").unwrap(); - } else { - print_unwind_info(&mut text, &sink.0); + impl UnwindCode { + fn from_slice(mem: &[u8]) -> Self { + let offset = mem[0]; + let op_and_info = mem[1]; + let op = UnwindOperation::from(op_and_info & 0xF); + let info = (op_and_info & 0xF0) >> 4; + + let value = match op { + UnwindOperation::LargeStackAlloc => match info { + 0 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), + 1 => UnwindValue::U32(LittleEndian::read_u32(&mem[2..])), + _ => panic!("unexpected stack alloc info value"), + }, + UnwindOperation::SaveNonvolatileRegister => { + UnwindValue::U16(LittleEndian::read_u16(&mem[2..])) + } + UnwindOperation::SaveNonvolatileRegisterFar => { + UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) + } + UnwindOperation::SaveXmm128 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), + UnwindOperation::SaveXmm128Far => { + UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) + } + _ => UnwindValue::None, + }; + + Self { + offset, + op, + info, + value, + } } + } - run_filecheck(&text, context) + #[derive(Debug)] + enum UnwindOperation { + PushNonvolatileRegister, + LargeStackAlloc, + SmallStackAlloc, + SetFramePointer, + SaveNonvolatileRegister, + SaveNonvolatileRegisterFar, + SaveXmm128, + SaveXmm128Far, + PushMachineFrame, } -} -fn print_unwind_info(text: &mut String, mem: &[u8]) { - let info = UnwindInfo::from_slice(mem); - - // Assert correct alignment and padding of the unwind information - assert!(mem.len() % 4 == 0); - assert_eq!( - mem.len(), - 4 + ((info.unwind_code_count_raw as usize) * 2) - + if (info.unwind_code_count_raw & 1) == 1 { - 2 - } else { - 0 + impl From for UnwindOperation { + fn from(value: u8) -> Self { + // The numerical value is specified as part of the Windows x64 ABI + match value { + 0 => Self::PushNonvolatileRegister, + 1 => Self::LargeStackAlloc, + 2 => Self::SmallStackAlloc, + 3 => Self::SetFramePointer, + 4 => Self::SaveNonvolatileRegister, + 5 => Self::SaveNonvolatileRegisterFar, + 6 => Self::SaveXmm128, + 7 => Self::SaveXmm128Far, + 8 => Self::PushMachineFrame, + _ => panic!("unsupported unwind operation"), } - ); + } + } - writeln!(text, "{:#?}", info).unwrap(); + #[derive(Debug)] + enum UnwindValue { + None, + U16(u16), + U32(u32), + } } -#[derive(Debug)] -struct UnwindInfo { - pub version: u8, - pub flags: u8, - pub prologue_size: u8, - pub unwind_code_count_raw: u8, - pub frame_register: u8, - pub frame_register_offset: u8, - pub unwind_codes: Vec, -} +mod systemv { + fn register_name<'a>(register: gimli::Register) -> std::borrow::Cow<'a, str> { + Cow::Owned(format!("r{}", register.0)) + } -impl UnwindInfo { - fn from_slice(mem: &[u8]) -> Self { - let version_and_flags = mem[0]; - let prologue_size = mem[1]; - let unwind_code_count_raw = mem[2]; - let frame_register_and_offset = mem[3]; - let mut unwind_codes = Vec::new(); - - let mut i = 0; - while i < unwind_code_count_raw { - let code = UnwindCode::from_slice(&mem[(4 + (i * 2) as usize)..]); - - i += match &code.value { - UnwindValue::None => 1, - UnwindValue::U16(_) => 2, - UnwindValue::U32(_) => 3, - }; + pub fn dump(text: &mut W, bytes: &[u8], address_size: u8) { + let mut eh_frame = gimli::EhFrame::new(bytes, gimli::LittleEndian); + eh_frame.set_address_size(address_size); + let bases = gimli::BaseAddresses::default(); + dump_eh_frame(text, &eh_frame, &bases, ®ister_name).unwrap(); + } + + // Remainder copied from https://github.com/gimli-rs/gimli/blob/1e49ffc9af4ec64a1b7316924d73c933dd7157c5/examples/dwarfdump.rs + use gimli::UnwindSection; + use std::borrow::Cow; + use std::collections::HashMap; + use std::fmt::{self, Debug, Write}; + use std::result; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum Error { + GimliError(gimli::Error), + IoError, + } - unwind_codes.push(code); + impl fmt::Display for Error { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> { + Debug::fmt(self, f) } + } - Self { - version: version_and_flags & 0x3, - flags: (version_and_flags & 0xF8) >> 3, - prologue_size, - unwind_code_count_raw, - frame_register: frame_register_and_offset & 0xF, - frame_register_offset: (frame_register_and_offset & 0xF0) >> 4, - unwind_codes, + impl From for Error { + fn from(err: gimli::Error) -> Self { + Self::GimliError(err) } } -} -#[derive(Debug)] -struct UnwindCode { - pub offset: u8, - pub op: UnwindOperation, - pub info: u8, - pub value: UnwindValue, -} + impl From for Error { + fn from(_: fmt::Error) -> Self { + Self::IoError + } + } + + pub(super) type Result = result::Result; + + pub(super) trait Reader: gimli::Reader + Send + Sync {} -impl UnwindCode { - fn from_slice(mem: &[u8]) -> Self { - let offset = mem[0]; - let op_and_info = mem[1]; - let op = UnwindOperation::from(op_and_info & 0xF); - let info = (op_and_info & 0xF0) >> 4; - - let value = match op { - UnwindOperation::LargeStackAlloc => match info { - 0 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), - 1 => UnwindValue::U32(LittleEndian::read_u32(&mem[2..])), - _ => panic!("unexpected stack alloc info value"), - }, - UnwindOperation::SaveNonvolatileRegister => { - UnwindValue::U16(LittleEndian::read_u16(&mem[2..])) + impl<'input, Endian> Reader for gimli::EndianSlice<'input, Endian> where + Endian: gimli::Endianity + Send + Sync + { + } + + pub(super) fn dump_eh_frame( + w: &mut W, + eh_frame: &gimli::EhFrame, + bases: &gimli::BaseAddresses, + register_name: &dyn Fn(gimli::Register) -> Cow<'static, str>, + ) -> Result<()> { + let mut cies = HashMap::new(); + + let mut entries = eh_frame.entries(bases); + loop { + match entries.next()? { + None => return Ok(()), + Some(gimli::CieOrFde::Cie(cie)) => { + writeln!(w, "{:#010x}: CIE", cie.offset())?; + writeln!(w, " length: {:#010x}", cie.entry_len())?; + // TODO: CIE_id + writeln!(w, " version: {:#04x}", cie.version())?; + // TODO: augmentation + writeln!(w, " code_align: {}", cie.code_alignment_factor())?; + writeln!(w, " data_align: {}", cie.data_alignment_factor())?; + writeln!(w, " ra_register: {:#x}", cie.return_address_register().0)?; + if let Some(encoding) = cie.lsda_encoding() { + writeln!(w, " lsda_encoding: {:#02x}", encoding.0)?; + } + if let Some((encoding, personality)) = cie.personality_with_encoding() { + write!(w, " personality: {:#02x} ", encoding.0)?; + dump_pointer(w, personality)?; + writeln!(w)?; + } + if let Some(encoding) = cie.fde_address_encoding() { + writeln!(w, " fde_encoding: {:#02x}", encoding.0)?; + } + dump_cfi_instructions( + w, + cie.instructions(eh_frame, bases), + true, + register_name, + )?; + writeln!(w)?; + } + Some(gimli::CieOrFde::Fde(partial)) => { + let mut offset = None; + let fde = partial.parse(|_, bases, o| { + offset = Some(o); + cies.entry(o) + .or_insert_with(|| eh_frame.cie_from_offset(bases, o)) + .clone() + })?; + + writeln!(w)?; + writeln!(w, "{:#010x}: FDE", fde.offset())?; + writeln!(w, " length: {:#010x}", fde.entry_len())?; + writeln!(w, " CIE_pointer: {:#010x}", offset.unwrap().0)?; + // TODO: symbolicate the start address like the canonical dwarfdump does. + writeln!(w, " start_addr: {:#018x}", fde.initial_address())?; + writeln!( + w, + " range_size: {:#018x} (end_addr = {:#018x})", + fde.len(), + fde.initial_address() + fde.len() + )?; + if let Some(lsda) = fde.lsda() { + write!(w, " lsda: ")?; + dump_pointer(w, lsda)?; + writeln!(w)?; + } + dump_cfi_instructions( + w, + fde.instructions(eh_frame, bases), + false, + register_name, + )?; + writeln!(w)?; + } + } + } + } + + fn dump_pointer(w: &mut W, p: gimli::Pointer) -> Result<()> { + match p { + gimli::Pointer::Direct(p) => { + write!(w, "{:#018x}", p)?; } - UnwindOperation::SaveNonvolatileRegisterFar => { - UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) + gimli::Pointer::Indirect(p) => { + write!(w, "({:#018x})", p)?; } - UnwindOperation::SaveXmm128 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), - UnwindOperation::SaveXmm128Far => UnwindValue::U32(LittleEndian::read_u32(&mem[2..])), - _ => UnwindValue::None, - }; - - Self { - offset, - op, - info, - value, } + Ok(()) } -} -#[derive(Debug)] -enum UnwindOperation { - PushNonvolatileRegister, - LargeStackAlloc, - SmallStackAlloc, - SetFramePointer, - SaveNonvolatileRegister, - SaveNonvolatileRegisterFar, - SaveXmm128, - SaveXmm128Far, - PushMachineFrame, -} + #[allow(clippy::unneeded_field_pattern)] + fn dump_cfi_instructions( + w: &mut W, + mut insns: gimli::CallFrameInstructionIter, + is_initial: bool, + register_name: &dyn Fn(gimli::Register) -> Cow<'static, str>, + ) -> Result<()> { + use gimli::CallFrameInstruction::*; -impl From for UnwindOperation { - fn from(value: u8) -> Self { - // The numerical value is specified as part of the Windows x64 ABI - match value { - 0 => Self::PushNonvolatileRegister, - 1 => Self::LargeStackAlloc, - 2 => Self::SmallStackAlloc, - 3 => Self::SetFramePointer, - 4 => Self::SaveNonvolatileRegister, - 5 => Self::SaveNonvolatileRegisterFar, - 6 => Self::SaveXmm128, - 7 => Self::SaveXmm128Far, - 8 => Self::PushMachineFrame, - _ => panic!("unsupported unwind operation"), + // TODO: we need to actually evaluate these instructions as we iterate them + // so we can print the initialized state for CIEs, and each unwind row's + // registers for FDEs. + // + // TODO: We should print DWARF expressions for the CFI instructions that + // embed DWARF expressions within themselves. + + if !is_initial { + writeln!(w, " Instructions:")?; } - } -} -#[derive(Debug)] -enum UnwindValue { - None, - U16(u16), - U32(u32), + loop { + match insns.next() { + Err(e) => { + writeln!(w, "Failed to decode CFI instruction: {}", e)?; + return Ok(()); + } + Ok(None) => { + if is_initial { + writeln!(w, " Instructions: Init State:")?; + } + return Ok(()); + } + Ok(Some(op)) => match op { + SetLoc { address } => { + writeln!(w, " DW_CFA_set_loc ({:#x})", address)?; + } + AdvanceLoc { delta } => { + writeln!(w, " DW_CFA_advance_loc ({})", delta)?; + } + DefCfa { register, offset } => { + writeln!( + w, + " DW_CFA_def_cfa ({}, {})", + register_name(register), + offset + )?; + } + DefCfaSf { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_def_cfa_sf ({}, {})", + register_name(register), + factored_offset + )?; + } + DefCfaRegister { register } => { + writeln!( + w, + " DW_CFA_def_cfa_register ({})", + register_name(register) + )?; + } + DefCfaOffset { offset } => { + writeln!(w, " DW_CFA_def_cfa_offset ({})", offset)?; + } + DefCfaOffsetSf { factored_offset } => { + writeln!( + w, + " DW_CFA_def_cfa_offset_sf ({})", + factored_offset + )?; + } + DefCfaExpression { expression: _ } => { + writeln!(w, " DW_CFA_def_cfa_expression (...)")?; + } + Undefined { register } => { + writeln!( + w, + " DW_CFA_undefined ({})", + register_name(register) + )?; + } + SameValue { register } => { + writeln!( + w, + " DW_CFA_same_value ({})", + register_name(register) + )?; + } + Offset { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_offset ({}, {})", + register_name(register), + factored_offset + )?; + } + OffsetExtendedSf { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_offset_extended_sf ({}, {})", + register_name(register), + factored_offset + )?; + } + ValOffset { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_val_offset ({}, {})", + register_name(register), + factored_offset + )?; + } + ValOffsetSf { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_val_offset_sf ({}, {})", + register_name(register), + factored_offset + )?; + } + Register { + dest_register, + src_register, + } => { + writeln!( + w, + " DW_CFA_register ({}, {})", + register_name(dest_register), + register_name(src_register) + )?; + } + Expression { + register, + expression: _, + } => { + writeln!( + w, + " DW_CFA_expression ({}, ...)", + register_name(register) + )?; + } + ValExpression { + register, + expression: _, + } => { + writeln!( + w, + " DW_CFA_val_expression ({}, ...)", + register_name(register) + )?; + } + Restore { register } => { + writeln!( + w, + " DW_CFA_restore ({})", + register_name(register) + )?; + } + RememberState => { + writeln!(w, " DW_CFA_remember_state")?; + } + RestoreState => { + writeln!(w, " DW_CFA_restore_state")?; + } + ArgsSize { size } => { + writeln!(w, " DW_CFA_GNU_args_size ({})", size)?; + } + Nop => { + writeln!(w, " DW_CFA_nop")?; + } + }, + } + } + } } diff --git a/crates/api/src/trampoline/func.rs b/crates/api/src/trampoline/func.rs index 4d64482deacc..b0b4bcdfd906 100644 --- a/crates/api/src/trampoline/func.rs +++ b/crates/api/src/trampoline/func.rs @@ -10,9 +10,7 @@ use std::mem; use std::panic::{self, AssertUnwindSafe}; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::isa::TargetIsa; -use wasmtime_environ::{ - ir, settings, CompiledFunction, CompiledFunctionUnwindInfo, Export, Module, -}; +use wasmtime_environ::{ir, settings, CompiledFunction, Export, Module}; use wasmtime_jit::trampoline::ir::{ ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind, }; @@ -112,7 +110,6 @@ fn make_trampoline( let mut context = Context::new(); context.func = Function::with_name_signature(ExternalName::user(0, 0), signature.clone()); - context.func.collect_frame_layout_info(); let ss = context.func.create_stack_slot(StackSlotData::new( StackSlotKind::ExplicitSlot, @@ -188,7 +185,7 @@ fn make_trampoline( .map_err(|error| pretty_error(&context.func, Some(isa), error)) .expect("compile_and_emit"); - let unwind_info = CompiledFunctionUnwindInfo::new(isa, &context); + let unwind_info = context.create_unwind_info(isa); code_memory .allocate_for_function(&CompiledFunction { @@ -249,7 +246,7 @@ pub fn create_handle_with_function( // Next up we wrap everything up into an `InstanceHandle` by publishing our // code memory (makes it executable) and ensuring all our various bits of // state make it into the instance constructors. - code_memory.publish(); + code_memory.publish(isa.as_ref()); let trampoline_state = TrampolineState { func, code_memory }; create_handle( module, diff --git a/crates/debug/src/frame.rs b/crates/debug/src/frame.rs deleted file mode 100644 index 566dc6d1a765..000000000000 --- a/crates/debug/src/frame.rs +++ /dev/null @@ -1,140 +0,0 @@ -use std::collections::HashMap; -use wasmtime_environ::entity::EntityRef; -use wasmtime_environ::isa::fde::map_reg; -use wasmtime_environ::isa::{CallConv, TargetIsa}; -use wasmtime_environ::wasm::DefinedFuncIndex; -use wasmtime_environ::{FrameLayoutChange, FrameLayouts}; - -use gimli::write::{ - Address, CallFrameInstruction, CommonInformationEntry as CIEEntry, Error, - FrameDescriptionEntry as FDEEntry, FrameTable, -}; -use gimli::{Encoding, Format, Register, X86_64}; - -fn to_cfi( - isa: &dyn TargetIsa, - change: &FrameLayoutChange, - cfa_def_reg: &mut Register, - cfa_def_offset: &mut i32, -) -> Option { - Some(match change { - FrameLayoutChange::CallFrameAddressAt { reg, offset } => { - let mapped = match map_reg(isa, *reg) { - Ok(r) => r, - Err(_) => return None, - }; - let offset = (*offset) as i32; - if mapped != *cfa_def_reg && offset != *cfa_def_offset { - *cfa_def_reg = mapped; - *cfa_def_offset = offset; - CallFrameInstruction::Cfa(mapped, offset) - } else if offset != *cfa_def_offset { - *cfa_def_offset = offset; - CallFrameInstruction::CfaOffset(offset) - } else if mapped != *cfa_def_reg { - *cfa_def_reg = mapped; - CallFrameInstruction::CfaRegister(mapped) - } else { - return None; - } - } - FrameLayoutChange::RegAt { reg, cfa_offset } => { - assert!(cfa_offset % -8 == 0); - let cfa_offset = *cfa_offset as i32; - let mapped = match map_reg(isa, *reg) { - Ok(r) => r, - Err(_) => return None, - }; - CallFrameInstruction::Offset(mapped, cfa_offset) - } - FrameLayoutChange::ReturnAddressAt { cfa_offset } => { - assert!(cfa_offset % -8 == 0); - let cfa_offset = *cfa_offset as i32; - CallFrameInstruction::Offset(X86_64::RA, cfa_offset) - } - FrameLayoutChange::Preserve => CallFrameInstruction::RememberState, - FrameLayoutChange::Restore => CallFrameInstruction::RestoreState, - }) -} - -pub fn get_debug_frame_bytes( - funcs: &[(*const u8, usize)], - isa: &dyn TargetIsa, - layouts: &FrameLayouts, -) -> Result, Error> { - // FIXME Only x86-64 at this moment. - if isa.name() != "x86" || isa.pointer_bits() != 64 { - return Ok(None); - } - - let address_size = isa.pointer_bytes(); - let encoding = Encoding { - format: Format::Dwarf64, - version: 4, - address_size, - }; - - let mut frames = FrameTable::default(); - - let mut cached_cies = HashMap::new(); - - for (i, f) in funcs.into_iter().enumerate() { - let layout = &layouts[DefinedFuncIndex::new(i)]; - - // FIXME Can only process functions with SystemV-like prologue. - if layout.call_conv != CallConv::Fast - && layout.call_conv != CallConv::Cold - && layout.call_conv != CallConv::SystemV - { - continue; - } - - // Caching CIE with similar initial_commands. - let (cie_id, mut cfa_def_reg, mut cfa_def_offset) = { - use std::collections::hash_map::Entry; - match cached_cies.entry(&layout.initial_commands) { - Entry::Occupied(o) => *o.get(), - Entry::Vacant(v) => { - // cfa_def_reg and cfa_def_offset initialized with some random values. - let mut cfa_def_reg = X86_64::RA; - let mut cfa_def_offset = 0i32; - - // TODO adjust code_alignment_factor and data_alignment_factor based on ISA. - let mut cie = CIEEntry::new( - encoding, - /* code_alignment_factor = */ 1, - /* data_alignment_factor = */ -8, - /* return_address_register = */ X86_64::RA, - ); - for cmd in layout.initial_commands.iter() { - if let Some(instr) = to_cfi(isa, cmd, &mut cfa_def_reg, &mut cfa_def_offset) - { - cie.add_instruction(instr); - } - } - let cie_id = frames.add_cie(cie); - *v.insert((cie_id, cfa_def_reg, cfa_def_offset)) - } - } - }; - - let f_len = f.1 as u32; - let mut fde = FDEEntry::new( - Address::Symbol { - symbol: i, - addend: 0, - }, - f_len, - ); - - for (offset, cmd) in layout.commands.into_iter() { - if let Some(instr) = to_cfi(isa, cmd, &mut cfa_def_reg, &mut cfa_def_offset) { - fde.add_instruction(*offset as u32, instr); - } - } - - frames.add_fde(cie_id, fde); - } - - Ok(Some(frames)) -} diff --git a/crates/debug/src/lib.rs b/crates/debug/src/lib.rs index 2ca0e747c12d..e831570cf0cc 100644 --- a/crates/debug/src/lib.rs +++ b/crates/debug/src/lib.rs @@ -2,19 +2,18 @@ #![allow(clippy::cast_ptr_alignment)] -use crate::frame::get_debug_frame_bytes; use anyhow::Error; use faerie::{Artifact, Decl}; +use gimli::write::{Address, FrameTable}; use more_asserts::assert_gt; use target_lexicon::BinaryFormat; -use wasmtime_environ::isa::TargetIsa; -use wasmtime_environ::{FrameLayouts, ModuleAddressMap, ModuleVmctxInfo, ValueLabelsRanges}; +use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; +use wasmtime_environ::{Compilation, ModuleAddressMap, ModuleVmctxInfo, ValueLabelsRanges}; pub use crate::read_debuginfo::{read_debuginfo, DebugInfoData, WasmFileInfo}; pub use crate::transform::transform_dwarf; pub use crate::write_debuginfo::{emit_dwarf, ResolvedSymbol, SymbolResolver}; -mod frame; mod gc; mod read_debuginfo; mod transform; @@ -28,6 +27,29 @@ impl SymbolResolver for FunctionRelocResolver { } } +fn create_frame_table<'a>( + isa: &dyn TargetIsa, + infos: impl Iterator>, +) -> Option { + let mut table = FrameTable::default(); + + let cie_id = table.add_cie(isa.create_systemv_cie()?); + + for (i, info) in infos.enumerate() { + if let Some(UnwindInfo::SystemV(info)) = info { + table.add_fde( + cie_id, + info.to_fde(Address::Symbol { + symbol: i, + addend: 0, + }), + ); + } + } + + Some(table) +} + pub fn emit_debugsections( obj: &mut Artifact, vmctx_info: &ModuleVmctxInfo, @@ -35,21 +57,13 @@ pub fn emit_debugsections( debuginfo_data: &DebugInfoData, at: &ModuleAddressMap, ranges: &ValueLabelsRanges, - frame_layouts: &FrameLayouts, + compilation: &Compilation, ) -> Result<(), Error> { let resolver = FunctionRelocResolver {}; let dwarf = transform_dwarf(isa, debuginfo_data, at, vmctx_info, ranges)?; + let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info)); - let max = at.values().map(|v| v.body_len).fold(0, usize::max); - let mut funcs_bodies = Vec::with_capacity(max as usize); - funcs_bodies.resize(max as usize, 0); - let funcs = at - .values() - .map(|v| (::std::ptr::null(), v.body_len)) - .collect::>(); - let frames = get_debug_frame_bytes(&funcs, isa, frame_layouts)?; - - emit_dwarf(obj, dwarf, &resolver, frames)?; + emit_dwarf(obj, dwarf, &resolver, frame_table)?; Ok(()) } @@ -70,8 +84,8 @@ pub fn emit_debugsections_image( vmctx_info: &ModuleVmctxInfo, at: &ModuleAddressMap, ranges: &ValueLabelsRanges, - frame_layouts: &FrameLayouts, funcs: &[(*const u8, usize)], + compilation: &Compilation, ) -> Result, Error> { let func_offsets = &funcs .iter() @@ -93,8 +107,8 @@ pub fn emit_debugsections_image( let body = unsafe { std::slice::from_raw_parts(segment_body.0, segment_body.1) }; obj.declare_with("all", Decl::function(), body.to_vec())?; - let frames = get_debug_frame_bytes(funcs, isa, frame_layouts)?; - emit_dwarf(&mut obj, dwarf, &resolver, frames)?; + let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info)); + emit_dwarf(&mut obj, dwarf, &resolver, frame_table)?; // LLDB is too "magical" about mach-o, generating elf let mut bytes = obj.emit_as(BinaryFormat::Elf)?; diff --git a/crates/debug/src/transform/expression.rs b/crates/debug/src/transform/expression.rs index f901b03575b0..02c0e2eda3fe 100644 --- a/crates/debug/src/transform/expression.rs +++ b/crates/debug/src/transform/expression.rs @@ -5,8 +5,7 @@ use more_asserts::{assert_le, assert_lt}; use std::collections::{HashMap, HashSet}; use wasmtime_environ::entity::EntityRef; use wasmtime_environ::ir::{StackSlots, ValueLabel, ValueLabelsRanges, ValueLoc}; -use wasmtime_environ::isa::fde::map_reg; -use wasmtime_environ::isa::TargetIsa; +use wasmtime_environ::isa::{unwind::systemv::x86::map_reg, TargetIsa}; use wasmtime_environ::wasm::{get_vmctx_value_label, DefinedFuncIndex}; use wasmtime_environ::ModuleMemoryOffset; diff --git a/crates/environ/src/cache.rs b/crates/environ/src/cache.rs index fc7e7c643d64..290a54033f88 100644 --- a/crates/environ/src/cache.rs +++ b/crates/environ/src/cache.rs @@ -1,6 +1,5 @@ use crate::address_map::{ModuleAddressMap, ValueLabelsRanges}; use crate::compilation::{Compilation, Relocations, Traps}; -use crate::frame_layout::FrameLayouts; use cranelift_codegen::ir; use cranelift_entity::PrimaryMap; use cranelift_wasm::DefinedFuncIndex; @@ -36,7 +35,6 @@ pub struct ModuleCacheData { value_ranges: ValueLabelsRanges, stack_slots: PrimaryMap, traps: Traps, - frame_layouts: FrameLayouts, } /// A type alias over the module cache data as a tuple. @@ -47,7 +45,6 @@ pub type ModuleCacheDataTupleType = ( ValueLabelsRanges, PrimaryMap, Traps, - FrameLayouts, ); struct Sha256Hasher(Sha256); @@ -207,7 +204,6 @@ impl ModuleCacheData { value_ranges: data.3, stack_slots: data.4, traps: data.5, - frame_layouts: data.6, } } @@ -219,7 +215,6 @@ impl ModuleCacheData { self.value_ranges, self.stack_slots, self.traps, - self.frame_layouts, ) } } diff --git a/crates/environ/src/cache/tests.rs b/crates/environ/src/cache/tests.rs index 35dc6c613e36..d454371178e0 100644 --- a/crates/environ/src/cache/tests.rs +++ b/crates/environ/src/cache/tests.rs @@ -100,6 +100,5 @@ fn new_module_cache_data() -> Result { PrimaryMap::new(), PrimaryMap::new(), PrimaryMap::new(), - PrimaryMap::new(), )) } diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compilation.rs index 0a896316e1e6..a6ef4beb4ec6 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compilation.rs @@ -5,136 +5,13 @@ use crate::cache::ModuleCacheDataTupleType; use crate::module; use crate::module_environ::FunctionBodyData; use crate::CacheConfig; -use cranelift_codegen::{binemit, ir, isa, Context}; +use cranelift_codegen::{binemit, ir, isa, isa::unwind::UnwindInfo}; use cranelift_entity::PrimaryMap; use cranelift_wasm::{DefinedFuncIndex, FuncIndex, ModuleTranslationState, WasmError}; use serde::{Deserialize, Serialize}; use std::ops::Range; use thiserror::Error; -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct FDERelocEntry(pub i64, pub usize, pub u8); - -/// Relocation entry for unwind info. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct CompiledFunctionUnwindInfoReloc { - /// Entry offest in the code block. - pub offset: u32, - /// Entry addend relative to the code block. - pub addend: u32, -} - -/// Compiled function unwind information. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub enum CompiledFunctionUnwindInfo { - /// No info. - None, - /// Windows UNWIND_INFO. - Windows(Vec), - /// Frame layout info. - FrameLayout(Vec, usize, Vec), -} - -impl CompiledFunctionUnwindInfo { - /// Constructs unwind info object. - pub fn new(isa: &dyn isa::TargetIsa, context: &Context) -> Self { - use cranelift_codegen::binemit::{ - FrameUnwindKind, FrameUnwindOffset, FrameUnwindSink, Reloc, - }; - use cranelift_codegen::isa::CallConv; - - struct Sink(Vec, usize, Vec); - impl FrameUnwindSink for Sink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() - } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); - } - fn reserve(&mut self, len: usize) { - self.0.reserve(len) - } - fn reloc(&mut self, r: Reloc, off: FrameUnwindOffset) { - self.2.push(FDERelocEntry( - 0, - off, - match r { - Reloc::Abs4 => 4, - Reloc::Abs8 => 8, - _ => { - panic!("unexpected reloc type"); - } - }, - )) - } - fn set_entry_offset(&mut self, off: FrameUnwindOffset) { - self.1 = off; - } - } - - let kind = match context.func.signature.call_conv { - CallConv::SystemV | CallConv::Fast | CallConv::Cold => FrameUnwindKind::Libunwind, - CallConv::WindowsFastcall => FrameUnwindKind::Fastcall, - _ => { - return CompiledFunctionUnwindInfo::None; - } - }; - - let mut sink = Sink(Vec::new(), 0, Vec::new()); - context.emit_unwind_info(isa, kind, &mut sink); - - let Sink(data, offset, relocs) = sink; - if data.is_empty() { - return CompiledFunctionUnwindInfo::None; - } - - match kind { - FrameUnwindKind::Fastcall => CompiledFunctionUnwindInfo::Windows(data), - FrameUnwindKind::Libunwind => { - CompiledFunctionUnwindInfo::FrameLayout(data, offset, relocs) - } - } - } - - /// Retuns true is no unwind info data. - pub fn is_empty(&self) -> bool { - match self { - CompiledFunctionUnwindInfo::None => true, - CompiledFunctionUnwindInfo::Windows(d) => d.is_empty(), - CompiledFunctionUnwindInfo::FrameLayout(c, _, _) => c.is_empty(), - } - } - - /// Returns size of serilized unwind info. - pub fn len(&self) -> usize { - match self { - CompiledFunctionUnwindInfo::None => 0, - CompiledFunctionUnwindInfo::Windows(d) => d.len(), - CompiledFunctionUnwindInfo::FrameLayout(c, _, _) => c.len(), - } - } - - /// Serializes data into byte array. - pub fn serialize(&self, dest: &mut [u8], relocs: &mut Vec) { - match self { - CompiledFunctionUnwindInfo::None => (), - CompiledFunctionUnwindInfo::Windows(d) => { - dest.copy_from_slice(d); - } - CompiledFunctionUnwindInfo::FrameLayout(code, _fde_offset, r) => { - dest.copy_from_slice(code); - r.iter().for_each(move |r| { - assert_eq!(r.2, 8); - relocs.push(CompiledFunctionUnwindInfoReloc { - offset: r.1 as u32, - addend: r.0 as u32, - }) - }); - } - } - } -} - /// Compiled function: machine code body, jump table offsets, and unwind information. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct CompiledFunction { @@ -145,7 +22,7 @@ pub struct CompiledFunction { pub jt_offsets: ir::JumpTableOffsets, /// The unwind information. - pub unwind_info: CompiledFunctionUnwindInfo, + pub unwind_info: Option, } type Functions = PrimaryMap; @@ -166,15 +43,15 @@ impl Compilation { /// Allocates the compilation result with the given function bodies. pub fn from_buffer( buffer: Vec, - functions: impl IntoIterator, ir::JumpTableOffsets, Range)>, + functions: impl IntoIterator, ir::JumpTableOffsets)>, ) -> Self { Self::new( functions .into_iter() - .map(|(body_range, jt_offsets, unwind_range)| CompiledFunction { + .map(|(body_range, jt_offsets)| CompiledFunction { body: buffer[body_range].to_vec(), jt_offsets, - unwind_info: CompiledFunctionUnwindInfo::Windows(buffer[unwind_range].to_vec()), + unwind_info: None, /* not implemented for lightbeam currently */ }) .collect(), ) diff --git a/crates/environ/src/cranelift.rs b/crates/environ/src/cranelift.rs index 5764ce14194c..eb7499744125 100644 --- a/crates/environ/src/cranelift.rs +++ b/crates/environ/src/cranelift.rs @@ -3,10 +3,8 @@ use crate::address_map::{FunctionAddressMap, InstructionAddressMap}; use crate::cache::{ModuleCacheDataTupleType, ModuleCacheEntry}; use crate::compilation::{ - Compilation, CompileError, CompiledFunction, CompiledFunctionUnwindInfo, Relocation, - RelocationTarget, TrapInformation, + Compilation, CompileError, CompiledFunction, Relocation, RelocationTarget, TrapInformation, }; -use crate::frame_layout::FrameLayout; use crate::func_environ::{get_func_name, FuncEnvironment}; use crate::module::{Module, ModuleLocal}; use crate::module_environ::FunctionBodyData; @@ -155,38 +153,6 @@ fn get_function_address_map<'data>( } } -fn get_frame_layout( - context: &Context, - isa: &dyn isa::TargetIsa, -) -> ( - Box<[ir::FrameLayoutChange]>, - Box<[(usize, ir::FrameLayoutChange)]>, -) { - let func = &context.func; - assert!(func.frame_layout.is_some(), "expected func.frame_layout"); - - let mut blocks = func.layout.blocks().collect::>(); - blocks.sort_by_key(|b| func.offsets[*b]); // Ensure inst offsets always increase - - let encinfo = isa.encoding_info(); - let mut last_offset = 0; - let mut commands = Vec::new(); - for b in blocks { - for (offset, inst, size) in func.inst_offsets(b, &encinfo) { - if let Some(cmds) = func.frame_layout.as_ref().unwrap().instructions.get(&inst) { - let address_offset = (offset + size) as usize; - assert!(last_offset < address_offset); - for cmd in cmds.iter() { - commands.push((address_offset, cmd.clone())); - } - last_offset = address_offset; - } - } - } - let initial = func.frame_layout.as_ref().unwrap().initial.clone(); - (initial, commands.into_boxed_slice()) -} - /// A compiler that compiles a WebAssembly module with Cranelift, translating the Wasm to Cranelift IR, /// optimizing it and then translating to assembly. pub struct Cranelift; @@ -239,7 +205,6 @@ fn compile( let mut value_ranges = PrimaryMap::with_capacity(function_body_inputs.len()); let mut stack_slots = PrimaryMap::with_capacity(function_body_inputs.len()); let mut traps = PrimaryMap::with_capacity(function_body_inputs.len()); - let mut frame_layouts = PrimaryMap::with_capacity(function_body_inputs.len()); function_body_inputs .into_iter() @@ -250,7 +215,6 @@ fn compile( let mut context = Context::new(); context.func.name = get_func_name(func_index); context.func.signature = module.signatures[module.functions[func_index]].clone(); - context.func.collect_frame_layout_info(); if generate_debug_info { context.func.collect_debug_info(); } @@ -279,7 +243,7 @@ fn compile( CompileError::Codegen(pretty_error(&context.func, Some(isa), error)) })?; - let unwind_info = CompiledFunctionUnwindInfo::new(isa, &context); + let unwind_info = context.create_unwind_info(isa); let address_transform = if generate_debug_info { let body_len = code_buf.len(); @@ -288,17 +252,6 @@ fn compile( None }; - let frame_layout = if generate_debug_info { - let (initial_commands, commands) = get_frame_layout(&context, isa); - Some(FrameLayout { - call_conv: context.func.signature.call_conv, - initial_commands, - commands, - }) - } else { - None - }; - let ranges = if generate_debug_info { let ranges = context.build_value_labels_ranges(isa).map_err(|error| { CompileError::Codegen(pretty_error(&context.func, Some(isa), error)) @@ -313,7 +266,6 @@ fn compile( context.func.jt_offsets, reloc_sink.func_relocs, address_transform, - frame_layout, ranges, context.func.stack_slots, trap_sink.traps, @@ -328,7 +280,6 @@ fn compile( func_jt_offsets, relocs, address_transform, - frame_layout, ranges, sss, function_traps, @@ -346,9 +297,6 @@ fn compile( value_ranges.push(ranges.unwrap_or_default()); stack_slots.push(sss); traps.push(function_traps); - if let Some(frame_layout) = frame_layout { - frame_layouts.push(frame_layout); - } }, ); @@ -361,7 +309,6 @@ fn compile( value_ranges, stack_slots, traps, - frame_layouts, )) } diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 59e5d60410c4..3fcfb3a54a5b 100755 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -13,10 +13,7 @@ pub mod settings { } pub mod isa { - pub use cranelift_codegen::isa::{CallConv, RegUnit, TargetFrontendConfig, TargetIsa}; - pub mod fde { - pub use cranelift_codegen::isa::fde::map_reg; - } + pub use cranelift_codegen::isa::{unwind, CallConv, RegUnit, TargetFrontendConfig, TargetIsa}; } pub mod entity { diff --git a/crates/environ/src/frame_layout.rs b/crates/environ/src/frame_layout.rs deleted file mode 100644 index 46f413279628..000000000000 --- a/crates/environ/src/frame_layout.rs +++ /dev/null @@ -1,21 +0,0 @@ -use cranelift_codegen::isa::CallConv; -use cranelift_entity::PrimaryMap; -use cranelift_wasm::DefinedFuncIndex; -use serde::{Deserialize, Serialize}; - -pub use cranelift_codegen::ir::FrameLayoutChange; - -/// Frame layout information: call convention and -/// registers save/restore commands. -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] -pub struct FrameLayout { - /// Call convention. - pub call_conv: CallConv, - /// Frame default/initial commands. - pub initial_commands: Box<[FrameLayoutChange]>, - /// Frame commands at specific offset. - pub commands: Box<[(usize, FrameLayoutChange)]>, -} - -/// Functions frame layouts. -pub type FrameLayouts = PrimaryMap; diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index a4bad0274d56..f259fa45d186 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -27,7 +27,6 @@ mod address_map; mod compilation; mod data_structures; -mod frame_layout; mod func_environ; mod module; mod module_environ; @@ -47,13 +46,11 @@ pub use crate::address_map::{ pub use crate::cache::create_new_config as cache_create_new_config; pub use crate::cache::CacheConfig; pub use crate::compilation::{ - Compilation, CompileError, CompiledFunction, CompiledFunctionUnwindInfo, - CompiledFunctionUnwindInfoReloc, Compiler, Relocation, RelocationTarget, Relocations, - TrapInformation, Traps, + Compilation, CompileError, CompiledFunction, Compiler, Relocation, RelocationTarget, + Relocations, TrapInformation, Traps, }; pub use crate::cranelift::Cranelift; pub use crate::data_structures::*; -pub use crate::frame_layout::{FrameLayout, FrameLayoutChange, FrameLayouts}; pub use crate::func_environ::BuiltinFunctionIndex; #[cfg(feature = "lightbeam")] pub use crate::lightbeam::Lightbeam; diff --git a/crates/environ/src/lightbeam.rs b/crates/environ/src/lightbeam.rs index 53e833e04a41..0d660ed7d029 100644 --- a/crates/environ/src/lightbeam.rs +++ b/crates/environ/src/lightbeam.rs @@ -61,7 +61,7 @@ impl crate::compilation::Compiler for Lightbeam { let code_section_ranges_and_jt = code_section .funcs() .into_iter() - .map(|r| (r, SecondaryMap::new(), 0..0)); + .map(|r| (r, SecondaryMap::new())); Ok(( Compilation::from_buffer(code_section.buffer().to_vec(), code_section_ranges_and_jt), diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index cf8a1844cf6d..5d2dde1e181d 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -29,6 +29,7 @@ more-asserts = "0.2.1" anyhow = "1.0" cfg-if = "0.1.9" log = "0.4" +gimli = { version = "0.20.0", default-features = false, features = ["write"] } [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3.8", features = ["winnt", "impl-default"] } diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index a31695527b16..6d90c85710f4 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -1,37 +1,33 @@ //! Memory management for executable code. -use crate::function_table::FunctionTable; +use crate::unwind::UnwindRegistry; use region; use std::mem::ManuallyDrop; use std::{cmp, mem}; -use wasmtime_environ::{Compilation, CompiledFunction}; +use wasmtime_environ::{ + isa::{unwind::UnwindInfo, TargetIsa}, + Compilation, CompiledFunction, +}; use wasmtime_runtime::{Mmap, VMFunctionBody}; struct CodeMemoryEntry { mmap: ManuallyDrop, - table: ManuallyDrop, + registry: ManuallyDrop, } impl CodeMemoryEntry { - fn new() -> Self { - Self { - mmap: ManuallyDrop::new(Mmap::new()), - table: ManuallyDrop::new(FunctionTable::new()), - } - } fn with_capacity(cap: usize) -> Result { - Ok(Self { - mmap: ManuallyDrop::new(Mmap::with_at_least(cap)?), - table: ManuallyDrop::new(FunctionTable::new()), - }) + let mmap = ManuallyDrop::new(Mmap::with_at_least(cap)?); + let registry = ManuallyDrop::new(UnwindRegistry::new(mmap.as_ptr() as usize)); + Ok(Self { mmap, registry }) } } impl Drop for CodeMemoryEntry { fn drop(&mut self) { unsafe { - // Table needs to be freed before mmap. - ManuallyDrop::drop(&mut self.table); + // The registry needs to be dropped before the mmap + ManuallyDrop::drop(&mut self.registry); ManuallyDrop::drop(&mut self.mmap); } } @@ -39,7 +35,7 @@ impl Drop for CodeMemoryEntry { /// Memory manager for executable code. pub struct CodeMemory { - current: CodeMemoryEntry, + current: Option, entries: Vec, position: usize, published: usize, @@ -54,7 +50,7 @@ impl CodeMemory { /// Create a new `CodeMemory` instance. pub fn new() -> Self { Self { - current: CodeMemoryEntry::new(), + current: None, entries: Vec::new(), position: 0, published: 0, @@ -70,16 +66,14 @@ impl CodeMemory { ) -> Result<&mut [VMFunctionBody], String> { let size = Self::function_allocation_size(func); - let (buf, table, start) = self.allocate(size)?; + let (buf, registry, start) = self.allocate(size)?; - let (_, _, _, vmfunc) = Self::copy_function(func, start as u32, buf, table); + let (_, _, vmfunc) = Self::copy_function(func, start as u32, buf, registry); Ok(vmfunc) } /// Allocate a continuous memory block for a compilation. - /// - /// Allocates memory for both the function bodies as well as function unwind data. pub fn allocate_for_compilation( &mut self, compilation: &Compilation, @@ -88,33 +82,35 @@ impl CodeMemory { .into_iter() .fold(0, |acc, func| acc + Self::function_allocation_size(func)); - let (mut buf, mut table, start) = self.allocate(total_len)?; + let (mut buf, registry, start) = self.allocate(total_len)?; let mut result = Vec::with_capacity(compilation.len()); let mut start = start as u32; for func in compilation.into_iter() { - let (next_start, next_buf, next_table, vmfunc) = - Self::copy_function(func, start, buf, table); + let (next_start, next_buf, vmfunc) = Self::copy_function(func, start, buf, registry); result.push(vmfunc); start = next_start; buf = next_buf; - table = next_table; } Ok(result.into_boxed_slice()) } /// Make all allocated memory executable. - pub fn publish(&mut self) { + pub fn publish(&mut self, isa: &dyn TargetIsa) { self.push_current(0) .expect("failed to push current memory map"); - for CodeMemoryEntry { mmap: m, table: t } in &mut self.entries[self.published..] { + for CodeMemoryEntry { + mmap: m, + registry: r, + } in &mut self.entries[self.published..] + { // Remove write access to the pages due to the relocation fixups. - t.publish(m.as_ptr() as u64) - .expect("failed to publish function table"); + r.publish(isa) + .expect("failed to publish function unwind registry"); if !m.is_empty() { unsafe { @@ -139,73 +135,79 @@ impl CodeMemory { /// * The offset within the current mmap that the slice starts at /// /// TODO: Add an alignment flag. - fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut FunctionTable, usize), String> { - if self.current.mmap.len() - self.position < size { + fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut UnwindRegistry, usize), String> { + assert!(size > 0); + + if match &self.current { + Some(e) => e.mmap.len() - self.position < size, + None => true, + } { self.push_current(cmp::max(0x10000, size))?; } let old_position = self.position; self.position += size; + let e = self.current.as_mut().unwrap(); + Ok(( - &mut self.current.mmap.as_mut_slice()[old_position..self.position], - &mut self.current.table, + &mut e.mmap.as_mut_slice()[old_position..self.position], + &mut e.registry, old_position, )) } /// Calculates the allocation size of the given compiled function. fn function_allocation_size(func: &CompiledFunction) -> usize { - if func.unwind_info.is_empty() { - func.body.len() - } else { - // Account for necessary unwind information alignment padding (32-bit) - ((func.body.len() + 3) & !3) + func.unwind_info.len() + match &func.unwind_info { + Some(UnwindInfo::WindowsX64(info)) => { + // Windows unwind information is required to be emitted into code memory + // This is because it must be a positive relative offset from the start of the memory + // Account for necessary unwind information alignment padding (32-bit alignment) + ((func.body.len() + 3) & !3) + info.emit_size() + } + _ => func.body.len(), } } /// Copies the data of the compiled function to the given buffer. /// - /// This will also add the function to the current function table. + /// This will also add the function to the current unwind registry. fn copy_function<'a>( func: &CompiledFunction, func_start: u32, buf: &'a mut [u8], - table: &'a mut FunctionTable, - ) -> ( - u32, - &'a mut [u8], - &'a mut FunctionTable, - &'a mut [VMFunctionBody], - ) { - let func_end = func_start + (func.body.len() as u32); - - let (body, remainder) = buf.split_at_mut(func.body.len()); + registry: &mut UnwindRegistry, + ) -> (u32, &'a mut [u8], &'a mut [VMFunctionBody]) { + let func_len = func.body.len(); + let mut func_end = func_start + (func_len as u32); + + let (body, mut remainder) = buf.split_at_mut(func_len); body.copy_from_slice(&func.body); let vmfunc = Self::view_as_mut_vmfunc_slice(body); - if func.unwind_info.is_empty() { - return (func_end, remainder, table, vmfunc); - } + if let Some(UnwindInfo::WindowsX64(info)) = &func.unwind_info { + // Windows unwind information is written following the function body + // Keep unwind information 32-bit aligned (round up to the nearest 4 byte boundary) + let unwind_start = (func_end + 3) & !3; + let unwind_size = info.emit_size(); + let padding = (unwind_start - func_end) as usize; - // Keep unwind information 32-bit aligned (round up to the nearest 4 byte boundary) - let padding = ((func.body.len() + 3) & !3) - func.body.len(); - let (unwind, remainder) = remainder.split_at_mut(padding + func.unwind_info.len()); - let mut relocs = Vec::new(); - func.unwind_info - .serialize(&mut unwind[padding..], &mut relocs); + let (slice, r) = remainder.split_at_mut(padding + unwind_size); - let unwind_start = func_end + (padding as u32); - let unwind_end = unwind_start + (func.unwind_info.len() as u32); + info.emit(&mut slice[padding..]); - relocs.iter_mut().for_each(move |r| { - r.offset += unwind_start; - r.addend += func_start; - }); + func_end = unwind_start + (unwind_size as u32); + remainder = r; + } - table.add_function(func_start, func_end, unwind_start, &relocs); + if let Some(info) = &func.unwind_info { + registry + .register(func_start, func_len as u32, info) + .expect("failed to register unwind information"); + } - (unwind_end, remainder, table, vmfunc) + (func_end, remainder, vmfunc) } /// Convert mut a slice from u8 to VMFunctionBody. @@ -215,21 +217,19 @@ impl CodeMemory { unsafe { &mut *body_ptr } } - /// Pushes the current Mmap (and function table) and allocates a new Mmap of the given size. + /// Pushes the current entry and allocates a new one with the given size. fn push_current(&mut self, new_size: usize) -> Result<(), String> { let previous = mem::replace( &mut self.current, if new_size == 0 { - CodeMemoryEntry::new() + None } else { - CodeMemoryEntry::with_capacity(cmp::max(0x10000, new_size))? + Some(CodeMemoryEntry::with_capacity(cmp::max(0x10000, new_size))?) }, ); - if !previous.mmap.is_empty() { - self.entries.push(previous); - } else { - assert_eq!(previous.table.len(), 0); + if let Some(e) = previous { + self.entries.push(e); } self.position = 0; diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index 5bf2fd95ff3b..0bc189f6ff95 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -18,9 +18,8 @@ use wasmtime_environ::isa::{TargetFrontendConfig, TargetIsa}; use wasmtime_environ::wasm::{DefinedFuncIndex, DefinedMemoryIndex, MemoryIndex}; use wasmtime_environ::RelocationTarget; use wasmtime_environ::{ - CacheConfig, CompileError, CompiledFunction, CompiledFunctionUnwindInfo, Compiler as _C, - FunctionBodyData, Module, ModuleMemoryOffset, ModuleVmctxInfo, Relocation, Relocations, Traps, - Tunables, VMOffsets, + CacheConfig, CompileError, CompiledFunction, Compiler as _C, FunctionBodyData, Module, + ModuleMemoryOffset, ModuleVmctxInfo, Relocation, Relocations, Traps, Tunables, VMOffsets, }; use wasmtime_runtime::{ InstantiationError, SignatureRegistry, TrapRegistration, TrapRegistry, VMFunctionBody, @@ -51,7 +50,6 @@ pub enum CompilationStrategy { /// TODO: Consider using cranelift-module. pub struct Compiler { isa: Box, - code_memory: CodeMemory, trap_registry: TrapRegistry, signatures: SignatureRegistry, @@ -107,40 +105,33 @@ impl Compiler { function_body_inputs: PrimaryMap>, debug_data: Option, ) -> Result { - let ( - compilation, - relocations, - address_transform, - value_ranges, - stack_slots, - traps, - frame_layouts, - ) = match self.strategy { - // For now, interpret `Auto` as `Cranelift` since that's the most stable - // implementation. - CompilationStrategy::Auto | CompilationStrategy::Cranelift => { - wasmtime_environ::cranelift::Cranelift::compile_module( - module, - module_translation, - function_body_inputs, - &*self.isa, - debug_data.is_some(), - &self.cache_config, - ) - } - #[cfg(feature = "lightbeam")] - CompilationStrategy::Lightbeam => { - wasmtime_environ::lightbeam::Lightbeam::compile_module( - module, - module_translation, - function_body_inputs, - &*self.isa, - debug_data.is_some(), - &self.cache_config, - ) + let (compilation, relocations, address_transform, value_ranges, stack_slots, traps) = + match self.strategy { + // For now, interpret `Auto` as `Cranelift` since that's the most stable + // implementation. + CompilationStrategy::Auto | CompilationStrategy::Cranelift => { + wasmtime_environ::cranelift::Cranelift::compile_module( + module, + module_translation, + function_body_inputs, + &*self.isa, + debug_data.is_some(), + &self.cache_config, + ) + } + #[cfg(feature = "lightbeam")] + CompilationStrategy::Lightbeam => { + wasmtime_environ::lightbeam::Lightbeam::compile_module( + module, + module_translation, + function_body_inputs, + &*self.isa, + debug_data.is_some(), + &self.cache_config, + ) + } } - } - .map_err(SetupError::Compile)?; + .map_err(SetupError::Compile)?; // Allocate all of the compiled functions into executable memory, // copying over their contents. @@ -218,8 +209,8 @@ impl Compiler { &module_vmctx_info, &address_transform, &value_ranges, - &frame_layouts, &funcs, + &compilation, ) .map_err(SetupError::DebugInfo)?; Some(bytes) @@ -242,7 +233,7 @@ impl Compiler { /// Make memory containing compiled code executable. pub(crate) fn publish_compiled_code(&mut self) { - self.code_memory.publish(); + self.code_memory.publish(self.isa.as_ref()); } /// Shared signature registry. @@ -284,7 +275,6 @@ pub fn make_trampoline( let mut context = Context::new(); context.func = ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wrapper_sig); - context.func.collect_frame_layout_info(); { let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx); @@ -363,7 +353,7 @@ pub fn make_trampoline( ))) })?; - let unwind_info = CompiledFunctionUnwindInfo::new(isa, &context); + let unwind_info = context.create_unwind_info(isa); let ptr = code_memory .allocate_for_function(&CompiledFunction { @@ -383,6 +373,10 @@ fn allocate_functions( code_memory: &mut CodeMemory, compilation: &wasmtime_environ::Compilation, ) -> Result, String> { + if compilation.is_empty() { + return Ok(PrimaryMap::new()); + } + let fat_ptrs = code_memory.allocate_for_compilation(compilation)?; // Second, create a PrimaryMap from result vector of pointers. @@ -391,6 +385,7 @@ fn allocate_functions( let fat_ptr: *mut [VMFunctionBody] = fat_ptrs[i]; result.push(fat_ptr); } + Ok(result) } diff --git a/crates/jit/src/function_table.rs b/crates/jit/src/function_table.rs deleted file mode 100644 index ed982cf7a2d4..000000000000 --- a/crates/jit/src/function_table.rs +++ /dev/null @@ -1,224 +0,0 @@ -//! Runtime function table. -//! -//! This module is primarily used to track JIT functions on Windows for stack walking and unwind. - -type FunctionTableReloc = wasmtime_environ::CompiledFunctionUnwindInfoReloc; - -/// Represents a runtime function table. -/// -/// This is used to register JIT code with the operating system to enable stack walking and unwinding. -#[cfg(all(target_os = "windows", target_arch = "x86_64"))] -pub(crate) struct FunctionTable { - functions: Vec, - published: bool, -} - -#[cfg(all(target_os = "windows", target_arch = "x86_64"))] -impl FunctionTable { - /// Creates a new function table. - pub fn new() -> Self { - Self { - functions: Vec::new(), - published: false, - } - } - - /// Returns the number of functions in the table, also referred to as its 'length'. - pub fn len(&self) -> usize { - self.functions.len() - } - - /// Adds a function to the table based off of the start offset, end offset, and unwind offset. - /// - /// The offsets are from the "module base", which is provided when the table is published. - pub fn add_function( - &mut self, - start: u32, - end: u32, - unwind: u32, - _relocs: &[FunctionTableReloc], - ) { - assert_eq!(_relocs.len(), 0); - use winapi::um::winnt; - - assert!(!self.published, "table has already been published"); - - let mut entry = winnt::RUNTIME_FUNCTION::default(); - - entry.BeginAddress = start; - entry.EndAddress = end; - - unsafe { - *entry.u.UnwindInfoAddress_mut() = unwind; - } - - self.functions.push(entry); - } - - /// Publishes the function table using the given base address. - /// - /// A published function table will automatically be deleted when it is dropped. - pub fn publish(&mut self, base_address: u64) -> Result<(), String> { - use winapi::um::winnt; - - if self.published { - return Err("function table was already published".into()); - } - - self.published = true; - - if self.functions.is_empty() { - return Ok(()); - } - - unsafe { - // Windows heap allocations are 32-bit aligned, but assert just in case - assert_eq!( - (self.functions.as_mut_ptr() as u64) % 4, - 0, - "function table allocation was not aligned" - ); - - if winnt::RtlAddFunctionTable( - self.functions.as_mut_ptr(), - self.functions.len() as u32, - base_address, - ) == 0 - { - return Err("failed to add function table".into()); - } - } - - Ok(()) - } -} - -#[cfg(target_os = "windows")] -impl Drop for FunctionTable { - fn drop(&mut self) { - use winapi::um::winnt; - - if self.published { - unsafe { - winnt::RtlDeleteFunctionTable(self.functions.as_mut_ptr()); - } - } - } -} - -/// Represents a runtime function table. -/// -/// This is used to register JIT code with the operating system to enable stack walking and unwinding. -#[cfg(unix)] -pub(crate) struct FunctionTable { - functions: Vec, - relocs: Vec, - published: Option>, -} - -#[cfg(unix)] -impl FunctionTable { - /// Creates a new function table. - pub fn new() -> Self { - Self { - functions: Vec::new(), - relocs: Vec::new(), - published: None, - } - } - - /// Returns the number of functions in the table, also referred to as its 'length'. - pub fn len(&self) -> usize { - self.functions.len() - } - - /// Adds a function to the table based off of the start offset, end offset, and unwind offset. - /// - /// The offsets are from the "module base", which is provided when the table is published. - pub fn add_function( - &mut self, - _start: u32, - _end: u32, - unwind: u32, - relocs: &[FunctionTableReloc], - ) { - assert!(self.published.is_none(), "table has already been published"); - self.functions.push(unwind); - self.relocs.extend_from_slice(relocs); - } - - /// Publishes the function table using the given base address. - /// - /// A published function table will automatically be deleted when it is dropped. - pub fn publish(&mut self, base_address: u64) -> Result<(), String> { - if self.published.is_some() { - return Err("function table was already published".into()); - } - - if self.functions.is_empty() { - assert_eq!(self.relocs.len(), 0); - self.published = Some(vec![]); - return Ok(()); - } - - extern "C" { - // libunwind import - fn __register_frame(fde: *const u8); - } - - for reloc in self.relocs.iter() { - let addr = base_address + (reloc.offset as u64); - let target = base_address + (reloc.addend as u64); - unsafe { - std::ptr::write(addr as *mut u64, target); - } - } - - let mut fdes = Vec::with_capacity(self.functions.len()); - for unwind_offset in self.functions.iter() { - let addr = base_address + (*unwind_offset as u64); - let off = unsafe { std::ptr::read::(addr as *const u32) } as usize + 4; - - let fde = (addr + off as u64) as usize; - unsafe { - __register_frame(fde as *const _); - } - fdes.push(fde); - } - - self.published = Some(fdes); - Ok(()) - } -} - -#[cfg(unix)] -impl Drop for FunctionTable { - fn drop(&mut self) { - extern "C" { - // libunwind import - fn __deregister_frame(fde: *const u8); - } - - if let Some(published) = &self.published { - unsafe { - // I'm not really sure why, but it appears to be way faster to - // unregister frames in reverse order rather than in-order. This - // way we're deregistering in LIFO order, and maybe there's some - // vec shifting or something like that in libgcc? - // - // Locally on Ubuntu 18.04 a wasm module with 40k empty - // functions takes 0.1s to compile and drop with reverse - // iteration. With forward iteration it takes 3s to compile and - // drop! - // - // Poking around libgcc sources seems to indicate that some sort - // of linked list is being traversed... We may need to figure - // out something else for backtraces in the future since this - // API may not be long-lived to keep calling. - for fde in published.iter().rev() { - __deregister_frame(*fde as *const _); - } - } - } - } -} diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index c524b31a091e..05ca60215c97 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -23,12 +23,12 @@ mod code_memory; mod compiler; -mod function_table; mod imports; mod instantiate; mod link; mod resolver; mod target_tunables; +mod unwind; pub mod native; pub mod trampoline; diff --git a/crates/jit/src/unwind.rs b/crates/jit/src/unwind.rs new file mode 100644 index 000000000000..8bb655137d74 --- /dev/null +++ b/crates/jit/src/unwind.rs @@ -0,0 +1,11 @@ +cfg_if::cfg_if! { + if #[cfg(all(windows, target_arch = "x86_64"))] { + mod winx64; + pub use self::winx64::*; + } else if #[cfg(unix)] { + mod systemv; + pub use self::systemv::*; + } else { + compile_error!("unsupported target platform for unwind"); + } +} diff --git a/crates/jit/src/unwind/systemv.rs b/crates/jit/src/unwind/systemv.rs new file mode 100644 index 000000000000..1e52d6f4dbfb --- /dev/null +++ b/crates/jit/src/unwind/systemv.rs @@ -0,0 +1,140 @@ +//! Module for System V ABI unwind registry. + +use anyhow::{bail, Result}; +use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; +use gimli::{ + write::{Address, EhFrame, EndianVec, FrameTable, Writer}, + RunTimeEndian, +}; + +/// Represents a registry of function unwind information for System V ABI. +pub struct UnwindRegistry { + base_address: usize, + functions: Vec, + frame_table: Vec, + registrations: Vec, + published: bool, +} + +extern "C" { + // libunwind import + fn __register_frame(fde: *const u8); + fn __deregister_frame(fde: *const u8); +} + +impl UnwindRegistry { + /// Creates a new unwind registry with the given base address. + pub fn new(base_address: usize) -> Self { + Self { + base_address, + functions: Vec::new(), + frame_table: Vec::new(), + registrations: Vec::new(), + published: false, + } + } + + /// Registers a function given the start offset, length, and unwind information. + pub fn register(&mut self, func_start: u32, _func_len: u32, info: &UnwindInfo) -> Result<()> { + if self.published { + bail!("unwind registry has already been published"); + } + + match info { + UnwindInfo::SystemV(info) => { + self.functions.push(info.to_fde(Address::Constant( + self.base_address as u64 + func_start as u64, + ))); + } + _ => bail!("unsupported unwind information"), + } + + Ok(()) + } + + /// Publishes all registered functions. + pub fn publish(&mut self, isa: &dyn TargetIsa) -> Result<()> { + if self.published { + bail!("unwind registry has already been published"); + } + + if self.functions.is_empty() { + self.published = true; + return Ok(()); + } + + self.set_frame_table(isa)?; + + unsafe { + self.register_frames(); + } + + self.published = true; + + Ok(()) + } + + fn set_frame_table(&mut self, isa: &dyn TargetIsa) -> Result<()> { + let mut table = FrameTable::default(); + let cie_id = table.add_cie(match isa.create_systemv_cie() { + Some(cie) => cie, + None => bail!("ISA does not support System V unwind information"), + }); + + let functions = std::mem::replace(&mut self.functions, Vec::new()); + + for func in functions { + table.add_fde(cie_id, func); + } + + let mut eh_frame = EhFrame(EndianVec::new(RunTimeEndian::default())); + table.write_eh_frame(&mut eh_frame).unwrap(); + + // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. + eh_frame.0.write_u32(0).unwrap(); + + self.frame_table = eh_frame.0.into_vec(); + + Ok(()) + } + + unsafe fn register_frames(&mut self) { + let start = self.frame_table.as_ptr(); + let end = start.add(self.frame_table.len()); + let mut current = start; + + // Walk all of the entries in the frame table and register them + while current < end { + let len = std::ptr::read::(current as *const u32) as usize; + + // Skip over the CIE + if current != start { + __register_frame(current); + self.registrations.push(current as usize); + } + + // Move to the next table entry (+4 because the length itself is not inclusive) + current = current.add(len + 4); + } + } +} + +impl Drop for UnwindRegistry { + fn drop(&mut self) { + if self.published { + unsafe { + // libgcc stores the frame entries as a linked list in decreasing sort order + // based on the PC value of the registered entry. + // + // As we store the registrations in increasing order, it would be O(N^2) to + // deregister in that order. + // + // To ensure that we just pop off the first element in the list upon every + // deregistration, walk our list of registrations backwards. + for fde in self.registrations.iter().rev() { + __deregister_frame(*fde as *const _); + } + } + } + } +} diff --git a/crates/jit/src/unwind/winx64.rs b/crates/jit/src/unwind/winx64.rs new file mode 100644 index 000000000000..763cc9ee85c7 --- /dev/null +++ b/crates/jit/src/unwind/winx64.rs @@ -0,0 +1,91 @@ +//! Module for Windows x64 ABI unwind registry. + +use anyhow::{bail, Result}; +use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; +use winapi::um::winnt; + +/// Represents a registry of function unwind information for Windows x64 ABI. +pub struct UnwindRegistry { + base_address: usize, + functions: Vec, + published: bool, +} + +impl UnwindRegistry { + /// Creates a new unwind registry with the given base address. + pub fn new(base_address: usize) -> Self { + Self { + base_address, + functions: Vec::new(), + published: false, + } + } + + /// Registers a function given the start offset, length, and unwind information. + pub fn register(&mut self, func_start: u32, func_len: u32, info: &UnwindInfo) -> Result<()> { + if self.published { + bail!("unwind registry has already been published"); + } + + match info { + UnwindInfo::WindowsX64(_) => { + let mut entry = winnt::RUNTIME_FUNCTION::default(); + + entry.BeginAddress = func_start; + entry.EndAddress = func_start + func_len; + + // The unwind information should be immediately following the function + // with padding for 4 byte alignment + unsafe { + *entry.u.UnwindInfoAddress_mut() = ((entry.EndAddress + 3) & !3); + } + + self.functions.push(entry); + + Ok(()) + } + _ => bail!("unsupported unwind information"), + } + } + + /// Publishes all registered functions. + pub fn publish(&mut self, isa: &dyn TargetIsa) -> Result<()> { + if self.published { + bail!("unwind registry has already been published"); + } + + self.published = true; + + if !self.functions.is_empty() { + // Windows heap allocations are 32-bit aligned, but assert just in case + assert_eq!( + (self.functions.as_mut_ptr() as u64) % 4, + 0, + "function table allocation was not aligned" + ); + + unsafe { + if winnt::RtlAddFunctionTable( + self.functions.as_mut_ptr(), + self.functions.len() as u32, + base_address as u64, + ) == 0 + { + bail!("failed to register function table"); + } + } + } + + Ok(()) + } +} + +impl Drop for UnwindRegistry { + fn drop(&mut self) { + if self.published { + unsafe { + winnt::RtlDeleteFunctionTable(self.functions.as_mut_ptr()); + } + } + } +} diff --git a/src/obj.rs b/src/obj.rs index 8684c500c55e..d07b6ab39a4d 100644 --- a/src/obj.rs +++ b/src/obj.rs @@ -79,37 +79,30 @@ pub fn compile_to_obj( }; // TODO: use the traps information - let ( - compilation, - relocations, - address_transform, - value_ranges, - stack_slots, - _traps, - frame_layouts, - ) = match strategy { - Strategy::Auto | Strategy::Cranelift => Cranelift::compile_module( - &module, - &module_translation, - lazy_function_body_inputs, - &*isa, - debug_info, - cache_config, - ), - #[cfg(feature = "lightbeam")] - Strategy::Lightbeam => Lightbeam::compile_module( - &module, - &module_translation, - lazy_function_body_inputs, - &*isa, - debug_info, - cache_config, - ), - #[cfg(not(feature = "lightbeam"))] - Strategy::Lightbeam => bail!("lightbeam support not enabled"), - other => bail!("unsupported compilation strategy {:?}", other), - } - .context("failed to compile module")?; + let (compilation, relocations, address_transform, value_ranges, stack_slots, _traps) = + match strategy { + Strategy::Auto | Strategy::Cranelift => Cranelift::compile_module( + &module, + &module_translation, + lazy_function_body_inputs, + &*isa, + debug_info, + cache_config, + ), + #[cfg(feature = "lightbeam")] + Strategy::Lightbeam => Lightbeam::compile_module( + &module, + &module_translation, + lazy_function_body_inputs, + &*isa, + debug_info, + cache_config, + ), + #[cfg(not(feature = "lightbeam"))] + Strategy::Lightbeam => bail!("lightbeam support not enabled"), + other => bail!("unsupported compilation strategy {:?}", other), + } + .context("failed to compile module")?; if compilation.is_empty() { bail!("no functions were found/compiled"); @@ -151,7 +144,7 @@ pub fn compile_to_obj( &debug_data, &address_transform, &value_ranges, - &frame_layouts, + &compilation, ) .context("failed to emit debug sections")?; }