From 1b16229ca649ee37adee6b6e1e4fbd5071a18305 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Tue, 17 Nov 2020 09:17:12 -0800 Subject: [PATCH] x64 and aarch64: carry MemFlags on loads/stores; don't emit trap info unless an op can trap. This end result was previously enacted by carrying a `SourceLoc` on every load/store, which was somewhat cumbersome, and only indirectly encoded metadata about a memory reference (can it trap) by its presence or absence. We have a type for this -- `MemFlags` -- that tells us everything we might want to know about a load or store, and we should plumb it through to code emission instead. This PR attaches a `MemFlags` to an `Amode` on x64, and puts it on load and store `Inst` variants on aarch64. These two choices seem to factor things out in the nicest way: there are relatively few load/store insts on aarch64 but many addressing modes, while the opposite is true on x64. --- cranelift/codegen/src/isa/aarch64/abi.rs | 16 ++- .../codegen/src/isa/aarch64/inst/emit.rs | 93 +++++++++----- .../src/isa/aarch64/inst/emit_tests.rs | 71 +++++++++++ cranelift/codegen/src/isa/aarch64/inst/mod.rs | 120 +++++++++++++++--- .../codegen/src/isa/aarch64/inst/unwind.rs | 2 + .../codegen/src/isa/aarch64/lower_inst.rs | 42 +++--- cranelift/codegen/src/isa/x64/abi.rs | 4 +- cranelift/codegen/src/isa/x64/inst/args.rs | 56 +++++++- cranelift/codegen/src/isa/x64/inst/emit.rs | 5 +- cranelift/codegen/src/isa/x64/inst/unwind.rs | 2 +- cranelift/codegen/src/isa/x64/lower.rs | 18 ++- 11 files changed, 340 insertions(+), 89 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 19f8b4534316..8496e51711f5 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -3,6 +3,7 @@ use crate::ir; use crate::ir::types; use crate::ir::types::*; +use crate::ir::MemFlags; use crate::isa; use crate::isa::aarch64::{inst::EmitState, inst::*}; use crate::machinst::*; @@ -312,11 +313,11 @@ impl ABIMachineSpec for AArch64MachineDeps { } fn gen_load_stack(mem: StackAMode, into_reg: Writable, ty: Type) -> Inst { - Inst::gen_load(into_reg, mem.into(), ty) + Inst::gen_load(into_reg, mem.into(), ty, MemFlags::trusted()) } fn gen_store_stack(mem: StackAMode, from_reg: Reg, ty: Type) -> Inst { - Inst::gen_store(mem.into(), from_reg, ty) + Inst::gen_store(mem.into(), from_reg, ty, MemFlags::trusted()) } fn gen_move(to_reg: Writable, from_reg: Reg, ty: Type) -> Inst { @@ -402,12 +403,12 @@ impl ABIMachineSpec for AArch64MachineDeps { fn gen_load_base_offset(into_reg: Writable, base: Reg, offset: i32, ty: Type) -> Inst { let mem = AMode::RegOffset(base, offset as i64, ty); - Inst::gen_load(into_reg, mem, ty) + Inst::gen_load(into_reg, mem, ty, MemFlags::trusted()) } fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Inst { let mem = AMode::RegOffset(base, offset as i64, ty); - Inst::gen_store(mem, from_reg, ty) + Inst::gen_store(mem, from_reg, ty, MemFlags::trusted()) } fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Inst; 2]> { @@ -464,6 +465,7 @@ impl ABIMachineSpec for AArch64MachineDeps { writable_stack_reg(), SImm7Scaled::maybe_from_i64(-16, types::I64).unwrap(), ), + flags: MemFlags::trusted(), }); // mov fp (x29), sp. This uses the ADDI rd, rs, 0 form of `MOV` because // the usual encoding (`ORR`) does not work with SP. @@ -500,6 +502,7 @@ impl ABIMachineSpec for AArch64MachineDeps { writable_stack_reg(), SImm7Scaled::maybe_from_i64(16, types::I64).unwrap(), ), + flags: MemFlags::trusted(), }); insts @@ -542,6 +545,7 @@ impl ABIMachineSpec for AArch64MachineDeps { stack_reg(), SImm7Scaled::maybe_from_i64((i * 16) as i64, types::I64).unwrap(), ), + flags: MemFlags::trusted(), }); } @@ -553,6 +557,7 @@ impl ABIMachineSpec for AArch64MachineDeps { stack_reg(), SImm9::maybe_from_i64((vec_offset + (i * 16)) as i64).unwrap(), ), + flags: MemFlags::trusted(), }); } @@ -591,6 +596,7 @@ impl ABIMachineSpec for AArch64MachineDeps { stack_reg(), SImm7Scaled::maybe_from_i64((i * 16) as i64, types::I64).unwrap(), ), + flags: MemFlags::trusted(), }); } @@ -601,6 +607,7 @@ impl ABIMachineSpec for AArch64MachineDeps { stack_reg(), SImm9::maybe_from_i64(((i * 16) + int_save_bytes) as i64).unwrap(), ), + flags: MemFlags::trusted(), }); } @@ -621,6 +628,7 @@ impl ABIMachineSpec for AArch64MachineDeps { writable_xreg(BALDRDASH_TLS_REG), AMode::UnsignedOffset(fp_reg(), UImm12Scaled::maybe_from_i64(off, I64).unwrap()), I64, + MemFlags::trusted(), )); } diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 597c9ac592e4..855eeb248ea4 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -3,7 +3,7 @@ use crate::binemit::{CodeOffset, Reloc, StackMap}; use crate::ir::constant::ConstantData; use crate::ir::types::*; -use crate::ir::TrapCode; +use crate::ir::{MemFlags, TrapCode}; use crate::isa::aarch64::inst::*; use crate::machinst::ty_bits; @@ -741,16 +741,18 @@ impl MachInstEmit for Inst { sink.put4(enc_bit_rr(size, op1, op2, rn, rd)) } - &Inst::ULoad8 { rd, ref mem } - | &Inst::SLoad8 { rd, ref mem } - | &Inst::ULoad16 { rd, ref mem } - | &Inst::SLoad16 { rd, ref mem } - | &Inst::ULoad32 { rd, ref mem } - | &Inst::SLoad32 { rd, ref mem } - | &Inst::ULoad64 { rd, ref mem, .. } - | &Inst::FpuLoad32 { rd, ref mem } - | &Inst::FpuLoad64 { rd, ref mem } - | &Inst::FpuLoad128 { rd, ref mem } => { + &Inst::ULoad8 { rd, ref mem, flags } + | &Inst::SLoad8 { rd, ref mem, flags } + | &Inst::ULoad16 { rd, ref mem, flags } + | &Inst::SLoad16 { rd, ref mem, flags } + | &Inst::ULoad32 { rd, ref mem, flags } + | &Inst::SLoad32 { rd, ref mem, flags } + | &Inst::ULoad64 { + rd, ref mem, flags, .. + } + | &Inst::FpuLoad32 { rd, ref mem, flags } + | &Inst::FpuLoad64 { rd, ref mem, flags } + | &Inst::FpuLoad128 { rd, ref mem, flags } => { let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state); for inst in mem_insts.into_iter() { @@ -778,7 +780,7 @@ impl MachInstEmit for Inst { }; let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if srcloc != SourceLoc::default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); } @@ -861,13 +863,13 @@ impl MachInstEmit for Inst { } } - &Inst::Store8 { rd, ref mem } - | &Inst::Store16 { rd, ref mem } - | &Inst::Store32 { rd, ref mem } - | &Inst::Store64 { rd, ref mem, .. } - | &Inst::FpuStore32 { rd, ref mem } - | &Inst::FpuStore64 { rd, ref mem } - | &Inst::FpuStore128 { rd, ref mem } => { + &Inst::Store8 { rd, ref mem, flags } + | &Inst::Store16 { rd, ref mem, flags } + | &Inst::Store32 { rd, ref mem, flags } + | &Inst::Store64 { rd, ref mem, flags } + | &Inst::FpuStore32 { rd, ref mem, flags } + | &Inst::FpuStore64 { rd, ref mem, flags } + | &Inst::FpuStore128 { rd, ref mem, flags } => { let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state); for inst in mem_insts.into_iter() { @@ -886,7 +888,7 @@ impl MachInstEmit for Inst { }; let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if srcloc != SourceLoc::default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); } @@ -943,21 +945,44 @@ impl MachInstEmit for Inst { } } - &Inst::StoreP64 { rt, rt2, ref mem } => match mem { - &PairAMode::SignedOffset(reg, simm7) => { - assert_eq!(simm7.scale_ty, I64); - sink.put4(enc_ldst_pair(0b1010100100, simm7, reg, rt, rt2)); + &Inst::StoreP64 { + rt, + rt2, + ref mem, + flags, + } => { + let srcloc = state.cur_srcloc(); + if srcloc != SourceLoc::default() && !flags.notrap() { + // Register the offset at which the actual load instruction starts. + sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); } - &PairAMode::PreIndexed(reg, simm7) => { - assert_eq!(simm7.scale_ty, I64); - sink.put4(enc_ldst_pair(0b1010100110, simm7, reg.to_reg(), rt, rt2)); + match mem { + &PairAMode::SignedOffset(reg, simm7) => { + assert_eq!(simm7.scale_ty, I64); + sink.put4(enc_ldst_pair(0b1010100100, simm7, reg, rt, rt2)); + } + &PairAMode::PreIndexed(reg, simm7) => { + assert_eq!(simm7.scale_ty, I64); + sink.put4(enc_ldst_pair(0b1010100110, simm7, reg.to_reg(), rt, rt2)); + } + &PairAMode::PostIndexed(reg, simm7) => { + assert_eq!(simm7.scale_ty, I64); + sink.put4(enc_ldst_pair(0b1010100010, simm7, reg.to_reg(), rt, rt2)); + } } - &PairAMode::PostIndexed(reg, simm7) => { - assert_eq!(simm7.scale_ty, I64); - sink.put4(enc_ldst_pair(0b1010100010, simm7, reg.to_reg(), rt, rt2)); + } + &Inst::LoadP64 { + rt, + rt2, + ref mem, + flags, + } => { + let srcloc = state.cur_srcloc(); + if srcloc != SourceLoc::default() && !flags.notrap() { + // Register the offset at which the actual load instruction starts. + sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); } - }, - &Inst::LoadP64 { rt, rt2, ref mem } => { + let rt = rt.to_reg(); let rt2 = rt2.to_reg(); match mem { @@ -1546,6 +1571,7 @@ impl MachInstEmit for Inst { let inst = Inst::FpuLoad64 { rd, mem: AMode::Label(MemLabel::PCRel(8)), + flags: MemFlags::trusted(), }; inst.emit(sink, emit_info, state); let inst = Inst::Jump { @@ -1558,6 +1584,7 @@ impl MachInstEmit for Inst { let inst = Inst::FpuLoad128 { rd, mem: AMode::Label(MemLabel::PCRel(8)), + flags: MemFlags::trusted(), }; inst.emit(sink, emit_info, state); let inst = Inst::Jump { @@ -2169,6 +2196,7 @@ impl MachInstEmit for Inst { I32, ExtendOp::UXTW, ), + flags: MemFlags::trusted(), }; inst.emit(sink, emit_info, state); // Add base of jump table to jump-table-sourced block offset @@ -2215,6 +2243,7 @@ impl MachInstEmit for Inst { let inst = Inst::ULoad64 { rd, mem: AMode::Label(MemLabel::PCRel(8)), + flags: MemFlags::trusted(), }; inst.emit(sink, emit_info, state); let inst = Inst::Jump { diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index 74aac428ef25..930de5b4cab0 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -1079,6 +1079,7 @@ fn test_aarch64_binemit() { Inst::ULoad8 { rd: writable_xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "41004038", "ldurb w1, [x2]", @@ -1087,6 +1088,7 @@ fn test_aarch64_binemit() { Inst::ULoad8 { rd: writable_xreg(1), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::zero(I8)), + flags: MemFlags::trusted(), }, "41004039", "ldrb w1, [x2]", @@ -1095,6 +1097,7 @@ fn test_aarch64_binemit() { Inst::ULoad8 { rd: writable_xreg(1), mem: AMode::RegReg(xreg(2), xreg(5)), + flags: MemFlags::trusted(), }, "41686538", "ldrb w1, [x2, x5]", @@ -1103,6 +1106,7 @@ fn test_aarch64_binemit() { Inst::SLoad8 { rd: writable_xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "41008038", "ldursb x1, [x2]", @@ -1111,6 +1115,7 @@ fn test_aarch64_binemit() { Inst::SLoad8 { rd: writable_xreg(1), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(63, I8).unwrap()), + flags: MemFlags::trusted(), }, "41FC8039", "ldrsb x1, [x2, #63]", @@ -1119,6 +1124,7 @@ fn test_aarch64_binemit() { Inst::SLoad8 { rd: writable_xreg(1), mem: AMode::RegReg(xreg(2), xreg(5)), + flags: MemFlags::trusted(), }, "4168A538", "ldrsb x1, [x2, x5]", @@ -1127,6 +1133,7 @@ fn test_aarch64_binemit() { Inst::ULoad16 { rd: writable_xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(5).unwrap()), + flags: MemFlags::trusted(), }, "41504078", "ldurh w1, [x2, #5]", @@ -1135,6 +1142,7 @@ fn test_aarch64_binemit() { Inst::ULoad16 { rd: writable_xreg(1), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8, I16).unwrap()), + flags: MemFlags::trusted(), }, "41104079", "ldrh w1, [x2, #8]", @@ -1143,6 +1151,7 @@ fn test_aarch64_binemit() { Inst::ULoad16 { rd: writable_xreg(1), mem: AMode::RegScaled(xreg(2), xreg(3), I16), + flags: MemFlags::trusted(), }, "41786378", "ldrh w1, [x2, x3, LSL #1]", @@ -1151,6 +1160,7 @@ fn test_aarch64_binemit() { Inst::SLoad16 { rd: writable_xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "41008078", "ldursh x1, [x2]", @@ -1159,6 +1169,7 @@ fn test_aarch64_binemit() { Inst::SLoad16 { rd: writable_xreg(28), mem: AMode::UnsignedOffset(xreg(20), UImm12Scaled::maybe_from_i64(24, I16).unwrap()), + flags: MemFlags::trusted(), }, "9C328079", "ldrsh x28, [x20, #24]", @@ -1167,6 +1178,7 @@ fn test_aarch64_binemit() { Inst::SLoad16 { rd: writable_xreg(28), mem: AMode::RegScaled(xreg(20), xreg(20), I16), + flags: MemFlags::trusted(), }, "9C7AB478", "ldrsh x28, [x20, x20, LSL #1]", @@ -1175,6 +1187,7 @@ fn test_aarch64_binemit() { Inst::ULoad32 { rd: writable_xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "410040B8", "ldur w1, [x2]", @@ -1183,6 +1196,7 @@ fn test_aarch64_binemit() { Inst::ULoad32 { rd: writable_xreg(12), mem: AMode::UnsignedOffset(xreg(0), UImm12Scaled::maybe_from_i64(204, I32).unwrap()), + flags: MemFlags::trusted(), }, "0CCC40B9", "ldr w12, [x0, #204]", @@ -1191,6 +1205,7 @@ fn test_aarch64_binemit() { Inst::ULoad32 { rd: writable_xreg(1), mem: AMode::RegScaled(xreg(2), xreg(12), I32), + flags: MemFlags::trusted(), }, "41786CB8", "ldr w1, [x2, x12, LSL #2]", @@ -1199,6 +1214,7 @@ fn test_aarch64_binemit() { Inst::SLoad32 { rd: writable_xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "410080B8", "ldursw x1, [x2]", @@ -1207,6 +1223,7 @@ fn test_aarch64_binemit() { Inst::SLoad32 { rd: writable_xreg(12), mem: AMode::UnsignedOffset(xreg(1), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()), + flags: MemFlags::trusted(), }, "2CFCBFB9", "ldrsw x12, [x1, #16380]", @@ -1215,6 +1232,7 @@ fn test_aarch64_binemit() { Inst::SLoad32 { rd: writable_xreg(1), mem: AMode::RegScaled(xreg(5), xreg(1), I32), + flags: MemFlags::trusted(), }, "A178A1B8", "ldrsw x1, [x5, x1, LSL #2]", @@ -1223,6 +1241,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "410040F8", "ldur x1, [x2]", @@ -1231,6 +1250,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(-256).unwrap()), + flags: MemFlags::trusted(), }, "410050F8", "ldur x1, [x2, #-256]", @@ -1239,6 +1259,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(255).unwrap()), + flags: MemFlags::trusted(), }, "41F04FF8", "ldur x1, [x2, #255]", @@ -1247,6 +1268,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()), + flags: MemFlags::trusted(), }, "41FC7FF9", "ldr x1, [x2, #32760]", @@ -1255,6 +1277,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::RegReg(xreg(2), xreg(3)), + flags: MemFlags::trusted(), }, "416863F8", "ldr x1, [x2, x3]", @@ -1263,6 +1286,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::RegScaled(xreg(2), xreg(3), I64), + flags: MemFlags::trusted(), }, "417863F8", "ldr x1, [x2, x3, LSL #3]", @@ -1271,6 +1295,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::SXTW), + flags: MemFlags::trusted(), }, "41D863F8", "ldr x1, [x2, w3, SXTW #3]", @@ -1279,6 +1304,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::SXTW), + flags: MemFlags::trusted(), }, "41C863F8", "ldr x1, [x2, w3, SXTW]", @@ -1287,6 +1313,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::Label(MemLabel::PCRel(64)), + flags: MemFlags::trusted(), }, "01020058", "ldr x1, pc+64", @@ -1295,6 +1322,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), + flags: MemFlags::trusted(), }, "410C41F8", "ldr x1, [x2, #16]!", @@ -1303,6 +1331,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), + flags: MemFlags::trusted(), }, "410441F8", "ldr x1, [x2], #16", @@ -1311,6 +1340,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::FPOffset(32768, I8), + flags: MemFlags::trusted(), }, "100090D2B063308B010240F9", "movz x16, #32768 ; add x16, fp, x16, UXTX ; ldr x1, [x16]", @@ -1319,6 +1349,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::FPOffset(-32768, I8), + flags: MemFlags::trusted(), }, "F0FF8F92B063308B010240F9", "movn x16, #32767 ; add x16, fp, x16, UXTX ; ldr x1, [x16]", @@ -1327,6 +1358,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::FPOffset(1048576, I8), // 2^20 + flags: MemFlags::trusted(), }, "1002A0D2B063308B010240F9", "movz x16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]", @@ -1335,6 +1367,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::FPOffset(1048576 + 1, I8), // 2^20 + 1 + flags: MemFlags::trusted(), }, "300080521002A072B063308B010240F9", "movz w16, #1 ; movk w16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]", @@ -1344,6 +1377,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::RegOffset(xreg(7), 8, I64), + flags: MemFlags::trusted(), }, "E18040F8", "ldur x1, [x7, #8]", @@ -1353,6 +1387,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::RegOffset(xreg(7), 1024, I64), + flags: MemFlags::trusted(), }, "E10042F9", "ldr x1, [x7, #1024]", @@ -1362,6 +1397,7 @@ fn test_aarch64_binemit() { Inst::ULoad64 { rd: writable_xreg(1), mem: AMode::RegOffset(xreg(7), 1048576, I64), + flags: MemFlags::trusted(), }, "1002A0D2F060308B010240F9", "movz x16, #16, LSL #16 ; add x16, x7, x16, UXTX ; ldr x1, [x16]", @@ -1371,6 +1407,7 @@ fn test_aarch64_binemit() { Inst::Store8 { rd: xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "41000038", "sturb w1, [x2]", @@ -1379,6 +1416,7 @@ fn test_aarch64_binemit() { Inst::Store8 { rd: xreg(1), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(4095, I8).unwrap()), + flags: MemFlags::trusted(), }, "41FC3F39", "strb w1, [x2, #4095]", @@ -1387,6 +1425,7 @@ fn test_aarch64_binemit() { Inst::Store16 { rd: xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "41000078", "sturh w1, [x2]", @@ -1395,6 +1434,7 @@ fn test_aarch64_binemit() { Inst::Store16 { rd: xreg(1), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8190, I16).unwrap()), + flags: MemFlags::trusted(), }, "41FC3F79", "strh w1, [x2, #8190]", @@ -1403,6 +1443,7 @@ fn test_aarch64_binemit() { Inst::Store32 { rd: xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "410000B8", "stur w1, [x2]", @@ -1411,6 +1452,7 @@ fn test_aarch64_binemit() { Inst::Store32 { rd: xreg(1), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()), + flags: MemFlags::trusted(), }, "41FC3FB9", "str w1, [x2, #16380]", @@ -1419,6 +1461,7 @@ fn test_aarch64_binemit() { Inst::Store64 { rd: xreg(1), mem: AMode::Unscaled(xreg(2), SImm9::zero()), + flags: MemFlags::trusted(), }, "410000F8", "stur x1, [x2]", @@ -1427,6 +1470,7 @@ fn test_aarch64_binemit() { Inst::Store64 { rd: xreg(1), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()), + flags: MemFlags::trusted(), }, "41FC3FF9", "str x1, [x2, #32760]", @@ -1435,6 +1479,7 @@ fn test_aarch64_binemit() { Inst::Store64 { rd: xreg(1), mem: AMode::RegReg(xreg(2), xreg(3)), + flags: MemFlags::trusted(), }, "416823F8", "str x1, [x2, x3]", @@ -1443,6 +1488,7 @@ fn test_aarch64_binemit() { Inst::Store64 { rd: xreg(1), mem: AMode::RegScaled(xreg(2), xreg(3), I64), + flags: MemFlags::trusted(), }, "417823F8", "str x1, [x2, x3, LSL #3]", @@ -1451,6 +1497,7 @@ fn test_aarch64_binemit() { Inst::Store64 { rd: xreg(1), mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::UXTW), + flags: MemFlags::trusted(), }, "415823F8", "str x1, [x2, w3, UXTW #3]", @@ -1459,6 +1506,7 @@ fn test_aarch64_binemit() { Inst::Store64 { rd: xreg(1), mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::UXTW), + flags: MemFlags::trusted(), }, "414823F8", "str x1, [x2, w3, UXTW]", @@ -1467,6 +1515,7 @@ fn test_aarch64_binemit() { Inst::Store64 { rd: xreg(1), mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), + flags: MemFlags::trusted(), }, "410C01F8", "str x1, [x2, #16]!", @@ -1475,6 +1524,7 @@ fn test_aarch64_binemit() { Inst::Store64 { rd: xreg(1), mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), + flags: MemFlags::trusted(), }, "410401F8", "str x1, [x2], #16", @@ -1485,6 +1535,7 @@ fn test_aarch64_binemit() { rt: xreg(8), rt2: xreg(9), mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::zero(I64)), + flags: MemFlags::trusted(), }, "482500A9", "stp x8, x9, [x10]", @@ -1494,6 +1545,7 @@ fn test_aarch64_binemit() { rt: xreg(8), rt2: xreg(9), mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(504, I64).unwrap()), + flags: MemFlags::trusted(), }, "48A51FA9", "stp x8, x9, [x10, #504]", @@ -1503,6 +1555,7 @@ fn test_aarch64_binemit() { rt: xreg(8), rt2: xreg(9), mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(-64, I64).unwrap()), + flags: MemFlags::trusted(), }, "48253CA9", "stp x8, x9, [x10, #-64]", @@ -1512,6 +1565,7 @@ fn test_aarch64_binemit() { rt: xreg(21), rt2: xreg(28), mem: PairAMode::SignedOffset(xreg(1), SImm7Scaled::maybe_from_i64(-512, I64).unwrap()), + flags: MemFlags::trusted(), }, "357020A9", "stp x21, x28, [x1, #-512]", @@ -1524,6 +1578,7 @@ fn test_aarch64_binemit() { writable_xreg(10), SImm7Scaled::maybe_from_i64(-64, I64).unwrap(), ), + flags: MemFlags::trusted(), }, "4825BCA9", "stp x8, x9, [x10, #-64]!", @@ -1536,6 +1591,7 @@ fn test_aarch64_binemit() { writable_xreg(20), SImm7Scaled::maybe_from_i64(504, I64).unwrap(), ), + flags: MemFlags::trusted(), }, "8FC29FA8", "stp x15, x16, [x20], #504", @@ -1546,6 +1602,7 @@ fn test_aarch64_binemit() { rt: writable_xreg(8), rt2: writable_xreg(9), mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::zero(I64)), + flags: MemFlags::trusted(), }, "482540A9", "ldp x8, x9, [x10]", @@ -1555,6 +1612,7 @@ fn test_aarch64_binemit() { rt: writable_xreg(8), rt2: writable_xreg(9), mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(504, I64).unwrap()), + flags: MemFlags::trusted(), }, "48A55FA9", "ldp x8, x9, [x10, #504]", @@ -1564,6 +1622,7 @@ fn test_aarch64_binemit() { rt: writable_xreg(8), rt2: writable_xreg(9), mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(-64, I64).unwrap()), + flags: MemFlags::trusted(), }, "48257CA9", "ldp x8, x9, [x10, #-64]", @@ -1573,6 +1632,7 @@ fn test_aarch64_binemit() { rt: writable_xreg(8), rt2: writable_xreg(9), mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(-512, I64).unwrap()), + flags: MemFlags::trusted(), }, "482560A9", "ldp x8, x9, [x10, #-512]", @@ -1585,6 +1645,7 @@ fn test_aarch64_binemit() { writable_xreg(10), SImm7Scaled::maybe_from_i64(-64, I64).unwrap(), ), + flags: MemFlags::trusted(), }, "4825FCA9", "ldp x8, x9, [x10, #-64]!", @@ -1597,6 +1658,7 @@ fn test_aarch64_binemit() { writable_xreg(12), SImm7Scaled::maybe_from_i64(504, I64).unwrap(), ), + flags: MemFlags::trusted(), }, "88E5DFA8", "ldp x8, x25, [x12], #504", @@ -4756,6 +4818,7 @@ fn test_aarch64_binemit() { Inst::FpuLoad32 { rd: writable_vreg(16), mem: AMode::RegScaled(xreg(8), xreg(9), F32), + flags: MemFlags::trusted(), }, "107969BC", "ldr s16, [x8, x9, LSL #2]", @@ -4765,6 +4828,7 @@ fn test_aarch64_binemit() { Inst::FpuLoad64 { rd: writable_vreg(16), mem: AMode::RegScaled(xreg(8), xreg(9), F64), + flags: MemFlags::trusted(), }, "107969FC", "ldr d16, [x8, x9, LSL #3]", @@ -4774,6 +4838,7 @@ fn test_aarch64_binemit() { Inst::FpuLoad128 { rd: writable_vreg(16), mem: AMode::RegScaled(xreg(8), xreg(9), I128), + flags: MemFlags::trusted(), }, "1079E93C", "ldr q16, [x8, x9, LSL #4]", @@ -4783,6 +4848,7 @@ fn test_aarch64_binemit() { Inst::FpuLoad32 { rd: writable_vreg(16), mem: AMode::Label(MemLabel::PCRel(8)), + flags: MemFlags::trusted(), }, "5000001C", "ldr s16, pc+8", @@ -4792,6 +4858,7 @@ fn test_aarch64_binemit() { Inst::FpuLoad64 { rd: writable_vreg(16), mem: AMode::Label(MemLabel::PCRel(8)), + flags: MemFlags::trusted(), }, "5000005C", "ldr d16, pc+8", @@ -4801,6 +4868,7 @@ fn test_aarch64_binemit() { Inst::FpuLoad128 { rd: writable_vreg(16), mem: AMode::Label(MemLabel::PCRel(8)), + flags: MemFlags::trusted(), }, "5000009C", "ldr q16, pc+8", @@ -4810,6 +4878,7 @@ fn test_aarch64_binemit() { Inst::FpuStore32 { rd: vreg(16), mem: AMode::RegScaled(xreg(8), xreg(9), F32), + flags: MemFlags::trusted(), }, "107929BC", "str s16, [x8, x9, LSL #2]", @@ -4819,6 +4888,7 @@ fn test_aarch64_binemit() { Inst::FpuStore64 { rd: vreg(16), mem: AMode::RegScaled(xreg(8), xreg(9), F64), + flags: MemFlags::trusted(), }, "107929FC", "str d16, [x8, x9, LSL #3]", @@ -4828,6 +4898,7 @@ fn test_aarch64_binemit() { Inst::FpuStore128 { rd: vreg(16), mem: AMode::RegScaled(xreg(8), xreg(9), I128), + flags: MemFlags::trusted(), }, "1079A93C", "str q16, [x8, x9, LSL #4]", diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 0652706c7c5e..627a02bdadb8 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -9,7 +9,7 @@ use crate::ir::types::{ F64X2, FFLAGS, I16, I16X4, I16X8, I32, I32X2, I32X4, I64, I64X2, I8, I8X16, I8X8, IFLAGS, R32, R64, }; -use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type}; +use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type}; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; @@ -523,57 +523,68 @@ pub enum Inst { ULoad8 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// A signed (sign-extending) 8-bit load. SLoad8 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// An unsigned (zero-extending) 16-bit load. ULoad16 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// A signed (sign-extending) 16-bit load. SLoad16 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// An unsigned (zero-extending) 32-bit load. ULoad32 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// A signed (sign-extending) 32-bit load. SLoad32 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// A 64-bit load. ULoad64 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// An 8-bit store. Store8 { rd: Reg, mem: AMode, + flags: MemFlags, }, /// A 16-bit store. Store16 { rd: Reg, mem: AMode, + flags: MemFlags, }, /// A 32-bit store. Store32 { rd: Reg, mem: AMode, + flags: MemFlags, }, /// A 64-bit store. Store64 { rd: Reg, mem: AMode, + flags: MemFlags, }, /// A store of a pair of registers. @@ -581,12 +592,14 @@ pub enum Inst { rt: Reg, rt2: Reg, mem: PairAMode, + flags: MemFlags, }, /// A load of a pair of registers. LoadP64 { rt: Writable, rt2: Writable, mem: PairAMode, + flags: MemFlags, }, /// A MOV instruction. These are encoded as ORR's (AluRRR form) but we @@ -782,31 +795,37 @@ pub enum Inst { FpuLoad32 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// Floating-point store, single-precision (32 bit). FpuStore32 { rd: Reg, mem: AMode, + flags: MemFlags, }, /// Floating-point load, double-precision (64 bit). FpuLoad64 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// Floating-point store, double-precision (64 bit). FpuStore64 { rd: Reg, mem: AMode, + flags: MemFlags, }, /// Floating-point/vector load, 128 bit. FpuLoad128 { rd: Writable, mem: AMode, + flags: MemFlags, }, /// Floating-point/vector store, 128 bit. FpuStore128 { rd: Reg, mem: AMode, + flags: MemFlags, }, LoadFpuConst64 { @@ -1411,24 +1430,48 @@ impl Inst { } /// Generic constructor for a load (zero-extending where appropriate). - pub fn gen_load(into_reg: Writable, mem: AMode, ty: Type) -> Inst { + pub fn gen_load(into_reg: Writable, mem: AMode, ty: Type, flags: MemFlags) -> Inst { match ty { - B1 | B8 | I8 => Inst::ULoad8 { rd: into_reg, mem }, - B16 | I16 => Inst::ULoad16 { rd: into_reg, mem }, - B32 | I32 | R32 => Inst::ULoad32 { rd: into_reg, mem }, - B64 | I64 | R64 => Inst::ULoad64 { rd: into_reg, mem }, - F32 => Inst::FpuLoad32 { rd: into_reg, mem }, - F64 => Inst::FpuLoad64 { rd: into_reg, mem }, + B1 | B8 | I8 => Inst::ULoad8 { + rd: into_reg, + mem, + flags, + }, + B16 | I16 => Inst::ULoad16 { + rd: into_reg, + mem, + flags, + }, + B32 | I32 | R32 => Inst::ULoad32 { + rd: into_reg, + mem, + flags, + }, + B64 | I64 | R64 => Inst::ULoad64 { + rd: into_reg, + mem, + flags, + }, + F32 => Inst::FpuLoad32 { + rd: into_reg, + mem, + flags, + }, + F64 => Inst::FpuLoad64 { + rd: into_reg, + mem, + flags, + }, _ => { if ty.is_vector() { let bits = ty_bits(ty); let rd = into_reg; if bits == 128 { - Inst::FpuLoad128 { rd, mem } + Inst::FpuLoad128 { rd, mem, flags } } else { assert_eq!(bits, 64); - Inst::FpuLoad64 { rd, mem } + Inst::FpuLoad64 { rd, mem, flags } } } else { unimplemented!("gen_load({})", ty); @@ -1438,24 +1481,48 @@ impl Inst { } /// Generic constructor for a store. - pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type) -> Inst { + pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlags) -> Inst { match ty { - B1 | B8 | I8 => Inst::Store8 { rd: from_reg, mem }, - B16 | I16 => Inst::Store16 { rd: from_reg, mem }, - B32 | I32 | R32 => Inst::Store32 { rd: from_reg, mem }, - B64 | I64 | R64 => Inst::Store64 { rd: from_reg, mem }, - F32 => Inst::FpuStore32 { rd: from_reg, mem }, - F64 => Inst::FpuStore64 { rd: from_reg, mem }, + B1 | B8 | I8 => Inst::Store8 { + rd: from_reg, + mem, + flags, + }, + B16 | I16 => Inst::Store16 { + rd: from_reg, + mem, + flags, + }, + B32 | I32 | R32 => Inst::Store32 { + rd: from_reg, + mem, + flags, + }, + B64 | I64 | R64 => Inst::Store64 { + rd: from_reg, + mem, + flags, + }, + F32 => Inst::FpuStore32 { + rd: from_reg, + mem, + flags, + }, + F64 => Inst::FpuStore64 { + rd: from_reg, + mem, + flags, + }, _ => { if ty.is_vector() { let bits = ty_bits(ty); let rd = from_reg; if bits == 128 { - Inst::FpuStore128 { rd, mem } + Inst::FpuStore128 { rd, mem, flags } } else { assert_eq!(bits, 64); - Inst::FpuStore64 { rd, mem } + Inst::FpuStore64 { rd, mem, flags } } } else { unimplemented!("gen_store({})", ty); @@ -2126,6 +2193,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rt, ref mut rt2, ref mut mem, + .. } => { map_use(mapper, rt); map_use(mapper, rt2); @@ -2135,6 +2203,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rt, ref mut rt2, ref mut mem, + .. } => { map_def(mapper, rt); map_def(mapper, rt2); @@ -2979,26 +3048,32 @@ impl Inst { &Inst::ULoad8 { rd, ref mem, + .. } | &Inst::SLoad8 { rd, ref mem, + .. } | &Inst::ULoad16 { rd, ref mem, + .. } | &Inst::SLoad16 { rd, ref mem, + .. } | &Inst::ULoad32 { rd, ref mem, + .. } | &Inst::SLoad32 { rd, ref mem, + .. } | &Inst::ULoad64 { rd, @@ -3035,14 +3110,17 @@ impl Inst { &Inst::Store8 { rd, ref mem, + .. } | &Inst::Store16 { rd, ref mem, + .. } | &Inst::Store32 { rd, ref mem, + .. } | &Inst::Store64 { rd, @@ -3070,13 +3148,13 @@ impl Inst { let mem = mem.show_rru(mb_rru); format!("{}{} {}, {}", mem_str, op, rd, mem) } - &Inst::StoreP64 { rt, rt2, ref mem } => { + &Inst::StoreP64 { rt, rt2, ref mem, .. } => { let rt = rt.show_rru(mb_rru); let rt2 = rt2.show_rru(mb_rru); let mem = mem.show_rru(mb_rru); format!("stp {}, {}, {}", rt, rt2, mem) } - &Inst::LoadP64 { rt, rt2, ref mem } => { + &Inst::LoadP64 { rt, rt2, ref mem, .. } => { let rt = rt.to_reg().show_rru(mb_rru); let rt2 = rt2.to_reg().show_rru(mb_rru); let mem = mem.show_rru(mb_rru); diff --git a/cranelift/codegen/src/isa/aarch64/inst/unwind.rs b/cranelift/codegen/src/isa/aarch64/inst/unwind.rs index ff6634ad47b6..698e0947956c 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/unwind.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/unwind.rs @@ -29,6 +29,7 @@ impl UnwindInfoGenerator for AArch64UnwindInfo { rt, rt2, mem: PairAMode::PreIndexed(rn, imm7), + .. } if *rt == regs::fp_reg() && *rt2 == regs::link_reg() && *rn == regs::writable_stack_reg() @@ -60,6 +61,7 @@ impl UnwindInfoGenerator for AArch64UnwindInfo { rt, rt2, mem: PairAMode::PreIndexed(rn, imm7), + .. } if rn.to_reg() == regs::stack_reg() && imm7.value % (pair_size as i16) == 0 => { // stp r1, r2, [sp, #(i * #16)] let stack_offset = imm7.value as u32; diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 5c295c07f94d..16c1944dca3a 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -1130,6 +1130,9 @@ pub(crate) fn lower_insn_to_regs>( | Opcode::Sload32Complex => true, _ => false, }; + let flags = ctx + .memflags(insn) + .expect("Load instruction should have memflags"); lower_load( ctx, @@ -1139,19 +1142,19 @@ pub(crate) fn lower_insn_to_regs>( |ctx, rd, elem_ty, mem| { let is_float = ty_has_float_or_vec_representation(elem_ty); ctx.emit(match (ty_bits(elem_ty), sign_extend, is_float) { - (1, _, _) => Inst::ULoad8 { rd, mem }, - (8, false, _) => Inst::ULoad8 { rd, mem }, - (8, true, _) => Inst::SLoad8 { rd, mem }, - (16, false, _) => Inst::ULoad16 { rd, mem }, - (16, true, _) => Inst::SLoad16 { rd, mem }, - (32, false, false) => Inst::ULoad32 { rd, mem }, - (32, true, false) => Inst::SLoad32 { rd, mem }, - (32, _, true) => Inst::FpuLoad32 { rd, mem }, - (64, _, false) => Inst::ULoad64 { rd, mem }, + (1, _, _) => Inst::ULoad8 { rd, mem, flags }, + (8, false, _) => Inst::ULoad8 { rd, mem, flags }, + (8, true, _) => Inst::SLoad8 { rd, mem, flags }, + (16, false, _) => Inst::ULoad16 { rd, mem, flags }, + (16, true, _) => Inst::SLoad16 { rd, mem, flags }, + (32, false, false) => Inst::ULoad32 { rd, mem, flags }, + (32, true, false) => Inst::SLoad32 { rd, mem, flags }, + (32, _, true) => Inst::FpuLoad32 { rd, mem, flags }, + (64, _, false) => Inst::ULoad64 { rd, mem, flags }, // Note that we treat some of the vector loads as scalar floating-point loads, // which is correct in a little endian environment. - (64, _, true) => Inst::FpuLoad64 { rd, mem }, - (128, _, _) => Inst::FpuLoad128 { rd, mem }, + (64, _, true) => Inst::FpuLoad64 { rd, mem, flags }, + (128, _, _) => Inst::FpuLoad128 { rd, mem, flags }, _ => panic!("Unsupported size in load"), }); @@ -1200,18 +1203,21 @@ pub(crate) fn lower_insn_to_regs>( _ => unreachable!(), }; let is_float = ty_has_float_or_vec_representation(elem_ty); + let flags = ctx + .memflags(insn) + .expect("Store instruction should have memflags"); let mem = lower_address(ctx, elem_ty, &inputs[1..], off); let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); ctx.emit(match (ty_bits(elem_ty), is_float) { - (1, _) | (8, _) => Inst::Store8 { rd, mem }, - (16, _) => Inst::Store16 { rd, mem }, - (32, false) => Inst::Store32 { rd, mem }, - (32, true) => Inst::FpuStore32 { rd, mem }, - (64, false) => Inst::Store64 { rd, mem }, - (64, true) => Inst::FpuStore64 { rd, mem }, - (128, _) => Inst::FpuStore128 { rd, mem }, + (1, _) | (8, _) => Inst::Store8 { rd, mem, flags }, + (16, _) => Inst::Store16 { rd, mem, flags }, + (32, false) => Inst::Store32 { rd, mem, flags }, + (32, true) => Inst::FpuStore32 { rd, mem, flags }, + (64, false) => Inst::Store64 { rd, mem, flags }, + (64, true) => Inst::FpuStore64 { rd, mem, flags }, + (128, _) => Inst::FpuStore128 { rd, mem, flags }, _ => panic!("Unsupported size in store"), }); } diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 2b9ce29bc8b5..4f84d75c1d87 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -1,7 +1,7 @@ //! Implementation of the standard x64 ABI. use crate::ir::types::*; -use crate::ir::{self, types, TrapCode, Type}; +use crate::ir::{self, types, MemFlags, TrapCode, Type}; use crate::isa; use crate::isa::{x64::inst::*, CallConv}; use crate::machinst::abi_impl::*; @@ -618,6 +618,7 @@ impl From for SyntheticAmode { SyntheticAmode::Real(Amode::ImmReg { simm32, base: regs::rbp(), + flags: MemFlags::trusted(), }) } StackAMode::NominalSPOffset(off, _ty) => { @@ -634,6 +635,7 @@ impl From for SyntheticAmode { SyntheticAmode::Real(Amode::ImmReg { simm32, base: regs::rsp(), + flags: MemFlags::trusted(), }) } } diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index bf209919bfeb..e992288560e9 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -3,6 +3,7 @@ use super::regs::{self, show_ireg_sized}; use super::EmitState; use crate::ir::condcodes::{FloatCC, IntCC}; +use crate::ir::MemFlags; use crate::isa::x64::inst::Inst; use crate::machinst::*; use regalloc::{ @@ -14,10 +15,14 @@ use std::string::String; /// A possible addressing mode (amode) that can be used in instructions. /// These denote a 64-bit value only. -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum Amode { /// Immediate sign-extended and a Register. - ImmReg { simm32: u32, base: Reg }, + ImmReg { + simm32: u32, + base: Reg, + flags: MemFlags, + }, /// sign-extend-32-to-64(Immediate) + Register1 + (Register2 << Shift) ImmRegRegShift { @@ -25,6 +30,7 @@ pub enum Amode { base: Reg, index: Reg, shift: u8, /* 0 .. 3 only */ + flags: MemFlags, }, /// sign-extend-32-to-64(Immediate) + RIP (instruction pointer). @@ -35,7 +41,11 @@ pub enum Amode { impl Amode { pub(crate) fn imm_reg(simm32: u32, base: Reg) -> Self { debug_assert!(base.get_class() == RegClass::I64); - Self::ImmReg { simm32, base } + Self::ImmReg { + simm32, + base, + flags: MemFlags::trusted(), + } } pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Reg, index: Reg, shift: u8) -> Self { @@ -47,6 +57,7 @@ impl Amode { base, index, shift, + flags: MemFlags::trusted(), } } @@ -54,6 +65,30 @@ impl Amode { Self::RipRelative { target } } + pub(crate) fn with_flags(&self, flags: MemFlags) -> Self { + match self { + &Self::ImmReg { simm32, base, .. } => Self::ImmReg { + simm32, + base, + flags, + }, + &Self::ImmRegRegShift { + simm32, + base, + index, + shift, + .. + } => Self::ImmRegRegShift { + simm32, + base, + index, + shift, + flags, + }, + _ => panic!("Amode {:?} cannot take memflags", self), + } + } + /// Add the regs mentioned by `self` to `collector`. pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { match self { @@ -69,12 +104,24 @@ impl Amode { } } } + + pub(crate) fn get_flags(&self) -> MemFlags { + match self { + Amode::ImmReg { flags, .. } => *flags, + Amode::ImmRegRegShift { flags, .. } => *flags, + Amode::RipRelative { .. } => MemFlags::trusted(), + } + } + + pub(crate) fn can_trap(&self) -> bool { + !self.get_flags().notrap() + } } impl PrettyPrint for Amode { fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { match self { - Amode::ImmReg { simm32, base } => { + Amode::ImmReg { simm32, base, .. } => { format!("{}({})", *simm32 as i32, base.show_rru(mb_rru)) } Amode::ImmRegRegShift { @@ -82,6 +129,7 @@ impl PrettyPrint for Amode { base, index, shift, + .. } => format!( "{}({},{},{})", *simm32 as i32, diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 24ff06488a90..7d15063ad428 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -194,14 +194,14 @@ fn emit_std_enc_mem( // expression. But `enc_g` can be derived from a register of any class. let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if srcloc != SourceLoc::default() && mem_e.can_trap() { sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); } prefixes.emit(sink); match mem_e { - Amode::ImmReg { simm32, base } => { + Amode::ImmReg { simm32, base, .. } => { // First, the REX byte. let enc_e = int_reg_enc(*base); rex.emit_two_op(sink, enc_g, enc_e); @@ -260,6 +260,7 @@ fn emit_std_enc_mem( base: reg_base, index: reg_index, shift, + .. } => { let enc_base = int_reg_enc(*reg_base); let enc_index = int_reg_enc(*reg_index); diff --git a/cranelift/codegen/src/isa/x64/inst/unwind.rs b/cranelift/codegen/src/isa/x64/inst/unwind.rs index 022b2e65fe65..ffe43930f055 100644 --- a/cranelift/codegen/src/isa/x64/inst/unwind.rs +++ b/cranelift/codegen/src/isa/x64/inst/unwind.rs @@ -61,7 +61,7 @@ impl UnwindInfoGenerator for X64UnwindInfo { } Inst::MovRM { src, - dst: SyntheticAmode::Real(Amode::ImmReg { simm32, base }), + dst: SyntheticAmode::Real(Amode::ImmReg { simm32, base, .. }), .. } if *base == regs::rsp() => { // `mov reg, imm(rsp)` diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index b2b1968c8f36..bbe886c24b9c 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -579,6 +579,10 @@ fn matches_small_constant_shift>( /// /// Note: the 32-bit offset in Cranelift has to be sign-extended, which maps x86's behavior. fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i32) -> Amode { + let flags = ctx + .memflags(spec.insn) + .expect("Instruction with amode should have memflags"); + // We now either have an add that we must materialize, or some other input; as well as the // final offset. if let Some(add) = matches_input(ctx, spec, Opcode::Iadd) { @@ -632,7 +636,7 @@ fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i let final_offset = (offset as i64).wrapping_add(uext_cst as i64); if low32_will_sign_extend_to_64(final_offset as u64) { let base = put_input_in_reg(ctx, add_inputs[1 - i]); - return Amode::imm_reg(final_offset as u32, base); + return Amode::imm_reg(final_offset as u32, base).with_flags(flags); } } } @@ -642,7 +646,7 @@ fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i let final_offset = (offset as i64).wrapping_add(cst as i64); if low32_will_sign_extend_to_64(final_offset as u64) { let base = put_input_in_reg(ctx, add_inputs[1 - i]); - return Amode::imm_reg(final_offset as u32, base); + return Amode::imm_reg(final_offset as u32, base).with_flags(flags); } } } @@ -654,11 +658,11 @@ fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i ) }; - return Amode::imm_reg_reg_shift(offset as u32, base, index, shift); + return Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags); } let input = put_input_in_reg(ctx, spec); - Amode::imm_reg(offset as u32, input) + Amode::imm_reg(offset as u32, input).with_flags(flags) } //============================================================================= @@ -3060,7 +3064,8 @@ fn lower_insn_to_regs>( let base = put_input_in_reg(ctx, inputs[0]); let index = put_input_in_reg(ctx, inputs[1]); let shift = 0; - Amode::imm_reg_reg_shift(offset as u32, base, index, shift) + let flags = ctx.memflags(insn).expect("load should have memflags"); + Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) } _ => unreachable!(), @@ -3132,7 +3137,8 @@ fn lower_insn_to_regs>( let base = put_input_in_reg(ctx, inputs[1]); let index = put_input_in_reg(ctx, inputs[2]); let shift = 0; - Amode::imm_reg_reg_shift(offset as u32, base, index, shift) + let flags = ctx.memflags(insn).expect("store should have memflags"); + Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags) } _ => unreachable!(),