Merge branch 'main' into bti

Cargo.toml

@@ -51,12 +51,12 @@ test-programs = { path = "crates/test-programs" }
 wasmtime-runtime = { path = "crates/runtime" }
 tokio = { version = "1.8.0", features = ["rt", "time", "macros", "rt-multi-thread"] }
 tracing-subscriber = "0.3.1"
-wast = "45.0.0"
+wast = "46.0.0"
 criterion = "0.3.4"
 num_cpus = "1.13.0"
 memchr = "2.4"
 async-trait = "0.1"
-wat = "1.0.47"
+wat = "1.0.48"
 once_cell = "1.9.0"
 rayon = "1.5.0"
 component-macro-test = { path = "crates/misc/component-macro-test" }

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -780,6 +780,11 @@
        (Pacisp
         (key APIKey))
 
+       ;; Strip pointer authentication code from instruction address in LR;
+       ;; equivalent to a no-op if Pointer authentication (FEAT_PAuth) is not
+       ;; supported.
+       (Xpaclri)
+
        ;; Branch target identification; equivalent to a no-op if Branch Target
        ;; Identification (FEAT_BTI) is not supported.
        (Bti
@@ -1370,6 +1375,9 @@
 ))
 
 ;; Extractors for target features ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(decl pure sign_return_address_disabled () Unit)
+(extern constructor sign_return_address_disabled sign_return_address_disabled)
+
 (decl use_lse () Inst)
 (extern extractor use_lse use_lse)
 
@@ -2591,4 +2599,11 @@
 
 (decl aarch64_link () Reg)
 (rule (aarch64_link)
+      (if (sign_return_address_disabled))
       (mov_preg (preg_link)))
+
+(rule (aarch64_link)
+      ;; This constructor is always used for non-leaf functions, so it is safe
+      ;; to clobber LR.
+      (let ((_ Unit (emit (MInst.Xpaclri))))
+           (mov_preg (preg_link))))

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -3121,6 +3121,7 @@ impl MachInstEmit for Inst {
 
                 sink.put4(0xd503233f | key << 6);
             }
+            &Inst::Xpaclri => sink.put4(0xd50320ff),
             &Inst::Bti { targets } => {
                 let targets = match targets {
                     BranchTargetType::None => 0b00,

cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs

@@ -57,6 +57,7 @@ fn test_aarch64_binemit() {
         "retab",
     ));
     insns.push((Inst::Pacisp { key: APIKey::B }, "7F2303D5", "pacibsp"));
+    insns.push((Inst::Xpaclri, "FF2003D5", "xpaclri"));
     insns.push((
         Inst::Bti {
             targets: BranchTargetType::J,

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -981,12 +981,7 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
             collector.reg_def(rd);
             collector.reg_use(rn);
         }
-        &Inst::Ret { ref rets } => {
-            for &ret in rets {
-                collector.reg_use(ret);
-            }
-        }
-        &Inst::AuthenticatedRet { ref rets, .. } => {
+        &Inst::Ret { ref rets } | &Inst::AuthenticatedRet { ref rets, .. } => {
             for &ret in rets {
                 collector.reg_use(ret);
             }
@@ -1039,7 +1034,10 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
             collector.reg_def(rd);
             memarg_operands(mem, collector);
         }
-        &Inst::Pacisp { .. } => {}
+        &Inst::Pacisp { .. } | &Inst::Xpaclri => {
+            // Neither LR nor SP is an allocatable register, so there is no need
+            // to do anything.
+        }
         &Inst::Bti { .. } => {}
         &Inst::VirtualSPOffsetAdj { .. } => {}
 
@@ -2729,6 +2727,7 @@ impl Inst {
 
                 "paci".to_string() + key + "sp"
             }
+            &Inst::Xpaclri => "xpaclri".to_string(),
             &Inst::Bti { targets } => {
                 let targets = match targets {
                     BranchTargetType::None => "",

cranelift/codegen/src/isa/aarch64/lower/isle.rs

@@ -71,6 +71,14 @@ pub struct SinkableAtomicLoad {
 impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
     isle_prelude_methods!();
 
+    fn sign_return_address_disabled(&mut self) -> Option<()> {
+        if self.isa_flags.sign_return_address() {
+            None
+        } else {
+            Some(())
+        }
+    }
+
     fn use_lse(&mut self, _: Inst) -> Option<()> {
         if self.isa_flags.has_lse() {
             Some(())

cranelift/codegen/src/isa/mod.rs

@@ -317,6 +317,15 @@ impl<'a> dyn TargetIsa + 'a {
         }
     }
 
+    /// Returns the minimum symbol alignment for this ISA.
+    pub fn symbol_alignment(&self) -> u64 {
+        match self.triple().architecture {
+            // All symbols need to be aligned to at least 2 on s390x.
+            Architecture::S390x => 2,
+            _ => 1,
+        }
+    }
+
     /// Get the pointer type of this ISA.
     pub fn pointer_type(&self) -> ir::Type {
         ir::Type::int(self.pointer_bits() as u16).unwrap()

cranelift/codegen/src/isa/s390x/inst/emit.rs

@@ -11,17 +11,28 @@ use crate::trace;
 use core::convert::TryFrom;
 use regalloc2::Allocation;
 
+/// Type(s) of memory instructions available for mem_finalize.
+pub struct MemInstType {
+    /// True if 12-bit unsigned displacement is supported.
+    pub have_d12: bool,
+    /// True if 20-bit signed displacement is supported.
+    pub have_d20: bool,
+    /// True if PC-relative addressing is supported (memory access).
+    pub have_pcrel: bool,
+    /// True if PC-relative addressing is supported (load address).
+    pub have_unaligned_pcrel: bool,
+    /// True if an index register is supported.
+    pub have_index: bool,
+}
+
 /// Memory addressing mode finalization: convert "special" modes (e.g.,
 /// generic arbitrary stack offset) into real addressing modes, possibly by
 /// emitting some helper instructions that come immediately before the use
 /// of this amode.
 pub fn mem_finalize(
     mem: &MemArg,
     state: &EmitState,
-    have_d12: bool,
-    have_d20: bool,
-    have_pcrel: bool,
-    have_index: bool,
+    mi: MemInstType,
 ) -> (SmallVec<[Inst; 4]>, MemArg) {
     let mut insts = SmallVec::new();
 
@@ -70,9 +81,10 @@ pub fn mem_finalize(
 
     // If this addressing mode cannot be handled by the instruction, use load-address.
     let need_load_address = match &mem {
-        &MemArg::Label { .. } | &MemArg::Symbol { .. } if !have_pcrel => true,
-        &MemArg::BXD20 { .. } if !have_d20 => true,
-        &MemArg::BXD12 { index, .. } | &MemArg::BXD20 { index, .. } if !have_index => {
+        &MemArg::Label { .. } | &MemArg::Symbol { .. } if !mi.have_pcrel => true,
+        &MemArg::Symbol { flags, .. } if !mi.have_unaligned_pcrel && !flags.aligned() => true,
+        &MemArg::BXD20 { .. } if !mi.have_d20 => true,
+        &MemArg::BXD12 { index, .. } | &MemArg::BXD20 { index, .. } if !mi.have_index => {
             index != zero_reg()
         }
         _ => false,
@@ -93,8 +105,8 @@ pub fn mem_finalize(
             index,
             disp,
             flags,
-        } if !have_d12 => {
-            assert!(have_d20);
+        } if !mi.have_d12 => {
+            assert!(mi.have_d20);
             MemArg::BXD20 {
                 base,
                 index,
@@ -122,10 +134,13 @@ pub fn mem_emit(
     let (mem_insts, mem) = mem_finalize(
         mem,
         state,
-        opcode_rx.is_some(),
-        opcode_rxy.is_some(),
-        opcode_ril.is_some(),
-        true,
+        MemInstType {
+            have_d12: opcode_rx.is_some(),
+            have_d20: opcode_rxy.is_some(),
+            have_pcrel: opcode_ril.is_some(),
+            have_unaligned_pcrel: opcode_ril.is_some() && !add_trap,
+            have_index: true,
+        },
     );
     for inst in mem_insts.into_iter() {
         inst.emit(&[], sink, emit_info, state);
@@ -190,10 +205,13 @@ pub fn mem_rs_emit(
     let (mem_insts, mem) = mem_finalize(
         mem,
         state,
-        opcode_rs.is_some(),
-        opcode_rsy.is_some(),
-        false,
-        false,
+        MemInstType {
+            have_d12: opcode_rs.is_some(),
+            have_d20: opcode_rsy.is_some(),
+            have_pcrel: false,
+            have_unaligned_pcrel: false,
+            have_index: false,
+        },
     );
     for inst in mem_insts.into_iter() {
         inst.emit(&[], sink, emit_info, state);
@@ -236,7 +254,17 @@ pub fn mem_imm8_emit(
     emit_info: &EmitInfo,
     state: &mut EmitState,
 ) {
-    let (mem_insts, mem) = mem_finalize(mem, state, true, true, false, false);
+    let (mem_insts, mem) = mem_finalize(
+        mem,
+        state,
+        MemInstType {
+            have_d12: true,
+            have_d20: true,
+            have_pcrel: false,
+            have_unaligned_pcrel: false,
+            have_index: false,
+        },
+    );
     for inst in mem_insts.into_iter() {
         inst.emit(&[], sink, emit_info, state);
     }
@@ -274,7 +302,17 @@ pub fn mem_imm16_emit(
     emit_info: &EmitInfo,
     state: &mut EmitState,
 ) {
-    let (mem_insts, mem) = mem_finalize(mem, state, true, false, false, false);
+    let (mem_insts, mem) = mem_finalize(
+        mem,
+        state,
+        MemInstType {
+            have_d12: true,
+            have_d20: false,
+            have_pcrel: false,
+            have_unaligned_pcrel: false,
+            have_index: false,
+        },
+    );
     for inst in mem_insts.into_iter() {
         inst.emit(&[], sink, emit_info, state);
     }
@@ -336,7 +374,17 @@ pub fn mem_vrx_emit(
     emit_info: &EmitInfo,
     state: &mut EmitState,
 ) {
-    let (mem_insts, mem) = mem_finalize(mem, state, true, false, false, true);
+    let (mem_insts, mem) = mem_finalize(
+        mem,
+        state,
+        MemInstType {
+            have_d12: true,
+            have_d20: false,
+            have_pcrel: false,
+            have_unaligned_pcrel: false,
+            have_index: true,
+        },
+    );
     for inst in mem_insts.into_iter() {
         inst.emit(&[], sink, emit_info, state);
     }