diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs index a04cd4735f478..82468030f7aca 100644 --- a/compiler/rustc_codegen_gcc/src/asm.rs +++ b/compiler/rustc_codegen_gcc/src/asm.rs @@ -611,9 +611,12 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister { } // They can be retrieved from https://gcc.gnu.org/onlinedocs/gcc/Machine-Constraints.html InlineAsmRegOrRegClass::RegClass(reg) => match reg { - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) + | InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) + | InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg) => "w", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) + | InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => "x", InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { unreachable!("clobber-only") } @@ -698,9 +701,12 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister { /// the type is, as long as it is valid for the constraint code. fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegClass) -> Type<'gcc> { match reg { - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) + | InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) - | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) + | InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg) + | InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => { cx.type_vector(cx.type_i64(), 2) } InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { @@ -863,9 +869,12 @@ fn modifier_to_gcc( // The modifiers can be retrieved from // https://gcc.gnu.org/onlinedocs/gcc/Modifiers.html#Modifiers match reg { - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) + | InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::reg) => modifier, InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) - | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) + | InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg) + | InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => { if modifier == Some('v') { None } else { modifier } } InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 82ca3f519f740..eff2f9fbc70fc 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -656,9 +656,12 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> // The constraints can be retrieved from // https://llvm.org/docs/LangRef.html#supported-constraint-code-list InlineAsmRegOrRegClass::RegClass(reg) => match reg { - AArch64(AArch64InlineAsmRegClass::reg) => "r", - AArch64(AArch64InlineAsmRegClass::vreg) => "w", - AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", + AArch64(AArch64InlineAsmRegClass::reg) | Arm64EC(Arm64ECInlineAsmRegClass::reg) => "r", + AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(Arm64ECInlineAsmRegClass::vreg) => { + "w" + } + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => "x", AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"), Arm(ArmInlineAsmRegClass::reg) => "r", Arm(ArmInlineAsmRegClass::sreg) @@ -734,8 +737,11 @@ fn modifier_to_llvm( // The modifiers can be retrieved from // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers match reg { - AArch64(AArch64InlineAsmRegClass::reg) => modifier, - AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + AArch64(AArch64InlineAsmRegClass::reg) | Arm64EC(Arm64ECInlineAsmRegClass::reg) => modifier, + AArch64(AArch64InlineAsmRegClass::vreg) + | AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => { if modifier == Some('v') { None } else { @@ -817,10 +823,13 @@ fn modifier_to_llvm( fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type { use InlineAsmRegClass::*; match reg { - AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), - AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => { - cx.type_vector(cx.type_i64(), 2) + AArch64(AArch64InlineAsmRegClass::reg) | Arm64EC(Arm64ECInlineAsmRegClass::reg) => { + cx.type_i32() } + AArch64(AArch64InlineAsmRegClass::vreg) + | AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => cx.type_vector(cx.type_i64(), 2), AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"), Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(), Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), @@ -922,7 +931,10 @@ fn llvm_fixup_input<'ll, 'tcx>( use InlineAsmRegClass::*; let dl = &bx.tcx.data_layout; match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + ( + AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(Arm64ECInlineAsmRegClass::vreg), + Abi::Scalar(s), + ) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) @@ -930,9 +942,11 @@ fn llvm_fixup_input<'ll, 'tcx>( value } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) - if s.primitive() != Primitive::Float(Float::F128) => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16), + Abi::Scalar(s), + ) if s.primitive() != Primitive::Float(Float::F128) => { let elem_ty = llvm_asm_scalar_type(bx.cx, s); let count = 16 / layout.size.bytes(); let vec_ty = bx.cx.type_vector(elem_ty, count); @@ -943,9 +957,11 @@ fn llvm_fixup_input<'ll, 'tcx>( } bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) - if layout.size.bytes() == 8 => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); let vec_ty = bx.cx.type_vector(elem_ty, count); let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); @@ -1064,25 +1080,32 @@ fn llvm_fixup_output<'ll, 'tcx>( ) -> &'ll Value { use InlineAsmRegClass::*; match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + ( + AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(Arm64ECInlineAsmRegClass::vreg), + Abi::Scalar(s), + ) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { bx.extract_element(value, bx.const_i32(0)) } else { value } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) - if s.primitive() != Primitive::Float(Float::F128) => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16), + Abi::Scalar(s), + ) if s.primitive() != Primitive::Float(Float::F128) => { value = bx.extract_element(value, bx.const_i32(0)); if let Primitive::Pointer(_) = s.primitive() { value = bx.inttoptr(value, layout.llvm_type(bx.cx)); } value } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) - if layout.size.bytes() == 8 => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); let vec_ty = bx.cx.type_vector(elem_ty, count * 2); let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); @@ -1195,23 +1218,30 @@ fn llvm_fixup_output_type<'ll, 'tcx>( ) -> &'ll Type { use InlineAsmRegClass::*; match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + ( + AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(Arm64ECInlineAsmRegClass::vreg), + Abi::Scalar(s), + ) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { cx.type_vector(cx.type_i8(), 8) } else { layout.llvm_type(cx) } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) - if s.primitive() != Primitive::Float(Float::F128) => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16), + Abi::Scalar(s), + ) if s.primitive() != Primitive::Float(Float::F128) => { let elem_ty = llvm_asm_scalar_type(cx, s); let count = 16 / layout.size.bytes(); cx.type_vector(elem_ty, count) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) - if layout.size.bytes() == 8 => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(cx, element); cx.type_vector(elem_ty, count * 2) } diff --git a/compiler/rustc_target/src/asm/aarch64.rs b/compiler/rustc_target/src/asm/aarch64.rs index 067f5c3a9568b..7f9447b63c251 100644 --- a/compiler/rustc_target/src/asm/aarch64.rs +++ b/compiler/rustc_target/src/asm/aarch64.rs @@ -88,20 +88,6 @@ fn reserved_x18( } } -fn restricted_for_arm64ec( - arch: InlineAsmArch, - _reloc_model: RelocModel, - _target_features: &FxIndexSet, - _target: &Target, - _is_clobber: bool, -) -> Result<(), &'static str> { - if arch == InlineAsmArch::Arm64EC { - Err("x13, x14, x23, x24, x28, v16-v31, z*, p*, ffr cannot be used for Arm64EC") - } else { - Ok(()) - } -} - def_regs! { AArch64 AArch64InlineAsmReg AArch64InlineAsmRegClass { x0: reg = ["x0", "w0"], @@ -117,8 +103,8 @@ def_regs! { x10: reg = ["x10", "w10"], x11: reg = ["x11", "w11"], x12: reg = ["x12", "w12"], - x13: reg = ["x13", "w13"] % restricted_for_arm64ec, - x14: reg = ["x14", "w14"] % restricted_for_arm64ec, + x13: reg = ["x13", "w13"], + x14: reg = ["x14", "w14"], x15: reg = ["x15", "w15"], x16: reg = ["x16", "w16"], x17: reg = ["x17", "w17"], @@ -126,12 +112,12 @@ def_regs! { x20: reg = ["x20", "w20"], x21: reg = ["x21", "w21"], x22: reg = ["x22", "w22"], - x23: reg = ["x23", "w23"] % restricted_for_arm64ec, - x24: reg = ["x24", "w24"] % restricted_for_arm64ec, + x23: reg = ["x23", "w23"], + x24: reg = ["x24", "w24"], x25: reg = ["x25", "w25"], x26: reg = ["x26", "w26"], x27: reg = ["x27", "w27"], - x28: reg = ["x28", "w28"] % restricted_for_arm64ec, + x28: reg = ["x28", "w28"], x30: reg = ["x30", "w30", "lr", "wlr"], // FIXME: z* cannot be used for Arm64EC v0: vreg, vreg_low16 = ["v0", "b0", "h0", "s0", "d0", "q0", "z0"], @@ -150,39 +136,39 @@ def_regs! { v13: vreg, vreg_low16 = ["v13", "b13", "h13", "s13", "d13", "q13", "z13"], v14: vreg, vreg_low16 = ["v14", "b14", "h14", "s14", "d14", "q14", "z14"], v15: vreg, vreg_low16 = ["v15", "b15", "h15", "s15", "d15", "q15", "z15"], - v16: vreg = ["v16", "b16", "h16", "s16", "d16", "q16", "z16"] % restricted_for_arm64ec, - v17: vreg = ["v17", "b17", "h17", "s17", "d17", "q17", "z17"] % restricted_for_arm64ec, - v18: vreg = ["v18", "b18", "h18", "s18", "d18", "q18", "z18"] % restricted_for_arm64ec, - v19: vreg = ["v19", "b19", "h19", "s19", "d19", "q19", "z19"] % restricted_for_arm64ec, - v20: vreg = ["v20", "b20", "h20", "s20", "d20", "q20", "z20"] % restricted_for_arm64ec, - v21: vreg = ["v21", "b21", "h21", "s21", "d21", "q21", "z21"] % restricted_for_arm64ec, - v22: vreg = ["v22", "b22", "h22", "s22", "d22", "q22", "z22"] % restricted_for_arm64ec, - v23: vreg = ["v23", "b23", "h23", "s23", "d23", "q23", "z23"] % restricted_for_arm64ec, - v24: vreg = ["v24", "b24", "h24", "s24", "d24", "q24", "z24"] % restricted_for_arm64ec, - v25: vreg = ["v25", "b25", "h25", "s25", "d25", "q25", "z25"] % restricted_for_arm64ec, - v26: vreg = ["v26", "b26", "h26", "s26", "d26", "q26", "z26"] % restricted_for_arm64ec, - v27: vreg = ["v27", "b27", "h27", "s27", "d27", "q27", "z27"] % restricted_for_arm64ec, - v28: vreg = ["v28", "b28", "h28", "s28", "d28", "q28", "z28"] % restricted_for_arm64ec, - v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29", "z29"] % restricted_for_arm64ec, - v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30", "z30"] % restricted_for_arm64ec, - v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31", "z31"] % restricted_for_arm64ec, - p0: preg = ["p0"] % restricted_for_arm64ec, - p1: preg = ["p1"] % restricted_for_arm64ec, - p2: preg = ["p2"] % restricted_for_arm64ec, - p3: preg = ["p3"] % restricted_for_arm64ec, - p4: preg = ["p4"] % restricted_for_arm64ec, - p5: preg = ["p5"] % restricted_for_arm64ec, - p6: preg = ["p6"] % restricted_for_arm64ec, - p7: preg = ["p7"] % restricted_for_arm64ec, - p8: preg = ["p8"] % restricted_for_arm64ec, - p9: preg = ["p9"] % restricted_for_arm64ec, - p10: preg = ["p10"] % restricted_for_arm64ec, - p11: preg = ["p11"] % restricted_for_arm64ec, - p12: preg = ["p12"] % restricted_for_arm64ec, - p13: preg = ["p13"] % restricted_for_arm64ec, - p14: preg = ["p14"] % restricted_for_arm64ec, - p15: preg = ["p15"] % restricted_for_arm64ec, - ffr: preg = ["ffr"] % restricted_for_arm64ec, + v16: vreg = ["v16", "b16", "h16", "s16", "d16", "q16", "z16"], + v17: vreg = ["v17", "b17", "h17", "s17", "d17", "q17", "z17"], + v18: vreg = ["v18", "b18", "h18", "s18", "d18", "q18", "z18"], + v19: vreg = ["v19", "b19", "h19", "s19", "d19", "q19", "z19"], + v20: vreg = ["v20", "b20", "h20", "s20", "d20", "q20", "z20"], + v21: vreg = ["v21", "b21", "h21", "s21", "d21", "q21", "z21"], + v22: vreg = ["v22", "b22", "h22", "s22", "d22", "q22", "z22"], + v23: vreg = ["v23", "b23", "h23", "s23", "d23", "q23", "z23"], + v24: vreg = ["v24", "b24", "h24", "s24", "d24", "q24", "z24"], + v25: vreg = ["v25", "b25", "h25", "s25", "d25", "q25", "z25"], + v26: vreg = ["v26", "b26", "h26", "s26", "d26", "q26", "z26"], + v27: vreg = ["v27", "b27", "h27", "s27", "d27", "q27", "z27"], + v28: vreg = ["v28", "b28", "h28", "s28", "d28", "q28", "z28"], + v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29", "z29"], + v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30", "z30"], + v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31", "z31"], + p0: preg = ["p0"], + p1: preg = ["p1"], + p2: preg = ["p2"], + p3: preg = ["p3"], + p4: preg = ["p4"], + p5: preg = ["p5"], + p6: preg = ["p6"], + p7: preg = ["p7"], + p8: preg = ["p8"], + p9: preg = ["p9"], + p10: preg = ["p10"], + p11: preg = ["p11"], + p12: preg = ["p12"], + p13: preg = ["p13"], + p14: preg = ["p14"], + p15: preg = ["p15"], + ffr: preg = ["ffr"], #error = ["x19", "w19"] => "x19 is used internally by LLVM and cannot be used as an operand for inline asm", #error = ["x29", "w29", "fp", "wfp"] => diff --git a/compiler/rustc_target/src/asm/arm64ec.rs b/compiler/rustc_target/src/asm/arm64ec.rs new file mode 100644 index 0000000000000..d3fd731abd2d5 --- /dev/null +++ b/compiler/rustc_target/src/asm/arm64ec.rs @@ -0,0 +1,185 @@ +use std::fmt; + +use rustc_data_structures::fx::FxIndexSet; +use rustc_span::Symbol; + +use super::aarch64::target_reserves_x18; +use super::{InlineAsmArch, InlineAsmType, ModifierInfo}; +use crate::spec::{RelocModel, Target}; + +def_reg_class! { + Arm64EC Arm64ECInlineAsmRegClass { + reg, + vreg, + vreg_low16, + } +} + +impl Arm64ECInlineAsmRegClass { + pub fn valid_modifiers(self, _arch: super::InlineAsmArch) -> &'static [char] { + match self { + Self::reg => &['w', 'x'], + Self::vreg | Self::vreg_low16 => &['b', 'h', 's', 'd', 'q', 'v'], + } + } + + pub fn suggest_class(self, _arch: InlineAsmArch, _ty: InlineAsmType) -> Option { + None + } + + pub fn suggest_modifier(self, _arch: InlineAsmArch, ty: InlineAsmType) -> Option { + match self { + Self::reg => match ty.size().bits() { + 64 => None, + _ => Some(('w', "w0", 32).into()), + }, + Self::vreg | Self::vreg_low16 => match ty.size().bits() { + 8 => Some(('b', "b0", 8).into()), + 16 => Some(('h', "h0", 16).into()), + 32 => Some(('s', "s0", 32).into()), + 64 => Some(('d', "d0", 64).into()), + 128 => Some(('q', "q0", 128).into()), + _ => None, + }, + } + } + + pub fn default_modifier(self, _arch: InlineAsmArch) -> Option { + match self { + Self::reg => Some(('x', "x0", 64).into()), + Self::vreg | Self::vreg_low16 => Some(('v', "v0", 128).into()), + } + } + + pub fn supported_types( + self, + _arch: InlineAsmArch, + ) -> &'static [(InlineAsmType, Option)] { + match self { + Self::reg => types! { _: I8, I16, I32, I64, F16, F32, F64; }, + Self::vreg | Self::vreg_low16 => types! { + neon: I8, I16, I32, I64, F16, F32, F64, F128, + VecI8(8), VecI16(4), VecI32(2), VecI64(1), VecF16(4), VecF32(2), VecF64(1), + VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF16(8), VecF32(4), VecF64(2); + }, + } + } +} + +fn reserved_x18( + _arch: InlineAsmArch, + _reloc_model: RelocModel, + _target_features: &FxIndexSet, + target: &Target, + _is_clobber: bool, +) -> Result<(), &'static str> { + if target_reserves_x18(target) { + Err("x18 is a reserved register on this target") + } else { + Ok(()) + } +} + +def_regs! { + Arm64EC Arm64ECInlineAsmReg Arm64ECInlineAsmRegClass { + x0: reg = ["x0", "w0"], + x1: reg = ["x1", "w1"], + x2: reg = ["x2", "w2"], + x3: reg = ["x3", "w3"], + x4: reg = ["x4", "w4"], + x5: reg = ["x5", "w5"], + x6: reg = ["x6", "w6"], + x7: reg = ["x7", "w7"], + x8: reg = ["x8", "w8"], + x9: reg = ["x9", "w9"], + x10: reg = ["x10", "w10"], + x11: reg = ["x11", "w11"], + x12: reg = ["x12", "w12"], + x15: reg = ["x15", "w15"], + x16: reg = ["x16", "w16"], + x17: reg = ["x17", "w17"], + x18: reg = ["x18", "w18"] % reserved_x18, + x20: reg = ["x20", "w20"], + x21: reg = ["x21", "w21"], + x22: reg = ["x22", "w22"], + x25: reg = ["x25", "w25"], + x26: reg = ["x26", "w26"], + x27: reg = ["x27", "w27"], + x30: reg = ["x30", "w30", "lr", "wlr"], + v0: vreg, vreg_low16 = ["v0", "b0", "h0", "s0", "d0", "q0"], + v1: vreg, vreg_low16 = ["v1", "b1", "h1", "s1", "d1", "q1"], + v2: vreg, vreg_low16 = ["v2", "b2", "h2", "s2", "d2", "q2"], + v3: vreg, vreg_low16 = ["v3", "b3", "h3", "s3", "d3", "q3"], + v4: vreg, vreg_low16 = ["v4", "b4", "h4", "s4", "d4", "q4"], + v5: vreg, vreg_low16 = ["v5", "b5", "h5", "s5", "d5", "q5"], + v6: vreg, vreg_low16 = ["v6", "b6", "h6", "s6", "d6", "q6"], + v7: vreg, vreg_low16 = ["v7", "b7", "h7", "s7", "d7", "q7"], + v8: vreg, vreg_low16 = ["v8", "b8", "h8", "s8", "d8", "q8"], + v9: vreg, vreg_low16 = ["v9", "b9", "h9", "s9", "d9", "q9"], + v10: vreg, vreg_low16 = ["v10", "b10", "h10", "s10", "d10", "q10"], + v11: vreg, vreg_low16 = ["v11", "b11", "h11", "s11", "d11", "q11"], + v12: vreg, vreg_low16 = ["v12", "b12", "h12", "s12", "d12", "q12"], + v13: vreg, vreg_low16 = ["v13", "b13", "h13", "s13", "d13", "q13"], + v14: vreg, vreg_low16 = ["v14", "b14", "h14", "s14", "d14", "q14"], + v15: vreg, vreg_low16 = ["v15", "b15", "h15", "s15", "d15", "q15"], + #error = ["x19", "w19"] => + "x19 is used internally by LLVM and cannot be used as an operand for inline asm", + #error = ["x29", "w29", "fp", "wfp"] => + "the frame pointer cannot be used as an operand for inline asm", + #error = ["sp", "wsp"] => + "the stack pointer cannot be used as an operand for inline asm", + #error = ["xzr", "wzr"] => + "the zero register cannot be used as an operand for inline asm", + #error = [ + "x13", "w13", + "x14", "w14", + "x23", "w23", + "x24", "w24", + "x28", "w28", + "v16", "b16", "h16", "s16", "d16", "q16", + "v17", "b17", "h17", "s17", "d17", "q17", + "v18", "b18", "h18", "s18", "d18", "q18", + "v19", "b19", "h19", "s19", "d19", "q19", + "v20", "b20", "h20", "s20", "d20", "q20", + "v21", "b21", "h21", "s21", "d21", "q21", + "v22", "b22", "h22", "s22", "d22", "q22", + "v23", "b23", "h23", "s23", "d23", "q23", + "v24", "b24", "h24", "s24", "d24", "q24", + "v25", "b25", "h25", "s25", "d25", "q25", + "v26", "b26", "h26", "s26", "d26", "q26", + "v27", "b27", "h27", "s27", "d27", "q27", + "v28", "b28", "h28", "s28", "d28", "q28", + "v29", "b29", "h29", "s29", "d29", "q29", + "v30", "b30", "h30", "s30", "d30", "q30", + "v31", "b31", "h31", "s31", "d31", "q31" + ] => + "x13, x14, x23, x24, x28, v16-v31 cannot be used for Arm64EC", + #error = [ + "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", + "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", + "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", + "z30", "z31", + "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", + "p10", "p11", "p12", "p13", "p14", "p15", + "ffr" + ] => + "SVE cannot be used for Arm64EC", + } +} + +impl Arm64ECInlineAsmReg { + pub fn emit( + self, + out: &mut dyn fmt::Write, + _arch: InlineAsmArch, + modifier: Option, + ) -> fmt::Result { + let (prefix, index) = if (self as u32) < Self::v0 as u32 { + (modifier.unwrap_or('x'), self as u32 - Self::x0 as u32) + } else { + (modifier.unwrap_or('v'), self as u32 - Self::v0 as u32) + }; + assert!(index < 32); + write!(out, "{prefix}{index}") + } +} diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs index ef5143d804be1..c604d5b3fbc07 100644 --- a/compiler/rustc_target/src/asm/mod.rs +++ b/compiler/rustc_target/src/asm/mod.rs @@ -179,6 +179,7 @@ macro_rules! types { mod aarch64; mod arm; +mod arm64ec; mod avr; mod bpf; mod csky; @@ -197,6 +198,7 @@ mod x86; pub use aarch64::{AArch64InlineAsmReg, AArch64InlineAsmRegClass}; pub use arm::{ArmInlineAsmReg, ArmInlineAsmRegClass}; +pub use arm64ec::{Arm64ECInlineAsmReg, Arm64ECInlineAsmRegClass}; pub use avr::{AvrInlineAsmReg, AvrInlineAsmRegClass}; pub use bpf::{BpfInlineAsmReg, BpfInlineAsmRegClass}; pub use csky::{CSKYInlineAsmReg, CSKYInlineAsmRegClass}; @@ -279,6 +281,7 @@ pub enum InlineAsmReg { X86(X86InlineAsmReg), Arm(ArmInlineAsmReg), AArch64(AArch64InlineAsmReg), + Arm64EC(Arm64ECInlineAsmReg), RiscV(RiscVInlineAsmReg), Nvptx(NvptxInlineAsmReg), PowerPC(PowerPCInlineAsmReg), @@ -303,6 +306,7 @@ impl InlineAsmReg { Self::X86(r) => r.name(), Self::Arm(r) => r.name(), Self::AArch64(r) => r.name(), + Self::Arm64EC(r) => r.name(), Self::RiscV(r) => r.name(), Self::PowerPC(r) => r.name(), Self::Hexagon(r) => r.name(), @@ -323,6 +327,7 @@ impl InlineAsmReg { Self::X86(r) => InlineAsmRegClass::X86(r.reg_class()), Self::Arm(r) => InlineAsmRegClass::Arm(r.reg_class()), Self::AArch64(r) => InlineAsmRegClass::AArch64(r.reg_class()), + Self::Arm64EC(r) => InlineAsmRegClass::Arm64EC(r.reg_class()), Self::RiscV(r) => InlineAsmRegClass::RiscV(r.reg_class()), Self::PowerPC(r) => InlineAsmRegClass::PowerPC(r.reg_class()), Self::Hexagon(r) => InlineAsmRegClass::Hexagon(r.reg_class()), @@ -345,9 +350,8 @@ impl InlineAsmReg { Ok(match arch { InlineAsmArch::X86 | InlineAsmArch::X86_64 => Self::X86(X86InlineAsmReg::parse(name)?), InlineAsmArch::Arm => Self::Arm(ArmInlineAsmReg::parse(name)?), - InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC => { - Self::AArch64(AArch64InlineAsmReg::parse(name)?) - } + InlineAsmArch::AArch64 => Self::AArch64(AArch64InlineAsmReg::parse(name)?), + InlineAsmArch::Arm64EC => Self::Arm64EC(Arm64ECInlineAsmReg::parse(name)?), InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { Self::RiscV(RiscVInlineAsmReg::parse(name)?) } @@ -385,6 +389,7 @@ impl InlineAsmReg { Self::X86(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::Arm(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::AArch64(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), + Self::Arm64EC(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::RiscV(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::PowerPC(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::Hexagon(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), @@ -414,6 +419,7 @@ impl InlineAsmReg { Self::X86(r) => r.emit(out, arch, modifier), Self::Arm(r) => r.emit(out, arch, modifier), Self::AArch64(r) => r.emit(out, arch, modifier), + Self::Arm64EC(r) => r.emit(out, arch, modifier), Self::RiscV(r) => r.emit(out, arch, modifier), Self::PowerPC(r) => r.emit(out, arch, modifier), Self::Hexagon(r) => r.emit(out, arch, modifier), @@ -434,6 +440,7 @@ impl InlineAsmReg { Self::X86(r) => r.overlapping_regs(|r| cb(Self::X86(r))), Self::Arm(r) => r.overlapping_regs(|r| cb(Self::Arm(r))), Self::AArch64(_) => cb(self), + Self::Arm64EC(_) => cb(self), Self::RiscV(_) => cb(self), Self::PowerPC(r) => r.overlapping_regs(|r| cb(Self::PowerPC(r))), Self::Hexagon(r) => r.overlapping_regs(|r| cb(Self::Hexagon(r))), @@ -456,6 +463,7 @@ pub enum InlineAsmRegClass { X86(X86InlineAsmRegClass), Arm(ArmInlineAsmRegClass), AArch64(AArch64InlineAsmRegClass), + Arm64EC(Arm64ECInlineAsmRegClass), RiscV(RiscVInlineAsmRegClass), Nvptx(NvptxInlineAsmRegClass), PowerPC(PowerPCInlineAsmRegClass), @@ -480,6 +488,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.name(), Self::Arm(r) => r.name(), Self::AArch64(r) => r.name(), + Self::Arm64EC(r) => r.name(), Self::RiscV(r) => r.name(), Self::Nvptx(r) => r.name(), Self::PowerPC(r) => r.name(), @@ -506,6 +515,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::X86), Self::Arm(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Arm), Self::AArch64(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::AArch64), + Self::Arm64EC(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Arm64EC), Self::RiscV(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::RiscV), Self::Nvptx(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Nvptx), Self::PowerPC(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::PowerPC), @@ -535,6 +545,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.suggest_modifier(arch, ty), Self::Arm(r) => r.suggest_modifier(arch, ty), Self::AArch64(r) => r.suggest_modifier(arch, ty), + Self::Arm64EC(r) => r.suggest_modifier(arch, ty), Self::RiscV(r) => r.suggest_modifier(arch, ty), Self::Nvptx(r) => r.suggest_modifier(arch, ty), Self::PowerPC(r) => r.suggest_modifier(arch, ty), @@ -564,6 +575,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.default_modifier(arch), Self::Arm(r) => r.default_modifier(arch), Self::AArch64(r) => r.default_modifier(arch), + Self::Arm64EC(r) => r.default_modifier(arch), Self::RiscV(r) => r.default_modifier(arch), Self::Nvptx(r) => r.default_modifier(arch), Self::PowerPC(r) => r.default_modifier(arch), @@ -592,6 +604,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.supported_types(arch), Self::Arm(r) => r.supported_types(arch), Self::AArch64(r) => r.supported_types(arch), + Self::Arm64EC(r) => r.supported_types(arch), Self::RiscV(r) => r.supported_types(arch), Self::Nvptx(r) => r.supported_types(arch), Self::PowerPC(r) => r.supported_types(arch), @@ -616,9 +629,8 @@ impl InlineAsmRegClass { Self::X86(X86InlineAsmRegClass::parse(name)?) } InlineAsmArch::Arm => Self::Arm(ArmInlineAsmRegClass::parse(name)?), - InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC => { - Self::AArch64(AArch64InlineAsmRegClass::parse(name)?) - } + InlineAsmArch::AArch64 => Self::AArch64(AArch64InlineAsmRegClass::parse(name)?), + InlineAsmArch::Arm64EC => Self::Arm64EC(Arm64ECInlineAsmRegClass::parse(name)?), InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { Self::RiscV(RiscVInlineAsmRegClass::parse(name)?) } @@ -651,6 +663,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.valid_modifiers(arch), Self::Arm(r) => r.valid_modifiers(arch), Self::AArch64(r) => r.valid_modifiers(arch), + Self::Arm64EC(r) => r.valid_modifiers(arch), Self::RiscV(r) => r.valid_modifiers(arch), Self::Nvptx(r) => r.valid_modifiers(arch), Self::PowerPC(r) => r.valid_modifiers(arch), @@ -803,11 +816,16 @@ pub fn allocatable_registers( arm::fill_reg_map(arch, reloc_model, target_features, target, &mut map); map } - InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC => { + InlineAsmArch::AArch64 => { let mut map = aarch64::regclass_map(); aarch64::fill_reg_map(arch, reloc_model, target_features, target, &mut map); map } + InlineAsmArch::Arm64EC => { + let mut map = arm64ec::regclass_map(); + arm64ec::fill_reg_map(arch, reloc_model, target_features, target, &mut map); + map + } InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { let mut map = riscv::regclass_map(); riscv::fill_reg_map(arch, reloc_model, target_features, target, &mut map); @@ -1050,7 +1068,7 @@ impl InlineAsmClobberAbi { } }, InlineAsmClobberAbi::Arm64EC => clobbered_regs! { - AArch64 AArch64InlineAsmReg { + Arm64EC Arm64ECInlineAsmReg { // x13 and x14 cannot be used in Arm64EC. x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x15, diff --git a/tests/ui/asm/aarch64/arm64ec-sve.rs b/tests/ui/asm/aarch64/arm64ec-sve.rs index 269362a41e865..96db2e35a40e0 100644 --- a/tests/ui/asm/aarch64/arm64ec-sve.rs +++ b/tests/ui/asm/aarch64/arm64ec-sve.rs @@ -22,11 +22,12 @@ macro_rules! asm { fn f(x: f64) { unsafe { - // FIXME: z* cannot be used for Arm64EC asm!("", out("z0") _, in("z1") x); + //~^ ERROR invalid register `z0`: SVE cannot be used for Arm64EC + //~^^ ERROR invalid register `z1`: SVE cannot be used for Arm64EC asm!("", out("p0") _); - //~^ ERROR cannot use register `p0` + //~^ ERROR invalid register `p0`: SVE cannot be used for Arm64EC asm!("", out("ffr") _); - //~^ ERROR cannot use register `ffr` + //~^ ERROR invalid register `ffr`: SVE cannot be used for Arm64EC } } diff --git a/tests/ui/asm/aarch64/arm64ec-sve.stderr b/tests/ui/asm/aarch64/arm64ec-sve.stderr index c88177d759ecf..4d2ec6833de13 100644 --- a/tests/ui/asm/aarch64/arm64ec-sve.stderr +++ b/tests/ui/asm/aarch64/arm64ec-sve.stderr @@ -1,14 +1,26 @@ -error: cannot use register `p0`: x13, x14, x23, x24, x28, v16-v31, z*, p*, ffr cannot be used for Arm64EC - --> $DIR/arm64ec-sve.rs:27:18 +error: invalid register `z0`: SVE cannot be used for Arm64EC + --> $DIR/arm64ec-sve.rs:25:18 + | +LL | asm!("", out("z0") _, in("z1") x); + | ^^^^^^^^^^^ + +error: invalid register `z1`: SVE cannot be used for Arm64EC + --> $DIR/arm64ec-sve.rs:25:31 + | +LL | asm!("", out("z0") _, in("z1") x); + | ^^^^^^^^^^ + +error: invalid register `p0`: SVE cannot be used for Arm64EC + --> $DIR/arm64ec-sve.rs:28:18 | LL | asm!("", out("p0") _); | ^^^^^^^^^^^ -error: cannot use register `ffr`: x13, x14, x23, x24, x28, v16-v31, z*, p*, ffr cannot be used for Arm64EC - --> $DIR/arm64ec-sve.rs:29:18 +error: invalid register `ffr`: SVE cannot be used for Arm64EC + --> $DIR/arm64ec-sve.rs:30:18 | LL | asm!("", out("ffr") _); | ^^^^^^^^^^^^ -error: aborting due to 2 previous errors +error: aborting due to 4 previous errors