Skip to content

Commit

Permalink
[GlobalISel] Don't form anyextending atomic loads.
Browse files Browse the repository at this point in the history
Until we can reliably check the legality and improve our selection of these,
don't form them at all.
  • Loading branch information
aemerson committed Apr 5, 2024
1 parent 4abb722 commit 60fc4ac
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 59 deletions.
4 changes: 2 additions & 2 deletions llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -597,8 +597,8 @@ bool CombinerHelper::matchCombineExtendingLoads(MachineInstr &MI,
UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
(UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
const auto &MMO = LoadMI->getMMO();
// For atomics, only form anyextending loads.
if (MMO.isAtomic() && UseMI.getOpcode() != TargetOpcode::G_ANYEXT)
// Don't do anything for atomics.
if (MMO.isAtomic())
continue;
// Check for legality.
if (!isPreLegalize()) {
Expand Down
55 changes: 40 additions & 15 deletions llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,24 @@ define i8 @load_atomic_i8_aligned_monotonic_const(ptr readonly %ptr) {
}

define i8 @load_atomic_i8_aligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_acquire:
; CHECK: ldapurb w0, [x0, #4]
; GISEL-LABEL: load_atomic_i8_aligned_acquire:
; GISEL: add x8, x0, #4
; GISEL: ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_aligned_acquire:
; SDAG: ldapurb w0, [x0, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
%r = load atomic i8, ptr %gep acquire, align 1
ret i8 %r
}

define i8 @load_atomic_i8_aligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_acquire_const:
; CHECK: ldapurb w0, [x0, #4]
; GISEL-LABEL: load_atomic_i8_aligned_acquire_const:
; GISEL: add x8, x0, #4
; GISEL: ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_aligned_acquire_const:
; SDAG: ldapurb w0, [x0, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
%r = load atomic i8, ptr %gep acquire, align 1
ret i8 %r
Expand Down Expand Up @@ -101,16 +109,24 @@ define i16 @load_atomic_i16_aligned_monotonic_const(ptr readonly %ptr) {
}

define i16 @load_atomic_i16_aligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_acquire:
; CHECK: ldapurh w0, [x0, #8]
; GISEL-LABEL: load_atomic_i16_aligned_acquire:
; GISEL: add x8, x0, #8
; GISEL: ldaprh w0, [x8]
;
; SDAG-LABEL: load_atomic_i16_aligned_acquire:
; SDAG: ldapurh w0, [x0, #8]
%gep = getelementptr inbounds i16, ptr %ptr, i32 4
%r = load atomic i16, ptr %gep acquire, align 2
ret i16 %r
}

define i16 @load_atomic_i16_aligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_acquire_const:
; CHECK: ldapurh w0, [x0, #8]
; GISEL-LABEL: load_atomic_i16_aligned_acquire_const:
; GISEL: add x8, x0, #8
; GISEL: ldaprh w0, [x8]
;
; SDAG-LABEL: load_atomic_i16_aligned_acquire_const:
; SDAG: ldapurh w0, [x0, #8]
%gep = getelementptr inbounds i16, ptr %ptr, i32 4
%r = load atomic i16, ptr %gep acquire, align 2
ret i16 %r
Expand Down Expand Up @@ -367,16 +383,24 @@ define i8 @load_atomic_i8_unaligned_monotonic_const(ptr readonly %ptr) {
}

define i8 @load_atomic_i8_unaligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_acquire:
; CHECK: ldapurb w0, [x0, #4]
; GISEL-LABEL: load_atomic_i8_unaligned_acquire:
; GISEL: add x8, x0, #4
; GISEL: ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_unaligned_acquire:
; SDAG: ldapurb w0, [x0, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
%r = load atomic i8, ptr %gep acquire, align 1
ret i8 %r
}

define i8 @load_atomic_i8_unaligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_acquire_const:
; CHECK: ldapurb w0, [x0, #4]
; GISEL-LABEL: load_atomic_i8_unaligned_acquire_const:
; GISEL: add x8, x0, #4
; GISEL: ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_unaligned_acquire_const:
; SDAG: ldapurb w0, [x0, #4]
%gep = getelementptr inbounds i8, ptr %ptr, i32 4
%r = load atomic i8, ptr %gep acquire, align 1
ret i8 %r
Expand Down Expand Up @@ -819,7 +843,8 @@ define i128 @load_atomic_i128_unaligned_seq_cst_const(ptr readonly %ptr) {
define i8 @load_atomic_i8_from_gep() {
; GISEL-LABEL: load_atomic_i8_from_gep:
; GISEL: bl init
; GISEL: ldapurb w0, [x8, #1]
; GISEL: add x8, x8, #1
; GISEL: ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_from_gep:
; SDAG: bl init
Expand All @@ -834,7 +859,8 @@ define i8 @load_atomic_i8_from_gep() {
define i16 @load_atomic_i16_from_gep() {
; GISEL-LABEL: load_atomic_i16_from_gep:
; GISEL: bl init
; GISEL: ldapurh w0, [x8, #2]
; GISEL: add x8, x8, #2
; GISEL: ldaprh w0, [x8]
;
; SDAG-LABEL: load_atomic_i16_from_gep:
; SDAG: bl init
Expand Down Expand Up @@ -884,7 +910,6 @@ define i128 @load_atomic_i128_from_gep() {
;
; SDAG-LABEL: load_atomic_i128_from_gep:
; SDAG: bl init
; SDAG: ldp x0, x1, [sp, #16]
; SDAG: dmb ishld
%a = alloca [3 x i128]
call void @init(ptr %a)
Expand Down
56 changes: 28 additions & 28 deletions llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll
Original file line number Diff line number Diff line change
Expand Up @@ -993,24 +993,24 @@ define i8 @atomic_load_relaxed_8(ptr %p, i32 %off32) #0 {
; CHECK-NOLSE-O1: ; %bb.0:
; CHECK-NOLSE-O1-NEXT: ldrb w8, [x0, #4095]
; CHECK-NOLSE-O1-NEXT: ldrb w9, [x0, w1, sxtw]
; CHECK-NOLSE-O1-NEXT: add x11, x0, #291, lsl #12 ; =1191936
; CHECK-NOLSE-O1-NEXT: ldurb w10, [x0, #-256]
; CHECK-NOLSE-O1-NEXT: add w8, w8, w9
; CHECK-NOLSE-O1-NEXT: ldrb w9, [x11]
; CHECK-NOLSE-O1-NEXT: add w8, w8, w10
; CHECK-NOLSE-O1-NEXT: add w0, w8, w9
; CHECK-NOLSE-O1-NEXT: add w8, w9, w8, uxtb
; CHECK-NOLSE-O1-NEXT: add x9, x0, #291, lsl #12 ; =1191936
; CHECK-NOLSE-O1-NEXT: ldrb w9, [x9]
; CHECK-NOLSE-O1-NEXT: add w8, w8, w10, uxtb
; CHECK-NOLSE-O1-NEXT: add w0, w8, w9, uxtb
; CHECK-NOLSE-O1-NEXT: ret
;
; CHECK-OUTLINE-O1-LABEL: atomic_load_relaxed_8:
; CHECK-OUTLINE-O1: ; %bb.0:
; CHECK-OUTLINE-O1-NEXT: ldrb w8, [x0, #4095]
; CHECK-OUTLINE-O1-NEXT: ldrb w9, [x0, w1, sxtw]
; CHECK-OUTLINE-O1-NEXT: add x11, x0, #291, lsl #12 ; =1191936
; CHECK-OUTLINE-O1-NEXT: ldurb w10, [x0, #-256]
; CHECK-OUTLINE-O1-NEXT: add w8, w8, w9
; CHECK-OUTLINE-O1-NEXT: ldrb w9, [x11]
; CHECK-OUTLINE-O1-NEXT: add w8, w8, w10
; CHECK-OUTLINE-O1-NEXT: add w0, w8, w9
; CHECK-OUTLINE-O1-NEXT: add w8, w9, w8, uxtb
; CHECK-OUTLINE-O1-NEXT: add x9, x0, #291, lsl #12 ; =1191936
; CHECK-OUTLINE-O1-NEXT: ldrb w9, [x9]
; CHECK-OUTLINE-O1-NEXT: add w8, w8, w10, uxtb
; CHECK-OUTLINE-O1-NEXT: add w0, w8, w9, uxtb
; CHECK-OUTLINE-O1-NEXT: ret
;
; CHECK-NOLSE-O0-LABEL: atomic_load_relaxed_8:
Expand Down Expand Up @@ -1045,12 +1045,12 @@ define i8 @atomic_load_relaxed_8(ptr %p, i32 %off32) #0 {
; CHECK-LSE-O1: ; %bb.0:
; CHECK-LSE-O1-NEXT: ldrb w8, [x0, #4095]
; CHECK-LSE-O1-NEXT: ldrb w9, [x0, w1, sxtw]
; CHECK-LSE-O1-NEXT: ldurb w10, [x0, #-256]
; CHECK-LSE-O1-NEXT: add w8, w8, w10
; CHECK-LSE-O1-NEXT: add w8, w8, w9
; CHECK-LSE-O1-NEXT: add w8, w9, w8, uxtb
; CHECK-LSE-O1-NEXT: ldurb w9, [x0, #-256]
; CHECK-LSE-O1-NEXT: add w8, w8, w9, uxtb
; CHECK-LSE-O1-NEXT: add x9, x0, #291, lsl #12 ; =1191936
; CHECK-LSE-O1-NEXT: ldrb w9, [x9]
; CHECK-LSE-O1-NEXT: add w0, w8, w9
; CHECK-LSE-O1-NEXT: add w0, w8, w9, uxtb
; CHECK-LSE-O1-NEXT: ret
;
; CHECK-LSE-O0-LABEL: atomic_load_relaxed_8:
Expand Down Expand Up @@ -1089,24 +1089,24 @@ define i16 @atomic_load_relaxed_16(ptr %p, i32 %off32) #0 {
; CHECK-NOLSE-O1: ; %bb.0:
; CHECK-NOLSE-O1-NEXT: ldrh w8, [x0, #8190]
; CHECK-NOLSE-O1-NEXT: ldrh w9, [x0, w1, sxtw #1]
; CHECK-NOLSE-O1-NEXT: add x11, x0, #291, lsl #12 ; =1191936
; CHECK-NOLSE-O1-NEXT: ldurh w10, [x0, #-256]
; CHECK-NOLSE-O1-NEXT: add w8, w8, w9
; CHECK-NOLSE-O1-NEXT: ldrh w9, [x11]
; CHECK-NOLSE-O1-NEXT: add w8, w8, w10
; CHECK-NOLSE-O1-NEXT: add w0, w8, w9
; CHECK-NOLSE-O1-NEXT: add w8, w9, w8, uxth
; CHECK-NOLSE-O1-NEXT: add x9, x0, #291, lsl #12 ; =1191936
; CHECK-NOLSE-O1-NEXT: ldrh w9, [x9]
; CHECK-NOLSE-O1-NEXT: add w8, w8, w10, uxth
; CHECK-NOLSE-O1-NEXT: add w0, w8, w9, uxth
; CHECK-NOLSE-O1-NEXT: ret
;
; CHECK-OUTLINE-O1-LABEL: atomic_load_relaxed_16:
; CHECK-OUTLINE-O1: ; %bb.0:
; CHECK-OUTLINE-O1-NEXT: ldrh w8, [x0, #8190]
; CHECK-OUTLINE-O1-NEXT: ldrh w9, [x0, w1, sxtw #1]
; CHECK-OUTLINE-O1-NEXT: add x11, x0, #291, lsl #12 ; =1191936
; CHECK-OUTLINE-O1-NEXT: ldurh w10, [x0, #-256]
; CHECK-OUTLINE-O1-NEXT: add w8, w8, w9
; CHECK-OUTLINE-O1-NEXT: ldrh w9, [x11]
; CHECK-OUTLINE-O1-NEXT: add w8, w8, w10
; CHECK-OUTLINE-O1-NEXT: add w0, w8, w9
; CHECK-OUTLINE-O1-NEXT: add w8, w9, w8, uxth
; CHECK-OUTLINE-O1-NEXT: add x9, x0, #291, lsl #12 ; =1191936
; CHECK-OUTLINE-O1-NEXT: ldrh w9, [x9]
; CHECK-OUTLINE-O1-NEXT: add w8, w8, w10, uxth
; CHECK-OUTLINE-O1-NEXT: add w0, w8, w9, uxth
; CHECK-OUTLINE-O1-NEXT: ret
;
; CHECK-NOLSE-O0-LABEL: atomic_load_relaxed_16:
Expand Down Expand Up @@ -1141,12 +1141,12 @@ define i16 @atomic_load_relaxed_16(ptr %p, i32 %off32) #0 {
; CHECK-LSE-O1: ; %bb.0:
; CHECK-LSE-O1-NEXT: ldrh w8, [x0, #8190]
; CHECK-LSE-O1-NEXT: ldrh w9, [x0, w1, sxtw #1]
; CHECK-LSE-O1-NEXT: ldurh w10, [x0, #-256]
; CHECK-LSE-O1-NEXT: add w8, w8, w10
; CHECK-LSE-O1-NEXT: add w8, w8, w9
; CHECK-LSE-O1-NEXT: add w8, w9, w8, uxth
; CHECK-LSE-O1-NEXT: ldurh w9, [x0, #-256]
; CHECK-LSE-O1-NEXT: add w8, w8, w9, uxth
; CHECK-LSE-O1-NEXT: add x9, x0, #291, lsl #12 ; =1191936
; CHECK-LSE-O1-NEXT: ldrh w9, [x9]
; CHECK-LSE-O1-NEXT: add w0, w8, w9
; CHECK-LSE-O1-NEXT: add w0, w8, w9, uxth
; CHECK-LSE-O1-NEXT: ret
;
; CHECK-LSE-O0-LABEL: atomic_load_relaxed_16:
Expand Down
28 changes: 14 additions & 14 deletions llvm/test/CodeGen/AArch64/GlobalISel/arm64-pcsections.ll
Original file line number Diff line number Diff line change
Expand Up @@ -377,13 +377,13 @@ define i8 @atomic_load_relaxed_8(ptr %p, i32 %off32) {
; CHECK-NEXT: liveins: $w1, $x0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $w8 = LDRBBui renamable $x0, 4095, pcsections !0 :: (load monotonic (s8) from %ir.ptr_unsigned)
; CHECK-NEXT: renamable $w9 = LDRBBroW renamable $x0, killed renamable $w1, 1, 0, pcsections !0 :: (load unordered (s8) from %ir.ptr_regoff)
; CHECK-NEXT: renamable $w10 = LDURBBi renamable $x0, -256, pcsections !0 :: (load monotonic (s8) from %ir.ptr_unscaled)
; CHECK-NEXT: renamable $x11 = ADDXri killed renamable $x0, 291, 12
; CHECK-NEXT: $w8 = ADDWrs killed renamable $w8, killed renamable $w9, 0, pcsections !0
; CHECK-NEXT: renamable $w9 = LDRBBui killed renamable $x11, 0, pcsections !0 :: (load unordered (s8) from %ir.ptr_random)
; CHECK-NEXT: $w8 = ADDWrs killed renamable $w8, killed renamable $w10, 0, pcsections !0
; CHECK-NEXT: $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0, pcsections !0
; CHECK-NEXT: renamable $w9 = LDRBBroW renamable $x0, killed renamable $w1, 1, 0 :: (load unordered (s8) from %ir.ptr_regoff)
; CHECK-NEXT: renamable $w10 = LDURBBi renamable $x0, -256 :: (load monotonic (s8) from %ir.ptr_unscaled)
; CHECK-NEXT: renamable $w8 = ADDWrx killed renamable $w9, killed renamable $w8, 0, pcsections !0
; CHECK-NEXT: renamable $x9 = ADDXri killed renamable $x0, 291, 12
; CHECK-NEXT: renamable $w8 = ADDWrx killed renamable $w8, killed renamable $w10, 0, pcsections !0
; CHECK-NEXT: renamable $w9 = LDRBBui killed renamable $x9, 0, pcsections !0 :: (load unordered (s8) from %ir.ptr_random)
; CHECK-NEXT: renamable $w0 = ADDWrx killed renamable $w8, killed renamable $w9, 0, pcsections !0
; CHECK-NEXT: RET undef $lr, implicit $w0
%ptr_unsigned = getelementptr i8, ptr %p, i32 4095
%val_unsigned = load atomic i8, ptr %ptr_unsigned monotonic, align 1, !pcsections !0
Expand All @@ -409,13 +409,13 @@ define i16 @atomic_load_relaxed_16(ptr %p, i32 %off32) {
; CHECK-NEXT: liveins: $w1, $x0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $w8 = LDRHHui renamable $x0, 4095, pcsections !0 :: (load monotonic (s16) from %ir.ptr_unsigned)
; CHECK-NEXT: renamable $w9 = LDRHHroW renamable $x0, killed renamable $w1, 1, 1, pcsections !0 :: (load unordered (s16) from %ir.ptr_regoff)
; CHECK-NEXT: renamable $w10 = LDURHHi renamable $x0, -256, pcsections !0 :: (load monotonic (s16) from %ir.ptr_unscaled)
; CHECK-NEXT: renamable $x11 = ADDXri killed renamable $x0, 291, 12
; CHECK-NEXT: $w8 = ADDWrs killed renamable $w8, killed renamable $w9, 0, pcsections !0
; CHECK-NEXT: renamable $w9 = LDRHHui killed renamable $x11, 0, pcsections !0 :: (load unordered (s16) from %ir.ptr_random)
; CHECK-NEXT: $w8 = ADDWrs killed renamable $w8, killed renamable $w10, 0, pcsections !0
; CHECK-NEXT: $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0, pcsections !0
; CHECK-NEXT: renamable $w9 = LDRHHroW renamable $x0, killed renamable $w1, 1, 1 :: (load unordered (s16) from %ir.ptr_regoff)
; CHECK-NEXT: renamable $w10 = LDURHHi renamable $x0, -256 :: (load monotonic (s16) from %ir.ptr_unscaled)
; CHECK-NEXT: renamable $w8 = ADDWrx killed renamable $w9, killed renamable $w8, 8, pcsections !0
; CHECK-NEXT: renamable $x9 = ADDXri killed renamable $x0, 291, 12
; CHECK-NEXT: renamable $w8 = ADDWrx killed renamable $w8, killed renamable $w10, 8, pcsections !0
; CHECK-NEXT: renamable $w9 = LDRHHui killed renamable $x9, 0, pcsections !0 :: (load unordered (s16) from %ir.ptr_random)
; CHECK-NEXT: renamable $w0 = ADDWrx killed renamable $w8, killed renamable $w9, 8, pcsections !0
; CHECK-NEXT: RET undef $lr, implicit $w0
%ptr_unsigned = getelementptr i16, ptr %p, i32 4095
%val_unsigned = load atomic i16, ptr %ptr_unsigned monotonic, align 2, !pcsections !0
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel -global-isel-abort=1 -O0 -o - %s | FileCheck %s
target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
target triple = "arm64e-apple-macosx14.0.0"

define void @test(ptr %0) {
; CHECK-LABEL: test:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: sub sp, sp, #144
; CHECK-NEXT: stp x29, x30, [sp, #128] ; 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 144
; CHECK-NEXT: .cfi_offset w30, -8
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ldar w8, [x0]
; CHECK-NEXT: str w8, [sp, #116] ; 4-byte Folded Spill
; CHECK-NEXT: mov x8, #0 ; =0x0
; CHECK-NEXT: str x8, [sp, #120] ; 8-byte Folded Spill
; CHECK-NEXT: blr x8
; CHECK-NEXT: ldr w11, [sp, #116] ; 4-byte Folded Reload
; CHECK-NEXT: ldr x8, [sp, #120] ; 8-byte Folded Reload
; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: str xzr, [x9]
; CHECK-NEXT: str xzr, [x9, #8]
; CHECK-NEXT: str xzr, [x9, #16]
; CHECK-NEXT: str xzr, [x9, #24]
; CHECK-NEXT: str xzr, [x9, #32]
; CHECK-NEXT: str xzr, [x9, #40]
; CHECK-NEXT: ; implicit-def: $x10
; CHECK-NEXT: mov x10, x11
; CHECK-NEXT: str x10, [x9, #48]
; CHECK-NEXT: str xzr, [x9, #56]
; CHECK-NEXT: str xzr, [x9, #64]
; CHECK-NEXT: str xzr, [x9, #72]
; CHECK-NEXT: str xzr, [x9, #80]
; CHECK-NEXT: str xzr, [x9, #88]
; CHECK-NEXT: str xzr, [x9, #96]
; CHECK-NEXT: mov x0, x8
; CHECK-NEXT: blr x8
; CHECK-NEXT: ldp x29, x30, [sp, #128] ; 16-byte Folded Reload
; CHECK-NEXT: add sp, sp, #144
; CHECK-NEXT: ret
entry:
%atomic-load = load atomic i32, ptr %0 seq_cst, align 4
%call10 = call ptr null()
call void (ptr, ...) null(ptr null, ptr null, i32 0, ptr null, ptr null, i32 0, i32 0, i32 %atomic-load, i32 0, i32 0, i32 0, i32 0, i64 0, ptr null)
ret void
}

0 comments on commit 60fc4ac

Please sign in to comment.