From a9ce1818174b3e690545b573cc5804d73491ca1a Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Tue, 20 Aug 2024 04:51:15 +0000 Subject: [PATCH] [NVPTX] Fix bugs involving maximum/minimum and bf16 We would crash on sufficiently old NV hardware (Volta or so) due to incorrectly marking certain operations legal. --- llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 76 +- llvm/test/CodeGen/NVPTX/bf16-instructions.ll | 1559 +++++++++++++++--- 2 files changed, 1382 insertions(+), 253 deletions(-) diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 577141299b94963..73791102fc04dee 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -429,29 +429,50 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM, auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action, LegalizeAction NoF16Action) { - setOperationAction(Op, VT, STI.allowFP16Math() ? Action : NoF16Action); + bool IsOpSupported = STI.allowFP16Math(); + switch (Op) { + // Several FP16 instructions are available on sm_80 only. + case ISD::FMINNUM: + case ISD::FMAXNUM: + case ISD::FMAXNUM_IEEE: + case ISD::FMINNUM_IEEE: + case ISD::FMAXIMUM: + case ISD::FMINIMUM: + IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70; + break; + } + setOperationAction(Op, VT, IsOpSupported ? Action : NoF16Action); }; auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action, LegalizeAction NoBF16Action) { bool IsOpSupported = STI.hasBF16Math(); - // Few instructions are available on sm_90 only - switch(Op) { - case ISD::FADD: - case ISD::FMUL: - case ISD::FSUB: - case ISD::SELECT: - case ISD::SELECT_CC: - case ISD::SETCC: - case ISD::FEXP2: - case ISD::FCEIL: - case ISD::FFLOOR: - case ISD::FNEARBYINT: - case ISD::FRINT: - case ISD::FROUNDEVEN: - case ISD::FTRUNC: - IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78; - break; + switch (Op) { + // Several BF16 instructions are available on sm_90 only. + case ISD::FADD: + case ISD::FMUL: + case ISD::FSUB: + case ISD::SELECT: + case ISD::SELECT_CC: + case ISD::SETCC: + case ISD::FEXP2: + case ISD::FCEIL: + case ISD::FFLOOR: + case ISD::FNEARBYINT: + case ISD::FRINT: + case ISD::FROUNDEVEN: + case ISD::FTRUNC: + IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78; + break; + // Several BF16 instructions are available on sm_80 only. + case ISD::FMINNUM: + case ISD::FMAXNUM: + case ISD::FMAXNUM_IEEE: + case ISD::FMINNUM_IEEE: + case ISD::FMAXIMUM: + case ISD::FMINIMUM: + IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70; + break; } setOperationAction( Op, VT, IsOpSupported ? Action : NoBF16Action); @@ -838,26 +859,23 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM, AddPromotedToType(Op, MVT::bf16, MVT::f32); } - // max.f16, max.f16x2 and max.NaN are supported on sm_80+. - auto GetMinMaxAction = [&](LegalizeAction NotSm80Action) { - bool IsAtLeastSm80 = STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70; - return IsAtLeastSm80 ? Legal : NotSm80Action; - }; for (const auto &Op : {ISD::FMINNUM, ISD::FMAXNUM}) { - setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Promote), Promote); setOperationAction(Op, MVT::f32, Legal); setOperationAction(Op, MVT::f64, Legal); - setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand); + setFP16OperationAction(Op, MVT::f16, Legal, Promote); + setFP16OperationAction(Op, MVT::v2f16, Legal, Expand); setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand); setBF16OperationAction(Op, MVT::bf16, Legal, Promote); if (getOperationAction(Op, MVT::bf16) == Promote) AddPromotedToType(Op, MVT::bf16, MVT::f32); } + bool SupportsF32MinMaxNaN = + STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70; for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) { - setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Expand), Expand); - setFP16OperationAction(Op, MVT::bf16, Legal, Expand); - setOperationAction(Op, MVT::f32, GetMinMaxAction(Expand)); - setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand); + setOperationAction(Op, MVT::f32, SupportsF32MinMaxNaN ? Legal : Expand); + setFP16OperationAction(Op, MVT::f16, Legal, Expand); + setFP16OperationAction(Op, MVT::v2f16, Legal, Expand); + setBF16OperationAction(Op, MVT::bf16, Legal, Expand); setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand); } diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll index 4cbab8bf24166c5..f029e5f322968b7 100644 --- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll +++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll @@ -1,344 +1,1455 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM70 %s ; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM80 %s ; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck --check-prefixes=CHECK,SM90 %s ; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | %ptxas-verify -arch=sm_80 %} ; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %} +target triple = "nvptx64-nvidia-cuda" + ; LDST: .b8 bfloat_array[8] = {1, 2, 3, 4, 5, 6, 7, 8}; @"bfloat_array" = addrspace(1) constant [4 x bfloat] [bfloat 0xR0201, bfloat 0xR0403, bfloat 0xR0605, bfloat 0xR0807] -; CHECK-LABEL: test_fadd( -; CHECK-DAG: ld.param.b16 [[A:%rs[0-9]+]], [test_fadd_param_0]; -; CHECK-DAG: ld.param.b16 [[B:%rs[0-9]+]], [test_fadd_param_1]; -; SM90: add.rn.bf16 [[R:%rs[0-9]+]], [[A]], [[B]]; -; -; SM80-DAG: cvt.f32.bf16 [[FA:%f[0-9]+]], [[A]]; -; SM80-DAG: cvt.f32.bf16 [[FB:%f[0-9]+]], [[B]]; -; SM80: add.rn.f32 [[FR:%f[0-9]+]], [[FA]], [[FB]]; -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]]; -; CHECK-NEXT: st.param.b16 [func_retval0+0], [[R]]; -; CHECK-NEXT: ret; - define bfloat @test_fadd(bfloat %0, bfloat %1) { +; SM70-LABEL: test_fadd( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<3>; +; SM70-NEXT: .reg .b32 %r<11>; +; SM70-NEXT: .reg .f32 %f<4>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %r1, [test_fadd_param_1]; +; SM70-NEXT: shl.b32 %r2, %r1, 16; +; SM70-NEXT: mov.b32 %f1, %r2; +; SM70-NEXT: ld.param.u16 %r3, [test_fadd_param_0]; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f2, %r4; +; SM70-NEXT: add.rn.f32 %f3, %f2, %f1; +; SM70-NEXT: mov.b32 %r5, %f3; +; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1; +; SM70-NEXT: add.s32 %r7, %r6, %r5; +; SM70-NEXT: add.s32 %r8, %r7, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f3, %f3; +; SM70-NEXT: or.b32 %r9, %r5, 4194304; +; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fadd( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<4>; +; SM80-NEXT: .reg .f32 %f<4>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b16 %rs1, [test_fadd_param_0]; +; SM80-NEXT: ld.param.b16 %rs2, [test_fadd_param_1]; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs2; +; SM80-NEXT: cvt.f32.bf16 %f2, %rs1; +; SM80-NEXT: add.rn.f32 %f3, %f2, %f1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs3, %f3; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fadd( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b16 %rs1, [test_fadd_param_0]; +; SM90-NEXT: ld.param.b16 %rs2, [test_fadd_param_1]; +; SM90-NEXT: add.rn.bf16 %rs3, %rs1, %rs2; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM90-NEXT: ret; %3 = fadd bfloat %0, %1 ret bfloat %3 } -; CHECK-LABEL: test_fsub( -; CHECK-DAG: ld.param.b16 [[A:%rs[0-9]+]], [test_fsub_param_0]; -; CHECK-DAG: ld.param.b16 [[B:%rs[0-9]+]], [test_fsub_param_1]; -; SM90: sub.rn.bf16 [[R:%rs[0-9]+]], [[A]], [[B]]; -; -; SM80-DAG: cvt.f32.bf16 [[FA:%f[0-9]+]], [[A]]; -; SM80-DAG: cvt.f32.bf16 [[FB:%f[0-9]+]], [[B]]; -; SM80: sub.rn.f32 [[FR:%f[0-9]+]], [[FA]], [[FB]]; -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]]; -; CHECK-NEXT: st.param.b16 [func_retval0+0], [[R]]; -; CHECK-NEXT: ret; - define bfloat @test_fsub(bfloat %0, bfloat %1) { +; SM70-LABEL: test_fsub( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<3>; +; SM70-NEXT: .reg .b32 %r<11>; +; SM70-NEXT: .reg .f32 %f<4>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %r1, [test_fsub_param_1]; +; SM70-NEXT: shl.b32 %r2, %r1, 16; +; SM70-NEXT: mov.b32 %f1, %r2; +; SM70-NEXT: ld.param.u16 %r3, [test_fsub_param_0]; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f2, %r4; +; SM70-NEXT: sub.rn.f32 %f3, %f2, %f1; +; SM70-NEXT: mov.b32 %r5, %f3; +; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1; +; SM70-NEXT: add.s32 %r7, %r6, %r5; +; SM70-NEXT: add.s32 %r8, %r7, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f3, %f3; +; SM70-NEXT: or.b32 %r9, %r5, 4194304; +; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fsub( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<4>; +; SM80-NEXT: .reg .f32 %f<4>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b16 %rs1, [test_fsub_param_0]; +; SM80-NEXT: ld.param.b16 %rs2, [test_fsub_param_1]; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs2; +; SM80-NEXT: cvt.f32.bf16 %f2, %rs1; +; SM80-NEXT: sub.rn.f32 %f3, %f2, %f1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs3, %f3; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fsub( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b16 %rs1, [test_fsub_param_0]; +; SM90-NEXT: ld.param.b16 %rs2, [test_fsub_param_1]; +; SM90-NEXT: sub.rn.bf16 %rs3, %rs1, %rs2; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM90-NEXT: ret; %3 = fsub bfloat %0, %1 ret bfloat %3 } -; CHECK-LABEL: test_faddx2( -; CHECK-DAG: ld.param.b32 [[A:%r[0-9]+]], [test_faddx2_param_0]; -; CHECK-DAG: ld.param.b32 [[B:%r[0-9]+]], [test_faddx2_param_1]; -; SM90: add.rn.bf16x2 [[R:%r[0-9]+]], [[A]], [[B]]; - -; SM80-DAG: mov.b32 {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]]; -; SM80-DAG: mov.b32 {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]]; -; SM80-DAG: cvt.f32.bf16 [[FA1:%f[0-9]+]], [[A1]]; -; SM80-DAG: cvt.f32.bf16 [[FA0:%f[0-9]+]], [[A0]]; -; SM80-DAG: cvt.f32.bf16 [[FB0:%f[0-9]+]], [[B0]]; -; SM80-DAG: cvt.f32.bf16 [[FB1:%f[0-9]+]], [[B1]]; -; SM80-DAG: add.rn.f32 [[FR0:%f[0-9]+]], [[FA0]], [[FB0]]; -; SM80-DAG: add.rn.f32 [[FR1:%f[0-9]+]], [[FA1]], [[FB1]]; -; SM80-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]]; -; SM80-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]]; -; SM80: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]}; -; CHECK: st.param.b32 [func_retval0+0], [[R]]; -; CHECK: ret; - define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 { +; SM70-LABEL: test_faddx2( +; SM70: { +; SM70-NEXT: .reg .pred %p<3>; +; SM70-NEXT: .reg .b16 %rs<13>; +; SM70-NEXT: .reg .b32 %r<24>; +; SM70-NEXT: .reg .f32 %f<7>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.b32 %r1, [test_faddx2_param_0]; +; SM70-NEXT: ld.param.b32 %r2, [test_faddx2_param_1]; +; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM70-NEXT: cvt.u32.u16 %r3, %rs2; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f1, %r4; +; SM70-NEXT: mov.b32 {%rs4, %rs5}, %r1; +; SM70-NEXT: cvt.u32.u16 %r5, %rs5; +; SM70-NEXT: shl.b32 %r6, %r5, 16; +; SM70-NEXT: mov.b32 %f2, %r6; +; SM70-NEXT: add.rn.f32 %f3, %f2, %f1; +; SM70-NEXT: mov.b32 %r7, %f3; +; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1; +; SM70-NEXT: add.s32 %r9, %r8, %r7; +; SM70-NEXT: add.s32 %r10, %r9, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f3, %f3; +; SM70-NEXT: or.b32 %r11, %r7, 4194304; +; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; } +; SM70-NEXT: cvt.u32.u16 %r13, %rs1; +; SM70-NEXT: shl.b32 %r14, %r13, 16; +; SM70-NEXT: mov.b32 %f4, %r14; +; SM70-NEXT: cvt.u32.u16 %r15, %rs4; +; SM70-NEXT: shl.b32 %r16, %r15, 16; +; SM70-NEXT: mov.b32 %f5, %r16; +; SM70-NEXT: add.rn.f32 %f6, %f5, %f4; +; SM70-NEXT: mov.b32 %r17, %f6; +; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1; +; SM70-NEXT: add.s32 %r19, %r18, %r17; +; SM70-NEXT: add.s32 %r20, %r19, 32767; +; SM70-NEXT: setp.nan.f32 %p2, %f6, %f6; +; SM70-NEXT: or.b32 %r21, %r17, 4194304; +; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; } +; SM70-NEXT: mov.b32 %r23, {%rs11, %rs7}; +; SM70-NEXT: st.param.b32 [func_retval0+0], %r23; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_faddx2( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<7>; +; SM80-NEXT: .reg .b32 %r<4>; +; SM80-NEXT: .reg .f32 %f<7>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b32 %r1, [test_faddx2_param_0]; +; SM80-NEXT: ld.param.b32 %r2, [test_faddx2_param_1]; +; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs2; +; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1; +; SM80-NEXT: cvt.f32.bf16 %f2, %rs4; +; SM80-NEXT: add.rn.f32 %f3, %f2, %f1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3; +; SM80-NEXT: cvt.f32.bf16 %f4, %rs1; +; SM80-NEXT: cvt.f32.bf16 %f5, %rs3; +; SM80-NEXT: add.rn.f32 %f6, %f5, %f4; +; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6; +; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5}; +; SM80-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_faddx2( +; SM90: { +; SM90-NEXT: .reg .b32 %r<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b32 %r1, [test_faddx2_param_1]; +; SM90-NEXT: ld.param.b32 %r2, [test_faddx2_param_0]; +; SM90-NEXT: add.rn.bf16x2 %r3, %r2, %r1; +; SM90-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM90-NEXT: ret; %r = fadd <2 x bfloat> %a, %b ret <2 x bfloat> %r } -; CHECK-LABEL: test_fsubx2( -; CHECK-DAG: ld.param.b32 [[A:%r[0-9]+]], [test_fsubx2_param_0]; -; CHECK-DAG: ld.param.b32 [[B:%r[0-9]+]], [test_fsubx2_param_1]; -; SM90: sub.rn.bf16x2 [[R:%r[0-9]+]], [[A]], [[B]]; - -; SM80-DAG: mov.b32 {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]]; -; SM80-DAG: mov.b32 {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]]; -; SM80-DAG: cvt.f32.bf16 [[FA1:%f[0-9]+]], [[A1]]; -; SM80-DAG: cvt.f32.bf16 [[FA0:%f[0-9]+]], [[A0]]; -; SM80-DAG: cvt.f32.bf16 [[FB0:%f[0-9]+]], [[B0]]; -; SM80-DAG: cvt.f32.bf16 [[FB1:%f[0-9]+]], [[B1]]; -; SM80-DAG: sub.rn.f32 [[FR0:%f[0-9]+]], [[FA0]], [[FB0]]; -; SM80-DAG: sub.rn.f32 [[FR1:%f[0-9]+]], [[FA1]], [[FB1]]; -; SM80-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]]; -; SM80-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]]; -; SM80: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]}; - -; CHECK: st.param.b32 [func_retval0+0], [[R]]; -; CHECK: ret; - define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 { +; SM70-LABEL: test_fsubx2( +; SM70: { +; SM70-NEXT: .reg .pred %p<3>; +; SM70-NEXT: .reg .b16 %rs<13>; +; SM70-NEXT: .reg .b32 %r<24>; +; SM70-NEXT: .reg .f32 %f<7>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0]; +; SM70-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1]; +; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM70-NEXT: cvt.u32.u16 %r3, %rs2; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f1, %r4; +; SM70-NEXT: mov.b32 {%rs4, %rs5}, %r1; +; SM70-NEXT: cvt.u32.u16 %r5, %rs5; +; SM70-NEXT: shl.b32 %r6, %r5, 16; +; SM70-NEXT: mov.b32 %f2, %r6; +; SM70-NEXT: sub.rn.f32 %f3, %f2, %f1; +; SM70-NEXT: mov.b32 %r7, %f3; +; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1; +; SM70-NEXT: add.s32 %r9, %r8, %r7; +; SM70-NEXT: add.s32 %r10, %r9, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f3, %f3; +; SM70-NEXT: or.b32 %r11, %r7, 4194304; +; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; } +; SM70-NEXT: cvt.u32.u16 %r13, %rs1; +; SM70-NEXT: shl.b32 %r14, %r13, 16; +; SM70-NEXT: mov.b32 %f4, %r14; +; SM70-NEXT: cvt.u32.u16 %r15, %rs4; +; SM70-NEXT: shl.b32 %r16, %r15, 16; +; SM70-NEXT: mov.b32 %f5, %r16; +; SM70-NEXT: sub.rn.f32 %f6, %f5, %f4; +; SM70-NEXT: mov.b32 %r17, %f6; +; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1; +; SM70-NEXT: add.s32 %r19, %r18, %r17; +; SM70-NEXT: add.s32 %r20, %r19, 32767; +; SM70-NEXT: setp.nan.f32 %p2, %f6, %f6; +; SM70-NEXT: or.b32 %r21, %r17, 4194304; +; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; } +; SM70-NEXT: mov.b32 %r23, {%rs11, %rs7}; +; SM70-NEXT: st.param.b32 [func_retval0+0], %r23; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fsubx2( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<7>; +; SM80-NEXT: .reg .b32 %r<4>; +; SM80-NEXT: .reg .f32 %f<7>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0]; +; SM80-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1]; +; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs2; +; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1; +; SM80-NEXT: cvt.f32.bf16 %f2, %rs4; +; SM80-NEXT: sub.rn.f32 %f3, %f2, %f1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3; +; SM80-NEXT: cvt.f32.bf16 %f4, %rs1; +; SM80-NEXT: cvt.f32.bf16 %f5, %rs3; +; SM80-NEXT: sub.rn.f32 %f6, %f5, %f4; +; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6; +; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5}; +; SM80-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fsubx2( +; SM90: { +; SM90-NEXT: .reg .b32 %r<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b32 %r1, [test_fsubx2_param_1]; +; SM90-NEXT: ld.param.b32 %r2, [test_fsubx2_param_0]; +; SM90-NEXT: sub.rn.bf16x2 %r3, %r2, %r1; +; SM90-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM90-NEXT: ret; %r = fsub <2 x bfloat> %a, %b ret <2 x bfloat> %r } -; CHECK-LABEL: test_fmulx2( -; CHECK-DAG: ld.param.b32 [[A:%r[0-9]+]], [test_fmulx2_param_0]; -; CHECK-DAG: ld.param.b32 [[B:%r[0-9]+]], [test_fmulx2_param_1]; -; SM90: mul.rn.bf16x2 [[R:%r[0-9]+]], [[A]], [[B]]; - -; SM80-DAG: mov.b32 {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]]; -; SM80-DAG: mov.b32 {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]]; -; SM80-DAG: cvt.f32.bf16 [[FA1:%f[0-9]+]], [[A1]]; -; SM80-DAG: cvt.f32.bf16 [[FA0:%f[0-9]+]], [[A0]]; -; SM80-DAG: cvt.f32.bf16 [[FB0:%f[0-9]+]], [[B0]]; -; SM80-DAG: cvt.f32.bf16 [[FB1:%f[0-9]+]], [[B1]]; -; SM80-DAG: mul.rn.f32 [[FR0:%f[0-9]+]], [[FA0]], [[FB0]]; -; SM80-DAG: mul.rn.f32 [[FR1:%f[0-9]+]], [[FA1]], [[FB1]]; -; SM80-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]]; -; SM80-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]]; -; SM80: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]}; - -; CHECK: st.param.b32 [func_retval0+0], [[R]]; -; CHECK: ret; - define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 { +; SM70-LABEL: test_fmulx2( +; SM70: { +; SM70-NEXT: .reg .pred %p<3>; +; SM70-NEXT: .reg .b16 %rs<13>; +; SM70-NEXT: .reg .b32 %r<24>; +; SM70-NEXT: .reg .f32 %f<7>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0]; +; SM70-NEXT: ld.param.b32 %r2, [test_fmulx2_param_1]; +; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM70-NEXT: cvt.u32.u16 %r3, %rs2; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f1, %r4; +; SM70-NEXT: mov.b32 {%rs4, %rs5}, %r1; +; SM70-NEXT: cvt.u32.u16 %r5, %rs5; +; SM70-NEXT: shl.b32 %r6, %r5, 16; +; SM70-NEXT: mov.b32 %f2, %r6; +; SM70-NEXT: mul.rn.f32 %f3, %f2, %f1; +; SM70-NEXT: mov.b32 %r7, %f3; +; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1; +; SM70-NEXT: add.s32 %r9, %r8, %r7; +; SM70-NEXT: add.s32 %r10, %r9, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f3, %f3; +; SM70-NEXT: or.b32 %r11, %r7, 4194304; +; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; } +; SM70-NEXT: cvt.u32.u16 %r13, %rs1; +; SM70-NEXT: shl.b32 %r14, %r13, 16; +; SM70-NEXT: mov.b32 %f4, %r14; +; SM70-NEXT: cvt.u32.u16 %r15, %rs4; +; SM70-NEXT: shl.b32 %r16, %r15, 16; +; SM70-NEXT: mov.b32 %f5, %r16; +; SM70-NEXT: mul.rn.f32 %f6, %f5, %f4; +; SM70-NEXT: mov.b32 %r17, %f6; +; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1; +; SM70-NEXT: add.s32 %r19, %r18, %r17; +; SM70-NEXT: add.s32 %r20, %r19, 32767; +; SM70-NEXT: setp.nan.f32 %p2, %f6, %f6; +; SM70-NEXT: or.b32 %r21, %r17, 4194304; +; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; } +; SM70-NEXT: mov.b32 %r23, {%rs11, %rs7}; +; SM70-NEXT: st.param.b32 [func_retval0+0], %r23; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fmulx2( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<7>; +; SM80-NEXT: .reg .b32 %r<4>; +; SM80-NEXT: .reg .f32 %f<7>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0]; +; SM80-NEXT: ld.param.b32 %r2, [test_fmulx2_param_1]; +; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs2; +; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1; +; SM80-NEXT: cvt.f32.bf16 %f2, %rs4; +; SM80-NEXT: mul.rn.f32 %f3, %f2, %f1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3; +; SM80-NEXT: cvt.f32.bf16 %f4, %rs1; +; SM80-NEXT: cvt.f32.bf16 %f5, %rs3; +; SM80-NEXT: mul.rn.f32 %f6, %f5, %f4; +; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6; +; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5}; +; SM80-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fmulx2( +; SM90: { +; SM90-NEXT: .reg .b32 %r<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b32 %r1, [test_fmulx2_param_1]; +; SM90-NEXT: ld.param.b32 %r2, [test_fmulx2_param_0]; +; SM90-NEXT: mul.rn.bf16x2 %r3, %r2, %r1; +; SM90-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM90-NEXT: ret; %r = fmul <2 x bfloat> %a, %b ret <2 x bfloat> %r } -; CHECK-LABEL: test_fdiv( -; CHECK-DAG: ld.param.b32 [[A:%r[0-9]+]], [test_fdiv_param_0]; -; CHECK-DAG: ld.param.b32 [[B:%r[0-9]+]], [test_fdiv_param_1]; -; CHECK-DAG: mov.b32 {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]] -; CHECK-DAG: mov.b32 {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]] -; CHECK-DAG: cvt.f32.bf16 [[FA0:%f[0-9]+]], [[A0]]; -; CHECK-DAG: cvt.f32.bf16 [[FA1:%f[0-9]+]], [[A1]]; -; CHECK-DAG: cvt.f32.bf16 [[FB0:%f[0-9]+]], [[B0]]; -; CHECK-DAG: cvt.f32.bf16 [[FB1:%f[0-9]+]], [[B1]]; -; CHECK-DAG: div.rn.f32 [[FR0:%f[0-9]+]], [[FA0]], [[FB0]]; -; CHECK-DAG: div.rn.f32 [[FR1:%f[0-9]+]], [[FA1]], [[FB1]]; -; CHECK-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]]; -; CHECK-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]]; -; CHECK-NEXT: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} -; CHECK-NEXT: st.param.b32 [func_retval0+0], [[R]]; -; CHECK-NEXT: ret; - define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 { +; SM70-LABEL: test_fdiv( +; SM70: { +; SM70-NEXT: .reg .pred %p<3>; +; SM70-NEXT: .reg .b16 %rs<13>; +; SM70-NEXT: .reg .b32 %r<24>; +; SM70-NEXT: .reg .f32 %f<7>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.b32 %r1, [test_fdiv_param_0]; +; SM70-NEXT: ld.param.b32 %r2, [test_fdiv_param_1]; +; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM70-NEXT: cvt.u32.u16 %r3, %rs2; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f1, %r4; +; SM70-NEXT: mov.b32 {%rs4, %rs5}, %r1; +; SM70-NEXT: cvt.u32.u16 %r5, %rs5; +; SM70-NEXT: shl.b32 %r6, %r5, 16; +; SM70-NEXT: mov.b32 %f2, %r6; +; SM70-NEXT: div.rn.f32 %f3, %f2, %f1; +; SM70-NEXT: mov.b32 %r7, %f3; +; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1; +; SM70-NEXT: add.s32 %r9, %r8, %r7; +; SM70-NEXT: add.s32 %r10, %r9, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f3, %f3; +; SM70-NEXT: or.b32 %r11, %r7, 4194304; +; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; } +; SM70-NEXT: cvt.u32.u16 %r13, %rs1; +; SM70-NEXT: shl.b32 %r14, %r13, 16; +; SM70-NEXT: mov.b32 %f4, %r14; +; SM70-NEXT: cvt.u32.u16 %r15, %rs4; +; SM70-NEXT: shl.b32 %r16, %r15, 16; +; SM70-NEXT: mov.b32 %f5, %r16; +; SM70-NEXT: div.rn.f32 %f6, %f5, %f4; +; SM70-NEXT: mov.b32 %r17, %f6; +; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1; +; SM70-NEXT: add.s32 %r19, %r18, %r17; +; SM70-NEXT: add.s32 %r20, %r19, 32767; +; SM70-NEXT: setp.nan.f32 %p2, %f6, %f6; +; SM70-NEXT: or.b32 %r21, %r17, 4194304; +; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; } +; SM70-NEXT: mov.b32 %r23, {%rs11, %rs7}; +; SM70-NEXT: st.param.b32 [func_retval0+0], %r23; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fdiv( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<7>; +; SM80-NEXT: .reg .b32 %r<4>; +; SM80-NEXT: .reg .f32 %f<7>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b32 %r1, [test_fdiv_param_0]; +; SM80-NEXT: ld.param.b32 %r2, [test_fdiv_param_1]; +; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs2; +; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1; +; SM80-NEXT: cvt.f32.bf16 %f2, %rs4; +; SM80-NEXT: div.rn.f32 %f3, %f2, %f1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3; +; SM80-NEXT: cvt.f32.bf16 %f4, %rs1; +; SM80-NEXT: cvt.f32.bf16 %f5, %rs3; +; SM80-NEXT: div.rn.f32 %f6, %f5, %f4; +; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6; +; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5}; +; SM80-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fdiv( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<7>; +; SM90-NEXT: .reg .b32 %r<4>; +; SM90-NEXT: .reg .f32 %f<7>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b32 %r1, [test_fdiv_param_0]; +; SM90-NEXT: ld.param.b32 %r2, [test_fdiv_param_1]; +; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM90-NEXT: cvt.f32.bf16 %f1, %rs2; +; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1; +; SM90-NEXT: cvt.f32.bf16 %f2, %rs4; +; SM90-NEXT: div.rn.f32 %f3, %f2, %f1; +; SM90-NEXT: cvt.rn.bf16.f32 %rs5, %f3; +; SM90-NEXT: cvt.f32.bf16 %f4, %rs1; +; SM90-NEXT: cvt.f32.bf16 %f5, %rs3; +; SM90-NEXT: div.rn.f32 %f6, %f5, %f4; +; SM90-NEXT: cvt.rn.bf16.f32 %rs6, %f6; +; SM90-NEXT: mov.b32 %r3, {%rs6, %rs5}; +; SM90-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM90-NEXT: ret; %r = fdiv <2 x bfloat> %a, %b ret <2 x bfloat> %r } -; CHECK-LABEL: test_extract_0( -; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_extract_0_param_0]; -; CHECK: st.param.b16 [func_retval0+0], [[A]]; -; CHECK: ret; - define bfloat @test_extract_0(<2 x bfloat> %a) #0 { +; CHECK-LABEL: test_extract_0( +; CHECK: { +; CHECK-NEXT: .reg .b16 %rs<2>; +; CHECK-EMPTY: +; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: ld.param.b16 %rs1, [test_extract_0_param_0]; +; CHECK-NEXT: st.param.b16 [func_retval0+0], %rs1; +; CHECK-NEXT: ret; %e = extractelement <2 x bfloat> %a, i32 0 ret bfloat %e } -; CHECK-LABEL: test_extract_1( -; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_extract_1_param_0+2]; -; CHECK: st.param.b16 [func_retval0+0], [[A]]; -; CHECK: ret; - define bfloat @test_extract_1(<2 x bfloat> %a) #0 { +; CHECK-LABEL: test_extract_1( +; CHECK: { +; CHECK-NEXT: .reg .b16 %rs<2>; +; CHECK-EMPTY: +; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: ld.param.b16 %rs1, [test_extract_1_param_0+2]; +; CHECK-NEXT: st.param.b16 [func_retval0+0], %rs1; +; CHECK-NEXT: ret; %e = extractelement <2 x bfloat> %a, i32 1 ret bfloat %e } -; CHECK-LABEL: test_fpext_float( -; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_fpext_float_param_0]; -; CHECK: cvt.f32.bf16 [[R:%f[0-9]+]], [[A]]; -; CHECK: st.param.f32 [func_retval0+0], [[R]]; -; CHECK: ret; define float @test_fpext_float(bfloat %a) #0 { +; SM70-LABEL: test_fpext_float( +; SM70: { +; SM70-NEXT: .reg .b32 %r<3>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %r1, [test_fpext_float_param_0]; +; SM70-NEXT: shl.b32 %r2, %r1, 16; +; SM70-NEXT: mov.b32 %f1, %r2; +; SM70-NEXT: st.param.f32 [func_retval0+0], %f1; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fpext_float( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<2>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b16 %rs1, [test_fpext_float_param_0]; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs1; +; SM80-NEXT: st.param.f32 [func_retval0+0], %f1; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fpext_float( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<2>; +; SM90-NEXT: .reg .f32 %f<2>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b16 %rs1, [test_fpext_float_param_0]; +; SM90-NEXT: cvt.f32.bf16 %f1, %rs1; +; SM90-NEXT: st.param.f32 [func_retval0+0], %f1; +; SM90-NEXT: ret; %r = fpext bfloat %a to float ret float %r } -; CHECK-LABEL: test_fptrunc_float( -; CHECK: ld.param.f32 [[A:%f[0-9]+]], [test_fptrunc_float_param_0]; -; CHECK: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[A]]; -; CHECK: st.param.b16 [func_retval0+0], [[R]]; -; CHECK: ret; define bfloat @test_fptrunc_float(float %a) #0 { +; SM70-LABEL: test_fptrunc_float( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<3>; +; SM70-NEXT: .reg .b32 %r<7>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.f32 %f1, [test_fptrunc_float_param_0]; +; SM70-NEXT: mov.b32 %r1, %f1; +; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1; +; SM70-NEXT: add.s32 %r3, %r2, %r1; +; SM70-NEXT: add.s32 %r4, %r3, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f1, %f1; +; SM70-NEXT: or.b32 %r5, %r1, 4194304; +; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r6; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fptrunc_float( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<2>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.f32 %f1, [test_fptrunc_float_param_0]; +; SM80-NEXT: cvt.rn.bf16.f32 %rs1, %f1; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fptrunc_float( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<2>; +; SM90-NEXT: .reg .f32 %f<2>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.f32 %f1, [test_fptrunc_float_param_0]; +; SM90-NEXT: cvt.rn.bf16.f32 %rs1, %f1; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM90-NEXT: ret; %r = fptrunc float %a to bfloat ret bfloat %r } -; CHECK-LABEL: test_fadd_imm_1( -; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_fadd_imm_1_param_0]; -; SM90: mov.b16 [[B:%rs[0-9]+]], 0x3F80; -; SM90: add.rn.bf16 [[R:%rs[0-9]+]], [[A]], [[B]]; - -; SM80-DAG: cvt.f32.bf16 [[FA:%f[0-9]+]], [[A]]; -; SM80: add.rn.f32 [[FR:%f[0-9]+]], [[FA]], 0f3F800000; -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]]; - -; CHECK: st.param.b16 [func_retval0+0], [[R]]; -; CHECK-NEXT: ret; define bfloat @test_fadd_imm_1(bfloat %a) #0 { +; SM70-LABEL: test_fadd_imm_1( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<3>; +; SM70-NEXT: .reg .b32 %r<9>; +; SM70-NEXT: .reg .f32 %f<3>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %r1, [test_fadd_imm_1_param_0]; +; SM70-NEXT: shl.b32 %r2, %r1, 16; +; SM70-NEXT: mov.b32 %f1, %r2; +; SM70-NEXT: add.rn.f32 %f2, %f1, 0f3F800000; +; SM70-NEXT: mov.b32 %r3, %f2; +; SM70-NEXT: bfe.u32 %r4, %r3, 16, 1; +; SM70-NEXT: add.s32 %r5, %r4, %r3; +; SM70-NEXT: add.s32 %r6, %r5, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f2, %f2; +; SM70-NEXT: or.b32 %r7, %r3, 4194304; +; SM70-NEXT: selp.b32 %r8, %r7, %r6, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fadd_imm_1( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<3>; +; SM80-NEXT: .reg .f32 %f<3>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b16 %rs1, [test_fadd_imm_1_param_0]; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs1; +; SM80-NEXT: add.rn.f32 %f2, %f1, 0f3F800000; +; SM80-NEXT: cvt.rn.bf16.f32 %rs2, %f2; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fadd_imm_1( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b16 %rs1, [test_fadd_imm_1_param_0]; +; SM90-NEXT: mov.b16 %rs2, 0x3F80; +; SM90-NEXT: add.rn.bf16 %rs3, %rs1, %rs2; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM90-NEXT: ret; %r = fadd bfloat %a, 1.0 ret bfloat %r } -; CHECK-LABEL: test_select_cc_bf16_f64( -; CHECK-DAG: ld.param.f64 [[A:%fd[0-9]+]], [test_select_cc_bf16_f64_param_0]; -; CHECK-DAG: ld.param.f64 [[B:%fd[0-9]+]], [test_select_cc_bf16_f64_param_1]; -; CHECK: setp.lt.f64 [[P:%p[0-9]+]], [[A]], [[B]]; -; CHECK-DAG: ld.param.b16 [[C:%rs[0-9]+]], [test_select_cc_bf16_f64_param_2]; -; CHECK-DAG: ld.param.b16 [[D:%rs[0-9]+]], [test_select_cc_bf16_f64_param_3]; -; CHECK: selp.b16 [[R:%rs[0-9]+]], [[C]], [[D]], [[P]]; -; CHECK-NEXT: st.param.b16 [func_retval0+0], [[R]]; -; CHECK-NEXT: ret; define bfloat @test_select_cc_bf16_f64(double %a, double %b, bfloat %c, bfloat %d) #0 { +; CHECK-LABEL: test_select_cc_bf16_f64( +; CHECK: { +; CHECK-NEXT: .reg .pred %p<2>; +; CHECK-NEXT: .reg .b16 %rs<4>; +; CHECK-NEXT: .reg .f64 %fd<3>; +; CHECK-EMPTY: +; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: ld.param.f64 %fd1, [test_select_cc_bf16_f64_param_0]; +; CHECK-NEXT: ld.param.f64 %fd2, [test_select_cc_bf16_f64_param_1]; +; CHECK-NEXT: setp.lt.f64 %p1, %fd1, %fd2; +; CHECK-NEXT: ld.param.b16 %rs1, [test_select_cc_bf16_f64_param_2]; +; CHECK-NEXT: ld.param.b16 %rs2, [test_select_cc_bf16_f64_param_3]; +; CHECK-NEXT: selp.b16 %rs3, %rs1, %rs2, %p1; +; CHECK-NEXT: st.param.b16 [func_retval0+0], %rs3; +; CHECK-NEXT: ret; %cc = fcmp olt double %a, %b %r = select i1 %cc, bfloat %c, bfloat %d ret bfloat %r } -; CHECK-LABEL: test_extload_bf16x8 -; CHECK: ld.shared.v4.b32 {%r -; CHECK: mov.b32 {%rs -; CHECK: mov.b32 {%rs -; CHECK: mov.b32 {%rs -; CHECK: mov.b32 {%rs -; SM80: cvt.f32.bf16 %f{{.*}}, %rs -; SM80: cvt.f32.bf16 %f{{.*}}, %rs -; SM80: cvt.f32.bf16 %f{{.*}}, %rs -; SM80: cvt.f32.bf16 %f{{.*}}, %rs -; SM80: cvt.f32.bf16 %f{{.*}}, %rs -; SM80: cvt.f32.bf16 %f{{.*}}, %rs -; SM80: cvt.f32.bf16 %f{{.*}}, %rs -; SM80: cvt.f32.bf16 %f{{.*}}, %rs define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 { +; SM70-LABEL: test_extload_bf16x8( +; SM70: { +; SM70-NEXT: .reg .b16 %rs<17>; +; SM70-NEXT: .reg .b32 %r<21>; +; SM70-NEXT: .reg .f32 %f<9>; +; SM70-NEXT: .reg .b64 %rd<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u64 %rd1, [test_extload_bf16x8_param_0]; +; SM70-NEXT: ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1]; +; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1; +; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2; +; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r3; +; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4; +; SM70-NEXT: cvt.u32.u16 %r5, %rs8; +; SM70-NEXT: shl.b32 %r6, %r5, 16; +; SM70-NEXT: mov.b32 %f1, %r6; +; SM70-NEXT: cvt.u32.u16 %r7, %rs7; +; SM70-NEXT: shl.b32 %r8, %r7, 16; +; SM70-NEXT: mov.b32 %f2, %r8; +; SM70-NEXT: cvt.u32.u16 %r9, %rs6; +; SM70-NEXT: shl.b32 %r10, %r9, 16; +; SM70-NEXT: mov.b32 %f3, %r10; +; SM70-NEXT: cvt.u32.u16 %r11, %rs5; +; SM70-NEXT: shl.b32 %r12, %r11, 16; +; SM70-NEXT: mov.b32 %f4, %r12; +; SM70-NEXT: cvt.u32.u16 %r13, %rs4; +; SM70-NEXT: shl.b32 %r14, %r13, 16; +; SM70-NEXT: mov.b32 %f5, %r14; +; SM70-NEXT: cvt.u32.u16 %r15, %rs3; +; SM70-NEXT: shl.b32 %r16, %r15, 16; +; SM70-NEXT: mov.b32 %f6, %r16; +; SM70-NEXT: cvt.u32.u16 %r17, %rs2; +; SM70-NEXT: shl.b32 %r18, %r17, 16; +; SM70-NEXT: mov.b32 %f7, %r18; +; SM70-NEXT: cvt.u32.u16 %r19, %rs1; +; SM70-NEXT: shl.b32 %r20, %r19, 16; +; SM70-NEXT: mov.b32 %f8, %r20; +; SM70-NEXT: st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5}; +; SM70-NEXT: st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1}; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_extload_bf16x8( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<9>; +; SM80-NEXT: .reg .b32 %r<5>; +; SM80-NEXT: .reg .f32 %f<9>; +; SM80-NEXT: .reg .b64 %rd<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.u64 %rd1, [test_extload_bf16x8_param_0]; +; SM80-NEXT: ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1]; +; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1; +; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r2; +; SM80-NEXT: mov.b32 {%rs5, %rs6}, %r3; +; SM80-NEXT: mov.b32 {%rs7, %rs8}, %r4; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs8; +; SM80-NEXT: cvt.f32.bf16 %f2, %rs7; +; SM80-NEXT: cvt.f32.bf16 %f3, %rs6; +; SM80-NEXT: cvt.f32.bf16 %f4, %rs5; +; SM80-NEXT: cvt.f32.bf16 %f5, %rs4; +; SM80-NEXT: cvt.f32.bf16 %f6, %rs3; +; SM80-NEXT: cvt.f32.bf16 %f7, %rs2; +; SM80-NEXT: cvt.f32.bf16 %f8, %rs1; +; SM80-NEXT: st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5}; +; SM80-NEXT: st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1}; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_extload_bf16x8( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<9>; +; SM90-NEXT: .reg .b32 %r<5>; +; SM90-NEXT: .reg .f32 %f<9>; +; SM90-NEXT: .reg .b64 %rd<2>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.u64 %rd1, [test_extload_bf16x8_param_0]; +; SM90-NEXT: ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1]; +; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1; +; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r2; +; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r3; +; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r4; +; SM90-NEXT: cvt.f32.bf16 %f1, %rs8; +; SM90-NEXT: cvt.f32.bf16 %f2, %rs7; +; SM90-NEXT: cvt.f32.bf16 %f3, %rs6; +; SM90-NEXT: cvt.f32.bf16 %f4, %rs5; +; SM90-NEXT: cvt.f32.bf16 %f5, %rs4; +; SM90-NEXT: cvt.f32.bf16 %f6, %rs3; +; SM90-NEXT: cvt.f32.bf16 %f7, %rs2; +; SM90-NEXT: cvt.f32.bf16 %f8, %rs1; +; SM90-NEXT: st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5}; +; SM90-NEXT: st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1}; +; SM90-NEXT: ret; %load = load <8 x bfloat>, ptr addrspace(3) %arg, align 16 %res = fpext <8 x bfloat> %load to <8 x float> ret <8 x float> %res } -; CHECK-LABEL: test_fptosi_i16( -; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_fptosi_i16_param_0]; -; SM80: cvt.f32.bf16 [[B:%f[0-9]+]], [[A]]; -; SM80: cvt.rzi.s16.f32 [[C:%rs[0-9]+]], [[B]]; -; SM80: cvt.u32.u16 [[R:%r[0-9]+]], [[C]]; -; SM90: cvt.rzi.s16.bf16 [[B:%rs[0-9]+]], [[A]]; -; SM90: cvt.u32.u16 [[R:%r[0-9]+]], [[B]]; -; CHECK: st.param.b32 [func_retval0+0], [[R]]; -; CHECK: ret; define i16 @test_fptosi_i16(bfloat %a) { +; SM70-LABEL: test_fptosi_i16( +; SM70: { +; SM70-NEXT: .reg .b16 %rs<2>; +; SM70-NEXT: .reg .b32 %r<4>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %r1, [test_fptosi_i16_param_0]; +; SM70-NEXT: shl.b32 %r2, %r1, 16; +; SM70-NEXT: mov.b32 %f1, %r2; +; SM70-NEXT: cvt.rzi.s16.f32 %rs1, %f1; +; SM70-NEXT: cvt.u32.u16 %r3, %rs1; +; SM70-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fptosi_i16( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<3>; +; SM80-NEXT: .reg .b32 %r<2>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b16 %rs1, [test_fptosi_i16_param_0]; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs1; +; SM80-NEXT: cvt.rzi.s16.f32 %rs2, %f1; +; SM80-NEXT: cvt.u32.u16 %r1, %rs2; +; SM80-NEXT: st.param.b32 [func_retval0+0], %r1; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fptosi_i16( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<3>; +; SM90-NEXT: .reg .b32 %r<2>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b16 %rs1, [test_fptosi_i16_param_0]; +; SM90-NEXT: cvt.rzi.s16.bf16 %rs2, %rs1; +; SM90-NEXT: cvt.u32.u16 %r1, %rs2; +; SM90-NEXT: st.param.b32 [func_retval0+0], %r1; +; SM90-NEXT: ret; %r = fptosi bfloat %a to i16 ret i16 %r } -; CHECK-LABEL: test_fptoui_i16( -; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_fptoui_i16_param_0]; -; SM80: cvt.f32.bf16 [[B:%f[0-9]+]], [[A]]; -; SM80: cvt.rzi.u16.f32 [[C:%rs[0-9]+]], [[B]]; -; SM80: cvt.u32.u16 [[R:%r[0-9]+]], [[C]]; -; SM90: cvt.rzi.u16.bf16 [[B:%rs[0-9]+]], [[A]]; -; SM90: cvt.u32.u16 [[R:%r[0-9]+]], [[B]]; -; CHECK: st.param.b32 [func_retval0+0], [[R]]; -; CHECK: ret; define i16 @test_fptoui_i16(bfloat %a) { +; SM70-LABEL: test_fptoui_i16( +; SM70: { +; SM70-NEXT: .reg .b16 %rs<2>; +; SM70-NEXT: .reg .b32 %r<4>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %r1, [test_fptoui_i16_param_0]; +; SM70-NEXT: shl.b32 %r2, %r1, 16; +; SM70-NEXT: mov.b32 %f1, %r2; +; SM70-NEXT: cvt.rzi.u16.f32 %rs1, %f1; +; SM70-NEXT: cvt.u32.u16 %r3, %rs1; +; SM70-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_fptoui_i16( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<3>; +; SM80-NEXT: .reg .b32 %r<2>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b16 %rs1, [test_fptoui_i16_param_0]; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs1; +; SM80-NEXT: cvt.rzi.u16.f32 %rs2, %f1; +; SM80-NEXT: cvt.u32.u16 %r1, %rs2; +; SM80-NEXT: st.param.b32 [func_retval0+0], %r1; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_fptoui_i16( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<3>; +; SM90-NEXT: .reg .b32 %r<2>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b16 %rs1, [test_fptoui_i16_param_0]; +; SM90-NEXT: cvt.rzi.u16.bf16 %rs2, %rs1; +; SM90-NEXT: cvt.u32.u16 %r1, %rs2; +; SM90-NEXT: st.param.b32 [func_retval0+0], %r1; +; SM90-NEXT: ret; %r = fptoui bfloat %a to i16 ret i16 %r } -; CHECK-LABEL: test_sitofp_i16( -; CHECK: ld.param.u16 [[A:%rs[0-9]+]], [test_sitofp_i16_param_0]; -; SM80: cvt.rn.f32.s16 [[B:%f[0-9]+]], [[A]]; -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; -; SM90: cvt.rn.bf16.s16 [[R:%rs[0-9]+]], [[A]]; -; CHECK: st.param.b16 [func_retval0+0], [[R]]; -; CHECK: ret; define bfloat @test_sitofp_i16(i16 %a) { +; SM70-LABEL: test_sitofp_i16( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<4>; +; SM70-NEXT: .reg .b32 %r<7>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %rs1, [test_sitofp_i16_param_0]; +; SM70-NEXT: cvt.rn.f32.s16 %f1, %rs1; +; SM70-NEXT: mov.b32 %r1, %f1; +; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1; +; SM70-NEXT: add.s32 %r3, %r2, %r1; +; SM70-NEXT: add.s32 %r4, %r3, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f1, %f1; +; SM70-NEXT: or.b32 %r5, %r1, 4194304; +; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_sitofp_i16( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<3>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.u16 %rs1, [test_sitofp_i16_param_0]; +; SM80-NEXT: cvt.rn.f32.s16 %f1, %rs1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs2, %f1; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_sitofp_i16( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<3>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.u16 %rs1, [test_sitofp_i16_param_0]; +; SM90-NEXT: cvt.rn.bf16.s16 %rs2, %rs1; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM90-NEXT: ret; %r = sitofp i16 %a to bfloat ret bfloat %r } -; CHECK-LABEL: test_uitofp_i8( -; CHECK: ld.param.u8 %rs1, [test_uitofp_i8_param_0]; -; SM80: cvt.rn.f32.u16 [[B:%f[0-9]+]], [[A]]; -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; -; SM90: cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]]; -; CHECK: st.param.b16 [func_retval0+0], [[R]]; -; CHECK: ret; define bfloat @test_uitofp_i8(i8 %a) { +; SM70-LABEL: test_uitofp_i8( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<4>; +; SM70-NEXT: .reg .b32 %r<7>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u8 %rs1, [test_uitofp_i8_param_0]; +; SM70-NEXT: cvt.rn.f32.u16 %f1, %rs1; +; SM70-NEXT: mov.b32 %r1, %f1; +; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1; +; SM70-NEXT: add.s32 %r3, %r2, %r1; +; SM70-NEXT: add.s32 %r4, %r3, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f1, %f1; +; SM70-NEXT: or.b32 %r5, %r1, 4194304; +; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_uitofp_i8( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<3>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.u8 %rs1, [test_uitofp_i8_param_0]; +; SM80-NEXT: cvt.rn.f32.u16 %f1, %rs1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs2, %f1; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_uitofp_i8( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<3>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.u8 %rs1, [test_uitofp_i8_param_0]; +; SM90-NEXT: cvt.rn.bf16.u16 %rs2, %rs1; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM90-NEXT: ret; %r = uitofp i8 %a to bfloat ret bfloat %r } -; CHECK-LABEL: test_uitofp_i1( -; CHECK: ld.param.u8 [[A:%rs[0-9]+]], [test_uitofp_i1_param_0]; -; CHECK: and.b16 [[B:%rs[0-9]+]], [[A]], 1; -; CHECK: setp.eq.b16 [[C:%p[0-9]+]], [[B]], 1; -; CHECK: selp.u32 [[D:%r[0-9]+]], 1, 0, [[C]]; -; SM80: cvt.rn.f32.u32 [[E:%f[0-9]+]], [[D]]; -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[E]]; -; SM90: cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[D]]; -; CHECK: st.param.b16 [func_retval0+0], [[R]]; -; CHECK: ret; define bfloat @test_uitofp_i1(i1 %a) { +; SM70-LABEL: test_uitofp_i1( +; SM70: { +; SM70-NEXT: .reg .pred %p<3>; +; SM70-NEXT: .reg .b16 %rs<5>; +; SM70-NEXT: .reg .b32 %r<8>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u8 %rs1, [test_uitofp_i1_param_0]; +; SM70-NEXT: and.b16 %rs2, %rs1, 1; +; SM70-NEXT: setp.eq.b16 %p1, %rs2, 1; +; SM70-NEXT: selp.u32 %r1, 1, 0, %p1; +; SM70-NEXT: cvt.rn.f32.u32 %f1, %r1; +; SM70-NEXT: mov.b32 %r2, %f1; +; SM70-NEXT: bfe.u32 %r3, %r2, 16, 1; +; SM70-NEXT: add.s32 %r4, %r3, %r2; +; SM70-NEXT: add.s32 %r5, %r4, 32767; +; SM70-NEXT: setp.nan.f32 %p2, %f1, %f1; +; SM70-NEXT: or.b32 %r6, %r2, 4194304; +; SM70-NEXT: selp.b32 %r7, %r6, %r5, %p2; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs3}, %r7; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_uitofp_i1( +; SM80: { +; SM80-NEXT: .reg .pred %p<2>; +; SM80-NEXT: .reg .b16 %rs<4>; +; SM80-NEXT: .reg .b32 %r<2>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.u8 %rs1, [test_uitofp_i1_param_0]; +; SM80-NEXT: and.b16 %rs2, %rs1, 1; +; SM80-NEXT: setp.eq.b16 %p1, %rs2, 1; +; SM80-NEXT: selp.u32 %r1, 1, 0, %p1; +; SM80-NEXT: cvt.rn.f32.u32 %f1, %r1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs3, %f1; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_uitofp_i1( +; SM90: { +; SM90-NEXT: .reg .pred %p<2>; +; SM90-NEXT: .reg .b16 %rs<4>; +; SM90-NEXT: .reg .b32 %r<2>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.u8 %rs1, [test_uitofp_i1_param_0]; +; SM90-NEXT: and.b16 %rs2, %rs1, 1; +; SM90-NEXT: setp.eq.b16 %p1, %rs2, 1; +; SM90-NEXT: selp.u32 %r1, 1, 0, %p1; +; SM90-NEXT: cvt.rn.bf16.u32 %rs3, %r1; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM90-NEXT: ret; %r = uitofp i1 %a to bfloat ret bfloat %r } -; CHECK-LABEL: test_uitofp_i16( -; CHECK: ld.param.u16 [[A:%rs[0-9]+]], [test_uitofp_i16_param_0]; -; SM80: cvt.rn.f32.u16 [[B:%f[0-9]+]], [[A]]; -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; -; SM90: cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]]; -; CHECK: st.param.b16 [func_retval0+0], [[R]]; -; CHECK: ret; define bfloat @test_uitofp_i16(i16 %a) { +; SM70-LABEL: test_uitofp_i16( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<4>; +; SM70-NEXT: .reg .b32 %r<7>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %rs1, [test_uitofp_i16_param_0]; +; SM70-NEXT: cvt.rn.f32.u16 %f1, %rs1; +; SM70-NEXT: mov.b32 %r1, %f1; +; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1; +; SM70-NEXT: add.s32 %r3, %r2, %r1; +; SM70-NEXT: add.s32 %r4, %r3, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f1, %f1; +; SM70-NEXT: or.b32 %r5, %r1, 4194304; +; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_uitofp_i16( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<3>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.u16 %rs1, [test_uitofp_i16_param_0]; +; SM80-NEXT: cvt.rn.f32.u16 %f1, %rs1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs2, %f1; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_uitofp_i16( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<3>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.u16 %rs1, [test_uitofp_i16_param_0]; +; SM90-NEXT: cvt.rn.bf16.u16 %rs2, %rs1; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM90-NEXT: ret; %r = uitofp i16 %a to bfloat ret bfloat %r } -; CHECK-LABEL: test_uitofp_i32( -; CHECK: ld.param.u32 [[A:%r[0-9]+]], [test_uitofp_i32_param_0]; -; SM80: cvt.rn.f32.u32 [[B:%f[0-9]+]], [[A]]; -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; -; SM90: cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[A]]; -; CHECK: st.param.b16 [func_retval0+0], [[R]]; -; CHECK: ret; define bfloat @test_uitofp_i32(i32 %a) { +; SM70-LABEL: test_uitofp_i32( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<3>; +; SM70-NEXT: .reg .b32 %r<8>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u32 %r1, [test_uitofp_i32_param_0]; +; SM70-NEXT: cvt.rn.f32.u32 %f1, %r1; +; SM70-NEXT: mov.b32 %r2, %f1; +; SM70-NEXT: bfe.u32 %r3, %r2, 16, 1; +; SM70-NEXT: add.s32 %r4, %r3, %r2; +; SM70-NEXT: add.s32 %r5, %r4, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f1, %f1; +; SM70-NEXT: or.b32 %r6, %r2, 4194304; +; SM70-NEXT: selp.b32 %r7, %r6, %r5, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_uitofp_i32( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<2>; +; SM80-NEXT: .reg .b32 %r<2>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.u32 %r1, [test_uitofp_i32_param_0]; +; SM80-NEXT: cvt.rn.f32.u32 %f1, %r1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs1, %f1; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_uitofp_i32( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<2>; +; SM90-NEXT: .reg .b32 %r<2>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.u32 %r1, [test_uitofp_i32_param_0]; +; SM90-NEXT: cvt.rn.bf16.u32 %rs1, %r1; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM90-NEXT: ret; %r = uitofp i32 %a to bfloat ret bfloat %r } -; CHECK-LABEL: test_uitofp_i64( -; CHECK: ld.param.u64 [[A:%rd[0-9]+]], [test_uitofp_i64_param_0]; -; SM80: cvt.rn.f32.u64 [[B:%f[0-9]+]], [[A]]; -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; -; SM90: cvt.rn.bf16.u64 [[R:%rs[0-9]+]], [[A]]; -; CHECK: st.param.b16 [func_retval0+0], [[R]]; -; CHECK: ret; define bfloat @test_uitofp_i64(i64 %a) { +; SM70-LABEL: test_uitofp_i64( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<3>; +; SM70-NEXT: .reg .b32 %r<7>; +; SM70-NEXT: .reg .f32 %f<2>; +; SM70-NEXT: .reg .b64 %rd<2>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u64 %rd1, [test_uitofp_i64_param_0]; +; SM70-NEXT: cvt.rn.f32.u64 %f1, %rd1; +; SM70-NEXT: mov.b32 %r1, %f1; +; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1; +; SM70-NEXT: add.s32 %r3, %r2, %r1; +; SM70-NEXT: add.s32 %r4, %r3, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f1, %f1; +; SM70-NEXT: or.b32 %r5, %r1, 4194304; +; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r6; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_uitofp_i64( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<2>; +; SM80-NEXT: .reg .f32 %f<2>; +; SM80-NEXT: .reg .b64 %rd<2>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.u64 %rd1, [test_uitofp_i64_param_0]; +; SM80-NEXT: cvt.rn.f32.u64 %f1, %rd1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs1, %f1; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_uitofp_i64( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<2>; +; SM90-NEXT: .reg .b64 %rd<2>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.u64 %rd1, [test_uitofp_i64_param_0]; +; SM90-NEXT: cvt.rn.bf16.u64 %rs1, %rd1; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM90-NEXT: ret; %r = uitofp i64 %a to bfloat ret bfloat %r } -; CHECK-LABEL: test_roundeven( -; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_roundeven_param_0]; -; SM80: cvt.rni.f32.f32 [[F:%f[0-9]+]] -; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[F]]; -; SM90: cvt.rni.bf16.bf16 [[R:%rs[0-9]+]], [[A]]; -; CHECK: st.param.b16 [func_retval0+0], [[R]]; -; CHECK: ret; define bfloat @test_roundeven(bfloat %a) { +; SM70-LABEL: test_roundeven( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<3>; +; SM70-NEXT: .reg .b32 %r<9>; +; SM70-NEXT: .reg .f32 %f<3>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %r1, [test_roundeven_param_0]; +; SM70-NEXT: shl.b32 %r2, %r1, 16; +; SM70-NEXT: mov.b32 %f1, %r2; +; SM70-NEXT: cvt.rni.f32.f32 %f2, %f1; +; SM70-NEXT: mov.b32 %r3, %f2; +; SM70-NEXT: bfe.u32 %r4, %r3, 16, 1; +; SM70-NEXT: add.s32 %r5, %r4, %r3; +; SM70-NEXT: add.s32 %r6, %r5, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f2, %f2; +; SM70-NEXT: or.b32 %r7, %r3, 4194304; +; SM70-NEXT: selp.b32 %r8, %r7, %r6, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_roundeven( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<3>; +; SM80-NEXT: .reg .f32 %f<3>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b16 %rs1, [test_roundeven_param_0]; +; SM80-NEXT: cvt.f32.bf16 %f1, %rs1; +; SM80-NEXT: cvt.rni.f32.f32 %f2, %f1; +; SM80-NEXT: cvt.rn.bf16.f32 %rs2, %f2; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_roundeven( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<3>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b16 %rs1, [test_roundeven_param_0]; +; SM90-NEXT: cvt.rni.bf16.bf16 %rs2, %rs1; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs2; +; SM90-NEXT: ret; %r = call bfloat @llvm.roundeven.bf16(bfloat %a) ret bfloat %r } + +define bfloat @test_maximum(bfloat %a, bfloat %b) { +; SM70-LABEL: test_maximum( +; SM70: { +; SM70-NEXT: .reg .pred %p<6>; +; SM70-NEXT: .reg .b16 %rs<11>; +; SM70-NEXT: .reg .b32 %r<7>; +; SM70-NEXT: .reg .f32 %f<4>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.b16 %rs1, [test_maximum_param_0]; +; SM70-NEXT: ld.param.b16 %rs3, [test_maximum_param_1]; +; SM70-NEXT: cvt.u32.u16 %r1, %rs3; +; SM70-NEXT: shl.b32 %r2, %r1, 16; +; SM70-NEXT: mov.b32 %f1, %r2; +; SM70-NEXT: cvt.u32.u16 %r3, %rs1; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f2, %r4; +; SM70-NEXT: setp.gt.f32 %p1, %f2, %f1; +; SM70-NEXT: selp.b16 %rs5, %rs1, %rs3, %p1; +; SM70-NEXT: setp.nan.f32 %p2, %f2, %f1; +; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2; +; SM70-NEXT: setp.eq.s16 %p3, %rs1, 0; +; SM70-NEXT: selp.b16 %rs7, %rs1, %rs6, %p3; +; SM70-NEXT: setp.eq.s16 %p4, %rs3, 0; +; SM70-NEXT: selp.b16 %rs8, %rs3, %rs7, %p4; +; SM70-NEXT: cvt.u32.u16 %r5, %rs6; +; SM70-NEXT: shl.b32 %r6, %r5, 16; +; SM70-NEXT: mov.b32 %f3, %r6; +; SM70-NEXT: setp.eq.f32 %p5, %f3, 0f00000000; +; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5; +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs10; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_maximum( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<4>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b16 %rs1, [test_maximum_param_0]; +; SM80-NEXT: ld.param.b16 %rs2, [test_maximum_param_1]; +; SM80-NEXT: max.NaN.bf16 %rs3, %rs1, %rs2; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_maximum( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b16 %rs1, [test_maximum_param_0]; +; SM90-NEXT: ld.param.b16 %rs2, [test_maximum_param_1]; +; SM90-NEXT: max.NaN.bf16 %rs3, %rs1, %rs2; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM90-NEXT: ret; + %r = call bfloat @llvm.maximum.bf16(bfloat %a, bfloat %b) + ret bfloat %r +} + +define bfloat @test_maxnum(bfloat %a, bfloat %b) { +; SM70-LABEL: test_maxnum( +; SM70: { +; SM70-NEXT: .reg .pred %p<2>; +; SM70-NEXT: .reg .b16 %rs<3>; +; SM70-NEXT: .reg .b32 %r<11>; +; SM70-NEXT: .reg .f32 %f<4>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.u16 %r1, [test_maxnum_param_1]; +; SM70-NEXT: shl.b32 %r2, %r1, 16; +; SM70-NEXT: mov.b32 %f1, %r2; +; SM70-NEXT: ld.param.u16 %r3, [test_maxnum_param_0]; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f2, %r4; +; SM70-NEXT: max.f32 %f3, %f2, %f1; +; SM70-NEXT: mov.b32 %r5, %f3; +; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1; +; SM70-NEXT: add.s32 %r7, %r6, %r5; +; SM70-NEXT: add.s32 %r8, %r7, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f3, %f3; +; SM70-NEXT: or.b32 %r9, %r5, 4194304; +; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; } +; SM70-NEXT: st.param.b16 [func_retval0+0], %rs1; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_maxnum( +; SM80: { +; SM80-NEXT: .reg .b16 %rs<4>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b16 %rs1, [test_maxnum_param_0]; +; SM80-NEXT: ld.param.b16 %rs2, [test_maxnum_param_1]; +; SM80-NEXT: max.bf16 %rs3, %rs1, %rs2; +; SM80-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_maxnum( +; SM90: { +; SM90-NEXT: .reg .b16 %rs<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b16 %rs1, [test_maxnum_param_0]; +; SM90-NEXT: ld.param.b16 %rs2, [test_maxnum_param_1]; +; SM90-NEXT: max.bf16 %rs3, %rs1, %rs2; +; SM90-NEXT: st.param.b16 [func_retval0+0], %rs3; +; SM90-NEXT: ret; + %r = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b) + ret bfloat %r +} + +define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) { +; SM70-LABEL: test_maximum_v2( +; SM70: { +; SM70-NEXT: .reg .pred %p<11>; +; SM70-NEXT: .reg .b16 %rs<21>; +; SM70-NEXT: .reg .b32 %r<16>; +; SM70-NEXT: .reg .f32 %f<7>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_0]; +; SM70-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_1]; +; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM70-NEXT: cvt.u32.u16 %r3, %rs2; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f1, %r4; +; SM70-NEXT: mov.b32 {%rs4, %rs5}, %r1; +; SM70-NEXT: cvt.u32.u16 %r5, %rs5; +; SM70-NEXT: shl.b32 %r6, %r5, 16; +; SM70-NEXT: mov.b32 %f2, %r6; +; SM70-NEXT: setp.gt.f32 %p1, %f2, %f1; +; SM70-NEXT: selp.b16 %rs7, %rs5, %rs2, %p1; +; SM70-NEXT: setp.nan.f32 %p2, %f2, %f1; +; SM70-NEXT: selp.b16 %rs8, 0x7FC0, %rs7, %p2; +; SM70-NEXT: setp.eq.s16 %p3, %rs5, 0; +; SM70-NEXT: selp.b16 %rs9, %rs5, %rs8, %p3; +; SM70-NEXT: setp.eq.s16 %p4, %rs2, 0; +; SM70-NEXT: selp.b16 %rs10, %rs2, %rs9, %p4; +; SM70-NEXT: cvt.u32.u16 %r7, %rs8; +; SM70-NEXT: shl.b32 %r8, %r7, 16; +; SM70-NEXT: mov.b32 %f3, %r8; +; SM70-NEXT: setp.eq.f32 %p5, %f3, 0f00000000; +; SM70-NEXT: selp.b16 %rs12, %rs10, %rs8, %p5; +; SM70-NEXT: cvt.u32.u16 %r9, %rs1; +; SM70-NEXT: shl.b32 %r10, %r9, 16; +; SM70-NEXT: mov.b32 %f4, %r10; +; SM70-NEXT: cvt.u32.u16 %r11, %rs4; +; SM70-NEXT: shl.b32 %r12, %r11, 16; +; SM70-NEXT: mov.b32 %f5, %r12; +; SM70-NEXT: setp.gt.f32 %p6, %f5, %f4; +; SM70-NEXT: selp.b16 %rs15, %rs4, %rs1, %p6; +; SM70-NEXT: setp.nan.f32 %p7, %f5, %f4; +; SM70-NEXT: selp.b16 %rs16, 0x7FC0, %rs15, %p7; +; SM70-NEXT: setp.eq.s16 %p8, %rs4, 0; +; SM70-NEXT: selp.b16 %rs17, %rs4, %rs16, %p8; +; SM70-NEXT: setp.eq.s16 %p9, %rs1, 0; +; SM70-NEXT: selp.b16 %rs18, %rs1, %rs17, %p9; +; SM70-NEXT: cvt.u32.u16 %r13, %rs16; +; SM70-NEXT: shl.b32 %r14, %r13, 16; +; SM70-NEXT: mov.b32 %f6, %r14; +; SM70-NEXT: setp.eq.f32 %p10, %f6, 0f00000000; +; SM70-NEXT: selp.b16 %rs20, %rs18, %rs16, %p10; +; SM70-NEXT: mov.b32 %r15, {%rs20, %rs12}; +; SM70-NEXT: st.param.b32 [func_retval0+0], %r15; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_maximum_v2( +; SM80: { +; SM80-NEXT: .reg .b32 %r<4>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_1]; +; SM80-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_0]; +; SM80-NEXT: max.NaN.bf16x2 %r3, %r2, %r1; +; SM80-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_maximum_v2( +; SM90: { +; SM90-NEXT: .reg .b32 %r<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_1]; +; SM90-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_0]; +; SM90-NEXT: max.NaN.bf16x2 %r3, %r2, %r1; +; SM90-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM90-NEXT: ret; + %r = call <2 x bfloat> @llvm.maximum.bf16(<2 x bfloat> %a, <2 x bfloat> %b) + ret <2 x bfloat> %r +} + +define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) { +; SM70-LABEL: test_maxnum_v2( +; SM70: { +; SM70-NEXT: .reg .pred %p<3>; +; SM70-NEXT: .reg .b16 %rs<13>; +; SM70-NEXT: .reg .b32 %r<24>; +; SM70-NEXT: .reg .f32 %f<7>; +; SM70-EMPTY: +; SM70-NEXT: // %bb.0: +; SM70-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_0]; +; SM70-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_1]; +; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2; +; SM70-NEXT: cvt.u32.u16 %r3, %rs2; +; SM70-NEXT: shl.b32 %r4, %r3, 16; +; SM70-NEXT: mov.b32 %f1, %r4; +; SM70-NEXT: mov.b32 {%rs4, %rs5}, %r1; +; SM70-NEXT: cvt.u32.u16 %r5, %rs5; +; SM70-NEXT: shl.b32 %r6, %r5, 16; +; SM70-NEXT: mov.b32 %f2, %r6; +; SM70-NEXT: max.f32 %f3, %f2, %f1; +; SM70-NEXT: mov.b32 %r7, %f3; +; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1; +; SM70-NEXT: add.s32 %r9, %r8, %r7; +; SM70-NEXT: add.s32 %r10, %r9, 32767; +; SM70-NEXT: setp.nan.f32 %p1, %f3, %f3; +; SM70-NEXT: or.b32 %r11, %r7, 4194304; +; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; } +; SM70-NEXT: cvt.u32.u16 %r13, %rs1; +; SM70-NEXT: shl.b32 %r14, %r13, 16; +; SM70-NEXT: mov.b32 %f4, %r14; +; SM70-NEXT: cvt.u32.u16 %r15, %rs4; +; SM70-NEXT: shl.b32 %r16, %r15, 16; +; SM70-NEXT: mov.b32 %f5, %r16; +; SM70-NEXT: max.f32 %f6, %f5, %f4; +; SM70-NEXT: mov.b32 %r17, %f6; +; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1; +; SM70-NEXT: add.s32 %r19, %r18, %r17; +; SM70-NEXT: add.s32 %r20, %r19, 32767; +; SM70-NEXT: setp.nan.f32 %p2, %f6, %f6; +; SM70-NEXT: or.b32 %r21, %r17, 4194304; +; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2; +; SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; } +; SM70-NEXT: mov.b32 %r23, {%rs11, %rs7}; +; SM70-NEXT: st.param.b32 [func_retval0+0], %r23; +; SM70-NEXT: ret; +; +; SM80-LABEL: test_maxnum_v2( +; SM80: { +; SM80-NEXT: .reg .b32 %r<4>; +; SM80-EMPTY: +; SM80-NEXT: // %bb.0: +; SM80-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_1]; +; SM80-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_0]; +; SM80-NEXT: max.bf16x2 %r3, %r2, %r1; +; SM80-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM80-NEXT: ret; +; +; SM90-LABEL: test_maxnum_v2( +; SM90: { +; SM90-NEXT: .reg .b32 %r<4>; +; SM90-EMPTY: +; SM90-NEXT: // %bb.0: +; SM90-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_1]; +; SM90-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_0]; +; SM90-NEXT: max.bf16x2 %r3, %r2, %r1; +; SM90-NEXT: st.param.b32 [func_retval0+0], %r3; +; SM90-NEXT: ret; + %r = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) + ret <2 x bfloat> %r +}