diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index bc60d14ee700fe..da4bcd7f0c66ed 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -372,6 +372,9 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM, // Handle prefetches with PFD or PFDRL. setOperationAction(ISD::PREFETCH, MVT::Other, Custom); + // Handle readcyclecounter with STCKF. + setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); + for (MVT VT : MVT::fixedlen_vector_valuetypes()) { // Assume by default that all vector operations need to be expanded. for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode) @@ -6077,6 +6080,27 @@ SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op, return getCCResult(DAG, Intr); } +SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op, + SelectionDAG &DAG) const { + SDLoc DL(Op); + SDValue Chain = Op.getOperand(0); + + // STCKF only supports a memory operand, so we have to use a temporary. + SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64); + int SPFI = cast(StackPtr.getNode())->getIndex(); + MachinePointerInfo MPI = + MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); + + // Use STCFK to store the TOD clock into the temporary. + SDValue StoreOps[] = {Chain, StackPtr}; + Chain = DAG.getMemIntrinsicNode( + SystemZISD::STCKF, DL, DAG.getVTList(MVT::Other), StoreOps, MVT::i64, + MPI, MaybeAlign(), MachineMemOperand::MOStore); + + // And read it back from there. + return DAG.getLoad(MVT::i64, DL, Chain, StackPtr, MPI); +} + SDValue SystemZTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { switch (Op.getOpcode()) { @@ -6199,6 +6223,8 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op, return lowerIS_FPCLASS(Op, DAG); case ISD::GET_ROUNDING: return lowerGET_ROUNDING(Op, DAG); + case ISD::READCYCLECOUNTER: + return lowerREADCYCLECOUNTER(Op, DAG); default: llvm_unreachable("Unexpected node to lower"); } @@ -6425,6 +6451,7 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { OPCODE(STRV); OPCODE(VLER); OPCODE(VSTER); + OPCODE(STCKF); OPCODE(PREFETCH); OPCODE(ADA_ENTRY); } @@ -6985,6 +7012,17 @@ SDValue SystemZTargetLowering::combineSTORE( } } + // Combine STORE (READCYCLECOUNTER) into STCKF. + if (!SN->isTruncatingStore() && + Op1.getOpcode() == ISD::READCYCLECOUNTER && + Op1.hasOneUse() && + N->getOperand(0).reachesChainWithoutSideEffects(SDValue(Op1.getNode(), 1))) { + SDValue Ops[] = { Op1.getOperand(0), N->getOperand(2) }; + return DAG.getMemIntrinsicNode(SystemZISD::STCKF, SDLoc(N), + DAG.getVTList(MVT::Other), + Ops, MemVT, SN->getMemOperand()); + } + // Transform a store of an i128 moved from GPRs into two separate stores. if (MemVT == MVT::i128 && SN->isSimple() && ISD::isNormalStore(SN)) { SDValue LoPart, HiPart; diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h index 406a13b9281ca8..4943c5cb703c33 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -378,6 +378,9 @@ enum NodeType : unsigned { // Element swapping load/store. Same operands as regular load/store. VLER, VSTER, + // Use STORE CLOCK FAST to store current TOD clock value. + STCKF, + // Prefetch from the second operand using the 4-bit control code in // the first operand. The code is 1 for a load prefetch and 2 for // a store prefetch. @@ -717,6 +720,7 @@ class SystemZTargetLowering : public TargetLowering { SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const; SDValue lowerIS_FPCLASS(SDValue Op, SelectionDAG &DAG) const; SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const; + SDValue lowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const; bool canTreatAsByteVector(EVT VT) const; SDValue combineExtract(const SDLoc &DL, EVT ElemVT, EVT VecVT, SDValue OrigOp, diff --git a/llvm/lib/Target/SystemZ/SystemZInstrSystem.td b/llvm/lib/Target/SystemZ/SystemZInstrSystem.td index e26417ddb72d99..497844bec85d17 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrSystem.td +++ b/llvm/lib/Target/SystemZ/SystemZInstrSystem.td @@ -339,7 +339,7 @@ let hasSideEffects = 1 in // Store clock (fast / extended). let hasSideEffects = 1, Defs = [CC] in { def STCK : StoreInherentS<"stck", 0xB205, null_frag, 8>; - def STCKF : StoreInherentS<"stckf", 0xB27C, null_frag, 8>; + def STCKF : StoreInherentS<"stckf", 0xB27C, z_stckf, 8>; def STCKE : StoreInherentS<"stcke", 0xB278, null_frag, 16>; } diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td index 6c4e33a6aa7fa8..1611436b01b7fa 100644 --- a/llvm/lib/Target/SystemZ/SystemZOperators.td +++ b/llvm/lib/Target/SystemZ/SystemZOperators.td @@ -132,6 +132,8 @@ def SDT_ZIPM : SDTypeProfile<1, 1, def SDT_ZPrefetch : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>]>; +def SDT_ZStoreInherent : SDTypeProfile<0, 1, + [SDTCisPtrTy<0>]>; def SDT_ZTBegin : SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>, @@ -307,6 +309,8 @@ def z_loadeswap : SDNode<"SystemZISD::VLER", SDTLoad, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; def z_storeeswap : SDNode<"SystemZISD::VSTER", SDTStore, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; +def z_stckf : SDNode<"SystemZISD::STCKF", SDT_ZStoreInherent, + [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; def z_tdc : SDNode<"SystemZISD::TDC", SDT_ZTest>; diff --git a/llvm/test/CodeGen/SystemZ/readcyclecounter.ll b/llvm/test/CodeGen/SystemZ/readcyclecounter.ll new file mode 100644 index 00000000000000..34b6d34143f89e --- /dev/null +++ b/llvm/test/CodeGen/SystemZ/readcyclecounter.ll @@ -0,0 +1,27 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=s390x-ibm-linux | FileCheck %s + +; Verify that we correctly lower ISD::READCYCLECOUNTER. + +define i64 @test_builtin_readcyclecounter1() { +; CHECK-LABEL: test_builtin_readcyclecounter1: +; CHECK: # %bb.0: +; CHECK-NEXT: aghi %r15, -168 +; CHECK-NEXT: .cfi_def_cfa_offset 328 +; CHECK-NEXT: stckf 160(%r15) +; CHECK-NEXT: lg %r2, 160(%r15) +; CHECK-NEXT: aghi %r15, 168 +; CHECK-NEXT: br %r14 + %1 = tail call i64 @llvm.readcyclecounter() + ret i64 %1 +} + +define void @test_builtin_readcyclecounter2(ptr %ptr) { +; CHECK-LABEL: test_builtin_readcyclecounter2: +; CHECK: # %bb.0: +; CHECK-NEXT: stckf 0(%r2) +; CHECK-NEXT: br %r14 + %1 = tail call i64 @llvm.readcyclecounter() + store i64 %1, ptr %ptr + ret void +}