From ef748583c24d7dd7ab1f904a27f3a1382cda22c6 Mon Sep 17 00:00:00 2001 From: Sanjay Patel Date: Tue, 13 Oct 2020 11:52:00 -0400 Subject: [PATCH] [CostModel] rearrange basic intrinsic cost implementation This is bigger/uglier than before, but it should allow fixing all of the broken paths more easily. Test coverage added with rGfab028b and other commits. This is not NFC - the scalable vector test would crash without this patch. --- llvm/include/llvm/CodeGen/BasicTTIImpl.h | 68 ++++++++++++------- .../Analysis/CostModel/AArch64/sve-math.ll | 21 ++++++ 2 files changed, 66 insertions(+), 23 deletions(-) diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index b833130f034838..742fded56a8ebe 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1125,32 +1125,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { if (BaseT::getIntrinsicInstrCost(ICA, CostKind) == 0) return 0; - // Special case some scalar intrinsics. - Intrinsic::ID IID = ICA.getID(); - if (CostKind != TTI::TCK_RecipThroughput) { - switch (IID) { - default: - break; - case Intrinsic::cttz: - if (getTLI()->isCheapToSpeculateCttz()) - return TargetTransformInfo::TCC_Basic; - break; - case Intrinsic::ctlz: - if (getTLI()->isCheapToSpeculateCtlz()) - return TargetTransformInfo::TCC_Basic; - break; - case Intrinsic::memcpy: - return thisT()->getMemcpyCost(ICA.getInst()); - // TODO: other libc intrinsics. - } - return BaseT::getIntrinsicInstrCost(ICA, CostKind); - } - - // TODO: Combine these two logic paths. if (ICA.isTypeBasedOnly()) return getTypeBasedIntrinsicInstrCost(ICA, CostKind); + // TODO: Handle scalable vectors? Type *RetTy = ICA.getReturnType(); + if (isa(RetTy)) + return BaseT::getIntrinsicInstrCost(ICA, CostKind); + unsigned VF = ICA.getVectorFactor(); unsigned RetVF = (RetTy->isVectorTy() ? cast(RetTy)->getNumElements() @@ -1159,11 +1141,42 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { const IntrinsicInst *I = ICA.getInst(); const SmallVectorImpl &Args = ICA.getArgs(); FastMathFlags FMF = ICA.getFlags(); - + Intrinsic::ID IID = ICA.getID(); switch (IID) { default: + // FIXME: all cost kinds should default to the same thing? + if (CostKind != TTI::TCK_RecipThroughput) + return BaseT::getIntrinsicInstrCost(ICA, CostKind); break; + + case Intrinsic::cttz: + // FIXME: all cost kinds should default to the same thing? + if (CostKind != TTI::TCK_RecipThroughput) { + if (getTLI()->isCheapToSpeculateCttz()) + return TargetTransformInfo::TCC_Basic; + return BaseT::getIntrinsicInstrCost(ICA, CostKind); + } + break; + + case Intrinsic::ctlz: + // FIXME: all cost kinds should default to the same thing? + if (CostKind != TTI::TCK_RecipThroughput) { + if (getTLI()->isCheapToSpeculateCtlz()) + return TargetTransformInfo::TCC_Basic; + return BaseT::getIntrinsicInstrCost(ICA, CostKind); + } + break; + + case Intrinsic::memcpy: + // FIXME: all cost kinds should default to the same thing? + if (CostKind != TTI::TCK_RecipThroughput) + return thisT()->getMemcpyCost(ICA.getInst()); + return BaseT::getIntrinsicInstrCost(ICA, CostKind); + case Intrinsic::masked_scatter: { + // FIXME: all cost kinds should default to the same thing? + if (CostKind != TTI::TCK_RecipThroughput) + return BaseT::getIntrinsicInstrCost(ICA, CostKind); assert(VF == 1 && "Can't vectorize types here."); const Value *Mask = Args[3]; bool VarMask = !isa(Mask); @@ -1173,6 +1186,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { VarMask, Alignment, CostKind, I); } case Intrinsic::masked_gather: { + // FIXME: all cost kinds should default to the same thing? + if (CostKind != TTI::TCK_RecipThroughput) + return BaseT::getIntrinsicInstrCost(ICA, CostKind); assert(VF == 1 && "Can't vectorize types here."); const Value *Mask = Args[2]; bool VarMask = !isa(Mask); @@ -1193,11 +1209,17 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { case Intrinsic::vector_reduce_fmin: case Intrinsic::vector_reduce_umax: case Intrinsic::vector_reduce_umin: { + // FIXME: all cost kinds should default to the same thing? + if (CostKind != TTI::TCK_RecipThroughput) + return BaseT::getIntrinsicInstrCost(ICA, CostKind); IntrinsicCostAttributes Attrs(IID, RetTy, Args[0]->getType(), FMF, 1, I); return getTypeBasedIntrinsicInstrCost(Attrs, CostKind); } case Intrinsic::fshl: case Intrinsic::fshr: { + // FIXME: all cost kinds should default to the same thing? + if (CostKind != TTI::TCK_RecipThroughput) + return BaseT::getIntrinsicInstrCost(ICA, CostKind); const Value *X = Args[0]; const Value *Y = Args[1]; const Value *Z = Args[2]; diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-math.ll b/llvm/test/Analysis/CostModel/AArch64/sve-math.ll index 4a4d7dc117f390..5769fe3b0ff23c 100644 --- a/llvm/test/Analysis/CostModel/AArch64/sve-math.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-math.ll @@ -26,3 +26,24 @@ define @fadd_v2f64( %a, %a, %b ret %r } + +define @sqrt_v2f64( %a) { +; THRU-LABEL: 'sqrt_v2f64' +; THRU-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) +; THRU-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret %r +; +; LATE-LABEL: 'sqrt_v2f64' +; LATE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) +; LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret %r +; +; SIZE-LABEL: 'sqrt_v2f64' +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) +; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret %r +; +; SIZE_LATE-LABEL: 'sqrt_v2f64' +; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %r = call @llvm.sqrt.nxv2f64( %a) +; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret %r +; + %r = call @llvm.sqrt.v2f64( %a) + ret %r +}