Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DAG][RISCV] Use vp_reduce_* when widening illegal types for reductions #105455

Merged
merged 5 commits into from
Aug 22, 2024
Merged

[DAG][RISCV] Use vp_reduce_* when widening illegal types for reductions #105455

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8d6c803
[DAG][RISCV] Use vp_reduce_* when widening illegal types for reductions
preames Aug 20, 2024
d2cac62
clang-format
preames Aug 21, 2024
36ae990
Add support for unordered FP types and a bit of code style cleanup
preames Aug 21, 2024
41dcac0
Merge branch 'main' into pr-riscv-reduce-widen-via-vp
preames Aug 21, 2024
7a7455f
Update comments
preames Aug 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions llvm/include/llvm/IR/VPIntrinsics.def
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,63 +651,64 @@ END_REGISTER_VP(vp_gather, VP_GATHER)
#error \
"The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!"
#endif
#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN) \
#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN, SDOPC) \
BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, 1) \
VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \
VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
VP_PROPERTY_REDUCTION(0, 1) \
END_REGISTER_VP(VPID, VPSD)

// llvm.vp.reduce.add(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD,
vector_reduce_add)
vector_reduce_add, VECREDUCE_ADD)

// llvm.vp.reduce.mul(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL,
vector_reduce_mul)
vector_reduce_mul, VECREDUCE_MUL)

// llvm.vp.reduce.and(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND,
vector_reduce_and)
vector_reduce_and, VECREDUCE_AND)

// llvm.vp.reduce.or(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR,
vector_reduce_or)
vector_reduce_or, VECREDUCE_OR)

// llvm.vp.reduce.xor(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR,
vector_reduce_xor)
vector_reduce_xor, VECREDUCE_XOR)

// llvm.vp.reduce.smax(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX,
vector_reduce_smax)
vector_reduce_smax, VECREDUCE_SMAX)

// llvm.vp.reduce.smin(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN,
vector_reduce_smin)
vector_reduce_smin, VECREDUCE_SMIN)

// llvm.vp.reduce.umax(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX,
vector_reduce_umax)
vector_reduce_umax, VECREDUCE_UMAX)

// llvm.vp.reduce.umin(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN,
vector_reduce_umin)
vector_reduce_umin, VECREDUCE_UMIN)

// llvm.vp.reduce.fmax(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX,
vector_reduce_fmax)
vector_reduce_fmax, VECREDUCE_FMAX)

// llvm.vp.reduce.fmin(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN,
vector_reduce_fmin)
vector_reduce_fmin, VECREDUCE_FMIN)

// llvm.vp.reduce.fmaximum(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmaximum, VP_REDUCE_FMAXIMUM,
vector_reduce_fmaximum)
vector_reduce_fmaximum, VECREDUCE_FMAXIMUM)

// llvm.vp.reduce.fminimum(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fminimum, VP_REDUCE_FMINIMUM,
vector_reduce_fminimum)
vector_reduce_fminimum, VECREDUCE_FMINIMUM)

#undef HELPER_REGISTER_REDUCTION_VP

Expand Down
43 changes: 41 additions & 2 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7259,9 +7259,29 @@ SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
return DAG.getBuildVector(VT, dl, Scalars);
}

static unsigned getExtendForIntVecReduction(unsigned Opc) {
switch (Opc) {
default:
llvm_unreachable("Expected integer vector reduction");
case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_MUL:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
return ISD::ANY_EXTEND;
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
return ISD::SIGN_EXTEND;
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_UMIN:
return ISD::ZERO_EXTEND;
}
}

SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
SDLoc dl(N);
SDValue Op = GetWidenedVector(N->getOperand(0));
EVT VT = N->getValueType(0);
EVT OrigVT = N->getOperand(0).getValueType();
EVT WideVT = Op.getValueType();
EVT ElemVT = OrigVT.getVectorElementType();
Expand All @@ -7276,6 +7296,25 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
unsigned OrigElts = OrigVT.getVectorMinNumElements();
unsigned WideElts = WideVT.getVectorMinNumElements();

// Generate a vp.reduce_op if it is custom/legal for the target. This avoids
// needing to pad the source vector, because the inactive lanes can simply be
// disabled and not contribute to the result.
// TODO: VECREDUCE_FADD, VECREDUCE_FMUL aren't currently mapped correctly,
// and thus don't take this path.
if (auto VPOpcode = ISD::getVPForBaseOpcode(Opc);
VPOpcode && TLI.isOperationLegalOrCustom(*VPOpcode, WideVT)) {
SDValue Start = NeutralElem;
if (VT.isInteger())
Start = DAG.getNode(getExtendForIntVecReduction(Opc), dl, VT, Start);
assert(Start.getValueType() == VT);
EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
WideVT.getVectorElementCount());
SDValue Mask = DAG.getAllOnesConstant(dl, WideMaskVT);
SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
OrigVT.getVectorElementCount());
return DAG.getNode(*VPOpcode, dl, VT, {Start, Op, Mask, EVL}, Flags);
}

if (WideVT.isScalableVector()) {
unsigned GCD = std::gcd(OrigElts, WideElts);
EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT,
Expand All @@ -7284,14 +7323,14 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral,
DAG.getVectorIdxConstant(Idx, dl));
return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
return DAG.getNode(Opc, dl, VT, Op, Flags);
}

for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
DAG.getVectorIdxConstant(Idx, dl));

return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
return DAG.getNode(Opc, dl, VT, Op, Flags);
}

SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
Expand Down
Loading
Loading