Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DAG][RISCV] Use vp_reduce_* when widening illegal types for reductions #105455

Merged
merged 5 commits into from
Aug 22, 2024
Merged

[DAG][RISCV] Use vp_reduce_* when widening illegal types for reductions #105455

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8d6c803
[DAG][RISCV] Use vp_reduce_* when widening illegal types for reductions
preames Aug 20, 2024
d2cac62
clang-format
preames Aug 21, 2024
36ae990
Add support for unordered FP types and a bit of code style cleanup
preames Aug 21, 2024
41dcac0
Merge branch 'main' into pr-riscv-reduce-widen-via-vp
preames Aug 21, 2024
7a7455f
Update comments
preames Aug 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions llvm/include/llvm/IR/VPIntrinsics.def
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,63 +651,64 @@ END_REGISTER_VP(vp_gather, VP_GATHER)
#error \
"The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!"
#endif
#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN) \
#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN, SDOPC) \
BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, 1) \
VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \
VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
VP_PROPERTY_REDUCTION(0, 1) \
END_REGISTER_VP(VPID, VPSD)

// llvm.vp.reduce.add(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD,
vector_reduce_add)
vector_reduce_add, VECREDUCE_ADD)

// llvm.vp.reduce.mul(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL,
vector_reduce_mul)
vector_reduce_mul, VECREDUCE_MUL)

// llvm.vp.reduce.and(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND,
vector_reduce_and)
vector_reduce_and, VECREDUCE_AND)

// llvm.vp.reduce.or(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR,
vector_reduce_or)
vector_reduce_or, VECREDUCE_OR)

// llvm.vp.reduce.xor(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR,
vector_reduce_xor)
vector_reduce_xor, VECREDUCE_XOR)

// llvm.vp.reduce.smax(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX,
vector_reduce_smax)
vector_reduce_smax, VECREDUCE_SMAX)

// llvm.vp.reduce.smin(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN,
vector_reduce_smin)
vector_reduce_smin, VECREDUCE_SMIN)

// llvm.vp.reduce.umax(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX,
vector_reduce_umax)
vector_reduce_umax, VECREDUCE_UMAX)

// llvm.vp.reduce.umin(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN,
vector_reduce_umin)
vector_reduce_umin, VECREDUCE_UMIN)

// llvm.vp.reduce.fmax(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX,
vector_reduce_fmax)
vector_reduce_fmax, VECREDUCE_FMAX)

// llvm.vp.reduce.fmin(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN,
vector_reduce_fmin)
vector_reduce_fmin, VECREDUCE_FMIN)

// llvm.vp.reduce.fmaximum(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmaximum, VP_REDUCE_FMAXIMUM,
vector_reduce_fmaximum)
vector_reduce_fmaximum, VECREDUCE_FMAXIMUM)

// llvm.vp.reduce.fminimum(start,x,mask,vlen)
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fminimum, VP_REDUCE_FMINIMUM,
vector_reduce_fminimum)
vector_reduce_fminimum, VECREDUCE_FMINIMUM)

#undef HELPER_REGISTER_REDUCTION_VP

Expand Down
52 changes: 50 additions & 2 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7259,9 +7259,38 @@ SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
return DAG.getBuildVector(VT, dl, Scalars);
}

static unsigned getExtendForIntVecReduction(SDNode *N) {
switch (N->getOpcode()) {
default:
llvm_unreachable("Expected integer vector reduction");
case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_MUL:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
case ISD::VP_REDUCE_ADD:
case ISD::VP_REDUCE_MUL:
case ISD::VP_REDUCE_AND:
case ISD::VP_REDUCE_OR:
case ISD::VP_REDUCE_XOR:
return ISD::ANY_EXTEND;
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VP_REDUCE_SMAX:
case ISD::VP_REDUCE_SMIN:
return ISD::SIGN_EXTEND;
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_UMIN:
case ISD::VP_REDUCE_UMAX:
case ISD::VP_REDUCE_UMIN:
return ISD::ZERO_EXTEND;
}
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this the same as getExtendForIntVecReduction in LegalizeIntegerTypes.cpp? If so is there a way to deduplicate it

SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
SDLoc dl(N);
SDValue Op = GetWidenedVector(N->getOperand(0));
EVT VT = N->getValueType(0);
EVT OrigVT = N->getOperand(0).getValueType();
EVT WideVT = Op.getValueType();
EVT ElemVT = OrigVT.getVectorElementType();
Expand All @@ -7276,6 +7305,25 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
unsigned OrigElts = OrigVT.getVectorMinNumElements();
unsigned WideElts = WideVT.getVectorMinNumElements();

// Generate a vp.reduce_op if it is custom/legal for the target. This avoids
// need pad the source vector, because the inactive lanes can simply be
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the need to pad?

// disabled and not contribute to the result. To avoid possible recursion,
// only do this if the widened mask type is legal.
if (auto VPOpcode = ISD::getVPForBaseOpcode(Opc);
VPOpcode && VT.isInteger() &&
TLI.isOperationLegalOrCustom(*VPOpcode, WideVT)) {
if (EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
WideVT.getVectorElementCount());
TLI.isTypeLegal(WideMaskVT)) {
SDValue Start =
DAG.getNode(getExtendForIntVecReduction(N), dl, VT, NeutralElem);
SDValue Mask = DAG.getAllOnesConstant(dl, WideMaskVT);
SDValue EVL = DAG.getElementCount(dl, TLI.getVPExplicitVectorLengthTy(),
OrigVT.getVectorElementCount());
return DAG.getNode(*VPOpcode, dl, VT, {Start, Op, Mask, EVL}, Flags);
}
}

if (WideVT.isScalableVector()) {
unsigned GCD = std::gcd(OrigElts, WideElts);
EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT,
Expand All @@ -7284,14 +7332,14 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral,
DAG.getVectorIdxConstant(Idx, dl));
return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
return DAG.getNode(Opc, dl, VT, Op, Flags);
}

for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
DAG.getVectorIdxConstant(Idx, dl));

return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
return DAG.getNode(Opc, dl, VT, Op, Flags);
}

SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
Expand Down
Loading
Loading