diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index ec136162fbb68a..e6ddce732dec39 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -469,7 +469,7 @@ class InnerLoopVectorizer { VPTransformState &State); /// Widen a single select instruction within the innermost loop. - void widenSelectInstruction(SelectInst &I, VPUser &Operands, + void widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands, bool InvariantCond, VPTransformState &State); /// Fix the vectorized code, taking care of header phi's, live-outs, and more. @@ -4686,7 +4686,7 @@ void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def, } } -void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, +void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands, bool InvariantCond, VPTransformState &State) { @@ -4705,7 +4705,7 @@ void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, Value *Op0 = State.get(Operands.getOperand(1), Part); Value *Op1 = State.get(Operands.getOperand(2), Part); Value *Sel = Builder.CreateSelect(Cond, Op0, Op1); - VectorLoopValueMap.setVectorValue(&I, Part, Sel); + State.set(VPDef, &I, Sel, Part); addMetadata(Sel, &I); } } @@ -7636,16 +7636,8 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, // If the tail is to be folded by masking, the primary induction variable, if // exists needs to be represented in VPlan for it to model early-exit masking. - // Also, both the Phi and the live-out instruction of each reduction are - // required in order to introduce a select between them in VPlan. - if (CM.foldTailByMasking()) { - if (Legal->getPrimaryInduction()) - NeedDef.insert(Legal->getPrimaryInduction()); - for (auto &Reduction : Legal->getReductionVars()) { - NeedDef.insert(Reduction.first); - NeedDef.insert(Reduction.second.getLoopExitInstr()); - } - } + if (CM.foldTailByMasking() && Legal->getPrimaryInduction()) + NeedDef.insert(Legal->getPrimaryInduction()); // Collect instructions from the original loop that will become trivially dead // in the vectorized loop. We don't need to vectorize these instructions. For @@ -7858,8 +7850,8 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes( for (auto &Reduction : Legal->getReductionVars()) { if (CM.isInLoopReduction(Reduction.first)) continue; - VPValue *Phi = Plan->getVPValue(Reduction.first); - VPValue *Red = Plan->getVPValue(Reduction.second.getLoopExitInstr()); + VPValue *Phi = Plan->getOrAddVPValue(Reduction.first); + VPValue *Red = Plan->getOrAddVPValue(Reduction.second.getLoopExitInstr()); Builder.createNaryOp(Instruction::Select, {Cond, Red, Phi}); } } @@ -8003,7 +7995,8 @@ void VPWidenCallRecipe::execute(VPTransformState &State) { } void VPWidenSelectRecipe::execute(VPTransformState &State) { - State.ILV->widenSelectInstruction(Ingredient, *this, InvariantCond, State); + State.ILV->widenSelectInstruction(*cast(getUnderlyingInstr()), + this, *this, InvariantCond, State); } void VPWidenRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 0656854c3b36b8..46ab2eae623c22 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -108,6 +108,8 @@ VPValue *VPRecipeBase::toVPValue() { return V; if (auto *V = dyn_cast(this)) return V; + if (auto *V = dyn_cast(this)) + return V; return nullptr; } @@ -118,6 +120,8 @@ const VPValue *VPRecipeBase::toVPValue() const { return V; if (auto *V = dyn_cast(this)) return V; + if (auto *V = dyn_cast(this)) + return V; return nullptr; } @@ -843,8 +847,15 @@ void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent, void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - O << "\"WIDEN-SELECT" << VPlanIngredient(&Ingredient) - << (InvariantCond ? " (condition is loop invariant)" : ""); + O << "\"WIDEN-SELECT "; + printAsOperand(O, SlotTracker); + O << " = select "; + getOperand(0)->printAsOperand(O, SlotTracker); + O << ", "; + getOperand(1)->printAsOperand(O, SlotTracker); + O << ", "; + getOperand(2)->printAsOperand(O, SlotTracker); + O << (InvariantCond ? " (condition is loop invariant)" : ""); } void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c70d82bfb137b7..4c7982c5fedfb0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -877,10 +877,7 @@ class VPWidenCallRecipe : public VPRecipeBase, public VPValue, public VPUser { }; /// A recipe for widening select instructions. -class VPWidenSelectRecipe : public VPRecipeBase, public VPUser { -private: - /// Hold the select to be widened. - SelectInst &Ingredient; +class VPWidenSelectRecipe : public VPRecipeBase, public VPValue, public VPUser { /// Is the condition of the select loop invariant? bool InvariantCond; @@ -889,7 +886,8 @@ class VPWidenSelectRecipe : public VPRecipeBase, public VPUser { template VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) - : VPRecipeBase(VPWidenSelectSC), VPUser(Operands), Ingredient(I), + : VPRecipeBase(VPRecipeBase::VPWidenSelectSC), + VPValue(VPValue::VPVWidenSelectSC, &I), VPUser(Operands), InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 503abec30c66b6..ce3b4b1452e4c6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -78,7 +78,13 @@ class VPValue { /// are actually instantiated. Values of this enumeration are kept in the /// SubclassID field of the VPValue objects. They are used for concrete /// type identification. - enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC }; + enum { + VPValueSC, + VPInstructionSC, + VPMemoryInstructionSC, + VPVWidenCallSC, + VPVWidenSelectSC + }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {} VPValue(const VPValue &) = delete; diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll index a5b57432ff6e57..e8e80d19bc759c 100644 --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -39,7 +39,7 @@ for.end: ; CHECK-NEXT: "loop:\n" + ; CHECK-NEXT: "WIDEN-INDUCTION %iv = phi 0, %iv.next\l" + ; CHECK-NEXT: "WIDEN\l"" %cond0 = icmp %iv, 13\l" + -; CHECK-NEXT: "WIDEN-SELECT%s = select %cond0, 10, 20\l" +; CHECK-NEXT: "WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20>\l" ; CHECK-NEXT: ] define void @test() { entry: diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll index 048e5dd40ded89..ddc4747141fd24 100644 --- a/llvm/test/Transforms/LoopVectorize/optsize.ll +++ b/llvm/test/Transforms/LoopVectorize/optsize.ll @@ -341,6 +341,29 @@ for.end: ; preds = %for.body ret void } +; Make sure we do not crash while building the VPlan for the loop with the +; select below. +define i32 @PR48142(i32* %ptr.start, i32* %ptr.end) optsize { +; CHECK-LABEL: PR48142 +; CHECK-NOT: vector.body +entry: + br label %for.body + +for.body: + %i.014 = phi i32 [ 20, %entry ], [ %cond, %for.body ] + %ptr.iv = phi i32* [ %ptr.start, %entry ], [ %ptr.next, %for.body ] + %cmp4 = icmp slt i32 %i.014, 99 + %cond = select i1 %cmp4, i32 99, i32 %i.014 + store i32 0, i32* %ptr.iv + %ptr.next = getelementptr inbounds i32, i32* %ptr.iv, i64 1 + %cmp.not = icmp eq i32* %ptr.next, %ptr.end + br i1 %cmp.not, label %exit, label %for.body + +exit: + %res = phi i32 [ %cond, %for.body ] + ret i32 %res +} + !llvm.module.flags = !{!0} !0 = !{i32 1, !"ProfileSummary", !1} !1 = !{!2, !3, !4, !5, !6, !7, !8, !9} diff --git a/mlir/docs/Passes.md b/mlir/docs/Passes.md index 1109fd9bc2850b..c7ac56f84ea9f9 100644 --- a/mlir/docs/Passes.md +++ b/mlir/docs/Passes.md @@ -12,6 +12,10 @@ This document describes the available MLIR passes and their contracts. [include "ConversionPasses.md"] +## `async` Dialect Passes + +[include "AsyncPasses.md"] + ## `affine` Dialect Passes [include "AffinePasses.md"] @@ -51,3 +55,7 @@ This document describes the available MLIR passes and their contracts. ## `standard` Dialect Passes [include "StandardPasses.md"] + +## TOSA Dialect Passes + +[include "TosaPasses.md"]