Merged master:a70b511e7802 into amd-gfx:f635e0030b6a

Local branch amd-gfx f635e00 Merged master:2104783d024c into amd-gfx:5adf44be80e9
Remote branch master a70b511 Recommit "[VPlan] Use VPValue def for VPWidenSelectRecipe."

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -469,7 +469,7 @@ class InnerLoopVectorizer {
                             VPTransformState &State);
 
   /// Widen a single select instruction within the innermost loop.
-  void widenSelectInstruction(SelectInst &I, VPUser &Operands,
+  void widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands,
                               bool InvariantCond, VPTransformState &State);
 
   /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
@@ -4686,7 +4686,7 @@ void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def,
   }
 }
 
-void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I,
+void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPValue *VPDef,
                                                  VPUser &Operands,
                                                  bool InvariantCond,
                                                  VPTransformState &State) {
@@ -4705,7 +4705,7 @@ void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I,
     Value *Op0 = State.get(Operands.getOperand(1), Part);
     Value *Op1 = State.get(Operands.getOperand(2), Part);
     Value *Sel = Builder.CreateSelect(Cond, Op0, Op1);
-    VectorLoopValueMap.setVectorValue(&I, Part, Sel);
+    State.set(VPDef, &I, Sel, Part);
     addMetadata(Sel, &I);
   }
 }
@@ -7636,16 +7636,8 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
 
   // If the tail is to be folded by masking, the primary induction variable, if
   // exists needs to be represented in VPlan for it to model early-exit masking.
-  // Also, both the Phi and the live-out instruction of each reduction are
-  // required in order to introduce a select between them in VPlan.
-  if (CM.foldTailByMasking()) {
-    if (Legal->getPrimaryInduction())
-      NeedDef.insert(Legal->getPrimaryInduction());
-    for (auto &Reduction : Legal->getReductionVars()) {
-      NeedDef.insert(Reduction.first);
-      NeedDef.insert(Reduction.second.getLoopExitInstr());
-    }
-  }
+  if (CM.foldTailByMasking() && Legal->getPrimaryInduction())
+    NeedDef.insert(Legal->getPrimaryInduction());
 
   // Collect instructions from the original loop that will become trivially dead
   // in the vectorized loop. We don't need to vectorize these instructions. For
@@ -7858,8 +7850,8 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
     for (auto &Reduction : Legal->getReductionVars()) {
       if (CM.isInLoopReduction(Reduction.first))
         continue;
-      VPValue *Phi = Plan->getVPValue(Reduction.first);
-      VPValue *Red = Plan->getVPValue(Reduction.second.getLoopExitInstr());
+      VPValue *Phi = Plan->getOrAddVPValue(Reduction.first);
+      VPValue *Red = Plan->getOrAddVPValue(Reduction.second.getLoopExitInstr());
       Builder.createNaryOp(Instruction::Select, {Cond, Red, Phi});
     }
   }
@@ -8003,7 +7995,8 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
 }
 
 void VPWidenSelectRecipe::execute(VPTransformState &State) {
-  State.ILV->widenSelectInstruction(Ingredient, *this, InvariantCond, State);
+  State.ILV->widenSelectInstruction(*cast<SelectInst>(getUnderlyingInstr()),
+                                    this, *this, InvariantCond, State);
 }
 
 void VPWidenRecipe::execute(VPTransformState &State) {

llvm/lib/Transforms/Vectorize/VPlan.cpp

@@ -108,6 +108,8 @@ VPValue *VPRecipeBase::toVPValue() {
     return V;
   if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
     return V;
+  if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
+    return V;
   return nullptr;
 }
 
@@ -118,6 +120,8 @@ const VPValue *VPRecipeBase::toVPValue() const {
     return V;
   if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
     return V;
+  if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
+    return V;
   return nullptr;
 }
 
@@ -843,8 +847,15 @@ void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
 
 void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent,
                                 VPSlotTracker &SlotTracker) const {
-  O << "\"WIDEN-SELECT" << VPlanIngredient(&Ingredient)
-    << (InvariantCond ? " (condition is loop invariant)" : "");
+  O << "\"WIDEN-SELECT ";
+  printAsOperand(O, SlotTracker);
+  O << " = select ";
+  getOperand(0)->printAsOperand(O, SlotTracker);
+  O << ", ";
+  getOperand(1)->printAsOperand(O, SlotTracker);
+  O << ", ";
+  getOperand(2)->printAsOperand(O, SlotTracker);
+  O << (InvariantCond ? " (condition is loop invariant)" : "");
 }
 
 void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -877,10 +877,7 @@ class VPWidenCallRecipe : public VPRecipeBase, public VPValue, public VPUser {
 };
 
 /// A recipe for widening select instructions.
-class VPWidenSelectRecipe : public VPRecipeBase, public VPUser {
-private:
-  /// Hold the select to be widened.
-  SelectInst &Ingredient;
+class VPWidenSelectRecipe : public VPRecipeBase, public VPValue, public VPUser {
 
   /// Is the condition of the select loop invariant?
   bool InvariantCond;
@@ -889,7 +886,8 @@ class VPWidenSelectRecipe : public VPRecipeBase, public VPUser {
   template <typename IterT>
   VPWidenSelectRecipe(SelectInst &I, iterator_range<IterT> Operands,
                       bool InvariantCond)
-      : VPRecipeBase(VPWidenSelectSC), VPUser(Operands), Ingredient(I),
+      : VPRecipeBase(VPRecipeBase::VPWidenSelectSC),
+        VPValue(VPValue::VPVWidenSelectSC, &I), VPUser(Operands),
         InvariantCond(InvariantCond) {}
 
   ~VPWidenSelectRecipe() override = default;

llvm/lib/Transforms/Vectorize/VPlanValue.h

@@ -78,7 +78,13 @@ class VPValue {
   /// are actually instantiated. Values of this enumeration are kept in the
   /// SubclassID field of the VPValue objects. They are used for concrete
   /// type identification.
-  enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC, VPVWidenCallSC };
+  enum {
+    VPValueSC,
+    VPInstructionSC,
+    VPMemoryInstructionSC,
+    VPVWidenCallSC,
+    VPVWidenSelectSC
+  };
 
   VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {}
   VPValue(const VPValue &) = delete;

llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll

@@ -39,7 +39,7 @@ for.end:
 ; CHECK-NEXT:    "loop:\n" +
 ; CHECK-NEXT:      "WIDEN-INDUCTION %iv = phi 0, %iv.next\l" +
 ; CHECK-NEXT:      "WIDEN\l""  %cond0 = icmp %iv, 13\l" +
-; CHECK-NEXT:      "WIDEN-SELECT%s = select %cond0, 10, 20\l"
+; CHECK-NEXT:      "WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20>\l"
 ; CHECK-NEXT:  ]
 define void @test() {
 entry:

llvm/test/Transforms/LoopVectorize/optsize.ll

@@ -341,6 +341,29 @@ for.end:                                        ; preds = %for.body
   ret void
 }
 
+; Make sure we do not crash while building the VPlan for the loop with the
+; select below.
+define i32 @PR48142(i32* %ptr.start, i32* %ptr.end) optsize {
+; CHECK-LABEL: PR48142
+; CHECK-NOT: vector.body
+entry:
+  br label %for.body
+
+for.body:
+  %i.014 = phi i32 [ 20, %entry ], [ %cond, %for.body ]
+  %ptr.iv = phi i32* [ %ptr.start, %entry ], [ %ptr.next, %for.body ]
+  %cmp4 = icmp slt i32 %i.014, 99
+  %cond = select i1 %cmp4, i32 99, i32 %i.014
+  store i32 0, i32* %ptr.iv
+  %ptr.next = getelementptr inbounds i32, i32* %ptr.iv, i64 1
+  %cmp.not = icmp eq i32* %ptr.next, %ptr.end
+  br i1 %cmp.not, label %exit, label %for.body
+
+exit:
+  %res = phi i32 [ %cond, %for.body ]
+  ret i32 %res
+}
+
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"ProfileSummary", !1}
 !1 = !{!2, !3, !4, !5, !6, !7, !8, !9}

mlir/docs/Passes.md

@@ -12,6 +12,10 @@ This document describes the available MLIR passes and their contracts.
 
 [include "ConversionPasses.md"]
 
+## `async` Dialect Passes
+
+[include "AsyncPasses.md"]
+
 ## `affine` Dialect Passes
 
 [include "AffinePasses.md"]
@@ -51,3 +55,7 @@ This document describes the available MLIR passes and their contracts.
 ## `standard` Dialect Passes
 
 [include "StandardPasses.md"]
+
+## TOSA Dialect Passes
+
+[include "TosaPasses.md"]