Skip to content

Commit

Permalink
[LoopVectorize] Refine runtime memory check costs when there is an ou…
Browse files Browse the repository at this point in the history 
…ter loop

When we generate runtime memory checks for an inner loop it's
possible that these checks are invariant in the outer loop and
so will get hoisted out. In such cases, the effective cost of
the checks should reduce to reflect the outer loop trip count.

This fixes a 25% performance regression introduced by commit

49b0e6d

when building the SPEC2017 x264 benchmark with PGO, where we
decided the inner loop trip count wasn't high enough to warrant
the (incorrect) high cost of the runtime checks. Also, when
runtime memory checks consist entirely of diff checks these are
likely to be outer loop invariant.
  • Loading branch information
@david-arm
david-arm committed Jan 23, 2024
1 parent ea50e94 commit a5e2f19
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 12 deletions.
63 changes: 57 additions & 6 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1957,6 +1957,8 @@ class GeneratedRTChecks {
bool CostTooHigh = false;
const bool AddBranchWeights;

Loop *OuterLoop = nullptr;

public:
GeneratedRTChecks(ScalarEvolution &SE, DominatorTree *DT, LoopInfo *LI,
TargetTransformInfo *TTI, const DataLayout &DL,
Expand Down Expand Up @@ -2053,6 +2055,9 @@ class GeneratedRTChecks {
DT->eraseNode(SCEVCheckBlock);
LI->removeBlock(SCEVCheckBlock);
}

// Outer loop is used as part of the later cost calculations.
OuterLoop = L->getParentLoop();
}

InstructionCost getCost() {
Expand All @@ -2076,16 +2081,62 @@ class GeneratedRTChecks {
LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
RTCheckCost += C;
}
if (MemCheckBlock)
if (MemCheckBlock) {
InstructionCost MemCheckCost = 0;
for (Instruction &I : *MemCheckBlock) {
if (MemCheckBlock->getTerminator() == &I)
continue;
InstructionCost C =
TTI->getInstructionCost(&I, TTI::TCK_RecipThroughput);
LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
RTCheckCost += C;
MemCheckCost += C;
}

// If the runtime memory checks are being created inside an outer loop
// we should find out if these checks are outer loop invariant. If so,
// the checks will likely be hoisted out and so the effective cost will
// reduce according to the outer loop trip count.
if (OuterLoop) {
ScalarEvolution *SE = MemCheckExp.getSE();
// TODO: If profitable, we could refine this further by analysing every
// individual memory check, since there could be a mixture of loop
// variant and invariant checks that mean the final condition is
// variant.
const SCEV *Cond = SE->getSCEV(MemRuntimeCheckCond);
if (SE->isLoopInvariant(Cond, OuterLoop)) {
// It seems reasonable to assume that we can reduce the effective
// cost of the checks even when we know nothing about the trip
// count. Here I've assumed that the outer loop executes at least
// twice.
unsigned BestTripCount = 2;

// If exact trip count is known use that.
if (unsigned SmallTC = SE->getSmallConstantTripCount(OuterLoop))
BestTripCount = SmallTC;
else if (LoopVectorizeWithBlockFrequency) {
// Else use profile data if available.
if (auto EstimatedTC = getLoopEstimatedTripCount(OuterLoop))
BestTripCount = *EstimatedTC;
}

InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;

// Let's ensure the cost is always at least 1.
NewMemCheckCost = std::max(*NewMemCheckCost.getValue(),
(InstructionCost::CostType)1);

LLVM_DEBUG(dbgs()
<< "We expect runtime memory checks to be hoisted "
<< "out of the outer loop. Cost reduced from "
<< MemCheckCost << " to " << NewMemCheckCost << '\n');

MemCheckCost = NewMemCheckCost;
}
}

RTCheckCost += MemCheckCost;
}

if (SCEVCheckBlock || MemCheckBlock)
LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
<< "\n");
Expand Down Expand Up @@ -2144,8 +2195,8 @@ class GeneratedRTChecks {

BranchInst::Create(LoopVectorPreHeader, SCEVCheckBlock);
// Create new preheader for vector loop.
if (auto *PL = LI->getLoopFor(LoopVectorPreHeader))
PL->addBasicBlockToLoop(SCEVCheckBlock, *LI);
if (OuterLoop)
OuterLoop->addBasicBlockToLoop(SCEVCheckBlock, *LI);

SCEVCheckBlock->getTerminator()->eraseFromParent();
SCEVCheckBlock->moveBefore(LoopVectorPreHeader);
Expand Down Expand Up @@ -2179,8 +2230,8 @@ class GeneratedRTChecks {
DT->changeImmediateDominator(LoopVectorPreHeader, MemCheckBlock);
MemCheckBlock->moveBefore(LoopVectorPreHeader);

if (auto *PL = LI->getLoopFor(LoopVectorPreHeader))
PL->addBasicBlockToLoop(MemCheckBlock, *LI);
if (OuterLoop)
OuterLoop->addBasicBlockToLoop(MemCheckBlock, *LI);

BranchInst &BI =
*BranchInst::Create(Bypass, LoopVectorPreHeader, MemRuntimeCheckCond);
Expand Down
17 changes: 11 additions & 6 deletions llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ inner.exit:
define void @outer_no_tc(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_no_tc'
; CHECK: Calculating cost of runtime checks:
; CHECK: Total cost of runtime checks: 6
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 3
; CHECK: Total cost of runtime checks: 3
; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
Expand Down Expand Up @@ -68,7 +69,8 @@ outer.exit:
define void @outer_known_tc3(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_known_tc3'
; CHECK: Calculating cost of runtime checks:
; CHECK: Total cost of runtime checks: 6
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 2
; CHECK: Total cost of runtime checks: 2
; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
Expand Down Expand Up @@ -104,7 +106,8 @@ outer.exit:
define void @outer_known_tc64(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_known_tc64'
; CHECK: Calculating cost of runtime checks:
; CHECK: Total cost of runtime checks: 6
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 1
; CHECK: Total cost of runtime checks: 1
; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
Expand Down Expand Up @@ -140,7 +143,8 @@ outer.exit:
define void @outer_pgo_3(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_pgo_3'
; CHECK: Calculating cost of runtime checks:
; CHECK: Total cost of runtime checks: 6
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 2
; CHECK: Total cost of runtime checks: 2
; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
Expand Down Expand Up @@ -176,8 +180,9 @@ outer.exit:
define void @outer_known_tc3_full_range_checks(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_known_tc3_full_range_checks'
; CHECK: Calculating cost of runtime checks:
; CHECK: Total cost of runtime checks: 6
; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:8
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 2
; CHECK: Total cost of runtime checks: 2
; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:4
entry:
br label %outer.loop

Expand Down

0 comments on commit a5e2f19

Please sign in to comment.