diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 411a14976907b..2d15aab8a9656 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6814,7 +6814,12 @@ class Compiler void optFindLoops(); void optFindNewLoops(); - bool optCanonicalizeLoops(FlowGraphNaturalLoops* loops); + bool optCanonicalizeLoops(); + bool optCompactLoops(); + bool optCompactLoop(FlowGraphNaturalLoop* loop); + BasicBlock* optFindLoopCompactionInsertionPoint(FlowGraphNaturalLoop* loop, BasicBlock* top); + BasicBlock* optTryAdvanceLoopCompactionInsertionPoint(FlowGraphNaturalLoop* loop, BasicBlock* insertionPoint, BasicBlock* top, BasicBlock* bottom); + bool optLoopCompactionFixupFallThrough(BasicBlock* block, BasicBlock* oldNext, BasicBlock* newNext); bool optCreatePreheader(FlowGraphNaturalLoop* loop); void optSetPreheaderWeight(FlowGraphNaturalLoop* loop, BasicBlock* preheader); diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index e26af00a4a504..a0f40efa97525 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -1746,15 +1746,6 @@ class LoopSearch return false; } - // Compact the loop (sweep through it and move out any blocks that aren't part of the - // flow cycle), and find the exits. - if (!MakeCompactAndFindExits()) - { - // Unable to preserve well-formed loop during compaction. - JITDUMP(" can't compact\n"); - return false; - } - // We have a valid loop. return true; } @@ -1949,516 +1940,6 @@ class LoopSearch } return block->bbNum; } - - //------------------------------------------------------------------------ - // MakeCompactAndFindExits: Compact the loop (sweep through it and move out - // any blocks that aren't part of the flow cycle), and find the exits (set - // lastExit and exitCount). - // - // Return Value: - // true - Loop successfully compacted (or `loopBlocks` expanded to - // include all blocks in the lexical range), exits enumerated. - // false - Loop cannot be made compact and remain well-formed. - // - bool MakeCompactAndFindExits() - { - // Compaction (if it needs to happen) will require an insertion point. - BasicBlock* moveAfter = nullptr; - - for (BasicBlock* previous = top->Prev(); previous != bottom;) - { - BasicBlock* block = previous->Next(); - - if (loopBlocks.IsMember(block->bbNum)) - { - // This block is a member of the loop. Check to see if it may exit the loop. - CheckForExit(block); - - // Done processing this block; move on to the next. - previous = block; - continue; - } - - // This block is lexically between TOP and BOTTOM, but it does not - // participate in the flow cycle. Check for a run of consecutive - // such blocks. - // - // If blocks have been reordered and bbNum no longer reflects bbNext ordering - // (say by a call to MakeCompactAndFindExits for an earlier loop or unsuccessful - // attempt to find a loop), the bottom block of this loop may now appear earlier - // in the bbNext chain than other loop blocks. So when the previous hasn't reached bottom - // and block is a non-loop block, and we walk the bbNext chain, we may reach the end. - // If so, give up on recognition of this loop. - // - BasicBlock* lastNonLoopBlock = block; - BasicBlock* nextLoopBlock = block->Next(); - while ((nextLoopBlock != nullptr) && !loopBlocks.IsMember(nextLoopBlock->bbNum)) - { - lastNonLoopBlock = nextLoopBlock; - nextLoopBlock = nextLoopBlock->Next(); - } - - if (nextLoopBlock == nullptr) - { - JITDUMP("Did not find expected loop block when walking from " FMT_BB "\n", lastNonLoopBlock->bbNum); - return false; - } - - // Choose an insertion point for non-loop blocks if we haven't yet done so. - if (moveAfter == nullptr) - { - moveAfter = FindInsertionPoint(); - } - - if (!BasicBlock::sameEHRegion(previous, nextLoopBlock) || !BasicBlock::sameEHRegion(previous, moveAfter)) - { - // EH regions would be ill-formed if we moved these blocks out. - // See if we can consider them loop blocks without introducing - // a side-entry. - if (CanTreatAsLoopBlocks(block, lastNonLoopBlock)) - { - // The call to `canTreatAsLoop` marked these blocks as part of the loop; - // iterate without updating `previous` so that we'll analyze them as part - // of the loop. - continue; - } - else - { - // We can't move these out of the loop or leave them in, so just give - // up on this loop. - return false; - } - } - - // Now physically move the blocks. - BasicBlock* moveBefore = moveAfter->Next(); - - comp->fgUnlinkRange(block, lastNonLoopBlock); - comp->fgMoveBlocksAfter(block, lastNonLoopBlock, moveAfter); - comp->ehUpdateLastBlocks(moveAfter, lastNonLoopBlock); - - // Apply any adjustments needed for fallthrough at the boundaries of the moved region. - FixupFallThrough(moveAfter, moveBefore, block); - FixupFallThrough(lastNonLoopBlock, nextLoopBlock, moveBefore); - // Also apply any adjustments needed where the blocks were snipped out of the loop. - BasicBlock* newBlock = FixupFallThrough(previous, block, nextLoopBlock); - if (newBlock != nullptr) - { - // This new block is in the loop and is a loop exit. - loopBlocks.Insert(newBlock->bbNum); - lastExit = newBlock; - ++exitCount; - } - - // Update moveAfter for the next insertion. - moveAfter = lastNonLoopBlock; - - // Note that we've changed the flow graph, and continue without updating - // `previous` so that we'll process nextLoopBlock. - changedFlowGraph = true; - } - - if ((exitCount == 1) && (lastExit == nullptr)) - { - // If we happen to have a loop with two exits, one of which goes to an - // infinite loop that's lexically nested inside it, where the inner loop - // can't be moved out, we can end up in this situation (because - // CanTreatAsLoopBlocks will have decremented the count expecting to find - // another exit later). Bump the exit count to 2, since downstream code - // will not be prepared for null lastExit with exitCount of 1. - assert(forgotExit); - exitCount = 2; - } - - // Loop compaction was successful - return true; - } - - //------------------------------------------------------------------------ - // FindInsertionPoint: Find an appropriate spot to which blocks that are - // lexically between TOP and BOTTOM but not part of the flow cycle - // can be moved. - // - // Return Value: - // Block after which to insert moved blocks. - // - BasicBlock* FindInsertionPoint() - { - // Find an insertion point for blocks we're going to move. Move them down - // out of the loop, and if possible find a spot that won't break up fall-through. - BasicBlock* moveAfter = bottom; - while (moveAfter->bbFallsThrough()) - { - // Keep looking for a better insertion point if we can. - BasicBlock* newMoveAfter = TryAdvanceInsertionPoint(moveAfter); - - if (newMoveAfter == nullptr) - { - // Ran out of candidate insertion points, so just split up the fall-through. - return moveAfter; - } - - moveAfter = newMoveAfter; - } - - return moveAfter; - } - - //------------------------------------------------------------------------ - // TryAdvanceInsertionPoint: Find the next legal insertion point after - // the given one, if one exists. - // - // Arguments: - // oldMoveAfter - Prior insertion point; find the next after this. - // - // Return Value: - // The next block after `oldMoveAfter` that is a legal insertion point - // (i.e. blocks being swept out of the loop can be moved immediately - // after it), if one exists, else nullptr. - // - BasicBlock* TryAdvanceInsertionPoint(BasicBlock* oldMoveAfter) - { - BasicBlock* newMoveAfter = oldMoveAfter->Next(); - - if (!BasicBlock::sameEHRegion(oldMoveAfter, newMoveAfter)) - { - // Don't cross an EH region boundary. - return nullptr; - } - - if (newMoveAfter->KindIs(BBJ_ALWAYS, BBJ_COND)) - { - unsigned int destNum = newMoveAfter->KindIs(BBJ_ALWAYS) ? newMoveAfter->GetTarget()->bbNum - : newMoveAfter->GetTrueTarget()->bbNum; - if ((destNum >= top->bbNum) && (destNum <= bottom->bbNum) && !loopBlocks.IsMember(destNum)) - { - // Reversing this branch out of block `newMoveAfter` could confuse this algorithm - // (in particular, the edge would still be numerically backwards but no longer be - // lexically backwards, so a lexical forward walk from TOP would not find BOTTOM), - // so don't do that. - // We're checking for BBJ_ALWAYS and BBJ_COND only here -- we don't need to - // check for BBJ_SWITCH because we'd never consider it a loop back-edge. - return nullptr; - } - } - - // Similarly check to see if advancing to `newMoveAfter` would reverse the lexical order - // of an edge from the run of blocks being moved to `newMoveAfter` -- doing so would - // introduce a new lexical back-edge, which could (maybe?) confuse the loop search - // algorithm, and isn't desirable layout anyway. - for (BasicBlock* const predBlock : newMoveAfter->PredBlocks()) - { - unsigned int predNum = predBlock->bbNum; - - if ((predNum >= top->bbNum) && (predNum <= bottom->bbNum) && !loopBlocks.IsMember(predNum)) - { - // Don't make this forward edge a backwards edge. - return nullptr; - } - } - - if (IsRecordedBottom(newMoveAfter)) - { - // This is the BOTTOM of another loop; don't move any blocks past it, to avoid moving them - // out of that loop (we should have already done so when processing that loop if it were legal). - return nullptr; - } - - // Advancing the insertion point is ok, except that we can't split up any call finally - // pair, so if we've got such a pair recurse to see if we can move past the whole thing. - return (newMoveAfter->isBBCallFinallyPair() ? TryAdvanceInsertionPoint(newMoveAfter) : newMoveAfter); - } - - //------------------------------------------------------------------------ - // isOuterBottom: Determine if the given block is the BOTTOM of a previously - // recorded loop. - // - // Arguments: - // block - Block to check for BOTTOM-ness. - // - // Return Value: - // true - The blocks was recorded as `bottom` of some earlier-processed loop. - // false - No loops yet recorded have this block as their `bottom`. - // - bool IsRecordedBottom(BasicBlock* block) - { - if (block->bbNum > oldBlockMaxNum) - { - // This is a new block, which can't be an outer bottom block because we only allow old blocks - // as BOTTOM. - return false; - } - return BlockSetOps::IsMember(comp, bottomBlocks, block->bbNum); - } - - //------------------------------------------------------------------------ - // CanTreatAsLoopBlocks: If the given range of blocks can be treated as - // loop blocks, add them to loopBlockSet and return true. Otherwise, - // return false. - // - // Arguments: - // firstNonLoopBlock - First block in the run to be subsumed. - // lastNonLoopBlock - Last block in the run to be subsumed. - // - // Return Value: - // true - The blocks from `fistNonLoopBlock` to `lastNonLoopBlock` were - // successfully added to `loopBlocks`. - // false - Treating the blocks from `fistNonLoopBlock` to `lastNonLoopBlock` - // would not be legal (it would induce a side-entry). - // - // Notes: - // `loopBlocks` may be modified even if `false` is returned. - // `exitCount` and `lastExit` may be modified if this process identifies - // in-loop edges that were previously counted as exits. - // - bool CanTreatAsLoopBlocks(BasicBlock* firstNonLoopBlock, BasicBlock* lastNonLoopBlock) - { - for (BasicBlock* const testBlock : comp->Blocks(firstNonLoopBlock, lastNonLoopBlock)) - { - for (BasicBlock* const testPred : testBlock->PredBlocks()) - { - unsigned int predPosNum = PositionNum(testPred); - unsigned int firstNonLoopPosNum = PositionNum(firstNonLoopBlock); - unsigned int lastNonLoopPosNum = PositionNum(lastNonLoopBlock); - - if (loopBlocks.IsMember(predPosNum) || - ((predPosNum >= firstNonLoopPosNum) && (predPosNum <= lastNonLoopPosNum))) - { - // This pred is in the loop (or what will be the loop if we determine this - // run of exit blocks doesn't include a side-entry). - - if (predPosNum < firstNonLoopPosNum) - { - // We've already counted this block as an exit, so decrement the count. - --exitCount; - if (lastExit == testPred) - { - // Erase this now-bogus `lastExit` entry. - lastExit = nullptr; - INDEBUG(forgotExit = true); - } - } - } - else - { - // This pred is not in the loop, so this constitutes a side-entry. - return false; - } - } - - // Either we're going to abort the loop on a subsequent testBlock, or this - // testBlock is part of the loop. - loopBlocks.Insert(testBlock->bbNum); - } - - // All blocks were ok to leave in the loop. - return true; - } - - //------------------------------------------------------------------------ - // FixupFallThrough: Re-establish any broken control flow connectivity - // and eliminate any "goto-next"s that were created by changing the - // given block's lexical follower. - // - // Arguments: - // block - Block whose `bbNext` has changed. - // oldNext - Previous value of `block->bbNext`. - // newNext - New value of `block->bbNext`. - // - // Return Value: - // If a new block is created to reconnect flow, the new block is - // returned; otherwise, nullptr. - // - BasicBlock* FixupFallThrough(BasicBlock* block, BasicBlock* oldNext, BasicBlock* newNext) - { - // If we create a new block, that will be our return value. - BasicBlock* newBlock = nullptr; - - if (block->bbFallsThrough()) - { - // Need to reconnect the flow from `block` to `oldNext`. - - if (block->KindIs(BBJ_COND) && block->TrueTargetIs(newNext)) - { - // Reverse the jump condition - GenTree* test = block->lastNode(); - noway_assert(test->OperIsConditionalJump()); - - if (test->OperGet() == GT_JTRUE) - { - GenTree* cond = comp->gtReverseCond(test->AsOp()->gtOp1); - assert(cond == test->AsOp()->gtOp1); // Ensure `gtReverseCond` did not create a new node. - test->AsOp()->gtOp1 = cond; - } - else - { - comp->gtReverseCond(test); - } - - // Redirect the Conditional JUMP to go to `oldNext` - block->SetTrueTarget(oldNext); - block->SetFalseTarget(newNext); - } - else - { - // Insert an unconditional jump to `oldNext` just after `block`. - newBlock = comp->fgConnectFallThrough(block, oldNext); - noway_assert((newBlock == nullptr) || loopBlocks.CanRepresent(newBlock->bbNum)); - } - } - else if (block->KindIs(BBJ_ALWAYS) && block->TargetIs(newNext)) - { - // If block is newNext's only predecessor, move the IR from block to newNext, - // but keep the now-empty block around. - // - // We move the IR because loop recognition has a very limited search capability and - // won't walk from one block's statements to another, even if the blocks form - // a linear chain. So this IR move enhances counted loop recognition. - // - // The logic here is essentially echoing fgCompactBlocks... but we don't call - // that here because we don't want to delete block and do the necessary updates - // to all the other data in flight, and we'd also prefer that newNext be the - // survivor, not block. - // - if ((newNext->bbRefs == 1) && comp->fgCanCompactBlocks(block, newNext)) - { - JITDUMP("Moving stmts from " FMT_BB " to " FMT_BB "\n", block->bbNum, newNext->bbNum); - Statement* stmtList1 = block->firstStmt(); - Statement* stmtList2 = newNext->firstStmt(); - - // Is there anything to move? - // - if (stmtList1 != nullptr) - { - // Append newNext stmts to block's stmts. - // - if (stmtList2 != nullptr) - { - Statement* stmtLast1 = block->lastStmt(); - Statement* stmtLast2 = newNext->lastStmt(); - - stmtLast1->SetNextStmt(stmtList2); - stmtList2->SetPrevStmt(stmtLast1); - stmtList1->SetPrevStmt(stmtLast2); - } - - // Move block's stmts to newNext - // - newNext->bbStmtList = stmtList1; - block->bbStmtList = nullptr; - - // Update newNext's block flags - // - newNext->CopyFlags(block, BBF_COMPACT_UPD); - } - } - } - - return newBlock; - } - - //------------------------------------------------------------------------ - // CheckForExit: Check if the given block has any successor edges that are - // loop exits, and update `lastExit` and `exitCount` if so. - // - // Arguments: - // block - Block whose successor edges are to be checked. - // - // Notes: - // If one block has multiple exiting successor edges, those are counted - // as multiple exits in `exitCount`. - // - void CheckForExit(BasicBlock* block) - { - assert(!block->HasTarget() || block->HasInitializedTarget()); - - switch (block->GetKind()) - { - case BBJ_ALWAYS: - { - BasicBlock* exitPoint = block->GetTarget(); - - if (!loopBlocks.IsMember(exitPoint->bbNum)) - { - // Exit from a block other than BOTTOM - CLANG_FORMAT_COMMENT_ANCHOR; - -#if !defined(FEATURE_EH_FUNCLETS) - // On non-funclet platforms (x86), the catch exit is a BBJ_ALWAYS, but we don't want that to - // be considered a loop exit block, as catch handlers don't have predecessor lists and don't - // show up as might be expected in the dominator tree. - if (!BasicBlock::sameHndRegion(block, exitPoint)) - { - break; - } -#endif // !defined(FEATURE_EH_FUNCLETS) - - lastExit = block; - exitCount++; - } - break; - } - case BBJ_COND: - if (!loopBlocks.IsMember(block->GetTrueTarget()->bbNum)) - { - lastExit = block; - exitCount++; - } - - if (!loopBlocks.IsMember(block->GetFalseTarget()->bbNum)) - { - lastExit = block; - exitCount++; - } - break; - case BBJ_CALLFINALLY: - // Check fallthrough successor - if (!loopBlocks.IsMember(block->Next()->bbNum)) - { - lastExit = block; - exitCount++; - } - - FALLTHROUGH; - case BBJ_CALLFINALLYRET: - case BBJ_EHCATCHRET: - if (!loopBlocks.IsMember(block->GetTarget()->bbNum)) - { - lastExit = block; - exitCount++; - } - break; - - case BBJ_EHFINALLYRET: - case BBJ_EHFAULTRET: - case BBJ_EHFILTERRET: - // The "try" associated with this "finally" must be in the same loop, so the - // finally block will return control inside the loop. - break; - - case BBJ_THROW: - case BBJ_RETURN: - // Those are exits from the loop - lastExit = block; - exitCount++; - break; - - case BBJ_SWITCH: - for (BasicBlock* const exitPoint : block->SwitchTargets()) - { - if (!loopBlocks.IsMember(exitPoint->bbNum)) - { - lastExit = block; - exitCount++; - } - } - break; - - default: - noway_assert(!"Unexpected bbKind"); - break; - } - } }; } // end (anonymous) namespace @@ -4921,15 +4402,22 @@ void Compiler::optFindNewLoops() { m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); - if (optCanonicalizeLoops(m_loops)) + if (optCompactLoops()) + { + fgInvalidateDfsTree(); + m_dfsTree = fgComputeDfs(); + m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); + } + + if (optCanonicalizeLoops()) { fgInvalidateDfsTree(); - fgRenumberBlocks(); m_dfsTree = fgComputeDfs(); - m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); } + fgRenumberBlocks(); + // Starting now, we require all loops to have pre-headers. optLoopsRequirePreHeaders = true; @@ -4956,10 +4444,10 @@ void Compiler::optFindNewLoops() // Remarks: // Guarantees that all natural loops have preheaders. // -bool Compiler::optCanonicalizeLoops(FlowGraphNaturalLoops* loops) +bool Compiler::optCanonicalizeLoops() { bool changed = false; - for (FlowGraphNaturalLoop* loop : loops->InReversePostOrder()) + for (FlowGraphNaturalLoop* loop : m_loops->InReversePostOrder()) { changed |= optCreatePreheader(loop); } @@ -4967,6 +4455,304 @@ bool Compiler::optCanonicalizeLoops(FlowGraphNaturalLoops* loops) return changed; } +//----------------------------------------------------------------------------- +// optCompactLoops: Compact loops to make their loop blocks lexical if possible. +// +// Returns: +// True if the flow graph was changed. +// +bool Compiler::optCompactLoops() +{ + bool changed = false; + for (FlowGraphNaturalLoop* loop : m_loops->InReversePostOrder()) + { + if (!loop->GetHeader()->HasFlag(BBF_OLD_LOOP_HEADER_QUIRK)) + { + continue; + } + + changed |= optCompactLoop(loop); + } + + return changed; +} + +//----------------------------------------------------------------------------- +// optCompactLoop: Compact a specific loop. +// +// Parameters: +// loop - The loop +// +// Returns: +// True if the flow graph was changed. +// +bool Compiler::optCompactLoop(FlowGraphNaturalLoop* loop) +{ + BasicBlock* insertionPoint = nullptr; + + BasicBlock* top = loop->GetLexicallyTopMostBlock(); + unsigned numLoopBlocks = loop->NumLoopBlocks(); + + BasicBlock* cur = top; + bool changedFlowGraph = false; + while (numLoopBlocks > 0) + { + if (loop->ContainsBlock(cur)) + { + numLoopBlocks--; + cur = cur->Next(); + continue; + } + + BasicBlock* lastNonLoopBlock = cur; + while (true) + { + // Should always have a "bottom" block of the loop where we stop. + assert(lastNonLoopBlock->Next() != nullptr); + if (loop->ContainsBlock(lastNonLoopBlock->Next())) + { + break; + } + + lastNonLoopBlock = lastNonLoopBlock->Next(); + } + + if (insertionPoint == nullptr) + { + insertionPoint = optFindLoopCompactionInsertionPoint(loop, top); + } + + BasicBlock* previous = cur->Prev(); + BasicBlock* nextLoopBlock = lastNonLoopBlock->Next(); + assert(previous != nullptr); + if (!BasicBlock::sameEHRegion(previous, nextLoopBlock) || !BasicBlock::sameEHRegion(previous, insertionPoint)) + { + // EH regions would be ill-formed if we moved these blocks out. + cur = nextLoopBlock; + continue; + } + + // Now physically move the blocks. + BasicBlock* moveBefore = insertionPoint->Next(); + + fgUnlinkRange(cur, lastNonLoopBlock); + fgMoveBlocksAfter(cur, lastNonLoopBlock, insertionPoint); + ehUpdateLastBlocks(insertionPoint, lastNonLoopBlock); + + // Apply any adjustments needed for fallthrough at the boundaries of the moved region. + changedFlowGraph |= optLoopCompactionFixupFallThrough(insertionPoint, moveBefore, cur); + changedFlowGraph |= optLoopCompactionFixupFallThrough(lastNonLoopBlock, nextLoopBlock, moveBefore); + // Also apply any adjustments needed where the blocks were snipped out of the loop. + changedFlowGraph |= optLoopCompactionFixupFallThrough(previous, cur, nextLoopBlock); + + // Update insertionPoint for the next insertion. + insertionPoint = lastNonLoopBlock; + + cur = nextLoopBlock; + } + + return changedFlowGraph; +} + +//----------------------------------------------------------------------------- +// optFindLoopCompactionInsertionPoint: Find a good insertion point at which to +// move blocks from the lexical range of "loop" that is not part of the loop. +// +// Parameters: +// loop - The loop +// top - Lexical top block of the loop. +// +// Returns: +// Non-null insertion point. +// +BasicBlock* Compiler::optFindLoopCompactionInsertionPoint(FlowGraphNaturalLoop* loop, BasicBlock* top) +{ + // Find an insertion point for blocks we're going to move. Move them down + // out of the loop, and if possible find a spot that won't break up fall-through. + BasicBlock* bottom = loop->GetLexicallyBottomMostBlock(); + BasicBlock* insertionPoint = bottom; + while (insertionPoint->bbFallsThrough()) + { + // Keep looking for a better insertion point if we can. + BasicBlock* newInsertionPoint = optTryAdvanceLoopCompactionInsertionPoint(loop, insertionPoint, top, bottom); + if (newInsertionPoint == nullptr) + { + // Ran out of candidate insertion points, so just split up the fall-through. + break; + } + + insertionPoint = newInsertionPoint; + } + + return insertionPoint; +} + +//----------------------------------------------------------------------------- +// optTryAdvanceLoopCompactionInsertionPoint: Advance the insertion point to +// avoid having to insert new blocks due to fallthrough. +// +// Parameters: +// loop - The loop +// insertionPoint - Current insertion point +// top - Lexical top block of the loop. +// bottom - Lexical bottom block of the loop. +// +// Returns: +// New insertion point. +// +BasicBlock* Compiler::optTryAdvanceLoopCompactionInsertionPoint(FlowGraphNaturalLoop* loop, + BasicBlock* insertionPoint, + BasicBlock* top, + BasicBlock* bottom) +{ + BasicBlock* newInsertionPoint = insertionPoint->Next(); + + if (!BasicBlock::sameEHRegion(insertionPoint, newInsertionPoint)) + { + // Don't cross an EH region boundary. + return nullptr; + } + + // TODO-Quirk: Compatibility with old compaction + if (newInsertionPoint->KindIs(BBJ_ALWAYS, BBJ_COND)) + { + BasicBlock* dest = + newInsertionPoint->KindIs(BBJ_ALWAYS) ? newInsertionPoint->GetTarget() : newInsertionPoint->GetTrueTarget(); + if ((dest->bbNum >= top->bbNum) && (dest->bbNum <= bottom->bbNum) && !loop->ContainsBlock(dest)) + { + return nullptr; + } + } + + // TODO-Quirk: Compatibility with old compaction + for (BasicBlock* const predBlock : newInsertionPoint->PredBlocks()) + { + if ((predBlock->bbNum >= top->bbNum) && (predBlock->bbNum <= bottom->bbNum) && !loop->ContainsBlock(predBlock)) + { + // Don't make this forward edge a backwards edge. + return nullptr; + } + } + + // Compaction runs on outer loops before inner loops. That means all + // unlexical blocks here are part of an ancestor loop (or trivial + // BBJ_ALWAYS exit blocks). To avoid breaking lexicality of ancestor loops + // we avoid moving any block past the bottom of an ancestor loop. + for (FlowGraphNaturalLoop* ancestor = loop->GetParent(); ancestor != nullptr; ancestor = ancestor->GetParent()) + { + if (newInsertionPoint == ancestor->GetLexicallyBottomMostBlock()) + { + return nullptr; + } + } + + // Advancing the insertion point is ok, except that we can't split up any call finally + // pair, so if we've got such a pair recurse to see if we can move past the whole thing. + return newInsertionPoint->isBBCallFinallyPair() + ? optTryAdvanceLoopCompactionInsertionPoint(loop, newInsertionPoint, top, bottom) + : newInsertionPoint; +} + +//----------------------------------------------------------------------------- +// optLoopCompactionFixupFallThrough: Fix up fallthrough introduced due to +// moving a range of blocks. +// +// Parameters: +// block - Block that may have fallthrough +// oldNext - The old block that was the fallthrough block +// newNext - The new block that was the fallthrough block +// +// Returns: +// True if the flow graph was changed by this function. +// +bool Compiler::optLoopCompactionFixupFallThrough(BasicBlock* block, BasicBlock* oldNext, BasicBlock* newNext) +{ + bool changed = false; + + if (block->bbFallsThrough()) + { + // Need to reconnect the flow from `block` to `oldNext`. + + if (block->KindIs(BBJ_COND) && (newNext != nullptr) && block->TrueTargetIs(newNext)) + { + // Reverse the jump condition + GenTree* test = block->lastNode(); + noway_assert(test->OperIsConditionalJump()); + + if (test->OperGet() == GT_JTRUE) + { + GenTree* cond = gtReverseCond(test->AsOp()->gtOp1); + assert(cond == test->AsOp()->gtOp1); // Ensure `gtReverseCond` did not create a new node. + test->AsOp()->gtOp1 = cond; + } + else + { + gtReverseCond(test); + } + + // Redirect the Conditional JUMP to go to `oldNext` + block->SetTrueTarget(oldNext); + block->SetFalseTarget(newNext); + } + else + { + // Insert an unconditional jump to `oldNext` just after `block`. + fgConnectFallThrough(block, oldNext); + } + + changed = true; + } + else if (block->KindIs(BBJ_ALWAYS) && block->TargetIs(newNext)) + { + // If block is newNext's only predecessor, move the IR from block to newNext, + // but keep the now-empty block around. + // + // We move the IR because loop recognition has a very limited search capability and + // won't walk from one block's statements to another, even if the blocks form + // a linear chain. So this IR move enhances counted loop recognition. + // + // The logic here is essentially echoing fgCompactBlocks... but we don't call + // that here because we don't want to delete block and do the necessary updates + // to all the other data in flight, and we'd also prefer that newNext be the + // survivor, not block. + // + if ((newNext->bbRefs == 1) && fgCanCompactBlocks(block, newNext)) + { + JITDUMP("Moving stmts from " FMT_BB " to " FMT_BB "\n", block->bbNum, newNext->bbNum); + Statement* stmtList1 = block->firstStmt(); + Statement* stmtList2 = newNext->firstStmt(); + + // Is there anything to move? + // + if (stmtList1 != nullptr) + { + // Append newNext stmts to block's stmts. + // + if (stmtList2 != nullptr) + { + Statement* stmtLast1 = block->lastStmt(); + Statement* stmtLast2 = newNext->lastStmt(); + + stmtLast1->SetNextStmt(stmtList2); + stmtList2->SetPrevStmt(stmtLast1); + stmtList1->SetPrevStmt(stmtLast2); + } + + // Move block's stmts to newNext + // + newNext->bbStmtList = stmtList1; + block->bbStmtList = nullptr; + + // Update newNext's block flags + // + newNext->CopyFlags(block, BBF_COMPACT_UPD); + } + } + } + + return changed; +} + //----------------------------------------------------------------------------- // optCreatePreheader: Create (or find) a preheader for a natural loop. //