Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Arm64: Add If conversion pass #73472

Merged
merged 41 commits into from
Nov 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
ea2f3d5
Arm64: Add If conversion pass
a74nh Jul 15, 2022
8ee167f
Minor review fixups
a74nh Aug 10, 2022
039de48
Return a PhaseStatus
a74nh Aug 10, 2022
ddfad68
Fix formatting
a74nh Aug 11, 2022
41c3a9e
Check for side effects on NOPs
a74nh Aug 11, 2022
4c93a4c
Add function block comments for the phase
a74nh Aug 11, 2022
891510d
Remove creation of AND chains from if conversion pass
a74nh Aug 12, 2022
c1281b5
Update middleBlock flow
a74nh Aug 12, 2022
b900344
Check for order side effects
a74nh Aug 12, 2022
a3e89b9
Remove COLON_COND check
a74nh Aug 31, 2022
441a60c
Remove flag toggling
a74nh Aug 31, 2022
962c83b
Move the conditional assignment to the JTRUE block
a74nh Aug 31, 2022
90bff39
Fix formatting
a74nh Sep 2, 2022
36cf21d
Allow conditions with side effects
a74nh Sep 7, 2022
127f9a6
Fix formatting
a74nh Sep 7, 2022
d1f0862
Correct all moved SSA statements
a74nh Sep 15, 2022
852a5af
Add size costing check
a74nh Sep 15, 2022
6690566
Only move middle block ssa defs
a74nh Sep 16, 2022
58372e2
Fix formatting
a74nh Sep 16, 2022
10a8f1f
Fewer SSA assumptions
a74nh Sep 20, 2022
97b1af1
Use implicit func for value numbering
a74nh Sep 21, 2022
3f60851
Update header for gtFoldExprConditional
a74nh Oct 6, 2022
0f054fe
Cost based on speed
a74nh Oct 6, 2022
6f0abdf
Merge branch 'main'
a74nh Oct 6, 2022
393fd39
Add Stress mode for inner loops
a74nh Oct 7, 2022
09d62fe
Rework costings
a74nh Oct 7, 2022
73007b2
Check for invalid VNs
a74nh Oct 10, 2022
61af92d
Ignore compares against zero
a74nh Oct 10, 2022
4fdabea
Ensure float compares are contained
a74nh Oct 12, 2022
be60d30
Allow if conversion of test compares
a74nh Oct 12, 2022
806e061
Do not contain test compares within compare chains
a74nh Oct 13, 2022
a508e85
Add float versions of the JIT/opt/Compares tests
a74nh Oct 13, 2022
ac2568f
Fix formatting
a74nh Oct 13, 2022
bad8097
Compare chains use CmpCompares, selects use Compares
a74nh Oct 14, 2022
ed06f67
Merge main
a74nh Oct 17, 2022
935716c
Fix flow checking for empty blocks
a74nh Oct 20, 2022
12bfe0a
Merge main
a74nh Oct 20, 2022
f2f3a02
Fix to contexts setting JitStdOutFile
jakobbotsch Oct 26, 2022
2e183bd
Temporary fix for SPMI problem
jakobbotsch Oct 26, 2022
b2772ff
Fix attr and reg producing in select generation
a74nh Oct 28, 2022
4281fc0
Revert SPMI changes
jakobbotsch Oct 28, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions src/coreclr/jit/codegenarm64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4613,6 +4613,9 @@ void CodeGen::genCodeForConditionalCompare(GenTreeOp* tree, GenCondition prevCon
// Should only be called on contained nodes.
assert(targetReg == REG_NA);

// Should not be called for test conditionals (Arm64 does not have a ctst).
assert(tree->OperIsCmpCompare());

// For the ccmp flags, invert the condition of the compare.
insCflags cflags = InsCflagsForCcmp(GenCondition::FromRelop(tree));

Expand Down Expand Up @@ -4711,7 +4714,7 @@ void CodeGen::genCodeForSelect(GenTreeConditional* tree)
GenTree* op2 = tree->gtOp2;
var_types op1Type = genActualType(op1->TypeGet());
var_types op2Type = genActualType(op2->TypeGet());
emitAttr cmpSize = EA_ATTR(genTypeSize(op1Type));
emitAttr attr = emitActualTypeSize(tree->TypeGet());

assert(!op1->isUsedFromMemory());
assert(genTypeSize(op1Type) == genTypeSize(op2Type));
Expand All @@ -4721,10 +4724,20 @@ void CodeGen::genCodeForSelect(GenTreeConditional* tree)
if (opcond->isContained())
{
// Generate the contained condition.
bool chain = false;
JITDUMP("Generating compare chain:\n");
genCodeForContainedCompareChain(opcond, &chain, &prevCond);
assert(chain);
if (opcond->OperIsCompare())
{
genCodeForCompare(opcond->AsOp());
prevCond = GenCondition::FromRelop(opcond);
}
else
{
// Condition is a compare chain. Try to contain it.
assert(opcond->OperIs(GT_AND));
bool chain = false;
JITDUMP("Generating compare chain:\n");
genCodeForContainedCompareChain(opcond, &chain, &prevCond);
assert(chain);
}
}
else
{
Expand All @@ -4738,8 +4751,9 @@ void CodeGen::genCodeForSelect(GenTreeConditional* tree)
regNumber srcReg2 = genConsumeReg(op2);
const GenConditionDesc& prevDesc = GenConditionDesc::Get(prevCond);

emit->emitIns_R_R_R_COND(INS_csel, cmpSize, targetReg, srcReg1, srcReg2, JumpKindToInsCond(prevDesc.jumpKind1));
emit->emitIns_R_R_R_COND(INS_csel, attr, targetReg, srcReg1, srcReg2, JumpKindToInsCond(prevDesc.jumpKind1));
regSet.verifyRegUsed(targetReg);
genProduceReg(tree);
}

//------------------------------------------------------------------------
Expand Down
5 changes: 3 additions & 2 deletions src/coreclr/jit/codegenlinear.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1634,9 +1634,10 @@ void CodeGen::genConsumeRegs(GenTree* tree)
assert(cast->isContained());
genConsumeAddress(cast->CastOp());
}
else if (tree->OperIsCmpCompare() || tree->OperIs(GT_AND))
else if (tree->OperIsCompare() || tree->OperIs(GT_AND))
{
// Compares and ANDs may be contained in a conditional chain.
// Compares can be contained by a SELECT.
// ANDs and Cmp Compares may be contained in a chain.
genConsumeRegs(tree->gtGetOp1());
genConsumeRegs(tree->gtGetOp2());
}
Expand Down
9 changes: 9 additions & 0 deletions src/coreclr/jit/compiler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4718,6 +4718,7 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
bool doCse = true;
bool doAssertionProp = true;
bool doRangeAnalysis = true;
bool doIfConversion = true;
int iterations = 1;

#if defined(OPT_CONFIG)
Expand All @@ -4730,6 +4731,7 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
doCse = doValueNum;
doAssertionProp = doValueNum && (JitConfig.JitDoAssertionProp() != 0);
doRangeAnalysis = doAssertionProp && (JitConfig.JitDoRangeAnalysis() != 0);
doIfConversion = doIfConversion && (JitConfig.JitDoIfConversion() != 0);

if (opts.optRepeat)
{
Expand Down Expand Up @@ -4798,6 +4800,13 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
DoPhase(this, PHASE_ASSERTION_PROP_MAIN, &Compiler::optAssertionPropMain);
}

if (doIfConversion)
{
// If conversion
//
DoPhase(this, PHASE_IF_CONVERSION, &Compiler::optIfConversion);
}

if (doRangeAnalysis)
{
// Bounds check elimination via range analysis
Expand Down
12 changes: 12 additions & 0 deletions src/coreclr/jit/compiler.h
Original file line number Diff line number Diff line change
Expand Up @@ -2354,6 +2354,9 @@ class Compiler
GenTreeLclVar* gtNewLclVarAddrNode(unsigned lclNum, var_types type = TYP_I_IMPL);
GenTreeLclFld* gtNewLclFldAddrNode(unsigned lclNum, unsigned lclOffs, var_types type = TYP_I_IMPL);

GenTreeConditional* gtNewConditionalNode(
genTreeOps oper, GenTree* cond, GenTree* op1, GenTree* op2, var_types type);

#ifdef FEATURE_SIMD
GenTreeSIMD* gtNewSIMDNode(
var_types type, GenTree* op1, SIMDIntrinsicID simdIntrinsicID, CorInfoType simdBaseJitType, unsigned simdSize);
Expand Down Expand Up @@ -2793,6 +2796,7 @@ class Compiler
GenTree* gtFoldExprSpecial(GenTree* tree);
GenTree* gtFoldBoxNullable(GenTree* tree);
GenTree* gtFoldExprCompare(GenTree* tree);
GenTree* gtFoldExprConditional(GenTree* tree);
GenTree* gtCreateHandleCompare(genTreeOps oper,
GenTree* op1,
GenTree* op2,
Expand Down Expand Up @@ -4836,6 +4840,9 @@ class Compiler
// Does value-numbering for a block assignment.
void fgValueNumberBlockAssignment(GenTree* tree);

// Does value-numbering for a variable definition that has SSA.
void fgValueNumberSsaVarDef(GenTreeLclVarCommon* lcl);

// Does value-numbering for a cast tree.
void fgValueNumberCastTree(GenTree* tree);

Expand Down Expand Up @@ -6046,6 +6053,7 @@ class Compiler
void optEnsureUniqueHead(unsigned loopInd, weight_t ambientWeight);
PhaseStatus optUnrollLoops(); // Unrolls loops (needs to have cost info)
void optRemoveRedundantZeroInits();
PhaseStatus optIfConversion(); // If conversion

protected:
// This enumeration describes what is killed by a call.
Expand Down Expand Up @@ -6434,6 +6442,7 @@ class Compiler
OptInvertCountTreeInfoType optInvertCountTreeInfo(GenTree* tree);

bool optInvertWhileLoop(BasicBlock* block);
bool optIfConvert(BasicBlock* block);

private:
static bool optIterSmallOverflow(int iterAtExit, var_types incrType);
Expand Down Expand Up @@ -7338,6 +7347,7 @@ class Compiler
GenTree* optAssertionProp_Cast(ASSERT_VALARG_TP assertions, GenTreeCast* cast, Statement* stmt);
GenTree* optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCall* call, Statement* stmt);
GenTree* optAssertionProp_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
GenTree* optAssertionProp_ConditionalOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
GenTree* optAssertionProp_Comma(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
GenTree* optAssertionProp_BndsChk(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
GenTree* optAssertionPropGlobal_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
Expand Down Expand Up @@ -9505,6 +9515,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
STRESS_MODE(EMITTER) \
STRESS_MODE(CHK_REIMPORT) \
STRESS_MODE(GENERIC_CHECK) \
STRESS_MODE(IF_CONVERSION_COST) \
STRESS_MODE(IF_CONVERSION_INNER_LOOPS) \
STRESS_MODE(COUNT)

enum compStressArea
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/compphases.h
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ CompPhaseNameMacro(PHASE_OPTIMIZE_VALNUM_CSES, "Optimize Valnum CSEs",
CompPhaseNameMacro(PHASE_VN_COPY_PROP, "VN based copy prop", false, -1, false)
CompPhaseNameMacro(PHASE_OPTIMIZE_BRANCHES, "Redundant branch opts", false, -1, false)
CompPhaseNameMacro(PHASE_ASSERTION_PROP_MAIN, "Assertion prop", false, -1, false)
CompPhaseNameMacro(PHASE_IF_CONVERSION, "If conversion", false, -1, false)
CompPhaseNameMacro(PHASE_OPT_UPDATE_FLOW_GRAPH, "Update flow graph opt pass", false, -1, false)
CompPhaseNameMacro(PHASE_COMPUTE_EDGE_WEIGHTS2, "Compute edge weights (2, false)",false, -1, false)
CompPhaseNameMacro(PHASE_INSERT_GC_POLLS, "Insert GC Polls", false, -1, true)
Expand Down
130 changes: 130 additions & 0 deletions src/coreclr/jit/gentree.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5818,6 +5818,9 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree)
level = max(level, lvl2);
costEx += tree->AsConditional()->gtOp2->GetCostEx();
costSz += tree->AsConditional()->gtOp2->GetCostSz();

costEx += 1;
costSz += 1;
break;

default:
Expand Down Expand Up @@ -7358,6 +7361,17 @@ GenTreeLclFld* Compiler::gtNewLclFldAddrNode(unsigned lclNum, unsigned lclOffs,
return node;
}

GenTreeConditional* Compiler::gtNewConditionalNode(
genTreeOps oper, GenTree* cond, GenTree* op1, GenTree* op2, var_types type)
{
assert(GenTree::OperIsConditional(oper));
GenTreeConditional* node = new (this, oper) GenTreeConditional(oper, type, cond, op1, op2);
node->gtFlags |= (cond->gtFlags & GTF_ALL_EFFECT);
node->gtFlags |= (op1->gtFlags & GTF_ALL_EFFECT);
node->gtFlags |= (op2->gtFlags & GTF_ALL_EFFECT);
return node;
}

GenTreeLclFld* Compiler::gtNewLclFldNode(unsigned lnum, var_types type, unsigned offset)
{
GenTreeLclFld* node = new (this, GT_LCL_FLD) GenTreeLclFld(GT_LCL_FLD, type, lnum, offset);
Expand Down Expand Up @@ -12608,6 +12622,10 @@ GenTree* Compiler::gtFoldExpr(GenTree* tree)

if (!(kind & GTK_SMPOP))
{
if (tree->OperIsConditional())
{
return gtFoldExprConditional(tree);
}
return tree;
}

Expand Down Expand Up @@ -12876,6 +12894,118 @@ GenTree* Compiler::gtFoldExprCompare(GenTree* tree)
return cons;
}

//------------------------------------------------------------------------
// gtFoldExprConditional: see if a conditional is foldable
//
// Arguments:
// tree - condition to examine
//
// Returns:
// The original call if no folding happened.
// An alternative tree if folding happens.
//
// Notes:
// Supporting foldings are:
// SELECT TRUE X Y -> X
// SELECT FALSE X Y -> Y
// SELECT COND X X -> X
//
GenTree* Compiler::gtFoldExprConditional(GenTree* tree)
a74nh marked this conversation as resolved.
Show resolved Hide resolved
{
GenTree* cond = tree->AsConditional()->gtCond;
GenTree* op1 = tree->AsConditional()->gtOp1;
GenTree* op2 = tree->AsConditional()->gtOp2;

assert(tree->OperIsConditional());

// Check for a constant conditional
if (cond->OperIsConst())
{
// Constant conditions must be folded away.

JITDUMP("\nFolding conditional op with constant condition:\n");
DISPTREE(tree);

assert(cond->TypeIs(TYP_INT));
assert((tree->gtFlags & GTF_SIDE_EFFECT & ~GTF_ASG) == 0);
assert((tree->gtFlags & GTF_ORDER_SIDEEFF) == 0);

GenTree* replacement = nullptr;
if (cond->IsIntegralConst(0))
{
JITDUMP("Bashed to false path:\n");
replacement = op2;
}
else
{
// Condition should never be a constant other than 0 or 1
assert(cond->IsIntegralConst(1));
JITDUMP("Bashed to true path:\n");
replacement = op1;
}

if (fgGlobalMorph)
{
fgMorphTreeDone(replacement);
}
else
{
replacement->gtNext = tree->gtNext;
replacement->gtPrev = tree->gtPrev;
}
DISPTREE(replacement);
JITDUMP("\n");

// If we bashed to a compare, try to fold that.
if (replacement->OperIsCompare())
{
return gtFoldExprCompare(replacement);
}

return replacement;
}

assert(cond->OperIsCompare());

if (((tree->gtFlags & GTF_SIDE_EFFECT) != 0) || !GenTree::Compare(op1, op2, true))
a74nh marked this conversation as resolved.
Show resolved Hide resolved
{
// No folding.
return tree;
}

// GTF_ORDER_SIDEEFF here may indicate volatile subtrees.
// Or it may indicate a non-null assertion prop into an indir subtree.
if ((tree->gtFlags & GTF_ORDER_SIDEEFF) != 0)
{
// If op1 is "volatile" and op2 is not, we can still fold.
const bool op1MayBeVolatile = (op1->gtFlags & GTF_ORDER_SIDEEFF) != 0;
const bool op2MayBeVolatile = (op2->gtFlags & GTF_ORDER_SIDEEFF) != 0;

if (!op1MayBeVolatile || op2MayBeVolatile)
{
// No folding.
return tree;
}
}

JITDUMP("Bashed to first of two identical paths:\n");
GenTree* replacement = op1;

if (fgGlobalMorph)
{
fgMorphTreeDone(replacement);
}
else
{
replacement->gtNext = tree->gtNext;
replacement->gtPrev = tree->gtPrev;
}
DISPTREE(replacement);
JITDUMP("\n");

return replacement;
}

//------------------------------------------------------------------------
// gtCreateHandleCompare: generate a type handle comparison
//
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/jitconfigvalues.h
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,7 @@ CONFIG_INTEGER(JitDoValueNumber, W("JitDoValueNumber"), 1) // Perform value numb

CONFIG_METHODSET(JitOptRepeat, W("JitOptRepeat")) // Runs optimizer multiple times on the method
CONFIG_INTEGER(JitOptRepeatCount, W("JitOptRepeatCount"), 2) // Number of times to repeat opts when repeating
CONFIG_INTEGER(JitDoIfConversion, W("JitDoIfConversion"), 1) // Perform If conversion
#endif // defined(OPT_CONFIG)

CONFIG_INTEGER(JitTelemetry, W("JitTelemetry"), 1) // If non-zero, gather JIT telemetry data
Expand Down
34 changes: 22 additions & 12 deletions src/coreclr/jit/lowerarmarch.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2327,8 +2327,7 @@ bool Lowering::ContainCheckCompareChain(GenTree* child, GenTree* parent, GenTree
child->SetContained();
return true;
}
else if (child->OperIsCmpCompare() && varTypeIsIntegral(child->gtGetOp1()) &&
varTypeIsIntegral(child->gtGetOp2()))
else if (child->OperIsCmpCompare())
{
child->AsOp()->SetContained();

Expand Down Expand Up @@ -2422,17 +2421,28 @@ void Lowering::ContainCheckSelect(GenTreeConditional* node)
return;
}

// Check if the compare does not need to be generated into a register.
GenTree* startOfChain = nullptr;
ContainCheckCompareChain(node->gtCond, node, &startOfChain);

if (startOfChain != nullptr)
if (node->gtCond->OperIsCompare())
{
// All compare node types (including TEST_) are containable.
if (IsSafeToContainMem(node, node->gtCond))
{
node->gtCond->AsOp()->SetContained();
}
}
else
{
// The earliest node in the chain will be generated as a standard compare.
assert(startOfChain->OperIsCmpCompare());
startOfChain->AsOp()->gtGetOp1()->ClearContained();
startOfChain->AsOp()->gtGetOp2()->ClearContained();
ContainCheckCompare(startOfChain->AsOp());
// Check for a compare chain and try to contain it.
GenTree* startOfChain = nullptr;
ContainCheckCompareChain(node->gtCond, node, &startOfChain);

if (startOfChain != nullptr)
{
// The earliest node in the chain will be generated as a standard compare.
assert(startOfChain->OperIsCmpCompare());
startOfChain->AsOp()->gtGetOp1()->ClearContained();
startOfChain->AsOp()->gtGetOp2()->ClearContained();
ContainCheckCompare(startOfChain->AsOp());
}
}
}

Expand Down
5 changes: 3 additions & 2 deletions src/coreclr/jit/lsrabuild.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3226,10 +3226,11 @@ int LinearScan::BuildOperandUses(GenTree* node, regMaskTP candidates)
}
#endif // FEATURE_HW_INTRINSICS
#ifdef TARGET_ARM64
if (node->OperIs(GT_MUL) || node->OperIsCmpCompare() || node->OperIs(GT_AND))
if (node->OperIs(GT_MUL) || node->OperIsCompare() || node->OperIs(GT_AND))
{
// MUL can be contained for madd or msub on arm64.
// Compare and AND may be contained due to If Conversion.
// Compares can be contained by a SELECT.
// ANDs and Cmp Compares may be contained in a chain.
return BuildBinaryUses(node->AsOp(), candidates);
}
if (node->OperIs(GT_NEG, GT_CAST, GT_LSH, GT_RSH, GT_RSZ))
Expand Down
Loading