diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index c294957a0cd2c..c92b12a382b3a 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1042,6 +1042,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genIntToFloatCast(GenTree* treeNode); void genCkfinite(GenTree* treeNode); void genCodeForCompare(GenTreeOp* tree); +#ifdef TARGET_ARM64 + void genCodeForConditional(GenTreeConditional* tree); +#endif void genIntrinsic(GenTree* treeNode); void genPutArgStk(GenTreePutArgStk* treeNode); void genPutArgReg(GenTreeOp* tree); @@ -1704,6 +1707,12 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX instruction genMapShiftInsToShiftByConstantIns(instruction ins, int shiftByValue); #endif // TARGET_XARCH +#ifdef TARGET_ARM64 + static insCond InsCondForCompareOp(GenTree* tree); + static insCond InvertInsCond(insCond cond); + static insCflags InsCflagsForCcmp(insCond cond); +#endif + #ifndef TARGET_LOONGARCH64 // Maps a GenCondition code to a sequence of conditional jumps or other conditional instructions // such as X86's SETcc. A sequence of instructions rather than just a single one is required for diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index b13c11c50d668..e2d261c21153a 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -4411,6 +4411,60 @@ void CodeGen::genCodeForCompare(GenTreeOp* tree) } } +//------------------------------------------------------------------------ +// genCodeForCompare: Produce code for a GT_CEQ/GT_CNE node. +// +// Arguments: +// tree - the node +// +void CodeGen::genCodeForConditional(GenTreeConditional* tree) +{ + emitter* emit = GetEmitter(); + + GenTree* opcond = tree->gtCond; + GenTree* op1 = tree->gtOp1; + GenTree* op2 = tree->gtOp2; + var_types op1Type = genActualType(op1->TypeGet()); + var_types op2Type = genActualType(op2->TypeGet()); + emitAttr cmpSize = EA_ATTR(genTypeSize(op1Type)); + insCond cond = InsCondForCompareOp(opcond); + + assert(!op1->isUsedFromMemory()); + assert(genTypeSize(op1Type) == genTypeSize(op2Type)); + + regNumber targetReg = tree->GetRegNum(); + regNumber srcReg1 = genConsumeReg(op1); + + if (tree->OperIs(GT_SELECT)) + { + regNumber srcReg2 = genConsumeReg(op2); + emit->emitIns_R_R_R_COND(INS_csel, cmpSize, targetReg, srcReg1, srcReg2, cond); + regSet.verifyRegUsed(targetReg); + } + else + { + assert(!varTypeIsFloating(op2Type)); + // We don't support swapping op1 and op2 to generate cmp reg, imm. + assert(!op1->isContainedIntOrIImmed()); + // This should not be generating into a register. + assert(targetReg == REG_NA); + + // For the ccmp flags, get the condition of the compare. + insCflags cflags = InsCflagsForCcmp(InsCondForCompareOp(tree)); + + if (op2->isContainedIntOrIImmed()) + { + GenTreeIntConCommon* intConst = op2->AsIntConCommon(); + emit->emitIns_R_I_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, (int)intConst->IconValue(), cflags, cond); + } + else + { + regNumber srcReg2 = genConsumeReg(op2); + emit->emitIns_R_R_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, srcReg2, cflags, cond); + } + } +} + //------------------------------------------------------------------------ // genCodeForJumpCompare: Generates code for jmpCompare statement. // @@ -10381,4 +10435,123 @@ void CodeGen::genCodeForCond(GenTreeOp* tree) genProduceReg(tree); } +//------------------------------------------------------------------------ +// InsCondForCompareOp: Map the condition in a Compare/Conditional op to a insCond. +// +// Arguments: +// tree - the node +// +insCond CodeGen::InsCondForCompareOp(GenTree* tree) +{ + assert(tree->OperIsCompare() || tree->OperIsConditionalCompare()); + + if (tree->OperIsCompare()) + { + switch (tree->AsOp()->OperGet()) + { + case GT_EQ: + case GT_TEST_EQ: + return INS_COND_EQ; + case GT_NE: + case GT_TEST_NE: + return INS_COND_NE; + case GT_GE: + return INS_COND_GE; + case GT_GT: + return INS_COND_GT; + case GT_LT: + return INS_COND_LT; + case GT_LE: + return INS_COND_LE; + default: + assert(false && "Invalid condition"); + return INS_COND_EQ; + } + } + else + { + switch (tree->AsConditional()->OperGet()) + { + case GT_CEQ: + return INS_COND_EQ; + case GT_CNE: + return INS_COND_NE; + case GT_CGE: + return INS_COND_GE; + case GT_CGT: + return INS_COND_GT; + case GT_CLT: + return INS_COND_LT; + case GT_CLE: + return INS_COND_LE; + default: + assert(false && "Invalid condition"); + return INS_COND_EQ; + } + } +} + +//------------------------------------------------------------------------ +// InvertInsCond: Invert an insCond +// +// Arguments: +// cond - the insCond. +// +insCond CodeGen::InvertInsCond(insCond cond) +{ + switch (cond) + { + case INS_COND_EQ: + return INS_COND_NE; + case INS_COND_NE: + return INS_COND_EQ; + case INS_COND_GE: + return INS_COND_LT; + case INS_COND_GT: + return INS_COND_LE; + case INS_COND_LT: + return INS_COND_GE; + case INS_COND_LE: + return INS_COND_GT; + default: + assert(false && "Invalid condition"); + return INS_COND_EQ; + } +} + +//------------------------------------------------------------------------ +// InsCflagsForCcmp: Get the Cflags for a required for a CCMP instruction. +// +// Consider: +// cmp w, x +// ccmp y, z, A, COND +// This is: compare w and x, if this matches condition COND, then compare y and z. +// Otherwise set flags to A - this should match the case where cmp failed. +// Given COND, this function returns A. +// +// Arguments: +// cond - the insCond. +// +insCflags CodeGen::InsCflagsForCcmp(insCond cond) +{ + switch (InvertInsCond(cond)) + { + case INS_COND_EQ: + return INS_FLAGS_Z; + case INS_COND_NE: + return INS_FLAGS_NONE; + case INS_COND_GE: + return INS_FLAGS_Z; + case INS_COND_GT: + return INS_FLAGS_NONE; + case INS_COND_LT: + return INS_FLAGS_NC; + case INS_COND_LE: + return INS_FLAGS_NZC; + default: + assert(false && "Invalid condition"); + return INS_FLAGS_NONE; + } +} + #endif // TARGET_ARM64 diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 41ab1420db07e..f84764fed1f2a 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -367,6 +367,18 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForCompare(treeNode->AsOp()); break; +#ifdef TARGET_ARM64 + case GT_SELECT: + case GT_CEQ: + case GT_CNE: + case GT_CLT: + case GT_CLE: + case GT_CGE: + case GT_CGT: + genCodeForConditional(treeNode->AsConditional()); + break; +#endif + case GT_JTRUE: genCodeForJumpTrue(treeNode->AsOp()); break; diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index b9026bfedd6f4..9119e8ef8b3b3 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -2628,7 +2628,16 @@ void CodeGen::genCodeForJumpTrue(GenTreeOp* jtrue) assert(compiler->compCurBB->bbJumpKind == BBJ_COND); assert(jtrue->OperIs(GT_JTRUE)); - GenTreeOp* relop = jtrue->gtGetOp1()->AsOp(); + GenTreeOp* relop; + if (jtrue->gtGetOp1()->OperIsCompare()) + { + relop = jtrue->gtGetOp1()->AsOp(); + } + else + { + assert(jtrue->gtGetOp1()->OperIsConditionalCompare()); + relop = jtrue->gtGetOp1()->AsConditional(); + } GenCondition condition = GenCondition::FromRelop(relop); if (condition.PreferSwap()) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 287a14c6d9435..0e93db80e7210 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -2365,6 +2365,12 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) return false; // not encodable } +// true if this 'imm' can be encoded as a input operand to a ccmp instruction +/*static*/ bool emitter::emitIns_valid_imm_for_ccmp(INT64 imm) +{ + return ((imm & 0x01f) == imm); +} + // true if 'imm' can be encoded as an offset in a ldp/stp instruction /*static*/ bool emitter::canEncodeLoadOrStorePairOffset(INT64 imm, emitAttr attr) { @@ -7437,7 +7443,7 @@ void emitter::emitIns_R_I_FLAGS_COND( ins = insReverse(ins); imm = -imm; } - if ((imm >= 0) && (imm <= 31)) + if (isValidUimm5(imm)) { cfi.imm5 = imm; cfi.flags = flags; @@ -12980,7 +12986,7 @@ void emitter::emitDispInsHelp( cfi.immCFVal = (unsigned)emitGetInsSC(id); emitDispImm(cfi.imm5, true); emitDispFlags(cfi.flags); - printf(","); + printf(", "); emitDispCond(cfi.cond); break; @@ -13050,7 +13056,7 @@ void emitter::emitDispInsHelp( emitDispReg(id->idReg2(), size, true); cfi.immCFVal = (unsigned)emitGetInsSC(id); emitDispFlags(cfi.flags); - printf(","); + printf(", "); emitDispCond(cfi.cond); break; diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 4068fe3c25408..276bf69339f34 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -357,6 +357,12 @@ static bool isStackRegister(regNumber reg) return (reg == REG_ZR) || (reg == REG_FP); } // ZR (R31) encodes the SP register +// Returns true if 'value' is a legal unsigned immediate 5 bit encoding (such as for CCMP). +static bool isValidUimm5(ssize_t value) +{ + return (0 <= value) && (value <= 0x1FLL); +}; + // Returns true if 'value' is a legal unsigned immediate 8 bit encoding (such as for fMOV). static bool isValidUimm8(ssize_t value) { @@ -489,6 +495,9 @@ static bool emitIns_valid_imm_for_alu(INT64 imm, emitAttr size); // true if this 'imm' can be encoded as the offset in a ldr/str instruction static bool emitIns_valid_imm_for_ldst_offset(INT64 imm, emitAttr size); +// true if this 'imm' can be encoded as a input operand to a ccmp instruction +static bool emitIns_valid_imm_for_ccmp(INT64 imm); + // true if 'imm' can be encoded as an offset in a ldp/stp instruction static bool canEncodeLoadOrStorePairOffset(INT64 imm, emitAttr size); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index f3dee9573eb53..26ba762c706e5 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -1135,8 +1135,8 @@ struct GenTree if (gtType == TYP_VOID) { // These are the only operators which can produce either VOID or non-VOID results. - assert(OperIs(GT_NOP, GT_CALL, GT_COMMA) || OperIsCompare() || OperIsLong() || OperIsSimdOrHWintrinsic() || - IsCnsVec()); + assert(OperIs(GT_NOP, GT_CALL, GT_COMMA) || OperIsCompare() || OperIsConditionalCompare() || OperIsLong() || + OperIsSimdOrHWintrinsic() || IsCnsVec()); return false; } @@ -1400,6 +1400,28 @@ struct GenTree return OperIsCompare(OperGet()); } + static bool OperIsConditional(genTreeOps gtOper) + { + static_assert_no_msg(AreContiguous(GT_SELECT, GT_CEQ, GT_CNE, GT_CLT, GT_CLE, GT_CGE, GT_CGT)); + return (GT_SELECT <= gtOper) && (gtOper <= GT_CGT); + } + + bool OperIsConditional() const + { + return OperIsConditional(OperGet()); + } + + static bool OperIsConditionalCompare(genTreeOps gtOper) + { + static_assert_no_msg(AreContiguous(GT_CEQ, GT_CNE, GT_CLT, GT_CLE, GT_CGE, GT_CGT)); + return (GT_CEQ <= gtOper) && (gtOper <= GT_CGT); + } + + bool OperIsConditionalCompare() const + { + return OperIsConditionalCompare(OperGet()); + } + static bool OperIsShift(genTreeOps gtOper) { return (gtOper == GT_LSH) || (gtOper == GT_RSH) || (gtOper == GT_RSZ); @@ -2806,6 +2828,7 @@ class GenTreeUseEdgeIterator final void AdvanceStoreDynBlk(); void AdvanceFieldList(); void AdvancePhi(); + void AdvanceConditional(); template void AdvanceBinOp(); @@ -3934,6 +3957,26 @@ struct GenTreeColon : public GenTreeOp } }; +// GenTreeConditional -- Conditionally do an operation + +struct GenTreeConditional : public GenTreeOp +{ + GenTree* gtCond; + + GenTreeConditional( + genTreeOps oper, var_types type, GenTree* cond, GenTree* op1, GenTree* op2 DEBUGARG(bool largeNode = false)) + : GenTreeOp(oper, type, op1, op2 DEBUGARG(largeNode)), gtCond(cond) + { + assert(cond != nullptr); + } + +#if DEBUGGABLE_GENTREE + GenTreeConditional() : GenTreeOp() + { + } +#endif +}; + // gtCall -- method call (GT_CALL) enum class InlineObservation; @@ -8140,7 +8183,7 @@ struct GenCondition static GenCondition FromRelop(GenTree* relop) { - assert(relop->OperIsCompare()); + assert(relop->OperIsCompare() || relop->OperIsConditionalCompare()); if (varTypeIsFloating(relop->gtGetOp1())) { @@ -8177,17 +8220,37 @@ struct GenCondition static GenCondition FromIntegralRelop(GenTree* relop) { - assert(!varTypeIsFloating(relop->gtGetOp1()) && !varTypeIsFloating(relop->gtGetOp2())); + if (relop->OperIsConditionalCompare()) + { + assert(!varTypeIsFloating(relop->AsConditional()->gtOp1) && + !varTypeIsFloating(relop->AsConditional()->gtOp2)); + } + else + { + assert(!varTypeIsFloating(relop->gtGetOp1()) && !varTypeIsFloating(relop->gtGetOp2())); + } return FromIntegralRelop(relop->OperGet(), relop->IsUnsigned()); } static GenCondition FromIntegralRelop(genTreeOps oper, bool isUnsigned) { - assert(GenTree::OperIsCompare(oper)); + assert(GenTree::OperIsCompare(oper) || GenTree::OperIsConditionalCompare(oper)); // GT_TEST_EQ/NE are special, they need to be mapped as GT_EQ/NE - unsigned code = oper - ((oper >= GT_TEST_EQ) ? GT_TEST_EQ : GT_EQ); + unsigned code; + if (oper >= GT_CEQ) + { + code = oper - GT_CEQ; + } + else if (oper >= GT_TEST_EQ) + { + code = oper - GT_TEST_EQ; + } + else + { + code = oper - GT_EQ; + } if (isUnsigned || (code <= 1)) // EQ/NE are treated as unsigned { diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 53a3618100b9e..32b56e04bfd8c 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -144,6 +144,14 @@ GTNODE(GT , GenTreeOp ,0,GTK_BINOP) GTNODE(TEST_EQ , GenTreeOp ,0,GTK_BINOP|DBK_NOTHIR) GTNODE(TEST_NE , GenTreeOp ,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(SELECT , GenTreeConditional ,0,GTK_SPECIAL) +GTNODE(CEQ , GenTreeConditional ,0,GTK_SPECIAL) +GTNODE(CNE , GenTreeConditional ,0,GTK_SPECIAL) +GTNODE(CLT , GenTreeConditional ,0,GTK_SPECIAL) +GTNODE(CLE , GenTreeConditional ,0,GTK_SPECIAL) +GTNODE(CGE , GenTreeConditional ,0,GTK_SPECIAL) +GTNODE(CGT , GenTreeConditional ,0,GTK_SPECIAL) + GTNODE(COMMA , GenTreeOp ,0,GTK_BINOP|DBK_NOTLIR) GTNODE(QMARK , GenTreeQmark ,0,GTK_BINOP|GTK_EXOP|DBK_NOTLIR) GTNODE(COLON , GenTreeColon ,0,GTK_BINOP|DBK_NOTLIR) diff --git a/src/coreclr/jit/gtstructs.h b/src/coreclr/jit/gtstructs.h index 038164cb2f7b8..8d7baa9da46f1 100644 --- a/src/coreclr/jit/gtstructs.h +++ b/src/coreclr/jit/gtstructs.h @@ -99,6 +99,7 @@ GTSTRUCT_1(PhiArg , GT_PHI_ARG) GTSTRUCT_1(Phi , GT_PHI) GTSTRUCT_1(StoreInd , GT_STOREIND) GTSTRUCT_N(Indir , GT_STOREIND, GT_IND, GT_NULLCHECK, GT_BLK, GT_STORE_BLK, GT_OBJ, GT_STORE_OBJ, GT_STORE_DYN_BLK) +GTSTRUCT_N(Conditional , GT_SELECT, GT_CEQ, GT_CNE, GT_CLT, GT_CLE, GT_CGE, GT_CGT) #if FEATURE_ARG_SPLIT GTSTRUCT_2_SPECIAL(PutArgStk, GT_PUTARG_STK, GT_PUTARG_SPLIT) GTSTRUCT_1(PutArgSplit , GT_PUTARG_SPLIT)