diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index 36e4f126c6d3f..8fe561e2f64e5 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -4914,6 +4914,9 @@ void CodeGen::genArm64EmitterUnitTestsSve() SVE_PATTERN_ALL); // PTRUES .{, } theEmitter->emitIns_R_PATTERN(INS_sve_ptrues, EA_SCALABLE, REG_P15, INS_OPTS_SCALABLE_D); // PTRUES .{, } + // IF_SVE_DF_2A + theEmitter->emitIns_R_R(INS_sve_pnext, EA_SCALABLE, REG_P0, REG_P15, + INS_OPTS_SCALABLE_B); // PNEXT ., , . // IF_SVE_DG_2A theEmitter->emitIns_R_R(INS_sve_rdffr, EA_SCALABLE, REG_P10, REG_P15, @@ -4921,6 +4924,19 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R(INS_sve_rdffrs, EA_SCALABLE, REG_P7, REG_P14, INS_OPTS_SCALABLE_B); // RDFFRS .B, /Z + // IF_SVE_DH_1A + theEmitter->emitIns_R(INS_sve_rdffr, EA_SCALABLE, REG_P8); // RDFFR .B + + // IF_SVE_DJ_1A + theEmitter->emitIns_R(INS_sve_pfalse, EA_SCALABLE, REG_P13); // PFALSE .B + + // IF_SVE_DI_2A + theEmitter->emitIns_R_R(INS_sve_ptest, EA_SCALABLE, REG_P2, REG_P14, INS_OPTS_SCALABLE_B); // PTEST , .B + + // IF_SVE_DK_3A + theEmitter->emitIns_R_R_R(INS_sve_cntp, EA_8BYTE, REG_R29, REG_P0, REG_P15, + INS_OPTS_SCALABLE_D); // CNTP , , . + // IF_SVE_EP_3A theEmitter->emitIns_R_R_R(INS_sve_shadd, EA_SCALABLE, REG_V15, REG_P0, REG_V10, INS_OPTS_SCALABLE_B); // SHADD ., /M, ., . diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index abbf4ea42cc22..1d7b9f7e95974 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -1212,6 +1212,31 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(insOptsScalableStandard(id->idInsOpt())); // xx break; + case IF_SVE_DF_2A: // ........xx...... .......VVVV.DDDD -- SVE predicate next active + case IF_SVE_DI_2A: // ................ ..gggg.NNNN..... -- SVE predicate test + elemsize = id->idOpSize(); + assert(isScalableVectorSize(elemsize)); + assert(insOptsScalableStandard(id->idInsOpt())); + assert(isPredicateRegister(id->idReg1())); // DDDD + assert(isPredicateRegister(id->idReg2())); // gggg + break; + + case IF_SVE_DH_1A: // ................ ............DDDD -- SVE predicate read from FFR (unpredicated) + case IF_SVE_DJ_1A: // ................ ............DDDD -- SVE predicate zero + elemsize = id->idOpSize(); + assert(isScalableVectorSize(elemsize)); + assert(id->idInsOpt() == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(id->idReg1())); // DDDD + break; + + case IF_SVE_DK_3A: // ........xx...... ..gggg.NNNNddddd -- SVE predicate count + assert(id->idOpSize() == EA_8BYTE); + assert(insOptsScalableStandard(id->idInsOpt())); + assert(isGeneralRegister(id->idReg1())); // ddddd + assert(isPredicateRegister(id->idReg2())); // gggg + assert(isPredicateRegister(id->idReg3())); // NNNN + break; + case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match elemsize = id->idOpSize(); assert(isScalableVectorSize(elemsize)); @@ -6176,34 +6201,41 @@ void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg, insOpts o case INS_br: case INS_ret: assert(isGeneralRegister(reg)); - id->idReg1(reg); fmt = IF_BR_1A; break; case INS_dczva: assert(isGeneralRegister(reg)); assert(attr == EA_8BYTE); - id->idReg1(reg); fmt = IF_SR_1A; break; case INS_mrs_tpid0: - id->idReg1(reg); fmt = IF_SR_1A; break; case INS_sve_aesmc: case INS_sve_aesimc: id->idInsOpt(INS_OPTS_SCALABLE_B); - id->idReg1(reg); assert(isVectorRegister(reg)); // ddddd assert(isScalableVectorSize(attr)); fmt = IF_SVE_GL_1A; break; + case INS_sve_rdffr: + id->idInsOpt(INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg)); // DDDD + fmt = IF_SVE_DH_1A; + break; + + case INS_sve_pfalse: + id->idInsOpt(INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg)); // DDDD + fmt = IF_SVE_DJ_1A; + break; + case INS_sve_wrffr: id->idInsOpt(INS_OPTS_SCALABLE_B); - id->idReg1(reg); assert(isPredicateRegister(reg)); // NNNN fmt = IF_SVE_DR_1A; break; @@ -6212,7 +6244,6 @@ void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg, insOpts o assert(insOptsScalableStandard(opt)); assert(isHighPredicateRegister(reg)); // DDD assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - id->idReg1(reg); id->idInsOpt(opt); fmt = IF_SVE_DZ_1A; break; @@ -6225,6 +6256,7 @@ void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg, insOpts o id->idIns(ins); id->idInsFmt(fmt); + id->idReg1(reg); dispIns(id); appendToCurIG(id); @@ -7539,6 +7571,14 @@ void emitter::emitIns_R_R(instruction ins, fmt = IF_SVE_DD_2A; break; + case INS_sve_pnext: + assert(insOptsScalableStandard(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // VVVV + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_DF_2A; + break; + case INS_sve_rdffr: case INS_sve_rdffrs: assert(opt == INS_OPTS_SCALABLE_B); @@ -7547,6 +7587,13 @@ void emitter::emitIns_R_R(instruction ins, fmt = IF_SVE_DG_2A; break; + case INS_sve_ptest: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // gggg + assert(isPredicateRegister(reg2)); // NNNN + fmt = IF_SVE_DI_2A; + break; + case INS_sve_cntp: assert(insOptsScalableStandard(opt)); assert(insScalableOptsWithVectorLength(sopt)); // l @@ -9694,6 +9741,15 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_SVE_DC_3A; break; + case INS_sve_cntp: + assert(size == EA_8BYTE); + assert(isGeneralRegister(reg1)); // ddddd + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // NNNN + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_DK_3A; + break; + case INS_sve_shadd: case INS_sve_shsub: case INS_sve_shsubr: @@ -15086,6 +15142,8 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_CF_2D: case IF_SVE_CI_3A: case IF_SVE_DE_1A: + case IF_SVE_DH_1A: + case IF_SVE_DJ_1A: case IF_SVE_DM_2A: case IF_SVE_DN_2A: case IF_SVE_DO_2A: @@ -15094,7 +15152,6 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_DT_3A: case IF_SVE_DU_3A: case IF_SVE_CK_2A: - case IF_SVE_DI_2A: return PREDICATE_SIZED; case IF_SVE_DB_3A: @@ -15153,7 +15210,6 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_CS_3A: case IF_SVE_CV_3A: case IF_SVE_CV_3B: - case IF_SVE_DK_3A: case IF_SVE_DW_2A: // [] case IF_SVE_DW_2B: // [] case IF_SVE_JC_4A: @@ -15168,16 +15224,11 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_AJ_3A: case IF_SVE_AL_3A: case IF_SVE_CL_3A: - case IF_SVE_DF_2A: case IF_SVE_GS_3A: case IF_SVE_HJ_3A: case IF_SVE_IY_4A: return PREDICATE_NONE; - case IF_SVE_DD_2A: - assert((regpos >= 1) && (regpos <= 3)); - return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); - case IF_SVE_CX_4A: case IF_SVE_CX_4A_A: case IF_SVE_CY_3A: @@ -15198,9 +15249,21 @@ void emitter::emitIns_Call(EmitCallType callType, assert((regpos >= 1) && (regpos <= 3)); return (regpos == 2 ? PREDICATE_MERGE : PREDICATE_SIZED); + case IF_SVE_DD_2A: + case IF_SVE_DF_2A: + assert((regpos >= 1) && (regpos <= 3)); + return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); + case IF_SVE_DG_2A: return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); + case IF_SVE_DI_2A: + return (regpos == 1 ? PREDICATE_NONE : PREDICATE_SIZED); + + case IF_SVE_DK_3A: + assert((regpos == 2) || (regpos == 3)); + return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); + default: break; } @@ -18054,6 +18117,37 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) dst += emitOutput_Instr(dst, code); break; + case IF_SVE_DF_2A: // ........xx...... .......VVVV.DDDD -- SVE predicate next active + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // VVVV + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DH_1A: // ................ ............DDDD -- SVE predicate read from FFR (unpredicated) + case IF_SVE_DJ_1A: // ................ ............DDDD -- SVE predicate zero + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DI_2A: // ................ ..gggg.NNNN..... -- SVE predicate test + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_13_to_10(id->idReg1()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DK_3A: // ........xx...... ..gggg.NNNNddddd -- SVE predicate count + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + case IF_SVE_GA_2A: // ............iiii ......nnnn.ddddd -- SME2 multi-vec shift narrow imm = emitGetInsSC(id); assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); @@ -21112,12 +21206,33 @@ void emitter::emitDispInsHelp( break; } + // ., , . + case IF_SVE_DF_2A: // ........xx...... .......VVVV.DDDD -- SVE predicate next active + emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt, 1), id->idInsOpt(), true); // DDDD + emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt, 2), id->idInsOpt(), true); // VVVV + emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt, 3), id->idInsOpt(), false); // DDDD + break; + // .B, /Z case IF_SVE_DG_2A: // ................ .......gggg.DDDD -- SVE predicate read from FFR (predicated) + case IF_SVE_DI_2A: // ................ ..gggg.NNNN..... -- SVE predicate test emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt, 1), id->idInsOpt(), true); // DDDD emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt, 2), id->idInsOpt(), false); // gggg break; + // .B + case IF_SVE_DH_1A: // ................ ............DDDD -- SVE predicate read from FFR (unpredicated) + case IF_SVE_DJ_1A: // ................ ............DDDD -- SVE predicate zero + emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), id->idInsOpt(), false); // DDDD + break; + + // , , . + case IF_SVE_DK_3A: // ........xx...... ..gggg.NNNNddddd -- SVE predicate count + emitDispReg(id->idReg1(), size, true); // ddddd + emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt, 2), id->idInsOpt(), true); // gggg + emitDispPredicateReg(id->idReg3(), insGetPredicateType(fmt, 3), id->idInsOpt(), false); // NNNN + break; + // ., /M, . case IF_SVE_EQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer pairwise add and accumulate long emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd @@ -23967,6 +24082,11 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } break; + case IF_SVE_DF_2A: // ........xx...... .......VVVV.DDDD -- SVE predicate next active + result.insLatency = PERFSCORE_LATENCY_3C; + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + break; + case IF_SVE_DG_2A: // ................ .......gggg.DDDD -- SVE predicate read from FFR (predicated) switch (ins) { @@ -23987,6 +24107,26 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } break; + case IF_SVE_DH_1A: // ................ ............DDDD -- SVE predicate read from FFR (unpredicated) + result.insLatency = PERFSCORE_LATENCY_3C; + result.insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + + case IF_SVE_DJ_1A: // ................ ............DDDD -- SVE predicate zero + result.insLatency = PERFSCORE_LATENCY_2C; + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + break; + + case IF_SVE_DI_2A: // ................ ..gggg.NNNN..... -- SVE predicate test + result.insLatency = PERFSCORE_LATENCY_1C; + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + break; + + case IF_SVE_DK_3A: // ........xx...... ..gggg.NNNNddddd -- SVE predicate count + result.insLatency = PERFSCORE_LATENCY_2C; + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + break; + case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match case IF_SVE_HT_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE floating-point compare vectors result.insLatency = PERFSCORE_LATENCY_2C;