diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index 0450bb6f1a0d3..90105004bb4be 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -4792,52 +4792,48 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_SCALABLE_OPTS_WIDE); // LSR ., /M, ., .D // IF_SVE_CE_2A - theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_P2, REG_V12, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .B, - theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_P7, REG_V2, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [0] + theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_P2, REG_V12, INS_OPTS_SCALABLE_B); // PMOV .B, + theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_P7, REG_V2, INS_OPTS_SCALABLE_H); // PMOV .H, [0] // IF_SVE_CE_2B - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P15, REG_V7, 7, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .D, [] - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P7, REG_V16, 0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .D, [] + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P15, REG_V7, 7, INS_OPTS_SCALABLE_D); // PMOV .D, + // [] + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P7, REG_V16, 0, INS_OPTS_SCALABLE_D); // PMOV .D, + // [] // IF_SVE_CE_2C - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P0, REG_V31, 1, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [] - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P1, REG_V1, 0, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [] + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P0, REG_V31, 1, INS_OPTS_SCALABLE_H); // PMOV .H, + // [] + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P1, REG_V1, 0, INS_OPTS_SCALABLE_H); // PMOV .H, + // [] // IF_SVE_CE_2D - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P3, REG_V9, 3, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .S, [] - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P10, REG_V4, 0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .S, [] + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P3, REG_V9, 3, INS_OPTS_SCALABLE_S); // PMOV .S, + // [] + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P10, REG_V4, 0, INS_OPTS_SCALABLE_S); // PMOV .S, + // [] // IF_SVE_CF_2A - theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_V11, REG_P12, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV , .B - theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_V2, REG_P7, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [0], .S + theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_V11, REG_P12, INS_OPTS_SCALABLE_B); // PMOV , .B + theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_V2, REG_P7, INS_OPTS_SCALABLE_S); // PMOV [0], .S // IF_SVE_CF_2B - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V6, REG_P8, 7, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .D - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V9, REG_P7, 0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .D + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V6, REG_P8, 7, INS_OPTS_SCALABLE_D); // PMOV [], + // .D + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V9, REG_P7, 0, INS_OPTS_SCALABLE_D); // PMOV [], + // .D // IF_SVE_CF_2C - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V8, REG_P4, 1, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .H - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V5, REG_P9, 0, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .H + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V8, REG_P4, 1, INS_OPTS_SCALABLE_H); // PMOV [], + // .H + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V5, REG_P9, 0, INS_OPTS_SCALABLE_H); // PMOV [], + // .H // IF_SVE_CF_2D - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V14, REG_P2, 3, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .S - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V3, REG_P15, 0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .S + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V14, REG_P2, 3, INS_OPTS_SCALABLE_S); // PMOV [], + // .S + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V3, REG_P15, 0, INS_OPTS_SCALABLE_S); // PMOV [], + // .S // IF_SVE_CJ_2A theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_P1, REG_P2, @@ -5393,54 +5389,54 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_D); // MSB ., /M, ., . // IF_SVE_AT_3A - theEmitter->emitIns_R_R_R(INS_sve_add, EA_SCALABLE, REG_V0, REG_V0, REG_V0, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); // ADD ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqadd, EA_SCALABLE, REG_V3, REG_V31, REG_V12, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // SQADD ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqsub, EA_SCALABLE, REG_V7, REG_V0, REG_V31, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // SQSUB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sub, EA_SCALABLE, REG_V19, REG_V7, REG_V13, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // SUB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uqadd, EA_SCALABLE, REG_V23, REG_V28, REG_V29, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); // UQADD ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uqsub, EA_SCALABLE, REG_V31, REG_V31, REG_V31, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // UQSUB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_mul, EA_SCALABLE, REG_V5, REG_V0, REG_V31, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); // MUL ., ., . - theEmitter->emitIns_R_R_R(INS_sve_smulh, EA_SCALABLE, REG_V0, REG_V31, REG_V5, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // SMULH ., ., . - theEmitter->emitIns_R_R_R(INS_sve_umulh, EA_SCALABLE, REG_V31, REG_V5, REG_V0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // UMULH ., ., . + theEmitter->emitIns_R_R_R(INS_sve_add, EA_SCALABLE, REG_V0, REG_V0, REG_V0, + INS_OPTS_SCALABLE_B); // ADD ., ., . + theEmitter->emitIns_R_R_R(INS_sve_sqadd, EA_SCALABLE, REG_V3, REG_V31, REG_V12, + INS_OPTS_SCALABLE_H); // SQADD ., ., . + theEmitter->emitIns_R_R_R(INS_sve_sqsub, EA_SCALABLE, REG_V7, REG_V0, REG_V31, + INS_OPTS_SCALABLE_S); // SQSUB ., ., . + theEmitter->emitIns_R_R_R(INS_sve_sub, EA_SCALABLE, REG_V19, REG_V7, REG_V13, + INS_OPTS_SCALABLE_D); // SUB ., ., . + theEmitter->emitIns_R_R_R(INS_sve_uqadd, EA_SCALABLE, REG_V23, REG_V28, REG_V29, + INS_OPTS_SCALABLE_B); // UQADD ., ., . + theEmitter->emitIns_R_R_R(INS_sve_uqsub, EA_SCALABLE, REG_V31, REG_V31, REG_V31, + INS_OPTS_SCALABLE_H); // UQSUB ., ., . + theEmitter->emitIns_R_R_R(INS_sve_mul, EA_SCALABLE, REG_V5, REG_V0, REG_V31, + INS_OPTS_SCALABLE_B); // MUL ., ., . + theEmitter->emitIns_R_R_R(INS_sve_smulh, EA_SCALABLE, REG_V0, REG_V31, REG_V5, + INS_OPTS_SCALABLE_H); // SMULH ., ., . + theEmitter->emitIns_R_R_R(INS_sve_umulh, EA_SCALABLE, REG_V31, REG_V5, REG_V0, + INS_OPTS_SCALABLE_D); // UMULH ., ., . theEmitter->emitIns_R_R_R(INS_sve_sqdmulh, EA_SCALABLE, REG_V7, REG_V28, REG_V0, INS_OPTS_SCALABLE_B); // SQDMULH ., ., . theEmitter->emitIns_R_R_R(INS_sve_sqrdmulh, EA_SCALABLE, REG_V23, REG_V3, REG_V31, INS_OPTS_SCALABLE_H); // SQRDMULH ., ., . theEmitter->emitIns_R_R_R(INS_sve_ftssel, EA_SCALABLE, REG_V17, REG_V16, REG_V15, INS_OPTS_SCALABLE_D); // FTSSEL ., ., . - theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_trn2, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_trn2, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_SCALABLE, REG_V15, REG_V16, REG_V17, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uzp2, EA_SCALABLE, REG_V18, REG_V19, REG_V20, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uzp2, EA_SCALABLE, REG_V21, REG_V22, REG_V23, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zip1, EA_SCALABLE, REG_V24, REG_V25, REG_V26, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zip1, EA_SCALABLE, REG_V27, REG_V28, REG_V29, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zip2, EA_SCALABLE, REG_V30, REG_V31, REG_V0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zip2, EA_SCALABLE, REG_V1, REG_V2, REG_V3, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP2 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_B); // TRN1 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_H); // TRN1 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_trn2, EA_SCALABLE, REG_V6, REG_V7, REG_V8, + INS_OPTS_SCALABLE_S); // TRN2 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_trn2, EA_SCALABLE, REG_V9, REG_V10, REG_V11, + INS_OPTS_SCALABLE_D); // TRN2 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_SCALABLE, REG_V12, REG_V13, REG_V14, + INS_OPTS_SCALABLE_B); // UZP1 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_SCALABLE, REG_V15, REG_V16, REG_V17, + INS_OPTS_SCALABLE_H); // UZP1 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_uzp2, EA_SCALABLE, REG_V18, REG_V19, REG_V20, + INS_OPTS_SCALABLE_S); // UZP2 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_uzp2, EA_SCALABLE, REG_V21, REG_V22, REG_V23, + INS_OPTS_SCALABLE_D); // UZP2 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_zip1, EA_SCALABLE, REG_V24, REG_V25, REG_V26, + INS_OPTS_SCALABLE_B); // ZIP1 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_zip1, EA_SCALABLE, REG_V27, REG_V28, REG_V29, + INS_OPTS_SCALABLE_H); // ZIP1 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_zip2, EA_SCALABLE, REG_V30, REG_V31, REG_V0, + INS_OPTS_SCALABLE_S); // ZIP2 ., ., . + theEmitter->emitIns_R_R_R(INS_sve_zip2, EA_SCALABLE, REG_V1, REG_V2, REG_V3, + INS_OPTS_SCALABLE_D); // ZIP2 ., ., . theEmitter->emitIns_R_R_R(INS_sve_tbxq, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_B); // TBXQ ., ., . theEmitter->emitIns_R_R_R(INS_sve_tbxq, EA_SCALABLE, REG_V3, REG_V4, REG_V5, @@ -5479,18 +5475,18 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_S); // BGRP ., ., . theEmitter->emitIns_R_R_R(INS_sve_bgrp, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_D); // BGRP ., ., . - theEmitter->emitIns_R_R_R(INS_sve_fadd, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // FADD ., ., . - theEmitter->emitIns_R_R_R(INS_sve_fmul, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // FMUL ., ., . - theEmitter->emitIns_R_R_R(INS_sve_frecps, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // FRECPS ., ., . - theEmitter->emitIns_R_R_R(INS_sve_frsqrts, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // FRSQRTS ., ., . - theEmitter->emitIns_R_R_R(INS_sve_fsub, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // FSUB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_ftsmul, EA_SCALABLE, REG_V15, REG_V16, REG_V17, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // FTSMUL ., ., . + theEmitter->emitIns_R_R_R(INS_sve_fadd, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_H); // FADD ., ., . + theEmitter->emitIns_R_R_R(INS_sve_fmul, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_S); // FMUL ., ., . + theEmitter->emitIns_R_R_R(INS_sve_frecps, EA_SCALABLE, REG_V6, REG_V7, REG_V8, + INS_OPTS_SCALABLE_D); // FRECPS ., ., . + theEmitter->emitIns_R_R_R(INS_sve_frsqrts, EA_SCALABLE, REG_V9, REG_V10, REG_V11, + INS_OPTS_SCALABLE_H); // FRSQRTS ., ., . + theEmitter->emitIns_R_R_R(INS_sve_fsub, EA_SCALABLE, REG_V12, REG_V13, REG_V14, + INS_OPTS_SCALABLE_S); // FSUB ., ., . + theEmitter->emitIns_R_R_R(INS_sve_ftsmul, EA_SCALABLE, REG_V15, REG_V16, REG_V17, + INS_OPTS_SCALABLE_D); // FTSMUL ., ., . // IF_SVE_BA_3A theEmitter->emitIns_R_R_R(INS_sve_index, EA_4BYTE, REG_V24, REG_ZR, REG_R9, @@ -5503,12 +5499,12 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_B); // PMUL .B, .B, .B // IF_SVE_BG_3A - theEmitter->emitIns_R_R_R(INS_sve_asr, EA_SCALABLE, REG_V9, REG_V31, REG_V2, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED_WIDE); // ASR ., ., .D - theEmitter->emitIns_R_R_R(INS_sve_lsl, EA_SCALABLE, REG_V19, REG_V0, REG_V12, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED_WIDE); // LSL ., ., .D - theEmitter->emitIns_R_R_R(INS_sve_lsr, EA_SCALABLE, REG_V29, REG_V10, REG_V22, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED_WIDE); // LSR ., ., .D + theEmitter->emitIns_R_R_R(INS_sve_asr, EA_SCALABLE, REG_V9, REG_V31, REG_V2, + INS_OPTS_SCALABLE_B); // ASR ., ., .D + theEmitter->emitIns_R_R_R(INS_sve_lsl, EA_SCALABLE, REG_V19, REG_V0, REG_V12, + INS_OPTS_SCALABLE_H); // LSL ., ., .D + theEmitter->emitIns_R_R_R(INS_sve_lsr, EA_SCALABLE, REG_V29, REG_V10, REG_V22, + INS_OPTS_SCALABLE_S); // LSR ., ., .D // IF_SVE_BH_3A theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V4, REG_V2, REG_V0, 0, INS_OPTS_SCALABLE_D, @@ -5529,18 +5525,18 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_D_UXTW); // ADR .D, [.D, .D, UXTW{}] // IF_SVE_BR_3B - theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN1 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_trn2, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN2 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP1 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_uzp2, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP2 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_zip1, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP1 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_zip2, EA_SCALABLE, REG_V15, REG_V16, REG_V17, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP2 .Q, .Q, .Q + theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_Q); // TRN1 .Q, + // .Q, .Q + theEmitter->emitIns_R_R_R(INS_sve_trn2, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_Q); // TRN2 .Q, + // .Q, .Q + theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_Q); // UZP1 .Q, + // .Q, .Q + theEmitter->emitIns_R_R_R(INS_sve_uzp2, EA_SCALABLE, REG_V9, REG_V10, REG_V11, + INS_OPTS_SCALABLE_Q); // UZP2 .Q, .Q, .Q + theEmitter->emitIns_R_R_R(INS_sve_zip1, EA_SCALABLE, REG_V12, REG_V13, REG_V14, + INS_OPTS_SCALABLE_Q); // ZIP1 .Q, .Q, .Q + theEmitter->emitIns_R_R_R(INS_sve_zip2, EA_SCALABLE, REG_V15, REG_V16, REG_V17, + INS_OPTS_SCALABLE_Q); // ZIP2 .Q, .Q, .Q // IF_SVE_BS_1A theEmitter->emitIns_R_I(INS_sve_and, EA_SCALABLE, REG_V0, 0x00000000000000AA, @@ -5650,26 +5646,26 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_Q); // MOV ., // MOV implementation should produce same output as DUP implementation with same parameters - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V1, REG_V16, 63, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V17, REG_V18, 31, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V9, REG_V11, 15, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V2, REG_V3, 7, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V3, REG_V8, 3, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V13, REG_V9, 0, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V12, REG_V6, 0, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V2, REG_V7, 0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V10, REG_V20, 0, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V1, REG_V16, 63, INS_OPTS_SCALABLE_B); // MOV ., + // .[] + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V17, REG_V18, 31, INS_OPTS_SCALABLE_H); // MOV ., + // .[] + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V9, REG_V11, 15, INS_OPTS_SCALABLE_S); // MOV ., + // .[] + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V2, REG_V3, 7, INS_OPTS_SCALABLE_D); // MOV ., + // .[] + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V3, REG_V8, 3, INS_OPTS_SCALABLE_Q); // MOV ., + // .[] + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V13, REG_V9, 0, INS_OPTS_SCALABLE_B); // MOV ., + // + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V12, REG_V6, 0, INS_OPTS_SCALABLE_H); // MOV ., + // + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V2, REG_V7, 0, INS_OPTS_SCALABLE_S); // MOV ., + // + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D); // MOV ., + // + theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V10, REG_V20, 0, INS_OPTS_SCALABLE_Q); // MOV ., + // // IF_SVE_BZ_3A theEmitter->emitIns_R_R_R(INS_sve_tbl, EA_SCALABLE, REG_V0, REG_V1, REG_V2, @@ -5882,32 +5878,32 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_H); // BFCLAMP .H, .H, .H // IF_SVE_HK_3B - theEmitter->emitIns_R_R_R(INS_sve_bfadd, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // BFADD .H, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfmul, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // BFMUL .H, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfsub, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // BFSUB .H, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfadd, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_H); // BFADD .H, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfmul, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_H); // BFMUL .H, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfsub, EA_SCALABLE, REG_V6, REG_V7, REG_V8, + INS_OPTS_SCALABLE_H); // BFSUB .H, .H, .H #ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED // IF_SVE_AT_3B - theEmitter->emitIns_R_R_R(INS_sve_addpt, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // ADDPT .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_subpt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // SUBPT .D, .D, .D -#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + theEmitter->emitIns_R_R_R(INS_sve_addpt, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_D); // ADDPT .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_subpt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_D); // SUBPT .D, .D, .D +#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED // IF_SVE_AU_3A - theEmitter->emitIns_R_R_R(INS_sve_and, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // AND .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_bic, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // BIC .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_eor, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // EOR .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_mov, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // MOV .D, .D - theEmitter->emitIns_R_R_R(INS_sve_orr, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // ORR .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_and, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_D); // AND .D, + // .D, .D + theEmitter->emitIns_R_R_R(INS_sve_bic, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_D); // BIC .D, + // .D, .D + theEmitter->emitIns_R_R_R(INS_sve_eor, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_D); // EOR .D, + // .D, .D + theEmitter->emitIns_R_R_R(INS_sve_mov, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_D); // MOV .D, + // .D + theEmitter->emitIns_R_R_R(INS_sve_orr, EA_SCALABLE, REG_V12, REG_V13, REG_V14, + INS_OPTS_SCALABLE_D); // ORR .D, .D, .D // IF_SVE_AV_3A theEmitter->emitIns_R_R_R(INS_sve_bcax, EA_SCALABLE, REG_V0, REG_V1, REG_V2, @@ -6255,10 +6251,10 @@ void CodeGen::genArm64EmitterUnitTestsSve() // IF_SVE_CW_4A theEmitter->emitIns_R_R_R(INS_sve_mov, EA_SCALABLE, REG_V0, REG_P0, REG_V30, INS_OPTS_SCALABLE_H, INS_SCALABLE_OPTS_PREDICATE_MERGE); // MOV ., /M, . - theEmitter->emitIns_R_R_R_R(INS_sve_sel, EA_SCALABLE, REG_V29, REG_P15, REG_V28, REG_V4, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // SEL ., , ., . - theEmitter->emitIns_R_R_R_R(INS_sve_sel, EA_SCALABLE, REG_V5, REG_P13, REG_V27, REG_V5, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // SEL ., , ., . + theEmitter->emitIns_R_R_R_R(INS_sve_sel, EA_SCALABLE, REG_V29, REG_P15, REG_V28, REG_V4, + INS_OPTS_SCALABLE_D); // SEL ., , ., . + theEmitter->emitIns_R_R_R_R(INS_sve_sel, EA_SCALABLE, REG_V5, REG_P13, REG_V27, REG_V5, + INS_OPTS_SCALABLE_S); // SEL ., , ., . // IF_SVE_EQ_3A // Note: Scalable size is the size of the destination , not the source . @@ -8517,29 +8513,19 @@ void CodeGen::genArm64EmitterUnitTestsSve() // IF_SVE_IE_2A // LDR , [{, #, MUL VL}] - theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, 0, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, 33, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, -173, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, -256, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, 255, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, 0, INS_OPTS_NONE); + theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, 33, INS_OPTS_NONE); + theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, -173, INS_OPTS_NONE); + theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, -256, INS_OPTS_NONE); + theEmitter->emitIns_R_R_I(INS_sve_ldr, EA_SCALABLE, REG_V3, REG_R4, 255, INS_OPTS_NONE); // IF_SVE_JH_2A // STR , [{, #, MUL VL}] - theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, 0, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, 71, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, -165, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, -256, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, 255, INS_OPTS_NONE, - INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, 0, INS_OPTS_NONE); + theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, 71, INS_OPTS_NONE); + theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, -165, INS_OPTS_NONE); + theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, -256, INS_OPTS_NONE); + theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, 255, INS_OPTS_NONE); #ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED // IF_SVE_GG_3A @@ -8870,17 +8856,13 @@ void CodeGen::genArm64EmitterUnitTestsSve() // IF_SVE_CG_2A // REV ., . - theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V2, REG_V3, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V2, REG_V3, INS_OPTS_SCALABLE_B); // REV ., . - theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V2, REG_V4, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V2, REG_V4, INS_OPTS_SCALABLE_H); // REV ., . - theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V7, REG_V1, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V7, REG_V1, INS_OPTS_SCALABLE_S); // REV ., . - theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V2, REG_V5, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V2, REG_V5, INS_OPTS_SCALABLE_D); // IF_SVE_CB_2A // Note: EA_4BYTE used for B and H (source register is W) @@ -8937,106 +8919,57 @@ void CodeGen::genArm64EmitterUnitTestsSve() // IF_SVE_BF_2A // ASR ., ., # - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); // LSL ., ., #emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 15, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 31, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 63, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 15, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 31, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 63, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); // LSR ., ., #emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); // IF_SVE_FT_2A // SLI ., ., # diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 720945451c139..964a386241f95 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -4252,8 +4252,7 @@ void emitter::emitIns_Mov( { if (isPredicateRegister(dstReg) && isPredicateRegister(srcReg)) { - assert(insOptsNone(opt)); - + assert((opt == INS_OPTS_SCALABLE_B) || insOptsNone(opt)); opt = INS_OPTS_SCALABLE_B; attr = EA_SCALABLE; @@ -7882,42 +7881,14 @@ void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber reg1, int va case INS_sve_ldr: { - assert(isVectorRegister(reg1)); isSimple = false; size = EA_SCALABLE; attr = size; - fmt = IF_SVE_IE_2A; - - // TODO-SVE: Don't assume 128bit vectors - scale = NaturalScale_helper(EA_16BYTE); - ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate - - if (((imm & mask) == 0) && (isValidSimm<9>(imm >> scale))) - { - imm >>= scale; // The immediate is scaled by the size of the ld/st - } - else - { - useRegForImm = true; - regNumber rsvdReg = codeGen->rsGetRsvdReg(); - codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); - } - } - break; - - // TODO-SVE: Fold into INS_sve_ldr once REG_V0 and REG_P0 are distinct - case INS_sve_ldr_mask: - { - assert(isPredicateRegister(reg1)); - isSimple = false; - size = EA_SCALABLE; - attr = size; - fmt = IF_SVE_ID_2A; - ins = INS_sve_ldr; + fmt = isVectorRegister(reg1) ? IF_SVE_IE_2A : IF_SVE_ID_2A; // TODO-SVE: Don't assume 128bit vectors // Predicate size is vector length / 8 - scale = NaturalScale_helper(EA_2BYTE); + scale = NaturalScale_helper(isVectorRegister(reg1) ? EA_16BYTE : EA_2BYTE); ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate if (((imm & mask) == 0) && (isValidSimm<9>(imm >> scale))) @@ -7930,8 +7901,8 @@ void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber reg1, int va regNumber rsvdReg = codeGen->rsGetRsvdReg(); codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); } + break; } - break; default: NYI("emitIns_R_S"); // FP locals? @@ -8161,42 +8132,14 @@ void emitter::emitIns_S_R(instruction ins, emitAttr attr, regNumber reg1, int va case INS_sve_str: { - assert(isVectorRegister(reg1)); - isSimple = false; - size = EA_SCALABLE; - attr = size; - fmt = IF_SVE_JH_2A; - - // TODO-SVE: Don't assume 128bit vectors - scale = NaturalScale_helper(EA_16BYTE); - ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate - - if (((imm & mask) == 0) && (isValidSimm<9>(imm >> scale))) - { - imm >>= scale; // The immediate is scaled by the size of the ld/st - } - else - { - useRegForImm = true; - regNumber rsvdReg = codeGen->rsGetRsvdReg(); - codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); - } - } - break; - - // TODO-SVE: Fold into INS_sve_str once REG_V0 and REG_P0 are distinct - case INS_sve_str_mask: - { - assert(isPredicateRegister(reg1)); isSimple = false; size = EA_SCALABLE; attr = size; - fmt = IF_SVE_JG_2A; - ins = INS_sve_str; + fmt = isVectorRegister(reg1) ? IF_SVE_JH_2A : IF_SVE_JG_2A; // TODO-SVE: Don't assume 128bit vectors // Predicate size is vector length / 8 - scale = NaturalScale_helper(EA_2BYTE); + scale = NaturalScale_helper(isVectorRegister(reg1) ? EA_16BYTE : EA_2BYTE); ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate if (((imm & mask) == 0) && (isValidSimm<9>(imm >> scale))) @@ -8209,6 +8152,7 @@ void emitter::emitIns_S_R(instruction ins, emitAttr attr, regNumber reg1, int va regNumber rsvdReg = codeGen->rsGetRsvdReg(); codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); } + break; } break; diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index e36e5cdb4d7d2..99e86bb38c15c 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -1926,22 +1926,17 @@ void emitter::emitInsSve_R_R(instruction ins, assert(insOptsScalableStandard(opt)); return emitInsSve_R_R_I(INS_sve_pmov, attr, reg1, reg2, 0, opt, sopt); } - if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) + if (isPredicateRegister(reg1)) { - assert(isPredicateRegister(reg1)); assert(isVectorRegister(reg2)); fmt = IF_SVE_CE_2A; } - else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) + else { assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); fmt = IF_SVE_CF_2A; } - else - { - assert(!"invalid instruction"); - } break; case INS_sve_movs: @@ -2031,17 +2026,16 @@ void emitter::emitInsSve_R_R(instruction ins, break; case INS_sve_rev: - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + assert(insScalableOptsNone(sopt)); + if (isVectorRegister(reg1)) { assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); assert(isVectorRegister(reg2)); assert(isScalableVectorSize(size)); fmt = IF_SVE_CG_2A; } else { - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isPredicateRegister(reg1)); // DDDD assert(isPredicateRegister(reg2)); // NNNN @@ -2366,17 +2360,15 @@ void emitter::emitInsSve_R_R_I(instruction ins, assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); assert(insOptsScalableStandard(opt)); assert(isScalableVectorSize(size)); - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + assert(insScalableOptsNone(sopt)); + if (isVectorRegister(reg2)) { assert((ins == INS_sve_asr) || (ins == INS_sve_lsl) || (ins == INS_sve_lsr)); assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); fmt = IF_SVE_BF_2A; } else { - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isLowPredicateRegister(reg2)); // ggg fmt = IF_SVE_AM_2A; @@ -2446,7 +2438,7 @@ void emitter::emitInsSve_R_R_I(instruction ins, break; case INS_sve_mov: - if (sopt == INS_SCALABLE_OPTS_BROADCAST) + if (isVectorRegister(reg2)) { return emitInsSve_R_R_I(INS_sve_dup, attr, reg1, reg2, imm, opt, sopt); } @@ -2489,9 +2481,8 @@ void emitter::emitInsSve_R_R_I(instruction ins, break; case INS_sve_pmov: - if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) + if (isPredicateRegister(reg1)) { - assert(isPredicateRegister(reg1)); assert(isVectorRegister(reg2)); switch (opt) { @@ -2511,7 +2502,7 @@ void emitter::emitInsSve_R_R_I(instruction ins, unreached(); } } - else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) + else { assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); @@ -2533,10 +2524,6 @@ void emitter::emitInsSve_R_R_I(instruction ins, unreached(); } } - else - { - unreached(); - } break; case INS_sve_sqrshrn: @@ -2674,14 +2661,13 @@ void emitter::emitInsSve_R_R_I(instruction ins, assert(isValidSimm<9>(imm)); // iii // iiiiii - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + assert(insScalableOptsNone(sopt)); + if (isVectorRegister(reg1)) { - assert(isVectorRegister(reg1)); fmt = IF_SVE_IE_2A; } else { - assert(insScalableOptsNone(sopt)); assert(isPredicateRegister(reg1)); fmt = IF_SVE_ID_2A; } @@ -2694,14 +2680,13 @@ void emitter::emitInsSve_R_R_I(instruction ins, assert(isValidSimm<9>(imm)); // iii // iiiiii - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + assert(insScalableOptsNone(sopt)); + if (isVectorRegister(reg1)) { - assert(isVectorRegister(reg1)); fmt = IF_SVE_JH_2A; } else { - assert(insScalableOptsNone(sopt)); assert(isPredicateRegister(reg1)); fmt = IF_SVE_JG_2A; } @@ -2917,19 +2902,18 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); // mmmmm assert(isVectorRegister(reg3)); // ddddd + assert(insScalableOptsNone(sopt)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + if (isVectorRegister(reg2)) { // The instruction only has a .D variant. However, this doesn't matter as // it operates on bits not lanes. Effectively this means all standard opt // sizes are supported. assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg2)); // nnnnn fmt = IF_SVE_AU_3A; } else { - assert(insScalableOptsNone(sopt)); assert(isLowPredicateRegister(reg2)); // ggg fmt = IF_SVE_AA_3A; } @@ -2941,16 +2925,15 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(isVectorRegister(reg1)); assert(isVectorRegister(reg3)); assert(insOptsScalableStandard(opt)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + assert(insScalableOptsNone(sopt)); + if (isVectorRegister(reg2)) { - assert(isVectorRegister(reg2)); assert(ins != INS_sve_subr); fmt = IF_SVE_AT_3A; } else { assert(isLowPredicateRegister(reg2)); - assert(insScalableOptsNone(sopt)); fmt = IF_SVE_AA_3A; } break; @@ -2961,15 +2944,14 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg3)); // mmmmm + assert(insScalableOptsNone(sopt)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + if (isVectorRegister(reg2)) { - assert(isVectorRegister(reg2)); // nnnnn fmt = IF_SVE_AT_3B; } else { - assert(insScalableOptsNone(sopt)); assert(isLowPredicateRegister(reg2)); // ggg fmt = IF_SVE_AB_3B; } @@ -3007,14 +2989,13 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(isVectorRegister(reg1)); assert(isVectorRegister(reg3)); assert(insOptsScalableStandard(opt)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + assert(insScalableOptsNone(sopt)); + if (isVectorRegister(reg2)) { - assert(isVectorRegister(reg2)); fmt = IF_SVE_AT_3A; } else { - assert(insScalableOptsNone(sopt)); assert(isLowPredicateRegister(reg2)); fmt = IF_SVE_AA_3A; } @@ -3138,9 +3119,9 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(insOptsScalableWide(opt)); fmt = IF_SVE_AO_3A; } - else if (sopt == INS_SCALABLE_OPTS_UNPREDICATED_WIDE) + else if (isVectorRegister(reg2)) { - assert(isVectorRegister(reg2)); + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWide(opt)); fmt = IF_SVE_BG_3A; } @@ -3160,10 +3141,10 @@ void emitter::emitInsSve_R_R_R(instruction ins, case INS_sve_trn2: case INS_sve_zip2: assert(insOptsScalable(opt)); + assert(insScalableOptsNone(sopt)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + if (isVectorRegister(reg1)) { - assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmmmm @@ -3179,7 +3160,6 @@ void emitter::emitInsSve_R_R_R(instruction ins, } else { - assert(insScalableOptsNone(sopt)); assert(isPredicateRegister(reg1)); // DDDD assert(isPredicateRegister(reg2)); // NNNN assert(isPredicateRegister(reg3)); // MMMM @@ -3604,10 +3584,10 @@ void emitter::emitInsSve_R_R_R(instruction ins, break; case INS_sve_not: - if (isPredicateRegister(reg1) && sopt != INS_SCALABLE_OPTS_UNPREDICATED) + assert(insScalableOptsNone(sopt)); + if (isPredicateRegister(reg1)) { assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // DDDD assert(isPredicateRegister(reg2)); // gggg assert(isPredicateRegister(reg3)); // NNNN fmt = IF_SVE_CZ_4A; @@ -3618,7 +3598,6 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); fmt = IF_SVE_AP_3A; } break; @@ -3761,59 +3740,56 @@ void emitter::emitInsSve_R_R_R(instruction ins, case INS_sve_cpy: case INS_sve_mov: assert(insOptsScalableStandard(opt)); - // TODO-SVE: Following checks can be simplified to check reg1 as predicate register only after adding - // definitions for predicate registers. Currently, predicate registers P0 to P15 are aliased to simd - // registers V0 to V15. - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + if (isVectorRegister(reg1)) // ddddd { - assert(ins == INS_sve_mov); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_AU_3A; - // ORR is an alias for MOV, and is always the preferred disassembly. - ins = INS_sve_orr; - } - else if (isPredicateRegister(reg3) && - (sopt == INS_SCALABLE_OPTS_NONE || sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE)) - { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // NNNN - fmt = sopt == INS_SCALABLE_OPTS_NONE ? IF_SVE_CZ_4A : IF_SVE_CZ_4A_K; - // MOV is an alias for CPY, and is always the preferred disassembly. - ins = INS_sve_mov; - } - else if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) - { - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_CW_4A; - } - else - { - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - if (isGeneralRegisterOrSP(reg3)) + if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) + { + assert(isPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + fmt = IF_SVE_CW_4A; + } + else if (sopt == INS_SCALABLE_OPTS_WITH_SIMD_SCALAR) + { + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + fmt = IF_SVE_CP_3A; + // MOV is an alias for CPY, and is always the preferred disassembly. + ins = INS_sve_mov; + } + else if (isLowPredicateRegister(reg2)) { + assert(isGeneralRegisterOrSP(reg3)); assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_CQ_3A; reg3 = encodingSPtoZR(reg3); + // MOV is an alias for CPY, and is always the preferred disassembly. + ins = INS_sve_mov; } else { - assert(sopt == INS_SCALABLE_OPTS_WITH_SIMD_SCALAR); - assert(isVectorRegister(reg3)); - fmt = IF_SVE_CP_3A; + assert(insScalableOptsNone(sopt)); + assert(ins == INS_sve_mov); + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_AU_3A; + // ORR is an alias for MOV, and is always the preferred disassembly. + ins = INS_sve_orr; } - + } + else if (isPredicateRegister(reg3)) // NNNN + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // gggg + fmt = sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE ? IF_SVE_CZ_4A_K : IF_SVE_CZ_4A; // MOV is an alias for CPY, and is always the preferred disassembly. ins = INS_sve_mov; } + else + { + unreached(); + } break; case INS_sve_lasta: @@ -3992,14 +3968,14 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(isVectorRegister(reg3)); assert(insOptsScalableStandard(opt)); assert(isScalableVectorSize(size)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + assert(insScalableOptsNone(sopt)); + + if (isVectorRegister(reg2)) { - assert(isVectorRegister(reg2)); fmt = IF_SVE_AT_3A; } else { - assert(insScalableOptsNone(sopt)); assert(isLowPredicateRegister(reg2)); fmt = IF_SVE_AA_3A; } @@ -4119,15 +4095,14 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg3)); // mmmmm assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + assert(insScalableOptsNone(sopt)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + if (isVectorRegister(reg2)) // nnnnn { - assert(isVectorRegister(reg2)); // nnnnn fmt = IF_SVE_AT_3A; } else { - assert(insScalableOptsNone(sopt)); assert(isLowPredicateRegister(reg2)); // ggg fmt = IF_SVE_HL_3A; } @@ -4172,15 +4147,14 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(opt == INS_OPTS_SCALABLE_H); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg3)); // mmmmm + assert(insScalableOptsNone(sopt)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + if (isVectorRegister(reg2)) // nnnnn { - assert(isVectorRegister(reg2)); // nnnnn fmt = IF_SVE_HK_3B; } else { - assert(insScalableOptsNone(sopt)); assert(isLowPredicateRegister(reg2)); // ggg fmt = IF_SVE_HL_3B; } @@ -5901,7 +5875,8 @@ void emitter::emitInsSve_R_R_R_R(instruction ins, switch (ins) { case INS_sve_sel: - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + assert(insScalableOptsNone(sopt)); + if (isVectorRegister(reg1)) { if (reg1 == reg4) { @@ -5909,9 +5884,7 @@ void emitter::emitInsSve_R_R_R_R(instruction ins, return emitInsSve_R_R_R(INS_sve_mov, attr, reg1, reg2, reg3, opt, INS_SCALABLE_OPTS_PREDICATE_MERGE); } - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd assert(isPredicateRegister(reg2)); // VVVV assert(isVectorRegister(reg3)); // nnnnn assert(isVectorRegister(reg4)); // mmmmm diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9a899443294b5..dfa9af2dcdc94 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27525,12 +27525,6 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend: case NI_Sve_GatherVectorUInt32ZeroExtend: addr = Op(2); - if (varTypeIsSIMD(addr->gtType)) - { - // The address is a vector of addresses. - // Return true, but do not set pAddr. - return true; - } break; #endif // TARGET_ARM64 @@ -27610,7 +27604,18 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const if (addr != nullptr) { +#ifdef TARGET_ARM64 + static_assert_no_msg( + AreContiguous(NI_Sve_GatherVector, NI_Sve_GatherVectorByteZeroExtend, NI_Sve_GatherVectorInt16SignExtend, + NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend, NI_Sve_GatherVectorInt32SignExtend, + NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend, NI_Sve_GatherVectorSByteSignExtend, + NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt16ZeroExtend, + NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt32ZeroExtend)); + assert(varTypeIsI(addr) || (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_GatherVector) && + (intrinsicId <= NI_Sve_GatherVectorUInt32ZeroExtend)))); +#else assert(varTypeIsI(addr)); +#endif return true; } diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 38a480ae77ef5..0d0bbc6cfc963 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -525,7 +525,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, opt); GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, - falseReg, opt, INS_SCALABLE_OPTS_UNPREDICATED); + falseReg, opt); break; } else @@ -569,7 +569,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) // If the embedded instruction supports optional mask operation, use the "unpredicated" // version of the instruction, followed by "sel" to select the active lanes. GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, embMaskOp1Reg, - embMaskOp2Reg, opt, INS_SCALABLE_OPTS_UNPREDICATED); + embMaskOp2Reg, opt); } else { @@ -588,7 +588,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, - falseReg, opt, INS_SCALABLE_OPTS_UNPREDICATED); + falseReg, opt); break; } else if (targetReg != embMaskOp1Reg) @@ -761,7 +761,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) // have the value from embMaskOp1Reg GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, embMaskOp1Reg, - falseReg, opt, INS_SCALABLE_OPTS_UNPREDICATED); + falseReg, opt); } } else if (targetReg != embMaskOp1Reg) @@ -816,8 +816,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { // This generates an unpredicated version // Implicitly predicated should be taken care above `intrin.op2->IsEmbMaskOp()` - GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt, - INS_SCALABLE_OPTS_UNPREDICATED); + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); } } else if (isRMW) @@ -852,8 +851,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } else { - GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, op3Reg, opt, - INS_SCALABLE_OPTS_UNPREDICATED); + GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, op3Reg, opt); } break; @@ -1924,7 +1922,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_ReverseElement: // Use non-predicated version explicitly - GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt, INS_SCALABLE_OPTS_UNPREDICATED); + GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt); break; case NI_Sve_StoreNarrowing: @@ -1939,8 +1937,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_ZipHigh: case NI_Sve_ZipLow: // Use non-predicated version explicitly - GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt, - INS_SCALABLE_OPTS_UNPREDICATED); + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); break; case NI_Sve_SaturatingDecrementBy16BitElementCountScalar: diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index 4799b8b333491..1ea14aaf6a38a 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -1873,7 +1873,7 @@ instruction CodeGenInterface::ins_Load(var_types srcType, bool aligned /*=false* #if defined(TARGET_XARCH) return INS_kmovq_msk; #elif defined(TARGET_ARM64) - return INS_sve_ldr_mask; + return INS_sve_ldr; #endif } #endif // FEATURE_MASKED_HW_INTRINSICS @@ -2194,7 +2194,7 @@ instruction CodeGenInterface::ins_Store(var_types dstType, bool aligned /*=false #if defined(TARGET_XARCH) return INS_kmovq_msk; #elif defined(TARGET_ARM64) - return INS_sve_str_mask; + return INS_sve_str; #endif } #endif // FEATURE_MASKED_HW_INTRINSICS diff --git a/src/coreclr/jit/instr.h b/src/coreclr/jit/instr.h index 421dee30ac754..47db1c8304a8a 100644 --- a/src/coreclr/jit/instr.h +++ b/src/coreclr/jit/instr.h @@ -379,13 +379,6 @@ enum insScalableOpts : unsigned INS_SCALABLE_OPTS_IMM_BITMASK, // Variants with an immediate that is a bitmask INS_SCALABLE_OPTS_IMM_FIRST, // Variants with an immediate and a register, where the immediate comes first - - // Removable once REG_V0 and REG_P0 are distinct - INS_SCALABLE_OPTS_UNPREDICATED, // Variants without a predicate (eg add) - INS_SCALABLE_OPTS_UNPREDICATED_WIDE, // Variants without a predicate and wide elements (eg asr) - INS_SCALABLE_OPTS_TO_PREDICATE, // Variants moving to a predicate from a vector (e.g. pmov) - INS_SCALABLE_OPTS_TO_VECTOR, // Variants moving to a vector from a predicate (e.g. pmov) - INS_SCALABLE_OPTS_BROADCAST, // Used to distinguish mov from cpy, where mov is an alias for both }; // Maps directly to the pattern used in SVE instructions such as cntb. diff --git a/src/coreclr/jit/instrsarm64sve.h b/src/coreclr/jit/instrsarm64sve.h index e29899d981b6d..52a01668ae5a6 100644 --- a/src/coreclr/jit/instrsarm64sve.h +++ b/src/coreclr/jit/instrsarm64sve.h @@ -2840,11 +2840,6 @@ INST1(ldnt1sw, "ldnt1sw", 0, IF_SV INST1(st1q, "st1q", 0, IF_SVE_IY_4A, 0xE4202000 ) // ST1Q {.Q }, , [.D{, }] SVE_IY_4A 11100100001mmmmm 001gggnnnnnttttt E420 2000 - -// TODO-SVE: Removable once REG_V0 and REG_P0 are distinct -INST1(str_mask, "str_mask", 0, IF_SN_0A, BAD_CODE) -INST1(ldr_mask, "ldr_mask", 0, IF_SN_0A, BAD_CODE) - // clang-format on /*****************************************************************************/