Skip to content

Commit

Permalink
[wasm] Simd refactoring (#84170)
Browse files Browse the repository at this point in the history 
* [wasm] Simd refactoring

Rename few opcodes to contain `_SIMD_`.

Use more SimdIntrinsic fields to simplify the code.

* Fix build

* Fix narrowing intrinsics

* Verbose aot compilation

* Feedback

* Make it verbose on helix

* Do not set op for narrowing methods

Because `emit_hardware_intrinsics` doesn't call custom emit in such case
and so we endup with code emitted with wrong zero c0.

* Feedback

* Be quiet again :-)
  • Loading branch information
@radekdoulik
radekdoulik authored Apr 5, 2023
1 parent 5711ca3 commit 6114f19
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 81 deletions.
36 changes: 18 additions & 18 deletions src/mono/mono/mini/mini-llvm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7671,30 +7671,30 @@ MONO_RESTORE_WARNING
break;
}
#if defined(TARGET_ARM64) || defined(TARGET_WASM)
case OP_FCVTL:
case OP_FCVTL2: {
case OP_SIMD_FCVTL:
case OP_SIMD_FCVTL2: {
LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass);
gboolean high = ins->opcode == OP_FCVTL2;
gboolean high = ins->opcode == OP_SIMD_FCVTL2;
LLVMValueRef result = lhs;
if (high)
result = extract_high_elements (ctx, result);
result = LLVMBuildFPExt (builder, result, ret_t, "fcvtl");
values [ins->dreg] = result;
break;
}
case OP_SHL:
case OP_SSHR:
case OP_SSRA:
case OP_USHR:
case OP_USRA: {
case OP_SIMD_SHL:
case OP_SIMD_SSHR:
case OP_SIMD_SSRA:
case OP_SIMD_USHR:
case OP_SIMD_USRA: {
gboolean right = FALSE;
gboolean add = FALSE;
gboolean arith = FALSE;
switch (ins->opcode) {
case OP_USHR: right = TRUE; break;
case OP_USRA: right = TRUE; add = TRUE; break;
case OP_SSHR: arith = TRUE; break;
case OP_SSRA: arith = TRUE; add = TRUE; break;
case OP_SIMD_USHR: right = TRUE; break;
case OP_SIMD_USRA: right = TRUE; add = TRUE; break;
case OP_SIMD_SSHR: arith = TRUE; break;
case OP_SIMD_SSRA: arith = TRUE; add = TRUE; break;
}
LLVMValueRef shiftarg = lhs;
LLVMValueRef shift = rhs;
Expand All @@ -7715,16 +7715,16 @@ MONO_RESTORE_WARNING
values [ins->dreg] = result;
break;
}
case OP_SSHLL:
case OP_SSHLL2:
case OP_USHLL:
case OP_USHLL2: {
case OP_SIMD_SSHLL:
case OP_SIMD_SSHLL2:
case OP_SIMD_USHLL:
case OP_SIMD_USHLL2: {
LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass);
gboolean high = FALSE;
gboolean is_unsigned = FALSE;
switch (ins->opcode) {
case OP_SSHLL2: high = TRUE; break;
case OP_USHLL2: high = TRUE; case OP_USHLL: is_unsigned = TRUE; break;
case OP_SIMD_SSHLL2: high = TRUE; break;
case OP_SIMD_USHLL2: high = TRUE; case OP_SIMD_USHLL: is_unsigned = TRUE; break;
}
LLVMValueRef result = lhs;
if (high)
Expand Down
22 changes: 11 additions & 11 deletions src/mono/mono/mini/mini-ops.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1772,17 +1772,17 @@ MINI_OP3(OP_ARM64_TBX_INDIRECT, "arm64_tbx_indirect", XREG, IREG, XREG, XREG)

#endif // TARGET_ARM64

MINI_OP(OP_FCVTL, "convert_to_higher_precision", XREG, XREG, NONE)
MINI_OP(OP_FCVTL2, "convert_to_higher_precision_2", XREG, XREG, NONE)
MINI_OP(OP_USHLL, "unsigned_shift_left_long", XREG, XREG, IREG)
MINI_OP(OP_USHLL2, "unsigned_shift_left_long_2", XREG, XREG, IREG)
MINI_OP(OP_SSHLL, "signed_shift_left_long", XREG, XREG, IREG)
MINI_OP(OP_SSHLL2, "signed_shift_left_long_2", XREG, XREG, IREG)
MINI_OP(OP_SHL, "shl", XREG, XREG, IREG)
MINI_OP(OP_SSHR, "sshr", XREG, XREG, IREG)
MINI_OP(OP_USHR, "ushr", XREG, XREG, IREG)
MINI_OP3(OP_USRA, "usra", XREG, XREG, XREG, IREG)
MINI_OP3(OP_SSRA, "ssra", XREG, XREG, XREG, IREG)
MINI_OP(OP_SIMD_FCVTL, "simd_convert_to_higher_precision", XREG, XREG, NONE)
MINI_OP(OP_SIMD_FCVTL2, "simd_convert_to_higher_precision_2", XREG, XREG, NONE)
MINI_OP(OP_SIMD_USHLL, "simd_unsigned_shift_left_long", XREG, XREG, IREG)
MINI_OP(OP_SIMD_USHLL2, "simd_unsigned_shift_left_long_2", XREG, XREG, IREG)
MINI_OP(OP_SIMD_SSHLL, "simd_signed_shift_left_long", XREG, XREG, IREG)
MINI_OP(OP_SIMD_SSHLL2, "simd_signed_shift_left_long_2", XREG, XREG, IREG)
MINI_OP(OP_SIMD_SHL, "simd_shl", XREG, XREG, IREG)
MINI_OP(OP_SIMD_SSHR, "simd_sshr", XREG, XREG, IREG)
MINI_OP(OP_SIMD_USHR, "simd_ushr", XREG, XREG, IREG)
MINI_OP3(OP_SIMD_USRA, "simd_usra", XREG, XREG, XREG, IREG)
MINI_OP3(OP_SIMD_SSRA, "simd_ssra", XREG, XREG, XREG, IREG)

#if defined(TARGET_WASM)
MINI_OP(OP_WASM_ONESCOMPLEMENT, "wasm_onescomplement", XREG, XREG, NONE)
Expand Down
102 changes: 50 additions & 52 deletions src/mono/mono/mini/simd-intrinsics.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1864,11 +1864,11 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
int op = id == SN_WidenLower ? OP_XLOWER : OP_XUPPER;
MonoInst *lower_or_upper_half = emit_simd_ins_for_sig (cfg, klass, op, 0, arg0_type, fsig, args);
if (type_enum_is_float (arg0_type)) {
return emit_simd_ins (cfg, klass, OP_FCVTL, lower_or_upper_half->dreg, -1);
return emit_simd_ins (cfg, klass, OP_SIMD_FCVTL, lower_or_upper_half->dreg, -1);
} else {
int zero = alloc_ireg (cfg);
MONO_EMIT_NEW_ICONST (cfg, zero, 0);
op = type_enum_is_unsigned (arg0_type) ? OP_USHLL : OP_SSHLL;
op = type_enum_is_unsigned (arg0_type) ? OP_SIMD_USHLL : OP_SIMD_SSHLL;
return emit_simd_ins (cfg, klass, op, lower_or_upper_half->dreg, zero);
}
#else
Expand Down Expand Up @@ -2932,9 +2932,9 @@ static SimdIntrinsic advsimd_methods [] = {
{SN_CompareLessThanScalar, OP_XCOMPARE_SCALAR, CMP_LT, OP_XCOMPARE_SCALAR, CMP_LT_UN, OP_XCOMPARE_FP_SCALAR, CMP_LT},
{SN_CompareTest, OP_ARM64_CMTST},
{SN_CompareTestScalar, OP_ARM64_CMTST},
{SN_ConvertToDouble, OP_CVT_SI_FP, None, OP_CVT_UI_FP, None, OP_FCVTL},
{SN_ConvertToDouble, OP_CVT_SI_FP, None, OP_CVT_UI_FP, None, OP_SIMD_FCVTL},
{SN_ConvertToDoubleScalar, OP_CVT_SI_FP_SCALAR, None, OP_CVT_UI_FP_SCALAR},
{SN_ConvertToDoubleUpper, OP_FCVTL2},
{SN_ConvertToDoubleUpper, OP_SIMD_FCVTL2},
{SN_ConvertToInt32RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS},
{SN_ConvertToInt32RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS},
{SN_ConvertToInt32RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS},
Expand Down Expand Up @@ -3166,14 +3166,14 @@ static SimdIntrinsic advsimd_methods [] = {
{SN_ShiftArithmeticScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SSHL},
{SN_ShiftLeftAndInsert, OP_ARM64_SLI},
{SN_ShiftLeftAndInsertScalar, OP_ARM64_SLI},
{SN_ShiftLeftLogical, OP_SHL},
{SN_ShiftLeftLogical, OP_SIMD_SHL},
{SN_ShiftLeftLogicalSaturate},
{SN_ShiftLeftLogicalSaturateScalar},
{SN_ShiftLeftLogicalSaturateUnsigned, OP_ARM64_SQSHLU},
{SN_ShiftLeftLogicalSaturateUnsignedScalar, OP_ARM64_SQSHLU_SCALAR},
{SN_ShiftLeftLogicalScalar, OP_SHL},
{SN_ShiftLeftLogicalWideningLower, OP_SSHLL, None, OP_USHLL},
{SN_ShiftLeftLogicalWideningUpper, OP_SSHLL2, None, OP_USHLL2},
{SN_ShiftLeftLogicalScalar, OP_SIMD_SHL},
{SN_ShiftLeftLogicalWideningLower, OP_SIMD_SSHLL, None, OP_SIMD_USHLL},
{SN_ShiftLeftLogicalWideningUpper, OP_SIMD_SSHLL2, None, OP_SIMD_USHLL2},
{SN_ShiftLogical, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_USHL},
{SN_ShiftLogicalRounded, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URSHL},
{SN_ShiftLogicalRoundedSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL},
Expand All @@ -3184,9 +3184,9 @@ static SimdIntrinsic advsimd_methods [] = {
{SN_ShiftLogicalScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_USHL},
{SN_ShiftRightAndInsert, OP_ARM64_SRI},
{SN_ShiftRightAndInsertScalar, OP_ARM64_SRI},
{SN_ShiftRightArithmetic, OP_SSHR},
{SN_ShiftRightArithmeticAdd, OP_SSRA},
{SN_ShiftRightArithmeticAddScalar, OP_SSRA},
{SN_ShiftRightArithmetic, OP_SIMD_SSHR},
{SN_ShiftRightArithmeticAdd, OP_SIMD_SSRA},
{SN_ShiftRightArithmeticAddScalar, OP_SIMD_SSRA},
{SN_ShiftRightArithmeticNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRN},
{SN_ShiftRightArithmeticNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRN},
{SN_ShiftRightArithmeticNarrowingSaturateUnsignedLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRUN},
Expand All @@ -3203,10 +3203,10 @@ static SimdIntrinsic advsimd_methods [] = {
{SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN},
{SN_ShiftRightArithmeticRoundedNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRN},
{SN_ShiftRightArithmeticRoundedScalar, OP_ARM64_SRSHR},
{SN_ShiftRightArithmeticScalar, OP_SSHR},
{SN_ShiftRightLogical, OP_USHR},
{SN_ShiftRightLogicalAdd, OP_USRA},
{SN_ShiftRightLogicalAddScalar, OP_USRA},
{SN_ShiftRightArithmeticScalar, OP_SIMD_SSHR},
{SN_ShiftRightLogical, OP_SIMD_USHR},
{SN_ShiftRightLogicalAdd, OP_SIMD_USRA},
{SN_ShiftRightLogicalAddScalar, OP_SIMD_USRA},
{SN_ShiftRightLogicalNarrowingLower, OP_ARM64_SHRN},
{SN_ShiftRightLogicalNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_UQSHRN},
{SN_ShiftRightLogicalNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQSHRN},
Expand All @@ -3221,7 +3221,7 @@ static SimdIntrinsic advsimd_methods [] = {
{SN_ShiftRightLogicalRoundedNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_UQRSHRN},
{SN_ShiftRightLogicalRoundedNarrowingUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_RSHRN},
{SN_ShiftRightLogicalRoundedScalar, OP_ARM64_URSHR},
{SN_ShiftRightLogicalScalar, OP_USHR},
{SN_ShiftRightLogicalScalar, OP_SIMD_USHR},
{SN_SignExtendWideningLower, OP_ARM64_SXTL},
{SN_SignExtendWideningUpper, OP_ARM64_SXTL2},
{SN_Sqrt, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FSQRT},
Expand Down Expand Up @@ -4750,24 +4750,24 @@ static SimdIntrinsic wasmbase_methods [] = {

static SimdIntrinsic packedsimd_methods [] = {
{SN_Add},
{SN_And},
{SN_Bitmask},
{SN_And, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_AND},
{SN_Bitmask, OP_WASM_SIMD_BITMASK},
{SN_CompareEqual},
{SN_CompareNotEqual},
{SN_ConvertNarrowingSignedSaturate},
{SN_ConvertNarrowingUnsignedSaturate},
{SN_Dot},
{SN_Dot, OP_XOP_X_X_X, INTRINS_WASM_DOT},
{SN_ExtractLane},
{SN_Multiply},
{SN_Negate},
{SN_ReplaceLane},
{SN_ShiftLeft},
{SN_ShiftRightArithmetic},
{SN_ShiftRightLogical},
{SN_Shuffle},
{SN_ShiftLeft, OP_SIMD_SHL},
{SN_ShiftRightArithmetic, OP_SIMD_SSHR},
{SN_ShiftRightLogical, OP_SIMD_USHR},
{SN_Shuffle, OP_WASM_SIMD_SHUFFLE},
{SN_Splat},
{SN_Subtract},
{SN_Swizzle},
{SN_Swizzle, OP_WASM_SIMD_SWIZZLE},
{SN_get_IsSupported},
};

Expand Down Expand Up @@ -4839,54 +4839,59 @@ emit_wasm_supported_intrinsics (
id == SN_Splat && !MONO_TYPE_IS_VECTOR_PRIMITIVE(fsig->params [0]))
return NULL;

uint16_t op = info->default_op;
uint16_t c0 = info->default_instc0;

switch (id) {
case SN_Add:
case SN_Subtract:
case SN_Multiply:
return emit_simd_ins_for_binary_op (cfg, klass, fsig, args, arg0_type, id);
case SN_Negate:
return emit_simd_ins_for_unary_op (cfg, klass, fsig, args, arg0_type, id);
case SN_And:
return emit_simd_ins_for_sig (cfg, klass, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_AND, arg0_type, fsig, args);
case SN_Bitmask:
return emit_simd_ins_for_sig (cfg, klass, OP_WASM_SIMD_BITMASK, -1, -1, fsig, args);
case SN_CompareEqual:
return emit_simd_ins_for_sig (cfg, klass, type_enum_is_float (arg0_type) ? OP_XCOMPARE_FP : OP_XCOMPARE, CMP_EQ, arg0_type, fsig, args);
case SN_CompareNotEqual:
return emit_simd_ins_for_sig (cfg, klass, type_enum_is_float (arg0_type) ? OP_XCOMPARE_FP : OP_XCOMPARE, CMP_NE, arg0_type, fsig, args);
case SN_ConvertNarrowingSignedSaturate: {
int intrins = -1;
op = OP_XOP_X_X_X;

switch (arg0_type) {
case MONO_TYPE_I2:
intrins = INTRINS_WASM_NARROW_SIGNED_V16;
c0 = INTRINS_WASM_NARROW_SIGNED_V16;
break;
case MONO_TYPE_I4:
intrins = INTRINS_WASM_NARROW_SIGNED_V8;
c0 = INTRINS_WASM_NARROW_SIGNED_V8;
break;
}
if (intrins != -1)
return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X_X, intrins, arg0_type, fsig, args);

// continue with default emit
if (c0 != 0)
break;

return NULL;
}
case SN_ConvertNarrowingUnsignedSaturate: {
int intrins = -1;
op = OP_XOP_X_X_X;

switch (arg0_type) {
case MONO_TYPE_I2:
intrins = INTRINS_WASM_NARROW_UNSIGNED_V16;
c0 = INTRINS_WASM_NARROW_UNSIGNED_V16;
break;
case MONO_TYPE_I4:
intrins = INTRINS_WASM_NARROW_UNSIGNED_V8;
c0 = INTRINS_WASM_NARROW_UNSIGNED_V8;
break;
}
if (intrins != -1)
return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X_X, intrins, arg0_type, fsig, args);

// continue with default emit
if (c0 != 0)
break;

return NULL;
}
case SN_ExtractLane: {
int extract_op = type_to_xextract_op (arg0_type);
return emit_simd_ins_for_sig (cfg, klass, extract_op, -1, arg0_type, fsig, args);
op = type_to_xextract_op (arg0_type);
break;
}
case SN_ReplaceLane: {
int insert_op = type_to_xinsert_op (arg0_type);
Expand All @@ -4895,25 +4900,18 @@ emit_wasm_supported_intrinsics (
ins->inst_c1 = arg0_type;
return ins;
}
case SN_ShiftLeft:
return emit_simd_ins_for_sig (cfg, klass, OP_SHL, -1, -1, fsig, args);
case SN_ShiftRightArithmetic:
return emit_simd_ins_for_sig (cfg, klass, OP_SSHR, -1, -1, fsig, args);
case SN_ShiftRightLogical:
return emit_simd_ins_for_sig (cfg, klass, OP_USHR, -1, -1, fsig, args);
case SN_Splat: {
MonoType *etype = get_vector_t_elem_type (fsig->ret);
g_assert (fsig->param_count == 1 && mono_metadata_type_equal (fsig->params [0], etype));
return emit_simd_ins (cfg, klass, type_to_expand_op (etype->type), args [0]->dreg, -1);
}
case SN_Dot:
return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X_X, INTRINS_WASM_DOT, -1, fsig, args);
case SN_Shuffle:
return emit_simd_ins_for_sig (cfg, klass, OP_WASM_SIMD_SHUFFLE, -1, -1, fsig, args);
case SN_Swizzle:
return emit_simd_ins_for_sig (cfg, klass, OP_WASM_SIMD_SWIZZLE, -1, -1, fsig, args);
}

// default emit path for cases with op set
if (op != 0)
return emit_simd_ins_for_sig (cfg, klass, op, c0, arg0_type, fsig, args);
}

g_assert_not_reached ();

return NULL;
Expand Down
1 change: 1 addition & 0 deletions src/mono/wasm/build/WasmApp.Native.targets
View file
Original file line number Diff line number Diff line change
Expand Up @@ -583,6 +583,7 @@
<MonoAOTCompilerDefaultAotArguments Include="direct-icalls" />
<MonoAOTCompilerDefaultAotArguments Include="deterministic" />
<MonoAOTCompilerDefaultAotArguments Include="mattr=simd" Condition="'$(WasmEnableSIMD)' == 'true'" />
<MonoAOTCompilerDefaultProcessArguments Include="-v" Condition="'$(WasmAOTCompilerVerbose)' == 'true'" />
<MonoAOTCompilerDefaultProcessArguments Include="--wasm-exceptions" Condition="'$(WasmEnableExceptionHandling)' == 'true'" />
<MonoAOTCompilerDefaultProcessArguments Include="--wasm-gc-safepoints" Condition="'$(WasmEnableThreads)' == 'true' or '$(WasmEnablePerfTracing)' == 'true'" />
<AotProfilePath Include="$(WasmAotProfilePath)"/>
Expand Down

0 comments on commit 6114f19

Please sign in to comment.