diff --git a/src/coreclr/src/jit/codegenarm64.cpp b/src/coreclr/src/jit/codegenarm64.cpp index 77e12efd4117e..7e0c05a694cd6 100644 --- a/src/coreclr/src/jit/codegenarm64.cpp +++ b/src/coreclr/src/jit/codegenarm64.cpp @@ -4080,13 +4080,13 @@ void CodeGen::genSIMDIntrinsicInit(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicInit); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); var_types targetType = simdNode->TypeGet(); - genConsumeOperands(simdNode); + genConsumeRegs(op1); regNumber op1Reg = op1->IsIntegralConst(0) ? REG_ZR : op1->GetRegNum(); // TODO-ARM64-CQ Add LD1R to allow SIMDIntrinsicInit from contained memory @@ -4149,14 +4149,11 @@ void CodeGen::genSIMDIntrinsicInitN(GenTreeSIMD* simdNode) // and record the registers. regNumber operandRegs[FP_REGSIZE_BYTES]; unsigned initCount = 0; - for (GenTree* list = simdNode->gtGetOp1(); list != nullptr; list = list->gtGetOp2()) - { - assert(list->OperGet() == GT_LIST); - GenTree* listItem = list->gtGetOp1(); - assert(listItem->TypeGet() == baseType); - assert(!listItem->isContained()); - regNumber operandReg = genConsumeReg(listItem); - operandRegs[initCount] = operandReg; + for (GenTreeSIMD::Use& use : simdNode->Uses()) + { + assert(use.GetNode()->TypeGet() == baseType); + assert(!use.GetNode()->isContained()); + operandRegs[initCount] = genConsumeReg(use.GetNode()); initCount++; } @@ -4209,13 +4206,13 @@ void CodeGen::genSIMDIntrinsicUnOp(GenTreeSIMD* simdNode) simdNode->gtSIMDIntrinsicID == SIMDIntrinsicConvertToDouble || simdNode->gtSIMDIntrinsicID == SIMDIntrinsicConvertToInt64); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); var_types targetType = simdNode->TypeGet(); - genConsumeOperands(simdNode); + genConsumeRegs(op1); regNumber op1Reg = op1->GetRegNum(); assert(genIsValidFloatReg(op1Reg)); @@ -4244,13 +4241,13 @@ void CodeGen::genSIMDIntrinsicWiden(GenTreeSIMD* simdNode) assert((simdNode->gtSIMDIntrinsicID == SIMDIntrinsicWidenLo) || (simdNode->gtSIMDIntrinsicID == SIMDIntrinsicWidenHi)); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); var_types simdType = simdNode->TypeGet(); - genConsumeOperands(simdNode); + genConsumeRegs(op1); regNumber op1Reg = op1->GetRegNum(); regNumber srcReg = op1Reg; emitAttr emitSize = emitActualTypeSize(simdType); @@ -4286,15 +4283,16 @@ void CodeGen::genSIMDIntrinsicNarrow(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicNarrow); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); var_types simdType = simdNode->TypeGet(); emitAttr emitSize = emitTypeSize(simdType); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); @@ -4372,14 +4370,15 @@ void CodeGen::genSIMDIntrinsicBinOp(GenTreeSIMD* simdNode) simdNode->gtSIMDIntrinsicID == SIMDIntrinsicLessThanOrEqual || simdNode->gtSIMDIntrinsicID == SIMDIntrinsicGreaterThanOrEqual); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); var_types targetType = simdNode->TypeGet(); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); @@ -4413,13 +4412,14 @@ void CodeGen::genSIMDIntrinsicRelOp(GenTreeSIMD* simdNode) assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicOpEquality || simdNode->gtSIMDIntrinsicID == SIMDIntrinsicOpInEquality); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); var_types targetType = simdNode->TypeGet(); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); regNumber otherReg = op2Reg; @@ -4470,8 +4470,8 @@ void CodeGen::genSIMDIntrinsicDotProduct(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicDotProduct); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; var_types simdType = op1->TypeGet(); @@ -4481,7 +4481,8 @@ void CodeGen::genSIMDIntrinsicDotProduct(GenTreeSIMD* simdNode) var_types targetType = simdNode->TypeGet(); assert(targetType == baseType); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); regNumber tmpReg = targetReg; @@ -4554,8 +4555,8 @@ void CodeGen::genSIMDIntrinsicGetItem(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicGetItem); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types simdType = op1->TypeGet(); assert(varTypeIsSIMD(simdType)); @@ -4574,7 +4575,8 @@ void CodeGen::genSIMDIntrinsicGetItem(GenTreeSIMD* simdNode) // GetItem has 2 operands: // - the source of SIMD type (op1) // - the index of the value to be returned. - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); emitAttr baseTypeSize = emitTypeSize(baseType); unsigned baseTypeScale = genLog2(EA_SIZE_IN_BYTES(baseTypeSize)); @@ -4739,8 +4741,8 @@ void CodeGen::genSIMDIntrinsicSetItem(GenTreeSIMD* simdNode) // op1 is the SIMD vector // op2 is the value to be set - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); @@ -4751,7 +4753,8 @@ void CodeGen::genSIMDIntrinsicSetItem(GenTreeSIMD* simdNode) assert(op2->TypeGet() == baseType); assert(simdNode->gtSIMDSize >= ((index + 1) * genTypeSize(baseType))); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); @@ -4799,7 +4802,7 @@ void CodeGen::genSIMDIntrinsicUpperSave(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicUpperSave); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); assert(op1->IsLocal()); assert(emitTypeSize(op1->TypeGet()) == 16); regNumber targetReg = simdNode->GetRegNum(); @@ -4848,7 +4851,7 @@ void CodeGen::genSIMDIntrinsicUpperRestore(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicUpperRestore); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); assert(op1->IsLocal()); assert(emitTypeSize(op1->TypeGet()) == 16); regNumber srcReg = simdNode->GetRegNum(); diff --git a/src/coreclr/src/jit/compiler.h b/src/coreclr/src/jit/compiler.h index bfc7bba9dfe1c..b0e2b290af024 100644 --- a/src/coreclr/src/jit/compiler.h +++ b/src/coreclr/src/jit/compiler.h @@ -2553,9 +2553,15 @@ class Compiler #ifdef FEATURE_SIMD GenTreeSIMD* gtNewSIMDNode( - var_types type, GenTree* op1, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size); + var_types type, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size, GenTree* op1); GenTreeSIMD* gtNewSIMDNode( - var_types type, GenTree* op1, GenTree* op2, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size); + var_types type, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size, GenTree* op1, GenTree* op2); + GenTreeSIMD* gtNewSIMDNode(var_types type, + SIMDIntrinsicID simdIntrinsicID, + var_types baseType, + unsigned size, + unsigned numOps, + GenTree** ops); void SetOpLclRelatedToSIMDIntrinsic(GenTree* op); #endif @@ -10011,6 +10017,35 @@ class GenTreeVisitor } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + if (TVisitor::UseExecutionOrder && node->AsSIMD()->IsBinary() && node->IsReverseOp()) + { + result = WalkTree(&node->AsSIMD()->GetUse(1).NodeRef(), node); + if (result == fgWalkResult::WALK_ABORT) + { + return result; + } + result = WalkTree(&node->AsSIMD()->GetUse(0).NodeRef(), node); + if (result == fgWalkResult::WALK_ABORT) + { + return result; + } + } + else + { + for (GenTreeSIMD::Use& use : node->AsSIMD()->Uses()) + { + result = WalkTree(&use.NodeRef(), node); + if (result == fgWalkResult::WALK_ABORT) + { + return result; + } + } + } + break; +#endif + case GT_CMPXCHG: { GenTreeCmpXchg* const cmpXchg = node->AsCmpXchg(); diff --git a/src/coreclr/src/jit/compiler.hpp b/src/coreclr/src/jit/compiler.hpp index eb73160b9ae74..209a21abe4287 100644 --- a/src/coreclr/src/jit/compiler.hpp +++ b/src/coreclr/src/jit/compiler.hpp @@ -4280,20 +4280,6 @@ void GenTree::VisitOperands(TVisitor visitor) return; // Variadic nodes -#ifdef FEATURE_SIMD - case GT_SIMD: - if (this->AsSIMD()->gtSIMDIntrinsicID == SIMDIntrinsicInitN) - { - assert(this->AsSIMD()->gtOp1 != nullptr); - this->AsSIMD()->gtOp1->VisitListOperands(visitor); - } - else - { - VisitBinOpOperands(visitor); - } - return; -#endif // FEATURE_SIMD - #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: if ((this->AsHWIntrinsic()->gtOp1 != nullptr) && this->AsHWIntrinsic()->gtOp1->OperIsList()) @@ -4328,6 +4314,18 @@ void GenTree::VisitOperands(TVisitor visitor) } return; +#ifdef FEATURE_SIMD + case GT_SIMD: + for (GenTreeSIMD::Use& use : AsSIMD()->Uses()) + { + if (visitor(use.GetNode()) == VisitResult::Abort) + { + break; + } + } + return; +#endif // FEATURE_SIMD + case GT_CMPXCHG: { GenTreeCmpXchg* const cmpXchg = this->AsCmpXchg(); diff --git a/src/coreclr/src/jit/decomposelongs.cpp b/src/coreclr/src/jit/decomposelongs.cpp index 8178bd20194f3..8e48f2432b310 100644 --- a/src/coreclr/src/jit/decomposelongs.cpp +++ b/src/coreclr/src/jit/decomposelongs.cpp @@ -1701,17 +1701,17 @@ GenTree* DecomposeLongs::DecomposeSimdGetItem(LIR::Use& use) assert(simdTree->gtSIMDIntrinsicID == SIMDIntrinsicGetItem); assert(varTypeIsLong(baseType)); assert(varTypeIsLong(simdTree)); - assert(varTypeIsSIMD(simdTree->AsOp()->gtOp1->gtType)); - assert(simdTree->AsOp()->gtOp2->gtType == TYP_INT); + assert(varTypeIsSIMD(simdTree->GetOp(0)->TypeGet())); + assert(simdTree->GetOp(1)->TypeGet() == TYP_INT); - bool indexIsConst = simdTree->AsOp()->gtOp2->IsCnsIntOrI(); + bool indexIsConst = simdTree->GetOp(1)->IsCnsIntOrI(); ssize_t index = 0; if (indexIsConst) { - index = simdTree->AsOp()->gtOp2->AsIntCon()->gtIconVal; + index = simdTree->GetOp(1)->AsIntCon()->gtIconVal; } - GenTree* simdTmpVar = RepresentOpAsLocalVar(simdTree->AsOp()->gtOp1, simdTree, &simdTree->AsOp()->gtOp1); + GenTree* simdTmpVar = RepresentOpAsLocalVar(simdTree->GetOp(0), simdTree, &simdTree->GetUse(0).NodeRef()); unsigned simdTmpVarNum = simdTmpVar->AsLclVarCommon()->GetLclNum(); JITDUMP("[DecomposeSimdGetItem]: Saving op1 tree to a temp var:\n"); DISPTREERANGE(Range(), simdTmpVar); @@ -1721,7 +1721,7 @@ GenTree* DecomposeLongs::DecomposeSimdGetItem(LIR::Use& use) unsigned indexTmpVarNum = 0; if (!indexIsConst) { - indexTmpVar = RepresentOpAsLocalVar(simdTree->AsOp()->gtOp2, simdTree, &simdTree->AsOp()->gtOp2); + indexTmpVar = RepresentOpAsLocalVar(simdTree->GetOp(1), simdTree, &simdTree->GetUse(1).NodeRef()); indexTmpVarNum = indexTmpVar->AsLclVarCommon()->GetLclNum(); JITDUMP("[DecomposeSimdGetItem]: Saving op2 tree to a temp var:\n"); DISPTREERANGE(Range(), indexTmpVar); @@ -1737,7 +1737,7 @@ GenTree* DecomposeLongs::DecomposeSimdGetItem(LIR::Use& use) if (indexIsConst) { // Reuse the existing index constant node. - indexTimesTwo1 = simdTree->AsOp()->gtOp2; + indexTimesTwo1 = simdTree->GetOp(1); Range().Remove(indexTimesTwo1); indexTimesTwo1->AsIntCon()->gtIconVal = index * 2; @@ -1752,13 +1752,13 @@ GenTree* DecomposeLongs::DecomposeSimdGetItem(LIR::Use& use) } GenTree* loResult = - m_compiler->gtNewSIMDNode(TYP_INT, simdTmpVar1, indexTimesTwo1, SIMDIntrinsicGetItem, TYP_INT, simdSize); + m_compiler->gtNewSIMDNode(TYP_INT, SIMDIntrinsicGetItem, TYP_INT, simdSize, simdTmpVar1, indexTimesTwo1); Range().InsertBefore(simdTree, loResult); // Create: // hiResult = GT_SIMD{get_item}[int](tmp_simd_var, index * 2 + 1) - GenTree* simdTmpVar2 = m_compiler->gtNewLclLNode(simdTmpVarNum, simdTree->AsOp()->gtOp1->gtType); + GenTree* simdTmpVar2 = m_compiler->gtNewLclLNode(simdTmpVarNum, simdTree->GetOp(0)->TypeGet()); GenTree* indexTimesTwoPlusOne; if (indexIsConst) @@ -1778,7 +1778,7 @@ GenTree* DecomposeLongs::DecomposeSimdGetItem(LIR::Use& use) } GenTree* hiResult = - m_compiler->gtNewSIMDNode(TYP_INT, simdTmpVar2, indexTimesTwoPlusOne, SIMDIntrinsicGetItem, TYP_INT, simdSize); + m_compiler->gtNewSIMDNode(TYP_INT, SIMDIntrinsicGetItem, TYP_INT, simdSize, simdTmpVar2, indexTimesTwoPlusOne); Range().InsertBefore(simdTree, hiResult); // Done with the original tree; remove it. diff --git a/src/coreclr/src/jit/flowgraph.cpp b/src/coreclr/src/jit/flowgraph.cpp index 080e2d5cdf330..ab0845f702b8b 100644 --- a/src/coreclr/src/jit/flowgraph.cpp +++ b/src/coreclr/src/jit/flowgraph.cpp @@ -18777,6 +18777,23 @@ void Compiler::fgSetTreeSeqHelper(GenTree* tree, bool isLIR) } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + if (tree->AsSIMD()->IsBinary() && tree->IsReverseOp()) + { + fgSetTreeSeqHelper(tree->AsSIMD()->GetOp(1), isLIR); + fgSetTreeSeqHelper(tree->AsSIMD()->GetOp(0), isLIR); + } + else + { + for (GenTreeSIMD::Use& use : tree->AsSIMD()->Uses()) + { + fgSetTreeSeqHelper(use.GetNode(), isLIR); + } + } + break; +#endif + case GT_CMPXCHG: // Evaluate the trees left to right fgSetTreeSeqHelper(tree->AsCmpXchg()->gtOpLocation, isLIR); @@ -19144,6 +19161,12 @@ GenTree* Compiler::fgGetFirstNode(GenTree* tree) { child = child->GetChild(1); } +#ifdef FEATURE_SIMD + else if (child->OperIsSIMD() && child->AsSIMD()->IsBinary() && child->IsReverseOp()) + { + child = child->GetChild(1); + } +#endif else { child = child->GetChild(0); @@ -21368,6 +21391,16 @@ void Compiler::fgDebugCheckFlags(GenTree* tree) } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + for (GenTreeSIMD::Use& use : tree->AsSIMD()->Uses()) + { + fgDebugCheckFlags(use.GetNode()); + chkFlags |= (use.GetNode()->gtFlags & GTF_ALL_EFFECT); + } + break; +#endif + case GT_CMPXCHG: chkFlags |= (GTF_GLOB_REF | GTF_ASG); diff --git a/src/coreclr/src/jit/gentree.cpp b/src/coreclr/src/jit/gentree.cpp index 8c0e992491ce8..f7c773b259650 100644 --- a/src/coreclr/src/jit/gentree.cpp +++ b/src/coreclr/src/jit/gentree.cpp @@ -1434,16 +1434,6 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) return false; } break; -#ifdef FEATURE_SIMD - case GT_SIMD: - if ((op1->AsSIMD()->gtSIMDIntrinsicID != op2->AsSIMD()->gtSIMDIntrinsicID) || - (op1->AsSIMD()->gtSIMDBaseType != op2->AsSIMD()->gtSIMDBaseType) || - (op1->AsSIMD()->gtSIMDSize != op2->AsSIMD()->gtSIMDSize)) - { - return false; - } - break; -#endif // FEATURE_SIMD #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: @@ -1574,6 +1564,11 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) case GT_FIELD_LIST: return GenTreeFieldList::Equals(op1->AsFieldList(), op2->AsFieldList()); +#ifdef FEATURE_SIMD + case GT_SIMD: + return GenTreeSIMD::Equals(op1->AsSIMD(), op2->AsSIMD()); +#endif + case GT_CMPXCHG: return Compare(op1->AsCmpXchg()->gtOpLocation, op2->AsCmpXchg()->gtOpLocation) && Compare(op1->AsCmpXchg()->gtOpValue, op2->AsCmpXchg()->gtOpValue) && @@ -1810,6 +1805,18 @@ bool Compiler::gtHasRef(GenTree* tree, ssize_t lclNum, bool defOnly) } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + for (GenTreeSIMD::Use& use : tree->AsSIMD()->Uses()) + { + if (gtHasRef(use.GetNode(), lclNum, defOnly)) + { + return true; + } + } + break; +#endif // FEATURE_SIMD + case GT_CMPXCHG: if (gtHasRef(tree->AsCmpXchg()->gtOpLocation, lclNum, defOnly)) { @@ -2107,14 +2114,6 @@ unsigned Compiler::gtHashValue(GenTree* tree) case GT_INDEX_ADDR: break; -#ifdef FEATURE_SIMD - case GT_SIMD: - hash += tree->AsSIMD()->gtSIMDIntrinsicID; - hash += tree->AsSIMD()->gtSIMDBaseType; - hash += tree->AsSIMD()->gtSIMDSize; - break; -#endif // FEATURE_SIMD - #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: hash += tree->AsHWIntrinsic()->gtHWIntrinsicId; @@ -2233,6 +2232,18 @@ unsigned Compiler::gtHashValue(GenTree* tree) } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + hash += tree->AsSIMD()->gtSIMDIntrinsicID; + hash += tree->AsSIMD()->gtSIMDBaseType; + hash += tree->AsSIMD()->gtSIMDSize; + for (GenTreeSIMD::Use& use : tree->AsSIMD()->Uses()) + { + hash = genTreeHashAdd(hash, gtHashValue(use.GetNode())); + } + break; +#endif // FEATURE_SIMD + case GT_CMPXCHG: hash = genTreeHashAdd(hash, gtHashValue(tree->AsCmpXchg()->gtOpLocation)); hash = genTreeHashAdd(hash, gtHashValue(tree->AsCmpXchg()->gtOpValue)); @@ -4431,6 +4442,57 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + if (tree->AsSIMD()->IsBinary()) + { + GenTree* op1 = tree->AsSIMD()->GetOp(0); + GenTree* op2 = tree->AsSIMD()->GetOp(1); + + if (tree->IsReverseOp()) + { + std::swap(op1, op2); + } + + level = gtSetEvalOrder(op1); + lvl2 = gtSetEvalOrder(op2); + + if ((fgOrder == FGOrderTree) && (level < lvl2) && gtCanSwapOrder(op1, op2)) + { + tree->gtFlags ^= GTF_REVERSE_OPS; + std::swap(level, lvl2); + } + + if (level == 0) + { + level = lvl2; + } + else if (level == lvl2) + { + level += 1; + } + + costEx = op1->GetCostEx() + op2->GetCostEx() + 1; + costSz = op1->GetCostSz() + op2->GetCostSz() + 1; + } + else + { + level = 0; + costEx = tree->AsSIMD()->GetNumOps(); + costSz = tree->AsSIMD()->GetNumOps(); + + for (GenTreeSIMD::Use& use : tree->AsSIMD()->Uses()) + { + level = max(level, gtSetEvalOrder(use.GetNode())); + costEx += use.GetNode()->GetCostEx(); + costSz += use.GetNode()->GetCostSz(); + } + + level++; + } + break; +#endif // FEATURE_SIMD + case GT_CMPXCHG: level = gtSetEvalOrder(tree->AsCmpXchg()->gtOpLocation); @@ -4745,6 +4807,18 @@ GenTree** GenTree::gtGetChildPointer(GenTree* parent) const } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + for (GenTreeSIMD::Use& use : parent->AsSIMD()->Uses()) + { + if (this == use.GetNode()) + { + return &use.NodeRef(); + } + } + break; +#endif // FEATURE_SIMD + case GT_CMPXCHG: if (this == parent->AsCmpXchg()->gtOpLocation) { @@ -4977,17 +5051,6 @@ bool GenTree::TryGetUse(GenTree* def, GenTree*** use) return false; #endif // FEATURE_ARG_SPLIT -#ifdef FEATURE_SIMD - case GT_SIMD: - if (this->AsSIMD()->gtSIMDIntrinsicID == SIMDIntrinsicInitN) - { - assert(this->AsSIMD()->gtOp1 != nullptr); - return this->AsSIMD()->gtOp1->TryGetUseList(def, use); - } - - return TryGetUseBinOp(def, use); -#endif // FEATURE_SIMD - #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: if ((this->AsHWIntrinsic()->gtOp1 != nullptr) && this->AsHWIntrinsic()->gtOp1->OperIsList()) @@ -5021,6 +5084,19 @@ bool GenTree::TryGetUse(GenTree* def, GenTree*** use) } return false; +#ifdef FEATURE_SIMD + case GT_SIMD: + for (GenTreeSIMD::Use& simdUse : AsSIMD()->Uses()) + { + if (simdUse.GetNode() == def) + { + *use = &simdUse.NodeRef(); + return true; + } + } + return false; +#endif // FEATURE_SIMD + case GT_CMPXCHG: { GenTreeCmpXchg* const cmpXchg = this->AsCmpXchg(); @@ -5981,7 +6057,7 @@ GenTree* Compiler::gtNewSIMDVectorZero(var_types simdType, var_types baseType, u baseType = genActualType(baseType); GenTree* initVal = gtNewZeroConNode(baseType); initVal->gtType = baseType; - return gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, baseType, size); + return gtNewSIMDNode(simdType, SIMDIntrinsicInit, baseType, size, initVal); } //--------------------------------------------------------------------- @@ -6015,7 +6091,7 @@ GenTree* Compiler::gtNewSIMDVectorOne(var_types simdType, var_types baseType, un baseType = genActualType(baseType); initVal->gtType = baseType; - return gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, baseType, size); + return gtNewSIMDNode(simdType, SIMDIntrinsicInit, baseType, size, initVal); } #endif // FEATURE_SIMD @@ -7366,16 +7442,6 @@ GenTree* Compiler::gtCloneExpr( } break; -#ifdef FEATURE_SIMD - case GT_SIMD: - { - GenTreeSIMD* simdOp = tree->AsSIMD(); - copy = gtNewSIMDNode(simdOp->TypeGet(), simdOp->gtGetOp1(), simdOp->gtGetOp2IfPresent(), - simdOp->gtSIMDIntrinsicID, simdOp->gtSIMDBaseType, simdOp->gtSIMDSize); - } - break; -#endif - #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: { @@ -7547,6 +7613,29 @@ GenTree* Compiler::gtCloneExpr( } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + { + GenTreeSIMD* simdOp = tree->AsSIMD(); + GenTreeSIMD::Use* uses = nullptr; + + if (simdOp->GetNumOps() > 3) + { + uses = getAllocator(CMK_ASTNode).allocate(simdOp->GetNumOps()); + } + + copy = new (this, GT_SIMD) GenTreeSIMD(simdOp->TypeGet(), simdOp->gtSIMDIntrinsicID, simdOp->gtSIMDBaseType, + simdOp->gtSIMDSize, simdOp->GetNumOps(), uses); + + for (unsigned i = 0; i < simdOp->GetNumOps(); i++) + { + copy->AsSIMD()->SetOp(i, gtCloneExpr(simdOp->GetOp(i), addFlags, deepVarNum, deepVarVal)); + copy->gtFlags |= (copy->AsSIMD()->GetOp(i)->gtFlags & GTF_ALL_EFFECT); + } + } + break; +#endif // FEATURE_SIMD + case GT_CMPXCHG: copy = new (this, GT_CMPXCHG) GenTreeCmpXchg(tree->TypeGet(), @@ -8341,6 +8430,11 @@ unsigned GenTree::NumChildren() return count; } +#ifdef FEATURE_SIMD + case GT_SIMD: + return AsSIMD()->GetNumOps(); +#endif // FEATURE_SIMD + case GT_CMPXCHG: return 3; @@ -8476,6 +8570,12 @@ GenTree* GenTree::GetChild(unsigned childNum) } unreached(); +#ifdef FEATURE_SIMD + case GT_SIMD: + noway_assert(childNum < AsSIMD()->GetNumOps()); + return AsSIMD()->GetOp(childNum); +#endif // FEATURE_SIMD + case GT_CMPXCHG: switch (childNum) { @@ -8731,19 +8831,6 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) return; // Variadic nodes -#ifdef FEATURE_SIMD - case GT_SIMD: - if (m_node->AsSIMD()->gtSIMDIntrinsicID == SIMDIntrinsicInitN) - { - SetEntryStateForList(m_node->AsSIMD()->gtOp1->AsArgList()); - } - else - { - SetEntryStateForBinOp(); - } - return; -#endif // FEATURE_SIMD - #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: if (m_node->AsHWIntrinsic()->gtOp1 == nullptr) @@ -8782,6 +8869,22 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) AdvanceFieldList(); return; +#ifdef FEATURE_SIMD + case GT_SIMD: + if (m_node->AsSIMD()->IsBinary() && m_node->IsReverseOp()) + { + m_edge = &m_node->AsSIMD()->GetUse(1).NodeRef(); + m_advance = &GenTreeUseEdgeIterator::AdvanceSIMDReverseOp; + } + else + { + m_statePtr = m_node->AsSIMD()->Uses().begin(); + m_advance = &GenTreeUseEdgeIterator::AdvanceSIMD; + AdvanceSIMD(); + } + return; +#endif // FEATURE_SIMD + case GT_PHI: m_statePtr = m_node->AsPhi()->gtUses; m_advance = &GenTreeUseEdgeIterator::AdvancePhi; @@ -9017,6 +9120,36 @@ void GenTreeUseEdgeIterator::AdvanceFieldList() } } +#ifdef FEATURE_SIMD +//------------------------------------------------------------------------ +// GenTreeUseEdgeIterator::AdvanceSIMD: produces the next operand of a SIMD node and advances the state. +// +void GenTreeUseEdgeIterator::AdvanceSIMD() +{ + assert(m_state == 0); + + if (m_statePtr == m_node->AsSIMD()->Uses().end()) + { + m_state = -1; + } + else + { + GenTreeSIMD::Use* currentUse = static_cast(m_statePtr); + m_edge = ¤tUse->NodeRef(); + m_statePtr = currentUse + 1; + } +} + +void GenTreeUseEdgeIterator::AdvanceSIMDReverseOp() +{ + assert(m_state == 0); + assert(m_edge == &m_node->AsSIMD()->GetUse(1).NodeRef()); + + m_edge = &m_node->AsSIMD()->GetUse(0).NodeRef(); + m_advance = &GenTreeUseEdgeIterator::Terminate; +} +#endif // FEATURE_SIMD + //------------------------------------------------------------------------ // GenTreeUseEdgeIterator::AdvancePhi: produces the next operand of a Phi node and advances the state. // @@ -11077,14 +11210,6 @@ void Compiler::gtDispTree(GenTree* tree, } } -#ifdef FEATURE_SIMD - if (tree->gtOper == GT_SIMD) - { - printf(" %s %s", varTypeName(tree->AsSIMD()->gtSIMDBaseType), - simdIntrinsicNames[tree->AsSIMD()->gtSIMDIntrinsicID]); - } -#endif // FEATURE_SIMD - #ifdef FEATURE_HW_INTRINSICS if (tree->gtOper == GT_HWINTRINSIC) { @@ -11175,6 +11300,24 @@ void Compiler::gtDispTree(GenTree* tree, } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + printf(" %s %s", varTypeName(tree->AsSIMD()->gtSIMDBaseType), + simdIntrinsicNames[tree->AsSIMD()->gtSIMDIntrinsicID]); + + gtDispCommonEndLine(tree); + + if (!topOnly) + { + for (unsigned i = 0; i < tree->AsSIMD()->GetNumOps(); i++) + { + gtDispChild(tree->AsSIMD()->GetOp(i), indentStack, + (i == tree->AsSIMD()->GetNumOps() - 1) ? IIArcBottom : IIArc); + } + } + break; +#endif // FEATURE_SIMD + case GT_PHI: gtDispCommonEndLine(tree); @@ -18016,24 +18159,43 @@ bool FieldSeqNode::IsPseudoField() const #ifdef FEATURE_SIMD GenTreeSIMD* Compiler::gtNewSIMDNode( - var_types type, GenTree* op1, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size) + var_types type, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size, GenTree* op1) { - assert(op1 != nullptr); SetOpLclRelatedToSIMDIntrinsic(op1); - - GenTreeSIMD* simdNode = new (this, GT_SIMD) GenTreeSIMD(type, op1, simdIntrinsicID, baseType, size); - return simdNode; + GenTreeSIMD* node = new (this, GT_SIMD) GenTreeSIMD(type, simdIntrinsicID, baseType, size, 1); + node->SetOp(0, op1); + node->gtFlags |= op1->gtFlags & GTF_ALL_EFFECT; + return node; } GenTreeSIMD* Compiler::gtNewSIMDNode( - var_types type, GenTree* op1, GenTree* op2, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size) + var_types type, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size, GenTree* op1, GenTree* op2) { - assert(op1 != nullptr); SetOpLclRelatedToSIMDIntrinsic(op1); SetOpLclRelatedToSIMDIntrinsic(op2); + GenTreeSIMD* node = new (this, GT_SIMD) GenTreeSIMD(type, simdIntrinsicID, baseType, size, 2); + node->SetOp(0, op1); + node->SetOp(1, op2); + node->gtFlags |= (op1->gtFlags | op2->gtFlags) & GTF_ALL_EFFECT; + return node; +} - GenTreeSIMD* simdNode = new (this, GT_SIMD) GenTreeSIMD(type, op1, op2, simdIntrinsicID, baseType, size); - return simdNode; +GenTreeSIMD* Compiler::gtNewSIMDNode( + var_types type, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size, unsigned numOps, GenTree** ops) +{ + GenTreeSIMD::Use* uses = nullptr; + if (numOps > 3) + { + uses = getAllocator(CMK_ASTNode).allocate(numOps); + } + GenTreeSIMD* node = new (this, GT_SIMD) GenTreeSIMD(type, simdIntrinsicID, baseType, size, numOps, uses); + for (unsigned i = 0; i < numOps; i++) + { + SetOpLclRelatedToSIMDIntrinsic(ops[i]); + node->SetOp(i, ops[i]); + node->gtFlags |= ops[i]->gtFlags & GTF_ALL_EFFECT; + } + return node; } //------------------------------------------------------------------- @@ -18061,7 +18223,6 @@ void Compiler::SetOpLclRelatedToSIMDIntrinsic(GenTree* op) bool GenTree::isCommutativeSIMDIntrinsic() { - assert(gtOper == GT_SIMD); switch (AsSIMD()->gtSIMDIntrinsicID) { case SIMDIntrinsicAdd: diff --git a/src/coreclr/src/jit/gentree.h b/src/coreclr/src/jit/gentree.h index 47c8a38f21aa6..4e8665d7d99f5 100644 --- a/src/coreclr/src/jit/gentree.h +++ b/src/coreclr/src/jit/gentree.h @@ -1603,10 +1603,6 @@ struct GenTree case GT_LIST: case GT_INTRINSIC: case GT_LEA: -#ifdef FEATURE_SIMD - case GT_SIMD: -#endif // !FEATURE_SIMD - #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: #endif // FEATURE_HW_INTRINSICS @@ -2619,6 +2615,10 @@ class GenTreeUseEdgeIterator final void AdvanceStoreDynBlk(); void AdvanceFieldList(); void AdvancePhi(); +#ifdef FEATURE_SIMD + void AdvanceSIMD(); + void AdvanceSIMDReverseOp(); +#endif template void AdvanceBinOp(); @@ -4492,48 +4492,184 @@ struct GenTreeIntrinsic : public GenTreeOp #endif }; -struct GenTreeJitIntrinsic : public GenTreeOp +class GenTreeUse { - var_types gtSIMDBaseType; // SIMD vector base type - unsigned gtSIMDSize; // SIMD vector size in bytes, use 0 for scalar intrinsics + GenTree* m_node; - GenTreeJitIntrinsic(genTreeOps oper, var_types type, GenTree* op1, GenTree* op2, var_types baseType, unsigned size) - : GenTreeOp(oper, type, op1, op2), gtSIMDBaseType(baseType), gtSIMDSize(size) +public: + GenTreeUse(GenTree* node) : m_node(node) { + assert(node != nullptr); } - bool isSIMD() const + GenTreeUse(const GenTreeUse&) = delete; + GenTreeUse& operator=(const GenTreeUse&) = delete; + + GenTree*& NodeRef() { - return gtSIMDSize != 0; + return m_node; } -#if DEBUGGABLE_GENTREE - GenTreeJitIntrinsic() : GenTreeOp() + GenTree* GetNode() const { + assert(m_node != nullptr); + return m_node; + } + + void SetNode(GenTree* node) + { + assert(node != nullptr); + m_node = node; } -#endif }; #ifdef FEATURE_SIMD -/* gtSIMD -- SIMD intrinsic (possibly-binary op [NULL op2 is allowed] with additional fields) */ -struct GenTreeSIMD : public GenTreeJitIntrinsic +struct GenTreeSIMD : public GenTree { + using Use = GenTreeUse; + SIMDIntrinsicID gtSIMDIntrinsicID; // operation Id + var_types gtSIMDBaseType; // SIMD vector base type + uint16_t gtSIMDSize; // SIMD vector size in bytes, use 0 for scalar intrinsics + +private: + uint16_t m_numOps; + union { + Use m_inlineUses[3]; + Use* m_uses; + }; + +public: + GenTreeSIMD(var_types type, + SIMDIntrinsicID simdIntrinsicID, + var_types baseType, + unsigned size, + unsigned numOps, + Use* uses = nullptr) + : GenTree(GT_SIMD, type) + , gtSIMDIntrinsicID(simdIntrinsicID) + , gtSIMDBaseType(baseType) + , gtSIMDSize(static_cast(size)) + , m_numOps(static_cast(numOps)) + { + assert(size < UINT16_MAX); + assert(numOps < UINT16_MAX); + + if (HasInlineUses()) + { + assert(uses == nullptr); + } + else + { + assert(uses != nullptr); + m_uses = uses; + } + } + + unsigned GetNumOps() const + { + return m_numOps; + } + + bool IsUnary() const + { + return m_numOps == 1; + } + + bool IsBinary() const + { + return m_numOps == 2; + } + + bool IsTernary() const + { + return m_numOps == 3; + } + + GenTree* GetOp(unsigned index) const + { + return GetUse(index).GetNode(); + } + + GenTree* GetLastOp() const + { + return m_numOps == 0 ? nullptr : GetOp(m_numOps - 1); + } + + void SetOp(unsigned index, GenTree* node) + { + assert(node != nullptr); + GetUse(index).SetNode(node); + } + + const Use& GetUse(unsigned index) const + { + assert(index < m_numOps); + return GetUses()[index]; + } + + Use& GetUse(unsigned index) + { + assert(index < m_numOps); + return GetUses()[index]; + } + + IteratorPair Uses() + { + Use* uses = GetUses(); + return MakeIteratorPair(uses, uses + GetNumOps()); + } + + static bool Equals(GenTreeSIMD* simd1, GenTreeSIMD* simd2) + { + if ((simd1->TypeGet() != simd2->TypeGet()) || (simd1->gtSIMDIntrinsicID != simd2->gtSIMDIntrinsicID) || + (simd1->gtSIMDBaseType != simd2->gtSIMDBaseType) || (simd1->gtSIMDSize != simd2->gtSIMDSize) || + (simd1->m_numOps != simd2->m_numOps)) + { + return false; + } + + for (unsigned i = 0; i < simd1->m_numOps; i++) + { + if (!Compare(simd1->GetOp(i), simd2->GetOp(i))) + { + return false; + } + } + + return true; + } + + // Delete some functions inherited from GenTree to avoid accidental use, at least + // when the node object is accessed via GenTreeSIMD* rather than GenTree*. + GenTree* gtGetOp1() const = delete; + GenTree* gtGetOp2() const = delete; + GenTree* gtGetOp2IfPresent() const = delete; + GenTreeUnOp* AsUnOp() = delete; + const GenTreeUnOp* AsUnOp() const = delete; + GenTreeOp* AsOp() = delete; + const GenTreeOp* AsOp() const = delete; + +private: + bool HasInlineUses() const + { + return m_numOps <= _countof(m_inlineUses); + } - GenTreeSIMD(var_types type, GenTree* op1, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size) - : GenTreeJitIntrinsic(GT_SIMD, type, op1, nullptr, baseType, size), gtSIMDIntrinsicID(simdIntrinsicID) + Use* GetUses() { + return HasInlineUses() ? m_inlineUses : m_uses; } - GenTreeSIMD( - var_types type, GenTree* op1, GenTree* op2, SIMDIntrinsicID simdIntrinsicID, var_types baseType, unsigned size) - : GenTreeJitIntrinsic(GT_SIMD, type, op1, op2, baseType, size), gtSIMDIntrinsicID(simdIntrinsicID) + const Use* GetUses() const { + return HasInlineUses() ? m_inlineUses : m_uses; } #if DEBUGGABLE_GENTREE - GenTreeSIMD() : GenTreeJitIntrinsic() +public: + GenTreeSIMD() : GenTree() { } #endif @@ -4541,23 +4677,37 @@ struct GenTreeSIMD : public GenTreeJitIntrinsic #endif // FEATURE_SIMD #ifdef FEATURE_HW_INTRINSICS -struct GenTreeHWIntrinsic : public GenTreeJitIntrinsic + +// Note that HW Instrinsic instructions are a sub class of GenTreeOp which only supports two operands +// However there are HW Instrinsic instructions that have 3 or even 4 operands and this is +// supported using a single op1 and using an ArgList for it: gtNewArgList(op1, op2, op3) + +struct GenTreeHWIntrinsic : public GenTreeOp { NamedIntrinsic gtHWIntrinsicId; + var_types gtSIMDBaseType; // SIMD vector base type var_types gtIndexBaseType; // for AVX2 Gather* intrinsics + uint16_t gtSIMDSize; // SIMD vector size in bytes, use 0 for scalar intrinsics GenTreeHWIntrinsic(var_types type, NamedIntrinsic hwIntrinsicID, var_types baseType, unsigned size) - : GenTreeJitIntrinsic(GT_HWINTRINSIC, type, nullptr, nullptr, baseType, size) + : GenTreeOp(GT_HWINTRINSIC, type, nullptr, nullptr) , gtHWIntrinsicId(hwIntrinsicID) + , gtSIMDBaseType(baseType) , gtIndexBaseType(TYP_UNKNOWN) + , gtSIMDSize(static_cast(size)) { + assert(size < UINT16_MAX); } GenTreeHWIntrinsic(var_types type, GenTree* op1, NamedIntrinsic hwIntrinsicID, var_types baseType, unsigned size) - : GenTreeJitIntrinsic(GT_HWINTRINSIC, type, op1, nullptr, baseType, size) + : GenTreeOp(GT_HWINTRINSIC, type, op1, nullptr) , gtHWIntrinsicId(hwIntrinsicID) + , gtSIMDBaseType(baseType) , gtIndexBaseType(TYP_UNKNOWN) + , gtSIMDSize(static_cast(size)) { + assert(size < UINT16_MAX); + if (OperIsMemoryStore()) { gtFlags |= (GTF_GLOB_REF | GTF_ASG); @@ -4566,19 +4716,24 @@ struct GenTreeHWIntrinsic : public GenTreeJitIntrinsic GenTreeHWIntrinsic( var_types type, GenTree* op1, GenTree* op2, NamedIntrinsic hwIntrinsicID, var_types baseType, unsigned size) - : GenTreeJitIntrinsic(GT_HWINTRINSIC, type, op1, op2, baseType, size) + : GenTreeOp(GT_HWINTRINSIC, type, op1, op2) , gtHWIntrinsicId(hwIntrinsicID) + , gtSIMDBaseType(baseType) , gtIndexBaseType(TYP_UNKNOWN) + , gtSIMDSize(static_cast(size)) { + assert(size < UINT16_MAX); + if (OperIsMemoryStore()) { gtFlags |= (GTF_GLOB_REF | GTF_ASG); } } - // Note that HW Instrinsic instructions are a sub class of GenTreeOp which only supports two operands - // However there are HW Instrinsic instructions that have 3 or even 4 operands and this is - // supported using a single op1 and using an ArgList for it: gtNewArgList(op1, op2, op3) + bool isSIMD() const + { + return gtSIMDSize != 0; + } bool OperIsMemoryLoad(); // Returns true for the HW Instrinsic instructions that have MemoryLoad semantics, // false otherwise @@ -4588,7 +4743,7 @@ struct GenTreeHWIntrinsic : public GenTreeJitIntrinsic // MemoryStore semantics, false otherwise #if DEBUGGABLE_GENTREE - GenTreeHWIntrinsic() : GenTreeJitIntrinsic() + GenTreeHWIntrinsic() : GenTreeOp() { } #endif @@ -6563,11 +6718,11 @@ inline bool GenTree::IsIntegralConstVector(ssize_t constVal) #ifdef FEATURE_SIMD // SIMDIntrinsicInit intrinsic with a const value as initializer // represents a const vector. - if ((gtOper == GT_SIMD) && (AsSIMD()->gtSIMDIntrinsicID == SIMDIntrinsicInit) && - gtGetOp1()->IsIntegralConst(constVal)) + if (OperIs(GT_SIMD) && (AsSIMD()->gtSIMDIntrinsicID == SIMDIntrinsicInit) && + AsSIMD()->GetOp(0)->IsIntegralConst(constVal)) { assert(varTypeIsIntegral(AsSIMD()->gtSIMDBaseType)); - assert(gtGetOp2IfPresent() == nullptr); + assert(AsSIMD()->IsUnary()); return true; } #endif diff --git a/src/coreclr/src/jit/gtlist.h b/src/coreclr/src/jit/gtlist.h index 31a301651ae5d..b601b8971d80d 100644 --- a/src/coreclr/src/jit/gtlist.h +++ b/src/coreclr/src/jit/gtlist.h @@ -205,7 +205,7 @@ GTNODE(RSH_LO , GenTreeOp ,0,GTK_BINOP) #endif // !defined(_TARGET_64BIT_) #ifdef FEATURE_SIMD -GTNODE(SIMD , GenTreeSIMD ,0,(GTK_BINOP|GTK_EXOP)) // SIMD functions/operators/intrinsics +GTNODE(SIMD , GenTreeSIMD ,0,GTK_SPECIAL) // SIMD functions/operators/intrinsics #endif // FEATURE_SIMD #ifdef FEATURE_HW_INTRINSICS diff --git a/src/coreclr/src/jit/lowerarmarch.cpp b/src/coreclr/src/jit/lowerarmarch.cpp index 639db6114ec7b..c6ed1255e7947 100644 --- a/src/coreclr/src/jit/lowerarmarch.cpp +++ b/src/coreclr/src/jit/lowerarmarch.cpp @@ -797,7 +797,7 @@ void Lowering::ContainCheckSIMD(GenTreeSIMD* simdNode) GenTree* op2; case SIMDIntrinsicInit: - op1 = simdNode->AsOp()->gtOp1; + op1 = simdNode->GetOp(0); if (op1->IsIntegralConst(0)) { MakeSrcContained(simdNode, op1); @@ -806,7 +806,7 @@ void Lowering::ContainCheckSIMD(GenTreeSIMD* simdNode) case SIMDIntrinsicInitArray: // We have an array and an index, which may be contained. - CheckImmedAndMakeContained(simdNode, simdNode->gtGetOp2()); + CheckImmedAndMakeContained(simdNode, simdNode->GetOp(1)); break; case SIMDIntrinsicOpEquality: @@ -820,8 +820,8 @@ void Lowering::ContainCheckSIMD(GenTreeSIMD* simdNode) // - the source SIMD struct // - index (which element to get) // The result is baseType of SIMD struct. - op1 = simdNode->AsOp()->gtOp1; - op2 = simdNode->AsOp()->gtOp2; + op1 = simdNode->GetOp(0); + op2 = simdNode->GetOp(1); // If the index is a constant, mark it as contained. if (op2->IsCnsIntOrI()) diff --git a/src/coreclr/src/jit/lowerxarch.cpp b/src/coreclr/src/jit/lowerxarch.cpp index 0e471faaa4291..84305656e2b1f 100644 --- a/src/coreclr/src/jit/lowerxarch.cpp +++ b/src/coreclr/src/jit/lowerxarch.cpp @@ -689,16 +689,14 @@ void Lowering::LowerSIMD(GenTreeSIMD* simdNode) int constArgCount = 0; float constArgValues[4]{0, 0, 0, 0}; - for (GenTreeArgList* list = simdNode->gtGetOp1()->AsArgList(); list != nullptr; list = list->Rest()) + for (GenTreeSIMD::Use& use : simdNode->Uses()) { - GenTree* arg = list->Current(); - - assert(arg->TypeGet() == simdNode->gtSIMDBaseType); + assert(use.GetNode()->TypeGet() == simdNode->gtSIMDBaseType); assert(argCount < _countof(constArgValues)); - if (arg->IsCnsFltOrDbl()) + if (use.GetNode()->IsCnsFltOrDbl()) { - constArgValues[constArgCount] = static_cast(arg->AsDblCon()->gtDconVal); + constArgValues[constArgCount] = static_cast(use.GetNode()->AsDblCon()->gtDconVal); constArgCount++; } @@ -707,9 +705,9 @@ void Lowering::LowerSIMD(GenTreeSIMD* simdNode) if (constArgCount == argCount) { - for (GenTreeArgList* list = simdNode->gtGetOp1()->AsArgList(); list != nullptr; list = list->Rest()) + for (GenTreeSIMD::Use& use : simdNode->Uses()) { - BlockRange().Remove(list->Current()); + BlockRange().Remove(use.GetNode()); } CORINFO_FIELD_HANDLE hnd = @@ -717,7 +715,7 @@ void Lowering::LowerSIMD(GenTreeSIMD* simdNode) GenTree* clsVarAddr = new (comp, GT_CLS_VAR_ADDR) GenTreeClsVar(GT_CLS_VAR_ADDR, TYP_I_IMPL, hnd, nullptr); BlockRange().InsertBefore(simdNode, clsVarAddr); simdNode->ChangeOper(GT_IND); - simdNode->gtOp1 = clsVarAddr; + simdNode->AsIndir()->Addr() = clsVarAddr; ContainCheckIndir(simdNode->AsIndir()); return; @@ -725,7 +723,7 @@ void Lowering::LowerSIMD(GenTreeSIMD* simdNode) } #ifdef _TARGET_XARCH_ - if ((simdNode->gtSIMDIntrinsicID == SIMDIntrinsicGetItem) && (simdNode->gtGetOp1()->OperGet() == GT_IND)) + if ((simdNode->gtSIMDIntrinsicID == SIMDIntrinsicGetItem) && (simdNode->GetOp(0)->OperGet() == GT_IND)) { // If SIMD vector is already in memory, we force its // addr to be evaluated into a reg. This would allow @@ -741,7 +739,7 @@ void Lowering::LowerSIMD(GenTreeSIMD* simdNode) // Ideally, we should be able to lower GetItem intrinsic // into GT_IND(newAddr) where newAddr combines // the addr of SIMD vector with the given index. - simdNode->gtOp1->gtFlags |= GTF_IND_REQ_ADDR_IN_REG; + simdNode->GetOp(0)->gtFlags |= GTF_IND_REQ_ADDR_IN_REG; } else if (simdNode->IsSIMDEqualityOrInequality()) { @@ -2440,7 +2438,7 @@ void Lowering::ContainCheckSIMD(GenTreeSIMD* simdNode) case SIMDIntrinsicInit: { - op1 = simdNode->AsOp()->gtOp1; + op1 = simdNode->GetOp(0); #ifndef _TARGET_64BIT_ if (op1->OperGet() == GT_LONG) { @@ -2476,7 +2474,7 @@ void Lowering::ContainCheckSIMD(GenTreeSIMD* simdNode) case SIMDIntrinsicInitArray: // We have an array and an index, which may be contained. - CheckImmedAndMakeContained(simdNode, simdNode->gtGetOp2()); + CheckImmedAndMakeContained(simdNode, simdNode->GetOp(1)); break; case SIMDIntrinsicOpEquality: @@ -2485,7 +2483,7 @@ void Lowering::ContainCheckSIMD(GenTreeSIMD* simdNode) // against zero using ptest. We can safely do this optimization // for integral vectors but not for floating-point for the reason // that we have +0.0 and -0.0 and +0.0 == -0.0 - op2 = simdNode->gtGetOp2(); + op2 = simdNode->GetOp(1); if ((comp->getSIMDSupportLevel() >= SIMD_SSE4_Supported) && op2->IsIntegralConstVector(0)) { MakeSrcContained(simdNode, op2); @@ -2498,8 +2496,8 @@ void Lowering::ContainCheckSIMD(GenTreeSIMD* simdNode) // - the source SIMD struct // - index (which element to get) // The result is baseType of SIMD struct. - op1 = simdNode->AsOp()->gtOp1; - op2 = simdNode->AsOp()->gtOp2; + op1 = simdNode->GetOp(0); + op2 = simdNode->GetOp(1); if (op1->OperGet() == GT_IND) { @@ -2522,8 +2520,8 @@ void Lowering::ContainCheckSIMD(GenTreeSIMD* simdNode) case SIMDIntrinsicShuffleSSE2: // Second operand is an integer constant and marked as contained. - assert(simdNode->AsOp()->gtOp2->IsCnsIntOrI()); - MakeSrcContained(simdNode, simdNode->AsOp()->gtOp2); + assert(simdNode->GetOp(1)->IsCnsIntOrI()); + MakeSrcContained(simdNode, simdNode->GetOp(1)); break; default: diff --git a/src/coreclr/src/jit/lsra.cpp b/src/coreclr/src/jit/lsra.cpp index 74755293e6029..c971129ef508b 100644 --- a/src/coreclr/src/jit/lsra.cpp +++ b/src/coreclr/src/jit/lsra.cpp @@ -6553,9 +6553,8 @@ void LinearScan::insertUpperVectorSave(GenTree* tree, saveLcl->SetRegNum(lclVarReg); SetLsraAdded(saveLcl); - GenTreeSIMD* simdNode = - new (compiler, GT_SIMD) GenTreeSIMD(LargeVectorSaveType, saveLcl, nullptr, SIMDIntrinsicUpperSave, - varDsc->lvBaseType, genTypeSize(varDsc->lvType)); + GenTreeSIMD* simdNode = compiler->gtNewSIMDNode(LargeVectorSaveType, SIMDIntrinsicUpperSave, varDsc->lvBaseType, + genTypeSize(varDsc->lvType), saveLcl); SetLsraAdded(simdNode); simdNode->SetRegNum(spillReg); if (spillToMem) @@ -6609,9 +6608,8 @@ void LinearScan::insertUpperVectorRestore(GenTree* tree, restoreLcl->SetRegNum(lclVarReg); SetLsraAdded(restoreLcl); - GenTreeSIMD* simdNode = - new (compiler, GT_SIMD) GenTreeSIMD(varDsc->lvType, restoreLcl, nullptr, SIMDIntrinsicUpperRestore, - varDsc->lvBaseType, genTypeSize(varDsc->lvType)); + GenTreeSIMD* simdNode = compiler->gtNewSIMDNode(varDsc->lvType, SIMDIntrinsicUpperRestore, varDsc->lvBaseType, + genTypeSize(varDsc->lvType), restoreLcl); regNumber restoreReg = upperVectorInterval->physReg; SetLsraAdded(simdNode); diff --git a/src/coreclr/src/jit/lsra.h b/src/coreclr/src/jit/lsra.h index 3ea2fd43236fd..e327895e100f3 100644 --- a/src/coreclr/src/jit/lsra.h +++ b/src/coreclr/src/jit/lsra.h @@ -1592,6 +1592,8 @@ class LinearScan : public LinearScanInterface #ifdef FEATURE_SIMD int BuildSIMD(GenTreeSIMD* tree); + int BuildSIMDUnaryRMWUses(GenTreeSIMD* node); + int BuildSIMDBinaryRMWUses(GenTreeSIMD* node); #endif // FEATURE_SIMD #ifdef FEATURE_HW_INTRINSICS diff --git a/src/coreclr/src/jit/lsraarm64.cpp b/src/coreclr/src/jit/lsraarm64.cpp index e028bf929cb5b..5c5a7c234c462 100644 --- a/src/coreclr/src/jit/lsraarm64.cpp +++ b/src/coreclr/src/jit/lsraarm64.cpp @@ -795,9 +795,6 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) bool buildUses = true; - GenTree* op1 = simdTree->gtGetOp1(); - GenTree* op2 = simdTree->gtGetOp2(); - switch (simdTree->gtSIMDIntrinsicID) { case SIMDIntrinsicInit: @@ -815,8 +812,8 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) case SIMDIntrinsicGetItem: { - op1 = simdTree->gtGetOp1(); - op2 = simdTree->gtGetOp2(); + GenTree* op1 = simdTree->GetOp(0); + GenTree* op2 = simdTree->GetOp(1); // We have an object and an index, either of which may be contained. bool setOp2DelayFree = false; @@ -874,41 +871,30 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) case SIMDIntrinsicSetZ: case SIMDIntrinsicSetW: case SIMDIntrinsicNarrow: - { // Op1 will write to dst before Op2 is free - BuildUse(op1); - RefPosition* op2Use = BuildUse(op2); - setDelayFree(op2Use); + BuildUse(simdTree->GetOp(0)); + setDelayFree(BuildUse(simdTree->GetOp(1))); srcCount = 2; buildUses = false; break; - } case SIMDIntrinsicInitN: - { - var_types baseType = simdTree->gtSIMDBaseType; - srcCount = (short)(simdTree->gtSIMDSize / genTypeSize(baseType)); + srcCount = static_cast(simdTree->GetNumOps()); + assert(srcCount == static_cast(simdTree->gtSIMDSize / genTypeSize(simdTree->gtSIMDBaseType))); if (varTypeIsFloating(simdTree->gtSIMDBaseType)) { // Need an internal register to stitch together all the values into a single vector in a SIMD reg. buildInternalFloatRegisterDefForNode(simdTree); } - int initCount = 0; - for (GenTree* list = op1; list != nullptr; list = list->gtGetOp2()) + for (GenTreeSIMD::Use& use : simdTree->Uses()) { - assert(list->OperGet() == GT_LIST); - GenTree* listItem = list->gtGetOp1(); - assert(listItem->TypeGet() == baseType); - assert(!listItem->isContained()); - BuildUse(listItem); - initCount++; + assert(use.GetNode()->TypeGet() == simdTree->gtSIMDBaseType); + assert(!use.GetNode()->isContained()); + BuildUse(use.GetNode()); } - assert(initCount == srcCount); buildUses = false; - break; - } case SIMDIntrinsicInitArray: // We have an array and an index, which may be contained. @@ -957,12 +943,15 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) } if (buildUses) { - assert(!op1->OperIs(GT_LIST)); + assert(simdTree->GetNumOps() <= 2); assert(srcCount == 0); - srcCount = BuildOperandUses(op1); - if ((op2 != nullptr) && !op2->isContained()) + + for (GenTreeSIMD::Use& use : simdTree->Uses()) { - srcCount += BuildOperandUses(op2); + if (!use.GetNode()->isContained()) + { + srcCount += BuildOperandUses(use.GetNode()); + } } } assert(internalCount <= MaxInternalCount); diff --git a/src/coreclr/src/jit/lsraxarch.cpp b/src/coreclr/src/jit/lsraxarch.cpp index 2c0cd97373e6c..b02079995dc5e 100644 --- a/src/coreclr/src/jit/lsraxarch.cpp +++ b/src/coreclr/src/jit/lsraxarch.cpp @@ -1841,7 +1841,7 @@ int LinearScan::BuildIntrinsic(GenTree* tree) // BuildSIMD: Set the NodeInfo for a GT_SIMD tree. // // Arguments: -// tree - The GT_SIMD node of interest +// simdTree - The GT_SIMD node of interest // // Return Value: // The number of sources consumed by this node. @@ -1864,9 +1864,8 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) (simdTree->gtSIMDIntrinsicID == SIMDIntrinsicOpInEquality)); } SetContainsAVXFlags(simdTree->gtSIMDSize); - GenTree* op1 = simdTree->gtGetOp1(); - GenTree* op2 = simdTree->gtGetOp2(); - int srcCount = 0; + + int srcCount = 0; switch (simdTree->gtSIMDIntrinsicID) { @@ -1881,7 +1880,8 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) CLANG_FORMAT_COMMENT_ANCHOR; #if !defined(_TARGET_64BIT_) - if (op1->OperGet() == GT_LONG) + GenTree* op1 = simdTree->GetOp(0); + if (op1->OperIs(GT_LONG)) { assert(op1->isContained()); GenTree* op1lo = op1->gtGetOp1(); @@ -1913,25 +1913,18 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) break; case SIMDIntrinsicInitN: - { - var_types baseType = simdTree->gtSIMDBaseType; - srcCount = (short)(simdTree->gtSIMDSize / genTypeSize(baseType)); + srcCount = static_cast(simdTree->GetNumOps()); + assert(srcCount == static_cast(simdTree->gtSIMDSize / genTypeSize(simdTree->gtSIMDBaseType))); // Need an internal register to stitch together all the values into a single vector in a SIMD reg. buildInternalFloatRegisterDefForNode(simdTree); - int initCount = 0; - for (GenTree* list = op1; list != nullptr; list = list->gtGetOp2()) + for (GenTreeSIMD::Use& use : simdTree->Uses()) { - assert(list->OperGet() == GT_LIST); - GenTree* listItem = list->gtGetOp1(); - assert(listItem->TypeGet() == baseType); - assert(!listItem->isContained()); - BuildUse(listItem); - initCount++; + assert(use.GetNode()->TypeGet() == simdTree->gtSIMDBaseType); + assert(!use.GetNode()->isContained()); + BuildUse(use.GetNode()); } - assert(initCount == srcCount); buildUses = false; - } - break; + break; case SIMDIntrinsicInitArray: // We have an array and an index, which may be contained. @@ -1994,7 +1987,7 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) case SIMDIntrinsicOpEquality: case SIMDIntrinsicOpInEquality: - if (simdTree->gtGetOp2()->isContained()) + if (simdTree->GetOp(1)->isContained()) { // If the second operand is contained then ContainCheckSIMD has determined // that PTEST can be used. We only need a single source register and no @@ -2029,7 +2022,7 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) if (varTypeIsFloating(simdTree->gtSIMDBaseType)) { if ((compiler->getSIMDSupportLevel() == SIMD_SSE2_Supported) || - (simdTree->gtGetOp1()->TypeGet() == TYP_SIMD32)) + (simdTree->GetOp(0)->TypeGet() == TYP_SIMD32)) { buildInternalFloatRegisterDefForNode(simdTree); setInternalRegsDelayFree = true; @@ -2059,8 +2052,8 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) // The result is baseType of SIMD struct. // op1 may be a contained memory op, but if so we will consume its address. // op2 may be a contained constant. - op1 = simdTree->gtGetOp1(); - op2 = simdTree->gtGetOp2(); + GenTree* op1 = simdTree->GetOp(0); + GenTree* op2 = simdTree->GetOp(1); if (!op1->isContained()) { @@ -2213,7 +2206,7 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) case SIMDIntrinsicShuffleSSE2: // Second operand is an integer constant and marked as contained. - assert(simdTree->gtGetOp2()->isContainedIntOrIImmed()); + assert(simdTree->GetOp(1)->isContainedIntOrIImmed()); break; case SIMDIntrinsicGetX: @@ -2233,11 +2226,18 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) } if (buildUses) { - assert(!op1->OperIs(GT_LIST)); assert(srcCount == 0); - // This is overly conservative, but is here for zero diffs. - srcCount = BuildRMWUses(simdTree); + + if (simdTree->IsUnary()) + { + srcCount = BuildSIMDUnaryRMWUses(simdTree); + } + else + { + srcCount = BuildSIMDBinaryRMWUses(simdTree); + } } + buildInternalRegisterUses(); if (dstCount == 1) { @@ -2249,6 +2249,125 @@ int LinearScan::BuildSIMD(GenTreeSIMD* simdTree) } return srcCount; } + +//------------------------------------------------------------------------ +// BuildSIMDUnaryRMWUses: Build uses for a RMW unary SIMD node. +// +// Arguments: +// node - The GT_SIMD node of interest +// +// Return Value: +// The number of sources consumed by this node. +// +// Notes: +// SSE unary instructions (sqrtps, cvttps2dq etc.) aren't really RMW, +// they have "dst, src" forms even when the VEX encoding is not available. +// However, it seems that SIMDIntrinsicConvertToSingle with UINT base type +// could benefit from getting the same register for both destination and +// source because its rather complicated codegen expansion starts by copying +// the source to the destination register. +// +int LinearScan::BuildSIMDUnaryRMWUses(GenTreeSIMD* node) +{ + assert(node->IsUnary()); + + GenTree* op1 = node->GetOp(0); + + if (op1->isContained()) + { + return BuildOperandUses(op1); + } + + tgtPrefUse = BuildUse(op1); + return 1; +} + +//------------------------------------------------------------------------ +// BuildSIMDBinaryRMWUses: Build uses for a RMW binary SIMD node. +// +// Arguments: +// node - The GT_SIMD node of interest +// +// Return Value: +// The number of sources consumed by this node. +// +int LinearScan::BuildSIMDBinaryRMWUses(GenTreeSIMD* node) +{ + assert(node->IsBinary()); + + GenTree* op1 = node->GetOp(0); + GenTree* op2 = node->GetOp(1); + + // Determine which operand, if any, should be delayRegFree. Normally, this would be op2, + // but if we have a commutative operator codegen can swap the operands, avoiding the need + // for delayRegFree. + // + // TODO-XArch-CQ: This should not be necessary when VEX encoding is available, at least + // for those intrinsics that directly map to SSE/AVX instructions. Intrinsics that require + // custom codegen expansion may still neeed this. + // + // Also, this doesn't check if the intrinsic is really RMW: + // - SIMDIntrinsicOpEquality/SIMDIntrinsicOpInEquality - they don't have a register def. + // - SIMDIntrinsicShuffleSSE2 - the second operand is always a contained immediate so they're + // really unary as far as the register allocator is concerned. + // - SIMDIntrinsicGetItem - the second operand is always an integer but the first may be a float + // and in that case delayRegFree is not needed. Either way, it's not a real RMW operation. It's + // also the only binary SIMD intrinsic that can have a contained op1. + + GenTree* delayUseOperand = op2; + + if (node->isCommutativeSIMDIntrinsic()) + { + if (op1->isContained()) + { + delayUseOperand = op1; + } + else if (!op2->isContained() || op2->IsCnsIntOrI()) + { + // If we have a commutative operator and op2 is not a memory op, we don't need + // to set delayRegFree on either operand because codegen can swap them. + delayUseOperand = nullptr; + } + } + else if (op1->isContained()) + { + delayUseOperand = nullptr; + } + + int srcCount = 0; + + // Build first use + if (!op1->isContained()) + { + tgtPrefUse = BuildUse(op1); + srcCount++; + } + else if (delayUseOperand == op1) + { + srcCount += BuildDelayFreeUses(op1); + } + else + { + srcCount += BuildOperandUses(op1); + } + + // Build second use + if (node->isCommutativeSIMDIntrinsic() && !op2->isContained()) + { + tgtPrefUse2 = BuildUse(op2); + srcCount++; + } + else if (delayUseOperand == op2) + { + srcCount += BuildDelayFreeUses(op2); + } + else + { + srcCount += BuildOperandUses(op2); + } + + return srcCount; +} #endif // FEATURE_SIMD #ifdef FEATURE_HW_INTRINSICS diff --git a/src/coreclr/src/jit/morph.cpp b/src/coreclr/src/jit/morph.cpp index 404cb60eeb28a..5993220182e1a 100644 --- a/src/coreclr/src/jit/morph.cpp +++ b/src/coreclr/src/jit/morph.cpp @@ -5919,7 +5919,6 @@ GenTree* Compiler::fgMorphField(GenTree* tree, MorphAddrContext* mac) GenTree* newTree = fgMorphFieldToSIMDIntrinsicGet(tree); if (newTree != tree) { - newTree = fgMorphSmpOp(newTree); return newTree; } } @@ -8979,7 +8978,7 @@ GenTree* Compiler::fgMorphOneAsgBlockOp(GenTree* tree) noway_assert(src->IsIntegralConst(0)); noway_assert(destVarDsc != nullptr); - src = new (this, GT_SIMD) GenTreeSIMD(asgType, src, SIMDIntrinsicInit, destVarDsc->lvBaseType, size); + src = gtNewSIMDNode(asgType, SIMDIntrinsicInit, destVarDsc->lvBaseType, size, src); } else #endif @@ -10743,10 +10742,8 @@ GenTree* Compiler::fgMorphFieldToSIMDIntrinsicGet(GenTree* tree) { assert(simdSize >= ((index + 1) * genTypeSize(baseType))); GenTree* op2 = gtNewIconNode(index); - tree = gtNewSIMDNode(baseType, simdStructNode, op2, SIMDIntrinsicGetItem, baseType, simdSize); -#ifdef DEBUG - tree->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED; -#endif + tree = gtNewSIMDNode(baseType, SIMDIntrinsicGetItem, baseType, simdSize, simdStructNode, op2); + tree = fgMorphTree(tree); } return tree; } @@ -10800,7 +10797,7 @@ GenTree* Compiler::fgMorphFieldAssignToSIMDIntrinsicSet(GenTree* tree) GenTree* target = gtClone(simdOp1Struct); assert(target != nullptr); var_types simdType = target->gtType; - GenTree* simdTree = gtNewSIMDNode(simdType, simdOp1Struct, op2, simdIntrinsicID, baseType, simdSize); + GenTree* simdTree = gtNewSIMDNode(simdType, simdIntrinsicID, baseType, simdSize, simdOp1Struct, op2); tree->AsOp()->gtOp1 = target; tree->AsOp()->gtOp2 = simdTree; @@ -14611,6 +14608,17 @@ GenTree* Compiler::fgMorphTree(GenTree* tree, MorphAddrContext* mac) } break; +#ifdef FEATURE_SIMD + case GT_SIMD: + tree->gtFlags &= ~GTF_ALL_EFFECT; + for (GenTreeSIMD::Use& use : tree->AsSIMD()->Uses()) + { + use.SetNode(fgMorphTree(use.GetNode())); + tree->gtFlags |= (use.GetNode()->gtFlags & GTF_ALL_EFFECT); + } + break; +#endif // FEATURE_SIMD + case GT_CMPXCHG: tree->AsCmpXchg()->gtOpLocation = fgMorphTree(tree->AsCmpXchg()->gtOpLocation); tree->AsCmpXchg()->gtOpValue = fgMorphTree(tree->AsCmpXchg()->gtOpValue); diff --git a/src/coreclr/src/jit/rationalize.cpp b/src/coreclr/src/jit/rationalize.cpp index 2c3efd4e60c39..57fd0ac6de458 100644 --- a/src/coreclr/src/jit/rationalize.cpp +++ b/src/coreclr/src/jit/rationalize.cpp @@ -340,8 +340,8 @@ void Rationalizer::RewriteAssignment(LIR::Use& use) var_types baseType = comp->getBaseTypeOfSIMDLocal(location); if (baseType != TYP_UNKNOWN) { - GenTreeSIMD* simdTree = new (comp, GT_SIMD) - GenTreeSIMD(simdType, initVal, SIMDIntrinsicInit, baseType, genTypeSize(simdType)); + GenTreeSIMD* simdTree = + comp->gtNewSIMDNode(simdType, SIMDIntrinsicInit, baseType, genTypeSize(simdType), initVal); assignment->AsOp()->gtOp2 = simdTree; value = simdTree; initVal->gtNext = simdTree; @@ -776,9 +776,17 @@ Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, Compiler::Ge { // Rewrite this as an explicit load. JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n"); - unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType); - GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2, - baseTypeSize, OFFSETOF__CORINFO_Array__data); + assert(simdNode->IsUnary() || simdNode->IsBinary()); + + GenTreeAddrMode* address = new (comp, GT_LEA) + GenTreeAddrMode(TYP_BYREF, simdNode->GetOp(0), nullptr, 0, OFFSETOF__CORINFO_Array__data); + + if (simdNode->IsBinary()) + { + address->SetIndex(simdNode->GetOp(1)); + address->SetScale(genTypeSize(simdNode->gtSIMDBaseType)); + } + GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address); BlockRange().InsertBefore(simdNode, address, ind); @@ -794,16 +802,12 @@ Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, Compiler::Ge // of a different width. If that assumption changes, we will EITHER have to make these type // transformations during importation, and plumb the types all the way through the JIT, // OR add a lot of special handling here. - GenTree* op1 = simdNode->gtGetOp1(); - if (op1 != nullptr && op1->gtType == TYP_STRUCT) + for (GenTreeSIMD::Use& use : simdNode->Uses()) { - op1->gtType = simdType; - } - - GenTree* op2 = simdNode->gtGetOp2IfPresent(); - if (op2 != nullptr && op2->gtType == TYP_STRUCT) - { - op2->gtType = simdType; + if (use.GetNode()->TypeGet() == TYP_STRUCT) + { + use.GetNode()->gtType = simdType; + } } } } diff --git a/src/coreclr/src/jit/simd.cpp b/src/coreclr/src/jit/simd.cpp index bef8a360550b4..57db2c6f0dad2 100644 --- a/src/coreclr/src/jit/simd.cpp +++ b/src/coreclr/src/jit/simd.cpp @@ -1133,7 +1133,7 @@ GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, u GenTree* op1 = impSIMDPopStack(simdType, true); GenTree* op2 = gtNewIconNode(index); - GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize); + GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, SIMDIntrinsicGetItem, baseType, simdSize, op1, op2); return simdTree; } @@ -1167,7 +1167,7 @@ SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd, // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements. // Compare vector as if they were vector and assign the result to a temp - GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size); + GenTree* compResult = gtNewSIMDNode(simdType, SIMDIntrinsicEqual, TYP_INT, size, *pOp1, *pOp2); unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Long ==")); lvaSetStruct(lclNum, typeHnd, false); GenTree* tmp = gtNewLclvNode(lclNum, simdType); @@ -1177,8 +1177,8 @@ SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd, // op2 = Shuffle(tmp, 0xB1) // IntrinsicId = BitwiseAnd *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp); - *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT), - SIMDIntrinsicShuffleSSE2, TYP_INT, size); + *pOp2 = gtNewSIMDNode(simdType, SIMDIntrinsicShuffleSSE2, TYP_INT, size, gtNewLclvNode(lclNum, simdType), + gtNewIconNode(SHUFFLE_ZWXY, TYP_INT)); return SIMDIntrinsicBitwiseAnd; } @@ -1255,23 +1255,23 @@ SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeH assert(*pOp1 != nullptr && *pOp2 != nullptr); // v1GreaterThanv2Signed - signed 32-bit comparison - GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size); + GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, SIMDIntrinsicGreaterThan, TYP_INT, size, *pOp1, *pOp2); // v1Equalsv2 - 32-bit equality - GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size); + GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, SIMDIntrinsicEqual, TYP_INT, size, dupOp1, dupOp2); // v1GreaterThanv2Unsigned - unsigned 32-bit comparison var_types tempBaseType = TYP_UINT; SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2); - GenTree* v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size); + GenTree* v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, sid, tempBaseType, size, dupDupOp1, dupDupOp2); - GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), - SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size); - GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT), - SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size); - GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2, - TYP_FLOAT, size); - GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size); + GenTree* z = gtNewSIMDNode(simdType, SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size, v1GreaterThanv2Signed, + gtNewIconNode(SHUFFLE_WWYY, TYP_INT)); + GenTree* t1 = gtNewSIMDNode(simdType, SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size, v1GreaterThanv2Unsigned, + gtNewIconNode(SHUFFLE_ZZXX, TYP_INT)); + GenTree* u1 = gtNewSIMDNode(simdType, SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size, v1Equalsv2, + gtNewIconNode(SHUFFLE_WWYY, TYP_INT)); + GenTree* w = gtNewSIMDNode(simdType, SIMDIntrinsicBitwiseAnd, TYP_INT, size, u1, t1); *pOp1 = z; *pOp2 = w; @@ -1325,11 +1325,11 @@ SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDL // (a==b) SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2); - *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size); + *pOp1 = gtNewSIMDNode(simdType, id, TYP_LONG, size, *pOp1, *pOp2); // (a > b) id = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2); - *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size); + *pOp2 = gtNewSIMDNode(simdType, id, TYP_LONG, size, dupOp1, dupOp2); return SIMDIntrinsicBitwiseOr; } @@ -1386,10 +1386,10 @@ SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual( assert(*pOp1 != nullptr && *pOp2 != nullptr); // (a==b) - *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size); + *pOp1 = gtNewSIMDNode(simdType, SIMDIntrinsicEqual, baseType, size, *pOp1, *pOp2); // (a > b) - *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size); + *pOp2 = gtNewSIMDNode(simdType, SIMDIntrinsicGreaterThan, baseType, size, dupOp1, dupOp2); return SIMDIntrinsicBitwiseOr; } @@ -1540,7 +1540,7 @@ SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID relOpIntrinsicId, initVal = gtNewIconNode((ssize_t)constVal); } initVal->gtType = tempBaseType; - GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size); + GenTree* constVector = gtNewSIMDNode(simdType, SIMDIntrinsicInit, tempBaseType, size, initVal); // Assign constVector to a temp, since we intend to use it more than once // TODO-CQ: We have quite a few such constant vectors constructed during @@ -1550,8 +1550,8 @@ SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID relOpIntrinsicId, // op1 = op1 - constVector // op2 = op2 - constVector - *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size); - *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size); + *pOp1 = gtNewSIMDNode(simdType, SIMDIntrinsicSub, baseType, size, *pOp1, constVector); + *pOp2 = gtNewSIMDNode(simdType, SIMDIntrinsicSub, baseType, size, *pOp2, tmp); } return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2); @@ -1653,7 +1653,7 @@ GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, var_types relOpBaseType = baseType; SIMDIntrinsicID relOpIntrinsic = impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2); - GenTree* bitVec = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size); + GenTree* bitVec = gtNewSIMDNode(simdType, relOpIntrinsic, relOpBaseType, size, bitVecOp1, bitVecOp2); unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec")); lvaSetStruct(bitVecLclNum, typeHnd, false); GenTree* bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec); @@ -1662,7 +1662,7 @@ GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, // Construct condSelectOp1 = vector.Zero - v GenTree* subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet()); GenTree* subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet()); - GenTree* negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size); + GenTree* negVec = gtNewSIMDNode(simdType, SIMDIntrinsicSub, baseType, size, subOp1, subOp2); // Construct ConditionalSelect(bitVec, vector.Zero - v, v) GenTree* vec = gtNewLclvNode(op1LclNum, op1->TypeGet()); @@ -1705,8 +1705,8 @@ GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, assert(bitMask != nullptr); bitMask->gtType = baseType; - GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size); - retVal = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size); + GenTree* bitMaskVector = gtNewSIMDNode(simdType, SIMDIntrinsicInit, baseType, size, bitMask); + retVal = gtNewSIMDNode(simdType, SIMDIntrinsicBitwiseAnd, baseType, size, op1, bitMaskVector); } else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG) { @@ -1718,7 +1718,7 @@ GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported); assert(baseType != TYP_LONG); - retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size); + retVal = gtNewSIMDNode(simdType, SIMDIntrinsicAbs, baseType, size, op1); } #elif defined(_TARGET_ARM64_) if (varTypeIsUnsigned(baseType)) @@ -1728,7 +1728,7 @@ GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, } else { - retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size); + retVal = gtNewSIMDNode(simdType, SIMDIntrinsicAbs, baseType, size, op1); } #else // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_) assert(!"Abs intrinsic on non-xarch target not implemented"); @@ -1776,17 +1776,17 @@ GenTree* Compiler::impSIMDSelect( asg = gtNewTempAssign(lclNum, op1); } - GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size); + GenTree* andExpr = gtNewSIMDNode(simdType, SIMDIntrinsicBitwiseAnd, baseType, size, op2, tmp); GenTree* dupOp1 = gtCloneExpr(tmp); assert(dupOp1 != nullptr); #ifdef _TARGET_ARM64_ // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right - GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size); + GenTree* andNotExpr = gtNewSIMDNode(simdType, SIMDIntrinsicBitwiseAndNot, baseType, size, op3, dupOp1); #else // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right - GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size); + GenTree* andNotExpr = gtNewSIMDNode(simdType, SIMDIntrinsicBitwiseAndNot, baseType, size, dupOp1, op3); #endif - GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size); + GenTree* simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicBitwiseOr, baseType, size, andExpr, andNotExpr); // If asg not null, create a GT_COMMA tree. if (asg != nullptr) @@ -1853,7 +1853,7 @@ GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId, (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT))) { // SSE2 or SSE4.1 has direct support - simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size); + simdTree = gtNewSIMDNode(simdType, intrinsicId, baseType, size, op1, op2); } else if (baseType == TYP_USHORT || baseType == TYP_BYTE) { @@ -1879,7 +1879,7 @@ GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId, } GenTree* initVal = gtNewIconNode(constVal); - GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size); + GenTree* constVector = gtNewSIMDNode(simdType, SIMDIntrinsicInit, TYP_INT, size, initVal); // Assign constVector to a temp, since we intend to use it more than once // TODO-CQ: We have quite a few such constant vectors constructed during @@ -1889,15 +1889,15 @@ GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId, // op1 = op1 - constVector // op2 = op2 - constVector - op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size); - op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size); + op1 = gtNewSIMDNode(simdType, operIntrinsic, baseType, size, op1, constVector); + op2 = gtNewSIMDNode(simdType, operIntrinsic, baseType, size, op2, tmp); // compute min/max of op1 and op2 considering them as if minMaxOperBaseType - simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size); + simdTree = gtNewSIMDNode(simdType, intrinsicId, minMaxOperBaseType, size, op1, op2); // re-adjust the value by adding or subtracting constVector tmp = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet()); - simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size); + simdTree = gtNewSIMDNode(simdType, adjustIntrinsic, baseType, size, simdTree, tmp); } #elif defined(_TARGET_ARM64_) // Arm64 has direct support for all types except int64/uint64 @@ -1909,7 +1909,7 @@ GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId, // Min/Max(op1, op2) = Select(compResult, op1, op2) if (baseType != TYP_ULONG && baseType != TYP_LONG) { - simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size); + simdTree = gtNewSIMDNode(simdType, intrinsicId, baseType, size, op1, op2); } #endif else @@ -1956,7 +1956,7 @@ GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId, assert(dupOp1 != nullptr); assert(dupOp2 != nullptr); relOpIntrinsic = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2); - GenTree* compResult = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size); + GenTree* compResult = gtNewSIMDNode(simdType, relOpIntrinsic, relOpBaseType, size, dupOp1, dupOp2); unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max")); lvaSetStruct(compResultLclNum, typeHnd, false); GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult); @@ -2472,95 +2472,31 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, { // Equivalent to (Vector) new Vector(0xffffffff); GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT); - simdTree = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size); + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicInit, TYP_INT, size, initVal); if (baseType != TYP_INT) { // cast it to required baseType if different from TYP_INT - simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size); + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicCast, baseType, size, simdTree); } retVal = simdTree; } break; case SIMDIntrinsicInit: - case SIMDIntrinsicInitN: { // SIMDIntrinsicInit: // op2 - the initializer value // op1 - byref of vector - // - // SIMDIntrinsicInitN - // op2 - list of initializer values stitched into a list - // op1 - byref of vector - bool initFromFirstArgIndir = false; - if (simdIntrinsicID == SIMDIntrinsicInit) - { - op2 = impSIMDPopStack(baseType); - } - else - { - assert(simdIntrinsicID == SIMDIntrinsicInitN); - assert(baseType == TYP_FLOAT); - - unsigned initCount = argCount - 1; - unsigned elementCount = getSIMDVectorLength(size, baseType); - noway_assert(initCount == elementCount); - - // Build a GT_LIST with the N values. - // We must maintain left-to-right order of the args, but we will pop - // them off in reverse order (the Nth arg was pushed onto the stack last). - - GenTree* list = nullptr; - GenTree* firstArg = nullptr; - GenTree* prevArg = nullptr; - bool areArgsContiguous = true; - for (unsigned i = 0; i < initCount; i++) - { - GenTree* nextArg = impSIMDPopStack(baseType); - if (areArgsContiguous) - { - GenTree* curArg = nextArg; - firstArg = curArg; - - if (prevArg != nullptr) - { - // Recall that we are popping the args off the stack in reverse order. - areArgsContiguous = areArgumentsContiguous(curArg, prevArg); - } - prevArg = curArg; - } - - list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list); - } - - if (areArgsContiguous && baseType == TYP_FLOAT) - { - // Since Vector2, Vector3 and Vector4's arguments type are only float, - // we intialize the vector from first argument address, only when - // the baseType is TYP_FLOAT and the arguments are located contiguously in memory - initFromFirstArgIndir = true; - GenTree* op2Address = createAddressNodeForSIMDInit(firstArg, size); - var_types simdType = getSIMDTypeForSize(size); - op2 = gtNewOperNode(GT_IND, simdType, op2Address); - } - else - { - op2 = list; - } - } - + op2 = impSIMDPopStack(baseType); op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd); assert(op1->TypeGet() == TYP_BYREF); - assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir); + assert(genActualType(op2->TypeGet()) == genActualType(baseType)); // For integral base types of size less than TYP_INT, expand the initializer // to fill size of TYP_INT bytes. if (varTypeIsSmallInt(baseType)) { - // This case should occur only for Init intrinsic. - assert(simdIntrinsicID == SIMDIntrinsicInit); - unsigned baseSize = genTypeSize(baseType); int multiplier; if (baseSize == 1) @@ -2601,27 +2537,69 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, op2 = gtNewOperNode(GT_MUL, TYP_INT, t1, t2); // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType - simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size); - simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size); + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicInit, TYP_INT, size, op2); + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicCast, baseType, size, simdTree); } else { + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicInit, baseType, size, op2); + } - if (initFromFirstArgIndir) + copyBlkDst = op1; + doCopyBlk = true; + } + break; + + case SIMDIntrinsicInitN: + { + // SIMDIntrinsicInitN + // op2 - list of initializer values stitched into a list + // op1 - byref of vector + assert(baseType == TYP_FLOAT); + + unsigned initCount = argCount - 1; + unsigned elementCount = getSIMDVectorLength(size, baseType); + noway_assert(initCount == elementCount); + + // We must maintain left-to-right order of the args, but we will pop + // them off in reverse order (the Nth arg was pushed onto the stack last). + + GenTree* args[SIMD_INTRINSIC_MAX_PARAM_COUNT - 1]; + + bool areArgsContiguous = true; + for (unsigned i = 0; i < initCount; i++) + { + args[initCount - 1 - i] = impSIMDPopStack(baseType); + + if (areArgsContiguous && (i > 0)) { - simdTree = op2; - if (op1->AsOp()->gtOp1->OperIsLocal()) - { - // label the dst struct's lclvar is used for SIMD intrinsic, - // so that this dst struct won't be promoted. - setLclRelatedToSIMDIntrinsic(op1->AsOp()->gtOp1); - } + // Recall that we are popping the args off the stack in reverse order. + areArgsContiguous = areArgumentsContiguous(args[initCount - 1 - i], args[initCount - 1 - i + 1]); } - else + } + + op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd); + assert(op1->TypeGet() == TYP_BYREF); + + if (areArgsContiguous) + { + // Since Vector2, Vector3 and Vector4's arguments type are only float, + // we intialize the vector from first argument address, only when + // the baseType is TYP_FLOAT and the arguments are located contiguously in memory + GenTree* op2Address = createAddressNodeForSIMDInit(args[0], size); + simdTree = gtNewOperNode(GT_IND, simdType, op2Address); + + if (op1->AsOp()->gtOp1->OperIsLocal()) { - simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size); + // label the dst struct's lclvar is used for SIMD intrinsic, + // so that this dst struct won't be promoted. + setLclRelatedToSIMDIntrinsic(op1->AsOp()->gtOp1); } } + else + { + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicInitN, baseType, size, initCount, args); + } copyBlkDst = op1; doCopyBlk = true; @@ -2745,8 +2723,15 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX) { - op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd); - simdTree = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size); + op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd); + if (op3 == nullptr) + { + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicInitArray, baseType, size, op2); + } + else + { + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicInitArray, baseType, size, op2, op3); + } copyBlkDst = op1; doCopyBlk = true; } @@ -2808,11 +2793,11 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, simdTree = op2; if (op3 != nullptr) { - simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size); + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicSetZ, baseType, size, simdTree, op3); } if (op4 != nullptr) { - simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size); + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicSetW, baseType, size, simdTree, op4); } copyBlkDst = op1; @@ -2829,7 +2814,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, assert(op1->TypeGet() == simdType); assert(op2->TypeGet() == simdType); - simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size); + simdTree = gtNewSIMDNode(genActualType(callType), SIMDIntrinsicOpEquality, baseType, size, op1, op2); if (simdType == TYP_SIMD12) { simdTree->gtFlags |= GTF_SIMD12_OP; @@ -2844,7 +2829,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, // op2 is the second operand op2 = impSIMDPopStack(simdType); op1 = impSIMDPopStack(simdType, instMethod); - simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size); + simdTree = gtNewSIMDNode(genActualType(callType), SIMDIntrinsicOpInEquality, baseType, size, op1, op2); if (simdType == TYP_SIMD12) { simdTree->gtFlags |= GTF_SIMD12_OP; @@ -2863,7 +2848,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, op1 = impSIMDPopStack(simdType, instMethod); SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2); - simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size); + simdTree = gtNewSIMDNode(genActualType(callType), intrinsicID, baseType, size, op1, op2); retVal = simdTree; } break; @@ -2936,7 +2921,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, } #endif // _TARGET_XARCH_ - simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size); + simdTree = gtNewSIMDNode(simdType, simdIntrinsicID, baseType, size, op1, op2); retVal = simdTree; } break; @@ -3002,7 +2987,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, assert(op1->TypeGet() == simdType); assert(op2->TypeGet() == TYP_INT); - simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size); + simdTree = gtNewSIMDNode(genActualType(callType), simdIntrinsicID, baseType, size, op1, op2); retVal = simdTree; } break; @@ -3023,7 +3008,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, op2 = impSIMDPopStack(simdType); op1 = impSIMDPopStack(simdType, instMethod); - simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size); + simdTree = gtNewSIMDNode(baseType, simdIntrinsicID, baseType, size, op1, op2); if (simdType == TYP_SIMD12) { simdTree->gtFlags |= GTF_SIMD12_OP; @@ -3047,7 +3032,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, op1 = impSIMDPopStack(simdType); - retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size); + retVal = gtNewSIMDNode(genActualType(callType), simdIntrinsicID, baseType, size, op1); } break; @@ -3096,7 +3081,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, GenTree* src = gtCloneExpr(op1); assert(src != nullptr); - simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size); + simdTree = gtNewSIMDNode(simdType, simdIntrinsicID, baseType, size, src, op2); copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1); doCopyBlk = true; @@ -3111,7 +3096,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, { op1 = impSIMDPopStack(simdType, instMethod); - simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size); + simdTree = gtNewSIMDNode(simdType, simdIntrinsicID, baseType, size, op1); retVal = simdTree; } break; @@ -3121,7 +3106,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, #ifdef _TARGET_64BIT_ op1 = impSIMDPopStack(simdType, instMethod); - simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size); + simdTree = gtNewSIMDNode(simdType, simdIntrinsicID, baseType, size, op1); retVal = simdTree; #else JITDUMP("SIMD Conversion to Int64 is not supported on this platform\n"); @@ -3136,7 +3121,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, op2 = impSIMDPopStack(simdType); op1 = impSIMDPopStack(simdType); // op1 and op2 are two input Vector. - simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size); + simdTree = gtNewSIMDNode(simdType, simdIntrinsicID, baseType, size, op1, op2); retVal = simdTree; } break; @@ -3151,7 +3136,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, GenTree* dupOp1 = fgInsertCommaFormTemp(&op1, op1Handle); // Widen the lower half and assign it to dstAddrLo. - simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size); + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicWidenLo, baseType, size, op1); // TODO-1stClassStructs: With the introduction of ClassLayout it would be preferrable to use // GT_OBJ instead of GT_BLK nodes to avoid losing information about the actual vector type. GenTree* loDest = new (this, GT_BLK) @@ -3162,7 +3147,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT); // Widen the upper half and assign it to dstAddrHi. - simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size); + simdTree = gtNewSIMDNode(simdType, SIMDIntrinsicWidenHi, baseType, size, dupOp1); GenTree* hiDest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, typGetBlkLayout(getSIMDTypeSizeInBytes(clsHnd))); GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, diff --git a/src/coreclr/src/jit/simd.h b/src/coreclr/src/jit/simd.h index 8874f733da99a..a9b60bda78711 100644 --- a/src/coreclr/src/jit/simd.h +++ b/src/coreclr/src/jit/simd.h @@ -43,7 +43,7 @@ enum SIMDLevel extern const char* const simdIntrinsicNames[]; #endif -enum SIMDIntrinsicID +enum SIMDIntrinsicID : uint16_t { #define SIMD_INTRINSIC(m, i, id, n, r, ac, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10) SIMDIntrinsic##id, #include "simdintrinsiclist.h" diff --git a/src/coreclr/src/jit/simdcodegenxarch.cpp b/src/coreclr/src/jit/simdcodegenxarch.cpp index 83d1093d9f382..bc99751316f36 100644 --- a/src/coreclr/src/jit/simdcodegenxarch.cpp +++ b/src/coreclr/src/jit/simdcodegenxarch.cpp @@ -752,7 +752,7 @@ void CodeGen::genSIMDIntrinsicInit(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicInit); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); @@ -981,14 +981,12 @@ void CodeGen::genSIMDIntrinsicInitN(GenTreeSIMD* simdNode) // and record the registers. regNumber operandRegs[SIMD_INTRINSIC_MAX_PARAM_COUNT]; unsigned initCount = 0; - for (GenTree* list = simdNode->gtGetOp1(); list != nullptr; list = list->gtGetOp2()) + for (GenTreeSIMD::Use& use : simdNode->Uses()) { - assert(list->OperGet() == GT_LIST); - GenTree* listItem = list->gtGetOp1(); - assert(listItem->TypeGet() == baseType); - assert(!listItem->isContained()); - regNumber operandReg = genConsumeReg(listItem); - operandRegs[initCount] = operandReg; + assert(use.GetNode()->TypeGet() == baseType); + assert(!use.GetNode()->isContained()); + assert(initCount < _countof(operandRegs)); + operandRegs[initCount] = genConsumeReg(use.GetNode()); initCount++; } @@ -1037,7 +1035,7 @@ void CodeGen::genSIMDIntrinsicUnOp(GenTreeSIMD* simdNode) assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicSqrt || simdNode->gtSIMDIntrinsicID == SIMDIntrinsicCast || simdNode->gtSIMDIntrinsicID == SIMDIntrinsicAbs); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); @@ -1066,7 +1064,7 @@ void CodeGen::genSIMDIntrinsic32BitConvert(GenTreeSIMD* simdNode) SIMDIntrinsicID intrinsicID = simdNode->gtSIMDIntrinsicID; assert((intrinsicID == SIMDIntrinsicConvertToSingle) || (intrinsicID == SIMDIntrinsicConvertToInt32)); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); @@ -1201,7 +1199,7 @@ void CodeGen::genSIMDIntrinsic64BitConvert(GenTreeSIMD* simdNode) SIMDIntrinsicID intrinsicID = simdNode->gtSIMDIntrinsicID; assert((intrinsicID == SIMDIntrinsicConvertToDouble) || (intrinsicID == SIMDIntrinsicConvertToInt64)); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); @@ -1522,14 +1520,14 @@ void CodeGen::genSIMDIntrinsicWiden(GenTreeSIMD* simdNode) assert((simdNode->gtSIMDIntrinsicID == SIMDIntrinsicWidenLo) || (simdNode->gtSIMDIntrinsicID == SIMDIntrinsicWidenHi)); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); var_types simdType = simdNode->TypeGet(); SIMDLevel level = compiler->getSIMDSupportLevel(); - genConsumeOperands(simdNode); + genConsumeRegs(op1); regNumber op1Reg = op1->GetRegNum(); regNumber srcReg = op1Reg; emitAttr emitSize = emitActualTypeSize(simdType); @@ -1595,8 +1593,8 @@ void CodeGen::genSIMDIntrinsicNarrow(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicNarrow); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); @@ -1604,7 +1602,8 @@ void CodeGen::genSIMDIntrinsicNarrow(GenTreeSIMD* simdNode) emitAttr emitSize = emitTypeSize(simdType); SIMDLevel level = compiler->getSIMDSupportLevel(); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); if (baseType == TYP_DOUBLE) @@ -1748,15 +1747,16 @@ void CodeGen::genSIMDIntrinsicBinOp(GenTreeSIMD* simdNode) simdNode->gtSIMDIntrinsicID == SIMDIntrinsicBitwiseXor || simdNode->gtSIMDIntrinsicID == SIMDIntrinsicMin || simdNode->gtSIMDIntrinsicID == SIMDIntrinsicMax); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); assert(targetReg != REG_NA); var_types targetType = simdNode->TypeGet(); SIMDLevel level = compiler->getSIMDSupportLevel(); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); regNumber otherReg = op2Reg; @@ -1928,14 +1928,15 @@ void CodeGen::genSIMDIntrinsicBinOp(GenTreeSIMD* simdNode) // void CodeGen::genSIMDIntrinsicRelOp(GenTreeSIMD* simdNode) { - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); var_types targetType = simdNode->TypeGet(); SIMDLevel level = compiler->getSIMDSupportLevel(); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); regNumber otherReg = op2Reg; @@ -2132,8 +2133,8 @@ void CodeGen::genSIMDIntrinsicDotProduct(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicDotProduct); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; var_types simdType = op1->TypeGet(); // TODO-1stClassStructs: Temporary to minimize asmDiffs @@ -2148,7 +2149,8 @@ void CodeGen::genSIMDIntrinsicDotProduct(GenTreeSIMD* simdNode) var_types targetType = simdNode->TypeGet(); assert(targetType == baseType); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); regNumber tmpReg1 = REG_NA; @@ -2416,8 +2418,8 @@ void CodeGen::genSIMDIntrinsicGetItem(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicGetItem); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types simdType = op1->TypeGet(); assert(varTypeIsSIMD(simdType)); @@ -2437,7 +2439,8 @@ void CodeGen::genSIMDIntrinsicGetItem(GenTreeSIMD* simdNode) // GetItem has 2 operands: // - the source of SIMD type (op1) // - the index of the value to be returned. - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber srcReg = op1->GetRegNum(); // Optimize the case of op1 is in memory and trying to access ith element. @@ -2717,8 +2720,8 @@ void CodeGen::genSIMDIntrinsicSetItem(GenTreeSIMD* simdNode) // op1 is the SIMD vector // op2 is the value to be set - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); var_types baseType = simdNode->gtSIMDBaseType; regNumber targetReg = simdNode->GetRegNum(); @@ -2732,7 +2735,8 @@ void CodeGen::genSIMDIntrinsicSetItem(GenTreeSIMD* simdNode) assert(op2->TypeGet() == baseType); assert(simdNode->gtSIMDSize >= ((index + 1) * genTypeSize(baseType))); - genConsumeOperands(simdNode); + genConsumeRegs(op1); + genConsumeRegs(op2); regNumber op1Reg = op1->GetRegNum(); regNumber op2Reg = op2->GetRegNum(); @@ -2792,8 +2796,8 @@ void CodeGen::genSIMDIntrinsicShuffleSSE2(GenTreeSIMD* simdNode) assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicShuffleSSE2); noway_assert(compiler->getSIMDSupportLevel() == SIMD_SSE2_Supported); - GenTree* op1 = simdNode->gtGetOp1(); - GenTree* op2 = simdNode->gtGetOp2(); + GenTree* op1 = simdNode->GetOp(0); + GenTree* op2 = simdNode->GetOp(1); assert(op2->isContained()); assert(op2->IsCnsIntOrI()); ssize_t shuffleControl = op2->AsIntConCommon()->IconValue(); @@ -3058,7 +3062,7 @@ void CodeGen::genSIMDIntrinsicUpperSave(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicUpperSave); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); assert(op1->IsLocal() && op1->TypeGet() == TYP_SIMD32); regNumber targetReg = simdNode->GetRegNum(); regNumber op1Reg = genConsumeReg(op1); @@ -3100,7 +3104,7 @@ void CodeGen::genSIMDIntrinsicUpperRestore(GenTreeSIMD* simdNode) { assert(simdNode->gtSIMDIntrinsicID == SIMDIntrinsicUpperRestore); - GenTree* op1 = simdNode->gtGetOp1(); + GenTree* op1 = simdNode->GetOp(0); assert(op1->IsLocal() && op1->TypeGet() == TYP_SIMD32); regNumber srcReg = simdNode->GetRegNum(); regNumber lclVarReg = genConsumeReg(op1);