diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index b60142a856953..3eb3c853e41a9 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -206,7 +206,7 @@ ABIPassingInformation::ABIPassingInformation(Compiler* comp, unsigned numSegment if (numSegments > 1) { - Segments = new (comp, CMK_ABI) ABIPassingSegment[numSegments]; + m_segments = new (comp, CMK_ABI) ABIPassingSegment[numSegments]; } } @@ -225,10 +225,10 @@ const ABIPassingSegment& ABIPassingInformation::Segment(unsigned index) const assert(index < NumSegments); if (NumSegments == 1) { - return SingleSegment; + return m_singleSegment; } - return Segments[index]; + return m_segments[index]; } //----------------------------------------------------------------------------- @@ -265,6 +265,25 @@ bool ABIPassingInformation::HasAnyRegisterSegment() const return false; } +//----------------------------------------------------------------------------- +// HasAnyFloatingRegisterSegment: +// Check if any part of this value is passed in a floating-point register. +// +// Return Value: +// True if so. +// +bool ABIPassingInformation::HasAnyFloatingRegisterSegment() const +{ + for (unsigned i = 0; i < NumSegments; i++) + { + if (Segment(i).IsPassedInRegister() && genIsValidFloatReg(Segment(i).GetRegister())) + { + return true; + } + } + return false; +} + //----------------------------------------------------------------------------- // HasAnyStackSegment: // Check if any part of this value is passed on the stack. @@ -348,8 +367,8 @@ bool ABIPassingInformation::IsSplitAcrossRegistersAndStack() const ABIPassingInformation ABIPassingInformation::FromSegment(Compiler* comp, const ABIPassingSegment& segment) { ABIPassingInformation info; - info.NumSegments = 1; - info.SingleSegment = segment; + info.NumSegments = 1; + info.m_singleSegment = segment; return info; } @@ -371,7 +390,7 @@ ABIPassingInformation ABIPassingInformation::FromSegments(Compiler* { ABIPassingInformation info; info.NumSegments = 2; - info.Segments = new (comp, CMK_ABI) ABIPassingSegment[2]{firstSegment, secondSegment}; + info.m_segments = new (comp, CMK_ABI) ABIPassingSegment[2]{firstSegment, secondSegment}; return info; } diff --git a/src/coreclr/jit/abi.h b/src/coreclr/jit/abi.h index 28a37d86e3d70..95c62e0b9f9c0 100644 --- a/src/coreclr/jit/abi.h +++ b/src/coreclr/jit/abi.h @@ -3,6 +3,9 @@ #pragma once +class ClassLayout; +enum class WellKnownArg : unsigned; + class ABIPassingSegment { regNumber m_register = REG_NA; @@ -42,8 +45,8 @@ struct ABIPassingInformation private: union { - ABIPassingSegment* Segments; - ABIPassingSegment SingleSegment; + ABIPassingSegment* m_segments; + ABIPassingSegment m_singleSegment; }; public: @@ -72,6 +75,7 @@ struct ABIPassingInformation ABIPassingSegment& Segment(unsigned index); bool HasAnyRegisterSegment() const; + bool HasAnyFloatingRegisterSegment() const; bool HasAnyStackSegment() const; bool HasExactlyOneRegisterSegment() const; bool HasExactlyOneStackSegment() const; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index fd9d2282c35cb..3f2ace8415866 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1258,87 +1258,6 @@ void CallArgABIInformation::SetHfaType(var_types type, unsigned hfaSlots) } } -//--------------------------------------------------------------- -// SetByteSize: Set information related to this argument's size and alignment. -// -// Arguments: -// byteSize - The size in bytes of the argument. -// byteAlignment - The alignment in bytes of the argument. -// isStruct - Whether this arg is a struct. -// isFloatHfa - Whether this is a float HFA. -// -// Remarks: -// This function will determine how the argument size needs to be rounded. On -// most ABIs all arguments are rounded to stack pointer size, but Apple arm64 -// ABI is an exception as it allows packing some small arguments into the -// same stack slot. -// -void CallArgABIInformation::SetByteSize(unsigned byteSize, unsigned byteAlignment, bool isStruct, bool isFloatHfa) -{ - unsigned roundedByteSize; - if (compAppleArm64Abi()) - { - // Only struct types need extension or rounding to pointer size, but HFA does not. - if (isStruct && !isFloatHfa) - { - roundedByteSize = roundUp(byteSize, TARGET_POINTER_SIZE); - } - else - { - roundedByteSize = byteSize; - } - } - else - { - roundedByteSize = roundUp(byteSize, TARGET_POINTER_SIZE); - } - -#if !defined(TARGET_ARM) - // Arm32 could have a struct with 8 byte alignment - // which rounded size % 8 is not 0. - assert(byteAlignment != 0); - assert(roundedByteSize % byteAlignment == 0); -#endif // TARGET_ARM - - ByteSize = roundedByteSize; - ByteAlignment = byteAlignment; -} - -//--------------------------------------------------------------- -// SetMultiRegsNumw: Set the registers for a multi-reg arg using 'sequential' registers. -// -// Remarks: -// This assumes that `NumRegs` and the first reg num has already been set and -// determines how many sequential registers are necessary to pass the -// argument. -// Note that on ARM the registers set may skip odd float registers if the arg -// is a HFA of doubles, since double and float registers overlap. -void CallArgABIInformation::SetMultiRegNums() -{ -#if FEATURE_MULTIREG_ARGS && !defined(UNIX_AMD64_ABI) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) - if (NumRegs == 1) - { - return; - } - - regNumber argReg = GetRegNum(0); -#ifdef TARGET_ARM - unsigned int regSize = (GetHfaType() == TYP_DOUBLE) ? 2 : 1; -#else - unsigned int regSize = 1; -#endif - - if (NumRegs > MAX_ARG_REG_COUNT) - NO_WAY("Multireg argument exceeds the maximum length"); - - for (unsigned int regIndex = 1; regIndex < NumRegs; regIndex++) - { - argReg = (regNumber)(argReg + regSize); - SetRegNum(regIndex, argReg); - } -#endif // FEATURE_MULTIREG_ARGS && !defined(UNIX_AMD64_ABI) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) -} - //--------------------------------------------------------------- // GetStackByteSize: Get the number of stack bytes used to pass this argument. // @@ -1468,7 +1387,7 @@ void CallArg::CheckIsStruct() CallArgs::CallArgs() : m_head(nullptr) , m_lateHead(nullptr) - , m_nextStackByteOffset(0) + , m_argsStackSize(0) #ifdef UNIX_X86_ABI , m_stkSizeBytes(0) , m_padStkAlign(0) @@ -9817,7 +9736,7 @@ void CallArgs::InternalCopyFrom(Compiler* comp, CallArgs* other, CopyNodeFunc co { assert((m_head == nullptr) && (m_lateHead == nullptr)); - m_nextStackByteOffset = other->m_nextStackByteOffset; + m_argsStackSize = other->m_argsStackSize; m_hasThisPointer = other->m_hasThisPointer; m_hasRetBuffer = other->m_hasRetBuffer; m_isVarArgs = other->m_isVarArgs; @@ -9845,6 +9764,7 @@ void CallArgs::InternalCopyFrom(Compiler* comp, CallArgs* other, CopyNodeFunc co carg->m_isTmp = arg.m_isTmp; carg->m_processed = arg.m_processed; carg->AbiInfo = arg.AbiInfo; + carg->NewAbiInfo = arg.NewAbiInfo; *tail = carg; tail = &carg->m_next; } @@ -13174,6 +13094,8 @@ const char* Compiler::gtGetWellKnownArgNameForArgMsg(WellKnownArg arg) return "swift error"; case WellKnownArg::SwiftSelf: return "swift self"; + case WellKnownArg::X86TailCallSpecialArg: + return "tail call"; default: return nullptr; } diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index a5e09db8a6d95..4092f40e7d2a6 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -18,6 +18,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #define _GENTREE_H_ /*****************************************************************************/ +#include "abi.h" #include "vartype.h" // For "var_types" #include "target.h" // For "regNumber" #include "ssaconfig.h" // For "SsaConfig::RESERVED_SSA_NUM" @@ -4480,6 +4481,7 @@ enum class WellKnownArg : unsigned DispatchIndirectCallTarget, SwiftError, SwiftSelf, + X86TailCallSpecialArg, }; #ifdef DEBUG @@ -4492,12 +4494,10 @@ struct CallArgABIInformation : NumRegs(0) , ByteOffset(0) , ByteSize(0) - , ByteAlignment(0) #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) , StructFloatFieldType() #endif , ArgType(TYP_UNDEF) - , IsBackFilled(false) , PassedByRef(false) #if FEATURE_ARG_SPLIT , m_isSplit(false) @@ -4523,7 +4523,6 @@ struct CallArgABIInformation unsigned NumRegs; unsigned ByteOffset; unsigned ByteSize; - unsigned ByteAlignment; #if defined(UNIX_AMD64_ABI) // Unix amd64 will split floating point types and integer types in structs // between floating point and general purpose registers. Keep track of that @@ -4541,9 +4540,6 @@ struct CallArgABIInformation // that type. Note that if a struct is passed by reference, this will still // be the struct type. var_types ArgType : 5; - // True when the argument fills a register slot skipped due to alignment - // requirements of previous arguments. - bool IsBackFilled : 1; // True iff the argument is passed by reference. bool PassedByRef : 1; @@ -4641,18 +4637,11 @@ struct CallArgABIInformation #endif // TARGET_LOONGARCH64 || TARGET_RISCV64 } - void SetByteSize(unsigned byteSize, unsigned byteAlignment, bool isStruct, bool isFloatHfa); - // Get the number of bytes that this argument is occupying on the stack, // including padding up to the target pointer size for platforms // where a stack argument can't take less. unsigned GetStackByteSize() const; - // Set the register numbers for a multireg argument. - // There's nothing to do on x64/Ux because the structDesc has already been used to set the - // register numbers. - void SetMultiRegNums(); - // Return number of stack slots that this argument is taking. // This value is not meaningful on Apple arm64 where multiple arguments can // be passed in the same stack slot. @@ -4755,6 +4744,7 @@ class CallArg public: CallArgABIInformation AbiInfo; + ABIPassingInformation NewAbiInfo; CallArg(const NewCallArg& arg) : CallArg() @@ -4817,7 +4807,7 @@ class CallArgs CallArg* m_head; CallArg* m_lateHead; - unsigned m_nextStackByteOffset; + unsigned m_argsStackSize; #ifdef UNIX_X86_ABI // Number of stack bytes pushed before we start pushing these arguments. unsigned m_stkSizeBytes; @@ -4846,7 +4836,6 @@ class CallArgs void AddedWellKnownArg(WellKnownArg arg); void RemovedWellKnownArg(WellKnownArg arg); regNumber GetCustomRegister(Compiler* comp, CorInfoCallConvExtension cc, WellKnownArg arg); - void SplitArg(CallArg* arg, unsigned numRegs, unsigned numSlots); void SortArgs(Compiler* comp, GenTreeCall* call, CallArg** sortedArgs); public: diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 687ac114e329f..84d4de9046e6b 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -1721,7 +1721,8 @@ void Compiler::lvaInitVarDsc(LclVarDsc* varDsc, template void Compiler::lvaClassifyParameterABI(Classifier& classifier) { - lvaParameterPassingInfo = new (this, CMK_LvaTable) ABIPassingInformation[info.compArgsCount]; + lvaParameterPassingInfo = + info.compArgsCount == 0 ? nullptr : new (this, CMK_LvaTable) ABIPassingInformation[info.compArgsCount]; for (unsigned i = 0; i < info.compArgsCount; i++) { @@ -1768,11 +1769,6 @@ void Compiler::lvaClassifyParameterABI(Classifier& classifier) // void Compiler::lvaClassifyParameterABI() { - if (info.compArgsCount == 0) - { - return; - } - ClassifierInfo cInfo; cInfo.CallConv = info.compCallConv; cInfo.IsVarArgs = info.compIsVarArgs; diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index c7790fb6199d6..a966813f04754 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -3448,10 +3448,13 @@ void Lowering::LowerCFGCall(GenTreeCall* call) call->gtArgs.PushLateBack(targetArg); // Set up ABI information for this arg. + targetArg->NewAbiInfo = + ABIPassingInformation::FromSegment(comp, ABIPassingSegment::InRegister(REG_DISPATCH_INDIRECT_CALL_ADDR, + 0, TARGET_POINTER_SIZE)); targetArg->AbiInfo.ArgType = callTarget->TypeGet(); targetArg->AbiInfo.SetRegNum(0, REG_DISPATCH_INDIRECT_CALL_ADDR); - targetArg->AbiInfo.NumRegs = 1; - targetArg->AbiInfo.SetByteSize(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE, false, false); + targetArg->AbiInfo.NumRegs = 1; + targetArg->AbiInfo.ByteSize = TARGET_POINTER_SIZE; // Lower the newly added args now that call is updated LowerArg(call, targetArg, true /* late */); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 54d801109d814..7b57180e59fa0 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -602,14 +602,7 @@ RefPosition* LinearScan::newRefPosition(Interval* theInterval, regNumber physicalReg = genRegNumFromMask(mask, theInterval->registerType); RefPosition* pos = newRefPosition(physicalReg, theLocation, RefTypeFixedReg, nullptr, mask); assert(theInterval != nullptr); -#if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - // The LoongArch64's ABI which the float args maybe passed by integer register - // when no float register left but free integer register. - assert((regType(theInterval->registerType) == FloatRegisterType) || - (allRegs(theInterval->registerType) & mask) != 0); -#else assert((allRegs(theInterval->registerType) & mask) != 0); -#endif } RefPosition* newRP = newRefPositionRaw(theLocation, theTreeNode, theRefType); diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index d35001114cdb4..1d2c876779bd4 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -786,6 +786,8 @@ const char* getWellKnownArgName(WellKnownArg arg) return "SwiftError"; case WellKnownArg::SwiftSelf: return "SwiftSelf"; + case WellKnownArg::X86TailCallSpecialArg: + return "X86TailCallSpecialArg"; } return "N/A"; @@ -811,7 +813,6 @@ void CallArg::Dump(Compiler* comp) { printf(", byteSize=%u, byteOffset=%u", AbiInfo.ByteSize, AbiInfo.ByteOffset); } - printf(", byteAlignment=%u", AbiInfo.ByteAlignment); if (GetLateNode() != nullptr) { printf(", isLate"); @@ -840,10 +841,6 @@ void CallArg::Dump(Compiler* comp) { printf(", isHfa(%s)", varTypeName(AbiInfo.GetHfaType())); } - if (AbiInfo.IsBackFilled) - { - printf(", isBackFilled"); - } if (m_wellKnownArg != WellKnownArg::None) { printf(", wellKnown[%s]", getWellKnownArgName(m_wellKnownArg)); @@ -852,37 +849,6 @@ void CallArg::Dump(Compiler* comp) } #endif -//------------------------------------------------------------------------ -// SplitArg: -// Record that the arg will be split over registers and stack, increasing the -// current stack usage. -// -// Parameters: -// arg - The argument. -// numRegs - The number of registers that will be used. -// numSlots - The number of stack slots that will be used. -// -void CallArgs::SplitArg(CallArg* arg, unsigned numRegs, unsigned numSlots) -{ - assert(numRegs > 0); - assert(numSlots > 0); - - if (m_argsComplete) - { - assert(arg->AbiInfo.IsSplit() == true); - assert(arg->AbiInfo.NumRegs == numRegs); - assert(m_hasStackArgs); - } - else - { - arg->AbiInfo.SetSplit(true); - arg->AbiInfo.NumRegs = numRegs; - arg->AbiInfo.ByteOffset = 0; - m_hasStackArgs = true; - } - m_nextStackByteOffset += numSlots * TARGET_POINTER_SIZE; -} - //------------------------------------------------------------------------ // SetTemp: Set that the specified argument was evaluated into a temp. // @@ -1951,23 +1917,6 @@ GenTree* Compiler::fgInsertCommaFormTemp(GenTree** ppTree) void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call) { assert(&call->gtArgs == this); - unsigned argIndex = 0; - unsigned intArgRegNum = 0; - unsigned fltArgRegNum = 0; - - bool callHasRetBuffArg = HasRetBuffer(); - bool callIsVararg = IsVarArgs(); - -#ifdef TARGET_ARM - regMaskTP argSkippedRegMask = RBM_NONE; - regMaskTP fltArgSkippedRegMask = RBM_NONE; -#endif // TARGET_ARM - -#if defined(TARGET_X86) - unsigned maxRegArgs = MAX_REG_ARG; // X86: non-const, must be calculated -#else - const unsigned maxRegArgs = MAX_REG_ARG; // other arch: fixed constant number -#endif if (IsAbiInformationDetermined()) { @@ -1976,13 +1925,12 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call } JITDUMP("Initializing arg info for %d.%s:\n", call->gtTreeID, GenTree::OpName(call->gtOper)); - m_nextStackByteOffset = INIT_ARG_STACK_SLOT * TARGET_POINTER_SIZE; - m_hasRegArgs = false; - m_hasStackArgs = false; + m_hasRegArgs = false; + m_hasStackArgs = false; // At this point, we should not have any late args, as this needs to be done before those are determined. assert(m_lateHead == nullptr); - if (TargetOS::IsUnix && callIsVararg) + if (TargetOS::IsUnix && IsVarArgs()) { // Currently native varargs is not implemented on non windows targets. // @@ -2119,102 +2067,15 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call } #endif - unsigned numArgs = CountArgs(); - -#ifdef TARGET_X86 - // Compute the maximum number of arguments that can be passed in registers. - // For X86 we handle the varargs and unmanaged calling conventions - -#ifndef UNIX_X86_ABI - if (call->gtFlags & GTF_CALL_POP_ARGS) - { - noway_assert(intArgRegNum < MAX_REG_ARG); - // No more register arguments for varargs (CALL_POP_ARGS) - maxRegArgs = intArgRegNum; - - // Add in this arg - if (HasThisPointer()) - { - maxRegArgs++; - } - // Add in the ret buff arg - if (callHasRetBuffArg) - { - maxRegArgs++; - } - } -#endif // UNIX_X86_ABI - - if (call->IsUnmanaged()) - { - noway_assert(intArgRegNum == 0); - - if (call->unmgdCallConv == CorInfoCallConvExtension::Thiscall) - { - noway_assert((call->gtArgs.GetArgByIndex(0)->GetEarlyNode() == nullptr) || - (call->gtArgs.GetArgByIndex(0)->GetEarlyNode()->TypeGet() == TYP_I_IMPL) || - (call->gtArgs.GetArgByIndex(0)->GetEarlyNode()->TypeGet() == TYP_BYREF)); - maxRegArgs = 1; - } - else - { - maxRegArgs = 0; - } -#ifdef UNIX_X86_ABI - // Add in the ret buff arg - if (callHasRetBuffArg && - call->unmgdCallConv != CorInfoCallConvExtension::C && // C and Stdcall calling conventions do not - call->unmgdCallConv != CorInfoCallConvExtension::Stdcall) // use registers to pass arguments. - maxRegArgs++; -#endif - } -#endif // TARGET_X86 - - /* Morph the user arguments */ - -#if defined(TARGET_ARM) - - // The ARM ABI has a concept of back-filling of floating-point argument registers, according - // to the "Procedure Call Standard for the ARM Architecture" document, especially - // section 6.1.2.3 "Parameter passing". Back-filling is where floating-point argument N+1 can - // appear in a lower-numbered register than floating point argument N. That is, argument - // register allocation is not strictly increasing. To support this, we need to keep track of unused - // floating-point argument registers that we can back-fill. We only support 4-byte float and - // 8-byte double types, and one to four element HFAs composed of these types. With this, we will - // only back-fill single registers, since there is no way with these types to create - // an alignment hole greater than one register. However, there can be up to 3 back-fill slots - // available (with 16 FP argument registers). Consider this code: - // - // struct HFA { float x, y, z; }; // a three element HFA - // void bar(float a1, // passed in f0 - // double a2, // passed in f2/f3; skip f1 for alignment - // HFA a3, // passed in f4/f5/f6 - // double a4, // passed in f8/f9; skip f7 for alignment. NOTE: it doesn't fit in the f1 back-fill slot - // HFA a5, // passed in f10/f11/f12 - // double a6, // passed in f14/f15; skip f13 for alignment. NOTE: it doesn't fit in the f1 or f7 back-fill - // // slots - // float a7, // passed in f1 (back-filled) - // float a8, // passed in f7 (back-filled) - // float a9, // passed in f13 (back-filled) - // float a10) // passed on the stack in [OutArg+0] - // - // Note that if we ever support FP types with larger alignment requirements, then there could - // be more than single register back-fills. - // - // Once we assign a floating-pointer register to the stack, they all must be on the stack. - // See "Procedure Call Standard for the ARM Architecture", section 6.1.2.3, "The back-filling - // continues only so long as no VFP CPRC has been allocated to a slot on the stack." - // We set anyFloatStackArgs to true when a floating-point argument has been assigned to the stack - // and prevent any additional floating-point arguments from going in registers. - - bool anyFloatStackArgs = false; - -#endif // TARGET_ARM - -#ifdef UNIX_AMD64_ABI - SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR structDesc; -#endif // UNIX_AMD64_ABI + ClassifierInfo info; + info.CallConv = call->GetUnmanagedCallConv(); + // X86 tailcall helper is considered varargs, but not for ABI classification purposes. + info.IsVarArgs = call->IsVarargs() && !call->IsTailCallViaJitHelper(); + info.HasThis = call->gtArgs.HasThisPointer(); + info.HasRetBuff = call->gtArgs.HasRetBuffer(); + PlatformClassifier classifier(info); + // Morph the user arguments for (CallArg& arg : Args()) { assert(arg.GetEarlyNode() != nullptr); @@ -2231,218 +2092,46 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call // where the "argx" node can legally have a type that is not ABI-compatible with the one in the signature. const var_types argSigType = arg.GetSignatureType(); const CORINFO_CLASS_HANDLE argSigClass = arg.GetSignatureClassHandle(); + ClassLayout* argLayout = argSigClass == NO_CLASS_HANDLE ? nullptr : comp->typGetObjLayout(argSigClass); - // Setup any HFA information about the argument. - bool isHfaArg = false; - var_types hfaType = TYP_UNDEF; - unsigned hfaSlots = 0; - - bool passUsingFloatRegs; - unsigned argAlignBytes = TARGET_POINTER_SIZE; - unsigned size = 0; - unsigned byteSize = 0; - - if (GlobalJitOptions::compFeatureHfa) - { - hfaType = comp->GetHfaType(argSigClass); - isHfaArg = varTypeIsValidHfaType(hfaType); + ABIPassingInformation abiInfo; - if (TargetOS::IsWindows && TargetArchitecture::IsArm64 && callIsVararg) - { - // Make sure for vararg methods isHfaArg is not true. - isHfaArg = false; - } - - if (isHfaArg) - { - hfaSlots = comp->GetHfaCount(argSigClass); - - // If we have a HFA struct it's possible we transition from a method that originally - // only had integer types to now start having FP types. We have to communicate this - // through this flag since LSRA later on will use this flag to determine whether - // or not to track the FP register set. - // - comp->compFloatingPointUsed = true; - } - } - - const bool isFloatHfa = (hfaType == TYP_FLOAT); - -#ifdef TARGET_ARM - passUsingFloatRegs = - !callIsVararg && (isHfaArg || varTypeUsesFloatReg(argSigType)) && !comp->opts.compUseSoftFP; - bool passUsingIntRegs = passUsingFloatRegs ? false : (intArgRegNum < MAX_REG_ARG); + // Some well known args have custom register assignment. + // These should not affect the placement of any other args or stack space required. + // Example: on AMD64 R10 and R11 are used for indirect VSD (generic interface) and cookie calls. + // TODO-Cleanup: Integrate this into the new style ABI classifiers. + regNumber nonStdRegNum = GetCustomRegister(comp, call->GetUnmanagedCallConv(), arg.GetWellKnownArg()); - // TODO-Cleanup: use "eeGetArgSizeAlignment" here. See also: https://github.com/dotnet/runtime/issues/46026. - if (varTypeIsStruct(argSigType)) + if (nonStdRegNum == REG_NA) { - argAlignBytes = comp->info.compCompHnd->getClassAlignmentRequirement(argSigClass); + abiInfo = classifier.Classify(comp, argSigType, argLayout, arg.GetWellKnownArg()); } else { - argAlignBytes = genTypeSize(argSigType); - } - - argAlignBytes = roundUp(argAlignBytes, TARGET_POINTER_SIZE); - - if (argAlignBytes == 2 * TARGET_POINTER_SIZE) - { - if (passUsingFloatRegs) - { - if (fltArgRegNum % 2 == 1) - { - fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_FLOAT); - fltArgRegNum++; - } - } - else if (passUsingIntRegs) - { - if (intArgRegNum % 2 == 1) - { - argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL); - intArgRegNum++; - } - } + ABIPassingSegment segment = ABIPassingSegment::InRegister(nonStdRegNum, 0, TARGET_POINTER_SIZE); + abiInfo = ABIPassingInformation::FromSegment(comp, segment); } -#elif defined(TARGET_ARM64) - - assert(!callIsVararg || !isHfaArg); - passUsingFloatRegs = !callIsVararg && (isHfaArg || varTypeUsesFloatReg(argSigType)); + JITDUMP("Argument %u ABI info: ", GetIndex(&arg)); + DBEXEC(VERBOSE, abiInfo.Dump()); -#elif defined(TARGET_AMD64) + arg.NewAbiInfo = abiInfo; + arg.AbiInfo = CallArgABIInformation(); - passUsingFloatRegs = varTypeIsFloating(argSigType); - -#elif defined(TARGET_X86) - - passUsingFloatRegs = false; - -#elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - - assert(!callIsVararg && !isHfaArg); - passUsingFloatRegs = varTypeUsesFloatReg(argSigType); - DWORD floatFieldFlags = STRUCT_NO_FLOAT_FIELD; - -#else -#error Unsupported or unset target architecture -#endif // TARGET* - - bool isBackFilled = false; - unsigned nextFltArgRegNum = fltArgRegNum; // This is the next floating-point argument register number to use - bool isStructArg = varTypeIsStruct(argSigType); - var_types structBaseType = TYP_STRUCT; - unsigned structSize = 0; - bool passStructByRef = false; - - // - // Figure out the size of the argument. This is either in number of registers, or number of - // TARGET_POINTER_SIZE stack slots, or the sum of these if the argument is split between the registers and - // the stack. - // - - if (isStructArg) - { - GenTree* actualArg = argx->gtEffectiveVal(); - - // Here we look at "actualArg" to avoid calling "getClassSize". - structSize = actualArg->TypeIs(TYP_STRUCT) ? actualArg->GetLayout(comp)->GetSize() : genTypeSize(actualArg); - - assert(structSize == comp->info.compCompHnd->getClassSize(argSigClass)); - } -#if defined(TARGET_AMD64) -#ifdef UNIX_AMD64_ABI - if (!isStructArg) - { - size = 1; // On AMD64, all primitives fit in a single (64-bit) 'slot' - byteSize = genTypeSize(argSigType); - } - else - { - size = (unsigned)(roundUp(structSize, TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE; - byteSize = structSize; - comp->eeGetSystemVAmd64PassStructInRegisterDescriptor(argSigClass, &structDesc); - } -#else // !UNIX_AMD64_ABI - size = 1; // On AMD64 Windows, all args fit in a single (64-bit) 'slot' - if (!isStructArg) - { - byteSize = genTypeSize(argSigType); - } - -#endif // UNIX_AMD64_ABI -#elif defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - if (isStructArg) - { - if (isHfaArg) - { - // HFA structs are passed by value in multiple registers. - // The "size" in registers may differ the size in pointer-sized units. - size = hfaSlots; - byteSize = structSize; - } - else - { - // Structs are either passed in 1 or 2 (64-bit) slots. - // Structs that are the size of 2 pointers are passed by value in multiple registers, - // if sufficient registers are available. - // Structs that are larger than 2 pointers (except for HFAs) are passed by - // reference (to a copy) - size = (unsigned)(roundUp(structSize, TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE; - byteSize = structSize; - if (size > 2) - { - size = 1; - } - } - // Note that there are some additional rules for multireg structs on ARM64. - // (i.e they cannot be split between registers and the stack) - } - else - { - size = 1; // Otherwise, all primitive types fit in a single (64-bit) 'slot' - byteSize = genTypeSize(argSigType); - } -#elif defined(TARGET_ARM) || defined(TARGET_X86) - if (isStructArg) - { - size = (unsigned)(roundUp(structSize, TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE; - byteSize = structSize; - } - else - { - // The typical case. - // Long/double type argument(s) will be modified as needed in Lowering. - size = genTypeStSz(argSigType); - byteSize = genTypeSize(argSigType); - } -#else -#error Unsupported or unset target architecture -#endif // TARGET_XXX - - if (isStructArg) + if (varTypeIsStruct(argSigType)) { assert(argx == arg.GetEarlyNode()); - assert(structSize != 0); Compiler::structPassingKind howToPassStruct; - structBaseType = comp->getArgTypeForStruct(argSigClass, &howToPassStruct, callIsVararg, structSize); - passStructByRef = (howToPassStruct == Compiler::SPK_ByReference); -#if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - if (!passStructByRef) + var_types structBaseType = + comp->getArgTypeForStruct(argSigClass, &howToPassStruct, IsVarArgs(), argLayout->GetSize()); +#if defined(TARGET_RISCV64) || defined(TARGET_LOONGARCH64) + if (arg.NewAbiInfo.HasAnyFloatingRegisterSegment()) { - assert((howToPassStruct == Compiler::SPK_ByValue) || (howToPassStruct == Compiler::SPK_PrimitiveType)); - -#if defined(TARGET_LOONGARCH64) - floatFieldFlags = comp->info.compCompHnd->getLoongArch64PassStructInRegisterFlags(argSigClass); -#else - floatFieldFlags = comp->info.compCompHnd->getRISCV64PassStructInRegisterFlags(argSigClass); -#endif - - passUsingFloatRegs = (floatFieldFlags & STRUCT_HAS_FLOAT_FIELDS_MASK) ? true : false; - comp->compFloatingPointUsed |= passUsingFloatRegs; - - if ((floatFieldFlags & (STRUCT_HAS_FLOAT_FIELDS_MASK ^ STRUCT_FLOAT_FIELD_ONLY_ONE)) != 0) + // Struct passed according to hardware floating-point calling convention + assert(arg.NewAbiInfo.NumSegments <= 2); + assert(!arg.NewAbiInfo.HasAnyStackSegment()); + if (arg.NewAbiInfo.NumSegments == 2) { // On LoongArch64, "getPrimitiveTypeForStruct" will incorrectly return "TYP_LONG" // for "struct { float, float }", and retyping to a primitive here will cause the @@ -2451,558 +2140,165 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call // TODO-LoongArch64: fix "getPrimitiveTypeForStruct". structBaseType = TYP_STRUCT; } - - if ((floatFieldFlags & (STRUCT_HAS_FLOAT_FIELDS_MASK ^ STRUCT_FLOAT_FIELD_ONLY_TWO)) != 0) - { - size = 1; - } - else if ((floatFieldFlags & STRUCT_FLOAT_FIELD_ONLY_TWO) != 0) + else { - size = 2; + assert(arg.NewAbiInfo.NumSegments == 1); + structBaseType = arg.NewAbiInfo.Segment(0).GetRegisterType(); } - } - else // if (passStructByRef) - { - size = 1; - byteSize = TARGET_POINTER_SIZE; - } -#else - if (howToPassStruct == Compiler::SPK_ByReference) - { - byteSize = TARGET_POINTER_SIZE; - } - else - { - byteSize = structSize; - } - if (howToPassStruct == Compiler::SPK_PrimitiveType) - { -#ifdef TARGET_ARM - // TODO-CQ: getArgTypeForStruct should *not* return TYP_DOUBLE for a double struct, - // or for a struct of two floats. This causes the struct to be address-taken. - if (structBaseType == TYP_DOUBLE) - { - size = 2; - } - else -#endif // TARGET_ARM + for (unsigned i = 0; i < arg.NewAbiInfo.NumSegments; ++i) { - size = 1; + arg.AbiInfo.StructFloatFieldType[i] = arg.NewAbiInfo.Segment(i).GetRegisterType(); } } - else if (passStructByRef) - { - size = 1; - } +#endif // defined(TARGET_RISCV64) || defined(TARGET_LOONGARCH64) + arg.AbiInfo.PassedByRef = howToPassStruct == Compiler::SPK_ByReference; + arg.AbiInfo.ArgType = structBaseType == TYP_UNKNOWN ? argx->TypeGet() : structBaseType; + +#ifdef UNIX_AMD64_ABI + comp->eeGetSystemVAmd64PassStructInRegisterDescriptor(argSigClass, &arg.AbiInfo.StructDesc); #endif } - - // The 'size' value has now must have been set. (the original value of zero is an invalid value) - assert(size != 0); - assert(byteSize != 0); - - if (compAppleArm64Abi()) + else { - // Arm64 Apple has a special ABI for passing small size arguments on stack, - // bytes are aligned to 1-byte, shorts to 2-byte, int/float to 4-byte, etc. - // It means passing 8 1-byte arguments on stack can take as small as 8 bytes. - argAlignBytes = comp->eeGetArgSizeAlignment(argSigType, isFloatHfa); + arg.AbiInfo.ArgType = argx->TypeGet(); } -#if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - regNumber nextOtherRegNum = REG_STK; -#endif - // - // Figure out if the argument will be passed in a register. - // - bool isRegArg = false; - regNumber nonStdRegNum = REG_NA; - - if (isRegParamType(genActualType(argSigType)) -#ifdef UNIX_AMD64_ABI - && (!isStructArg || structDesc.passedInRegisters) -#elif defined(TARGET_X86) - || (isStructArg && comp->isTrivialPointerSizedStruct(argSigClass)) -#endif - ) + if (abiInfo.IsSplitAcrossRegistersAndStack()) { -#ifdef TARGET_ARM - if (passUsingFloatRegs) - { - // First, see if it can be back-filled - if (!anyFloatStackArgs && // Is it legal to back-fill? (We haven't put any FP args on the stack yet) - (fltArgSkippedRegMask != RBM_NONE) && // Is there an available back-fill slot? - (size == 1)) // The size to back-fill is one float register - { - // Back-fill the register. - isBackFilled = true; - regMaskTP backFillBitMask = genFindLowestBit(fltArgSkippedRegMask); - fltArgSkippedRegMask &= - ~backFillBitMask; // Remove the back-filled register(s) from the skipped mask - nextFltArgRegNum = genMapFloatRegNumToRegArgNum(genRegNumFromMask(backFillBitMask)); - assert(nextFltArgRegNum < MAX_FLOAT_REG_ARG); - } - - // Does the entire float, double, or HFA fit in the FP arg registers? - // Check if the last register needed is still in the argument register range. - isRegArg = (nextFltArgRegNum + size - 1) < MAX_FLOAT_REG_ARG; - - if (!isRegArg) - { - anyFloatStackArgs = true; - } - } - else - { - isRegArg = intArgRegNum < MAX_REG_ARG; - } -#elif defined(TARGET_ARM64) - if (passUsingFloatRegs) - { - // Check if the last register needed is still in the fp argument register range. - isRegArg = (nextFltArgRegNum + (size - 1)) < MAX_FLOAT_REG_ARG; - - // Do we have a HFA arg that we wanted to pass in registers, but we ran out of FP registers? - if (isHfaArg && !isRegArg) - { - // recompute the 'size' so that it represent the number of stack slots rather than the number of - // registers - // - unsigned roundupSize = (unsigned)roundUp(structSize, TARGET_POINTER_SIZE); - size = roundupSize / TARGET_POINTER_SIZE; + m_hasStackArgs = true; + m_hasRegArgs = true; - // We also must update fltArgRegNum so that we no longer try to - // allocate any new floating point registers for args - // This prevents us from backfilling a subsequent arg into d7 - // - fltArgRegNum = MAX_FLOAT_REG_ARG; - } - } - else + arg.AbiInfo.SetSplit(true); + arg.AbiInfo.ByteOffset = 0; + unsigned regNumIndex = 0; + for (unsigned i = 0; i < abiInfo.NumSegments; i++) { - // Check if the last register needed is still in the int argument register range. - isRegArg = (intArgRegNum + (size - 1)) < maxRegArgs; - - // Did we run out of registers when we had a 16-byte struct (size===2) ? - // (i.e we only have one register remaining but we needed two registers to pass this arg) - // This prevents us from backfilling a subsequent arg into x7 - // - if (!isRegArg && (size > 1)) + const ABIPassingSegment& segment = abiInfo.Segment(i); + if (segment.IsPassedInRegister()) { - // Arm64 windows native varargs allows splitting a 16 byte struct (or SIMD type) between stack - // and the last general purpose register. - if (TargetOS::IsWindows && callIsVararg) + if (regNumIndex < MAX_ARG_REG_COUNT) { - // Override the decision and force a split. - isRegArg = (intArgRegNum + (size - 1)) <= maxRegArgs; + arg.AbiInfo.SetRegNum(regNumIndex, segment.GetRegister()); + regNumIndex++; } - else - { - // We also must update intArgRegNum so that we no longer try to - // allocate any new general purpose registers for args - // - intArgRegNum = maxRegArgs; - } - } - } -#elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - if (passUsingFloatRegs) - { - // Check if the last register needed is still in the fp argument register range. - passUsingFloatRegs = isRegArg = (nextFltArgRegNum + (size - 1)) < MAX_FLOAT_REG_ARG; - - if (isStructArg) + arg.AbiInfo.NumRegs++; + } + else { - if ((floatFieldFlags & (STRUCT_FLOAT_FIELD_FIRST | STRUCT_FLOAT_FIELD_SECOND)) && - passUsingFloatRegs) - { - passUsingFloatRegs = isRegArg = intArgRegNum < maxRegArgs; - } - - if (!passUsingFloatRegs) - { - size = structSize > 8 ? 2 : 1; - structBaseType = structSize <= 8 ? TYP_I_IMPL : TYP_STRUCT; - floatFieldFlags = 0; - } - else if (passUsingFloatRegs) - { - if ((floatFieldFlags & STRUCT_FLOAT_FIELD_ONLY_TWO) != 0) - { - nextOtherRegNum = genMapFloatRegArgNumToRegNum(nextFltArgRegNum + 1); - } - else if ((floatFieldFlags & STRUCT_FLOAT_FIELD_SECOND) != 0) - { - assert(size == 1); - size = 2; - passUsingFloatRegs = false; - nextOtherRegNum = genMapFloatRegArgNumToRegNum(nextFltArgRegNum); - } - else if ((floatFieldFlags & STRUCT_FLOAT_FIELD_FIRST) != 0) - { - assert(size == 1); - size = 2; - nextOtherRegNum = genMapIntRegArgNumToRegNum(intArgRegNum, call->GetUnmanagedCallConv()); - } - } + assert(segment.GetStackOffset() == 0); } - - assert(!isHfaArg); // LoongArch64 does not support HFA. } + } + else if (abiInfo.HasAnyRegisterSegment()) + { + // This is a register argument + m_hasRegArgs = true; - // if we run out of floating-point argument registers, try the int argument registers. - if (!isRegArg) + unsigned regNumIndex = 0; + for (unsigned i = 0; i < abiInfo.NumSegments; i++) { - // Check if the last register needed is still in the int argument register range. - isRegArg = (intArgRegNum + (size - 1)) < maxRegArgs; - if (!passUsingFloatRegs && isRegArg && (size > 1)) - { - nextOtherRegNum = genMapIntRegArgNumToRegNum(intArgRegNum + 1, call->GetUnmanagedCallConv()); - } + const ABIPassingSegment& segment = abiInfo.Segment(i); - // Did we run out of registers when we had a 16-byte struct (size===2) ? - // (i.e we only have one register remaining but we needed two registers to pass this arg) - // - if (!isRegArg && (size > 1)) + if (regNumIndex < MAX_ARG_REG_COUNT) { - // We also must update intArgRegNum so that we no longer try to - // allocate any new general purpose registers for args - // - isRegArg = intArgRegNum < maxRegArgs; // the split-struct case. - nextOtherRegNum = REG_STK; + arg.AbiInfo.SetRegNum(regNumIndex, segment.GetRegister()); + regNumIndex++; } - } -#else // not TARGET_ARM or TARGET_ARM64 or TARGET_LOONGARCH64 or TARGET_RISCV64 -#if defined(UNIX_AMD64_ABI) + arg.AbiInfo.NumRegs++; - // Here a struct can be passed in register following the classifications of its members and size. - // Now make sure there are actually enough registers to do so. - if (isStructArg) - { - unsigned int structFloatRegs = 0; - unsigned int structIntRegs = 0; - for (unsigned int i = 0; i < structDesc.eightByteCount; i++) +#ifdef TARGET_ARM + // Old style ABI info expects two registers counted for these segments. + if (segment.GetRegisterType() == TYP_DOUBLE) { - if (structDesc.IsIntegralSlot(i)) - { - structIntRegs++; - } - else if (structDesc.IsSseSlot(i)) + arg.AbiInfo.NumRegs++; + + if (argSigType == TYP_DOUBLE) { - structFloatRegs++; + arg.AbiInfo.SetRegNum(regNumIndex, REG_NEXT(segment.GetRegister())); + regNumIndex++; } } - - isRegArg = ((nextFltArgRegNum + structFloatRegs) <= MAX_FLOAT_REG_ARG) && - ((intArgRegNum + structIntRegs) <= MAX_REG_ARG); - } - else - { - if (passUsingFloatRegs) - { - isRegArg = nextFltArgRegNum < MAX_FLOAT_REG_ARG; - } - else - { - isRegArg = intArgRegNum < MAX_REG_ARG; - } +#endif } -#else // !defined(UNIX_AMD64_ABI) - isRegArg = (intArgRegNum + (size - 1)) < maxRegArgs; -#endif // !defined(UNIX_AMD64_ABI) -#endif // TARGET_ARM + +#if defined(UNIX_AMD64_ABI) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) + INDEBUG(arg.CheckIsStruct()); +#endif } else { - isRegArg = false; + assert(abiInfo.HasAnyStackSegment()); + // We only expect to see one stack segment in these cases. + assert(abiInfo.NumSegments == 1); + // This is a stack argument + m_hasStackArgs = true; + const ABIPassingSegment& segment = abiInfo.Segment(0); + arg.AbiInfo.SetRegNum(0, REG_STK); + arg.AbiInfo.ByteOffset = segment.GetStackOffset(); } - // Some well known args have custom register assignment. - // These should not affect the placement of any other args or stack space required. - // Example: on AMD64 R10 and R11 are used for indirect VSD (generic interface) and cookie calls. - nonStdRegNum = GetCustomRegister(comp, call->GetUnmanagedCallConv(), arg.GetWellKnownArg()); - if (nonStdRegNum != REG_NA) - { - isRegArg = true; - } - else if (call->IsTailCallViaJitHelper()) + // TODO-Cleanup: remove HFA information from VarDsc. + var_types hfaType = TYP_UNDEF; + bool isHfaArg = false; + unsigned hfaSlots = 0; + + if (GlobalJitOptions::compFeatureHfa) { - // We have already (before calling fgMorphArgs()) appended the 4 special args - // required by the x86 tailcall helper. These args are required to go on the - // stack. Force them to the stack here. - assert(numArgs >= 4); - if (argIndex >= numArgs - 4) + hfaType = comp->GetHfaType(argSigClass); + isHfaArg = varTypeIsValidHfaType(hfaType); + + if (TargetOS::IsWindows && TargetArchitecture::IsArm64 && IsVarArgs()) { - isRegArg = false; + // Make sure for vararg methods isHfaArg is not true. + isHfaArg = false; } - } - // Now we know if the argument goes in registers or not and how big it is. - -#ifdef TARGET_ARM - // If we ever allocate a floating point argument to the stack, then all - // subsequent HFA/float/double arguments go on the stack. - if (!isRegArg && passUsingFloatRegs) - { - for (; fltArgRegNum < MAX_FLOAT_REG_ARG; ++fltArgRegNum) + if (isHfaArg) { - fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_FLOAT); + hfaSlots = comp->GetHfaCount(argSigClass); + + // If we have a HFA struct it's possible we transition from a method that originally + // only had integer types to now start having FP types. We have to communicate this + // through this flag since LSRA later on will use this flag to determine whether + // or not to track the FP register set. + // + comp->compFloatingPointUsed = true; } } - // If we think we're going to split a struct between integer registers and the stack, check to - // see if we've already assigned a floating-point arg to the stack. - if (isRegArg && // We decided above to use a register for the argument - !passUsingFloatRegs && // We're using integer registers - (intArgRegNum + size > MAX_REG_ARG) && // We're going to split a struct type onto registers and stack - anyFloatStackArgs) // We've already used the stack for a floating-point argument + if (arg.AbiInfo.PassedByRef) { - isRegArg = false; // Change our mind; don't pass this struct partially in registers - - // Skip the rest of the integer argument registers - for (; intArgRegNum < MAX_REG_ARG; ++intArgRegNum) - { - argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL); - } + arg.AbiInfo.ByteSize = TARGET_POINTER_SIZE; } -#endif // TARGET_ARM - - arg.AbiInfo = CallArgABIInformation(); - arg.AbiInfo.ArgType = argx->TypeGet(); - - if (isRegArg) + else { - regNumber nextRegNum = REG_STK; + unsigned size = argLayout != nullptr ? argLayout->GetSize() : genTypeSize(argSigType); -#if defined(UNIX_AMD64_ABI) - regNumber nextOtherRegNum = REG_STK; - unsigned int structFloatRegs = 0; - unsigned int structIntRegs = 0; -#endif // defined(UNIX_AMD64_ABI) - - if (nonStdRegNum != REG_NA) + // Apple arm64 reuses the same stack slot for multiple args in some + // cases; old ABI info reflects that in the size. + // Primitives and float HFAs do not necessarily take up full stack + // slots. + if (compAppleArm64Abi() && (!varTypeIsStruct(argSigType) || (isHfaArg && (hfaType == TYP_FLOAT)))) { - nextRegNum = nonStdRegNum; - } -#if defined(UNIX_AMD64_ABI) - else if (isStructArg && structDesc.passedInRegisters) - { - // It is a struct passed in registers. Assign the next available register. - assert((structDesc.eightByteCount <= 2) && "Too many eightbytes."); - regNumber* nextRegNumPtrs[2] = {&nextRegNum, &nextOtherRegNum}; - for (unsigned int i = 0; i < structDesc.eightByteCount; i++) - { - if (structDesc.IsIntegralSlot(i)) - { - *nextRegNumPtrs[i] = - genMapIntRegArgNumToRegNum(intArgRegNum + structIntRegs, call->GetUnmanagedCallConv()); - ++structIntRegs; - } - else if (structDesc.IsSseSlot(i)) - { - *nextRegNumPtrs[i] = genMapFloatRegArgNumToRegNum(nextFltArgRegNum + structFloatRegs); - ++structFloatRegs; - } - } + arg.AbiInfo.ByteSize = size; } -#endif // defined(UNIX_AMD64_ABI) else { - // fill in or update the argInfo table - nextRegNum = passUsingFloatRegs - ? genMapFloatRegArgNumToRegNum(nextFltArgRegNum) - : genMapIntRegArgNumToRegNum(intArgRegNum, call->GetUnmanagedCallConv()); + arg.AbiInfo.ByteSize = roundUp(size, TARGET_POINTER_SIZE); } - -#ifdef WINDOWS_AMD64_ABI - assert(size == 1); -#endif - - // This is a register argument - m_hasRegArgs = true; - arg.AbiInfo.SetRegNum(0, nextRegNum); - arg.AbiInfo.NumRegs = size; - arg.AbiInfo.SetByteSize(byteSize, argAlignBytes, isStructArg, isFloatHfa); -#ifdef UNIX_AMD64_ABI - - if (isStructArg) - { - arg.AbiInfo.StructDesc.CopyFrom(structDesc); - } -#endif - -#if defined(UNIX_AMD64_ABI) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - assert(size <= 2); - - if (size == 2) - { - arg.AbiInfo.SetRegNum(1, nextOtherRegNum); - } - - INDEBUG(arg.CheckIsStruct()); -#endif - - arg.AbiInfo.IsBackFilled = isBackFilled; - - // Set up the next intArgRegNum and fltArgRegNum values. - if (!isBackFilled) - { -#if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - // Increment intArgRegNum by 'size' registers - if (nonStdRegNum == REG_NA) - { - if ((size > 1) && ((intArgRegNum + 1) == maxRegArgs) && (nextOtherRegNum == REG_STK)) - { - // This indicates a partial enregistration of a struct type - assert((isStructArg) || argx->OperIs(GT_FIELD_LIST) || argx->OperIsCopyBlkOp() || - (argx->gtOper == GT_COMMA && (argx->gtFlags & GTF_ASG))); - unsigned numRegsPartial = MAX_REG_ARG - intArgRegNum; - assert((unsigned char)numRegsPartial == numRegsPartial); - SplitArg(&arg, numRegsPartial, size - numRegsPartial); - assert(!passUsingFloatRegs); - assert(size == 2); - intArgRegNum = maxRegArgs; - } - else if ((floatFieldFlags & STRUCT_HAS_FLOAT_FIELDS_MASK) == 0x0) - { - if (passUsingFloatRegs) - { - fltArgRegNum += 1; - } - else - { - intArgRegNum += size; - } - } - else if ((floatFieldFlags & STRUCT_FLOAT_FIELD_ONLY_ONE) != 0) - { - structBaseType = structSize == 8 ? TYP_DOUBLE : TYP_FLOAT; - fltArgRegNum += 1; - arg.AbiInfo.StructFloatFieldType[0] = structBaseType; - } - else if ((floatFieldFlags & (STRUCT_FLOAT_FIELD_FIRST | STRUCT_FLOAT_FIELD_SECOND)) != 0) - { - fltArgRegNum += 1; - intArgRegNum += 1; - if ((floatFieldFlags & STRUCT_FLOAT_FIELD_FIRST) != 0) - { - arg.AbiInfo.StructFloatFieldType[0] = - (floatFieldFlags & STRUCT_FIRST_FIELD_SIZE_IS8) ? TYP_DOUBLE : TYP_FLOAT; - arg.AbiInfo.StructFloatFieldType[1] = - (floatFieldFlags & STRUCT_SECOND_FIELD_SIZE_IS8) ? TYP_LONG : TYP_INT; - } - else - { - arg.AbiInfo.StructFloatFieldType[0] = - (floatFieldFlags & STRUCT_FIRST_FIELD_SIZE_IS8) ? TYP_LONG : TYP_INT; - arg.AbiInfo.StructFloatFieldType[1] = - (floatFieldFlags & STRUCT_SECOND_FIELD_SIZE_IS8) ? TYP_DOUBLE : TYP_FLOAT; - } - } - else if ((floatFieldFlags & STRUCT_FLOAT_FIELD_ONLY_TWO) != 0) - { - fltArgRegNum += 2; - arg.AbiInfo.StructFloatFieldType[0] = - (floatFieldFlags & STRUCT_FIRST_FIELD_SIZE_IS8) ? TYP_DOUBLE : TYP_FLOAT; - arg.AbiInfo.StructFloatFieldType[1] = - (floatFieldFlags & STRUCT_SECOND_FIELD_SIZE_IS8) ? TYP_DOUBLE : TYP_FLOAT; - } - } -#else - -#if defined(UNIX_AMD64_ABI) - if (isStructArg) - { - // For this case, we've already set the regNums in the argTabEntry - intArgRegNum += structIntRegs; - fltArgRegNum += structFloatRegs; - } - else -#endif // defined(UNIX_AMD64_ABI) - { - if (nonStdRegNum == REG_NA) - { -#if FEATURE_ARG_SPLIT - // Check for a split (partially enregistered) struct - if (compFeatureArgSplit() && !passUsingFloatRegs && ((intArgRegNum + size) > MAX_REG_ARG)) - { - // This indicates a partial enregistration of a struct type - assert((isStructArg) || argx->OperIs(GT_FIELD_LIST) || argx->OperIsCopyBlkOp() || - (argx->gtOper == GT_COMMA && (argx->gtFlags & GTF_ASG))); - unsigned numRegsPartial = MAX_REG_ARG - intArgRegNum; - assert((unsigned char)numRegsPartial == numRegsPartial); - SplitArg(&arg, numRegsPartial, size - numRegsPartial); - } -#endif // FEATURE_ARG_SPLIT - - if (passUsingFloatRegs) - { - fltArgRegNum += size; - -#ifdef WINDOWS_AMD64_ABI - // Whenever we pass an integer register argument - // we skip the corresponding floating point register argument - intArgRegNum = min(intArgRegNum + size, (unsigned)MAX_REG_ARG); -#endif // WINDOWS_AMD64_ABI - // No supported architecture supports partial structs using float registers. - assert(fltArgRegNum <= MAX_FLOAT_REG_ARG); - } - else - { - // Increment intArgRegNum by 'size' registers - intArgRegNum += size; - -#ifdef WINDOWS_AMD64_ABI - fltArgRegNum = min(fltArgRegNum + size, (unsigned)MAX_FLOAT_REG_ARG); -#endif // WINDOWS_AMD64_ABI - } - } - } -#endif // defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - } - } - else // We have an argument that is not passed in a register - { - // This is a stack argument - m_hasStackArgs = true; - arg.AbiInfo.SetRegNum(0, REG_STK); - m_nextStackByteOffset = roundUp(m_nextStackByteOffset, argAlignBytes); - arg.AbiInfo.ByteOffset = m_nextStackByteOffset; - arg.AbiInfo.SetByteSize(byteSize, argAlignBytes, isStructArg, isFloatHfa); - - m_nextStackByteOffset += arg.AbiInfo.ByteSize; -#ifdef UNIX_AMD64_ABI - // TODO-Amd64-Unix-CQ: This is temporary (see also in fgMorphArgs). - if (structDesc.passedInRegisters) - { - arg.AbiInfo.StructDesc.CopyFrom(structDesc); - } -#endif } if (isHfaArg) { arg.AbiInfo.SetHfaType(hfaType, hfaSlots); } - - arg.AbiInfo.SetMultiRegNums(); - - if (varTypeIsStruct(arg.GetSignatureType())) - { - arg.AbiInfo.PassedByRef = passStructByRef; - arg.AbiInfo.ArgType = (structBaseType == TYP_UNKNOWN) ? argx->TypeGet() : structBaseType; - } - else - { - arg.AbiInfo.ArgType = argx->TypeGet(); - } - - argIndex++; } // end foreach argument loop + m_argsStackSize = classifier.StackSize(); + #ifdef DEBUG if (VERBOSE) { @@ -3029,7 +2325,7 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call // unsigned CallArgs::OutgoingArgsStackSize() const { - unsigned aligned = Compiler::GetOutgoingArgByteSize(m_nextStackByteOffset); + unsigned aligned = Compiler::GetOutgoingArgByteSize(m_argsStackSize); return max(aligned, (unsigned)MIN_ARG_AREA_FOR_CALL); } @@ -5356,8 +4652,8 @@ bool Compiler::fgCanFastTailCall(GenTreeCall* callee, const char** failReason) callee->gtArgs.AddFinalArgsAndDetermineABIInfo(this, callee); - unsigned calleeArgStackSize = 0; - unsigned callerArgStackSize = roundUp(info.compArgStackSize, TARGET_POINTER_SIZE); + unsigned calleeArgStackSize = callee->gtArgs.OutgoingArgsStackSize(); + unsigned callerArgStackSize = roundUp(lvaParameterStackSize, TARGET_POINTER_SIZE); auto reportFastTailCallDecision = [&](const char* thisFailReason) { if (failReason != nullptr) @@ -5409,25 +4705,16 @@ bool Compiler::fgCanFastTailCall(GenTreeCall* callee, const char** failReason) #endif // DEBUG }; +#if defined(TARGET_ARM) || defined(TARGET_RISCV64) for (CallArg& arg : callee->gtArgs.Args()) { - unsigned stackSize = arg.AbiInfo.GetStackByteSize(); - if (stackSize > 0) - { - calleeArgStackSize = roundUp(calleeArgStackSize, arg.AbiInfo.ByteAlignment); - calleeArgStackSize += stackSize; - } - -#if defined(TARGET_ARM) || defined(TARGET_RISCV64) - if (arg.AbiInfo.IsSplit()) + if (arg.NewAbiInfo.IsSplitAcrossRegistersAndStack()) { reportFastTailCallDecision("Argument splitting in callee is not supported on " TARGET_READABLE_NAME); return false; } -#endif // TARGET_ARM || TARGET_RISCV64 } - - calleeArgStackSize = GetOutgoingArgByteSize(calleeArgStackSize); +#endif // TARGET_ARM || TARGET_RISCV64 #if defined(TARGET_ARM) || defined(TARGET_RISCV64) if (compHasSplitParam) @@ -7156,19 +6443,26 @@ void Compiler::fgMorphTailCallViaJitHelper(GenTreeCall* call) unsigned nOldStkArgsWords = (compArgSize - (codeGen->intRegState.rsCalleeRegArgCount * REGSIZE_BYTES)) / REGSIZE_BYTES; GenTree* arg3Node = gtNewIconNode((ssize_t)nOldStkArgsWords, TYP_I_IMPL); - CallArg* arg3 = call->gtArgs.PushBack(this, NewCallArg::Primitive(arg3Node)); + CallArg* arg3 = + call->gtArgs.PushBack(this, NewCallArg::Primitive(arg3Node).WellKnown(WellKnownArg::X86TailCallSpecialArg)); // Inject a placeholder for the count of outgoing stack arguments that the Lowering phase will generate. // The constant will be replaced. GenTree* arg2Node = gtNewIconNode(9, TYP_I_IMPL); - CallArg* arg2 = call->gtArgs.InsertAfter(this, arg3, NewCallArg::Primitive(arg2Node)); + CallArg* arg2 = + call->gtArgs.InsertAfter(this, arg3, + NewCallArg::Primitive(arg2Node).WellKnown(WellKnownArg::X86TailCallSpecialArg)); // Inject a placeholder for the flags. // The constant will be replaced. GenTree* arg1Node = gtNewIconNode(8, TYP_I_IMPL); - CallArg* arg1 = call->gtArgs.InsertAfter(this, arg2, NewCallArg::Primitive(arg1Node)); + CallArg* arg1 = + call->gtArgs.InsertAfter(this, arg2, + NewCallArg::Primitive(arg1Node).WellKnown(WellKnownArg::X86TailCallSpecialArg)); // Inject a placeholder for the real call target that the Lowering phase will generate. // The constant will be replaced. GenTree* arg0Node = gtNewIconNode(7, TYP_I_IMPL); - CallArg* arg0 = call->gtArgs.InsertAfter(this, arg1, NewCallArg::Primitive(arg0Node)); + CallArg* arg0 = + call->gtArgs.InsertAfter(this, arg1, + NewCallArg::Primitive(arg0Node).WellKnown(WellKnownArg::X86TailCallSpecialArg)); // It is now a varargs tail call. call->gtArgs.SetIsVarArgs(); diff --git a/src/coreclr/jit/targetx86.cpp b/src/coreclr/jit/targetx86.cpp index eb6ced79ce1eb..e00bba78a3a53 100644 --- a/src/coreclr/jit/targetx86.cpp +++ b/src/coreclr/jit/targetx86.cpp @@ -84,7 +84,7 @@ ABIPassingInformation X86Classifier::Classify(Compiler* comp, unsigned numSlots = (size + TARGET_POINTER_SIZE - 1) / TARGET_POINTER_SIZE; bool canEnreg = false; - if (m_regs.Count() >= numSlots) + if ((m_regs.Count() >= numSlots) && (wellKnownParam != WellKnownArg::X86TailCallSpecialArg)) { switch (type) {