From 982daf3f4a0e641f63891a09f3049267368416aa Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 21 Oct 2022 16:39:02 -0700 Subject: [PATCH 01/25] Experimental --- src/coreclr/vm/threads.cpp | 24 ------------------------ src/coreclr/vm/threads.h | 1 - 2 files changed, 25 deletions(-) diff --git a/src/coreclr/vm/threads.cpp b/src/coreclr/vm/threads.cpp index d4919f4e1120f..20438e25103a1 100644 --- a/src/coreclr/vm/threads.cpp +++ b/src/coreclr/vm/threads.cpp @@ -3954,30 +3954,6 @@ DWORD Thread::Wait(CLREvent *pEvent, INT32 timeOut, PendingSync *syncInfo) return dwResult; } -void Thread::Wake(SyncBlock *psb) -{ - WRAPPER_NO_CONTRACT; - - CLREvent* hEvent = NULL; - WaitEventLink *walk = &m_WaitEventLink; - while (walk->m_Next) { - if (walk->m_Next->m_WaitSB == psb) { - hEvent = walk->m_Next->m_EventWait; - // We are guaranteed that only one thread can change walk->m_Next->m_WaitSB - // since the thread is helding the syncblock. - walk->m_Next->m_WaitSB = (SyncBlock*)((DWORD_PTR)walk->m_Next->m_WaitSB | 1); - break; - } -#ifdef _DEBUG - else if ((SyncBlock*)((DWORD_PTR)walk->m_Next & ~1) == psb) { - _ASSERTE (!"Can not wake a thread on the same SyncBlock more than once"); - } -#endif - } - PREFIX_ASSUME (hEvent != NULL); - hEvent->Set(); -} - #define WAIT_INTERRUPT_THREADABORT 0x1 #define WAIT_INTERRUPT_INTERRUPT 0x2 #define WAIT_INTERRUPT_OTHEREXCEPTION 0x4 diff --git a/src/coreclr/vm/threads.h b/src/coreclr/vm/threads.h index c312981b70771..1665565c5e640 100644 --- a/src/coreclr/vm/threads.h +++ b/src/coreclr/vm/threads.h @@ -3170,7 +3170,6 @@ class Thread // Support for Wait/Notify BOOL Block(INT32 timeOut, PendingSync *syncInfo); - void Wake(SyncBlock *psb); DWORD Wait(HANDLE *objs, int cntObjs, INT32 timeOut, PendingSync *syncInfo); DWORD Wait(CLREvent* pEvent, INT32 timeOut, PendingSync *syncInfo); From f443bfa26e5b6c5aa2ad96666991d04ad67ff963 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 21 Oct 2022 16:39:24 -0700 Subject: [PATCH 02/25] Experiment --- temp/RangeListMap.cpp | 309 ++++++++++++++++++++++++++++++ temp/RangeListMap.vcxproj | 135 +++++++++++++ temp/RangeListMap.vcxproj.filters | 22 +++ 3 files changed, 466 insertions(+) create mode 100644 temp/RangeListMap.cpp create mode 100644 temp/RangeListMap.vcxproj create mode 100644 temp/RangeListMap.vcxproj.filters diff --git a/temp/RangeListMap.cpp b/temp/RangeListMap.cpp new file mode 100644 index 0000000000000..cce008ebc3841 --- /dev/null +++ b/temp/RangeListMap.cpp @@ -0,0 +1,309 @@ +// RangeListMap.cpp : This file contains the 'main' function. Program execution begins and ends there. +// + +#include +#include + +#define TARGET_64BIT + + +class Range +{ +public: + void* begin; + void* end; + bool InRange(void* address) + { + return address >= begin && address <= end; + } +}; + +class RangeList +{ +public: + Range range; + RangeList* pRangeListNextForDelete = nullptr; // Used for adding to the cleanup list +}; + +// Unlike a RangeList, a RangeListMini cannot span multiple elements of the last level of the SegmentMap +// Always allocated via calloc +class RangeListMini +{ +public: + RangeListMini* pRangeListMiniNext; + RangeListMini* pRangeListMiniNextForFree; // Used for adding to the cleanup list + Range range; + RangeList* pRangeList; + bool isPrimaryRangeListMini; // RangeListMini are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. +}; + + +// For 64bit, we work with 8KB chunks of memory holding pointers to the next level. This provides 10 bits of address resolution per level. +// For *reasons* the X64 hardware is limited to 57bits of addressable address space, and the minimum granularity that makes sense for range lists is 64KB (or every 2^16 bits) +// Similarly the Arm64 specification requires addresses to use at most 52 bits. Thus we use the maximum addressable range of X64 to provide the real max range +// So the first level is bits [56:47] -> L4 +// Then [46:37] -> L3 +// [36:27] -> L2 +// [26:17] -> L1 +// This leaves 17 bits of the address to be handled by the RangeList linked list +// +// For 32bit VA processes, use 1KB chunks holding pointers to the next level. This provides 8 bites of address resolution per level. [31:24] and [23:16]. + +// The memory safety model for segment maps is that the pointers held within the individual segments can never change other than to go from NULL to a meaningful pointer, +// except for the final level, which is only permitted to change when CleanupWhileNoThreadMayLookupRangeLists is in use. + + + +class SingleElementSegmentMap +{ + +}; + + +template +void VolatileWrite(T* ptr, T val) +{ + *ptr = val; +} + +template +T VolatileRead(T* ptr) +{ + return *ptr; +} + +class RangeListMap +{ +#ifdef TARGET_64BIT + static const uintptr_t entriesPerMapLevel = 1024; + static const uintptr_t mapLevels = 4; + static const uintptr_t maxSetBit = 56; // This is 0 indexed + static const uintptr_t bitsPerLevel = 10; + RangeListMini***** _rangeListL4; + void** GetTopLevelAddress() { return reinterpret_cast(&_rangeListL4); } +#else + static const uintptr_t entriesPerMapLevel = 256; + static const uintptr_t mapLevels = 2; + static const uintptr_t maxSetBit = 31; // This is 0 indexed + static const uintptr_t bitsPerLevel = 8; + + RangeListMini** _rangeListL2; + void** GetTopLevelAddress() { return reinterpret_cast(&_rangeListL2); } +#endif + RangeListMini* pRangeListMinisReadyToDelete = nullptr; + + const uintptr_t bitsAtLastLevel = maxSetBit - (bitsPerLevel * mapLevels) + 1; + const uintptr_t bytesAtLastLevel = (1 << (bitsAtLastLevel - 1)); + + void* AllocateLevel() { return calloc(entriesPerMapLevel, sizeof(void*)); } + + uintptr_t EffectiveBitsForLevel(void* address, uintptr_t level) + { + uintptr_t addressAsInt = (uintptr_t)address; + uintptr_t addressBitsUsedInMap = addressAsInt >> (maxSetBit - (mapLevels * bitsPerLevel)); + uintptr_t addressBitsShifted = addressBitsUsedInMap >> ((level - 1) * bitsPerLevel); + uintptr_t addressBitsUsedInLevel = (entriesPerMapLevel - 1) & addressBitsShifted; + return addressBitsUsedInLevel; + } + + template + T EnsureLevel(T* outerLevel, uintptr_t level) + { + uintptr_t index = EffectiveBitsForLevel(address, level); + T rangeListResult = outerLevel[index]; + if (rangeListResult == NULL) + { + T rangeListNew = static_cast(AllocateLevel()); + T rangeListOld = InterlockedCompareExchangePointer((volatile PVOID*)&outerLevel[index], (PVOID)rangeListNew, NULL); + + if (rangeListOld != NULL) + { + rangeListResult = rangeListOld; + free(rangeListNew); + } + else + { + rangeListResult = rangeListNew; + } + } + + return rangeListResult; + } + + // Returns pointer to address in last level map that actually points at RangeList space. + RangeListMini** EnsureMapsForAddress(void* address) + { + uintptr_t index; + void* newRangeList; +#ifdef TARGET_64BIT + RangeListMini**** _rangeListL3 = EnsureLevel(_rangeListL4, 4); + if (_rangeListL3 == NULL) + return NULL; // Failure case + RangeListMini*** _rangeListL2 = EnsureLevel(_rangeListL3, 3); + if (_rangeListL2 == NULL) + return NULL; // Failure case +#endif + RangeListMini** _rangeListL1 = EnsureLevel(_rangeListL2, 2); + if (_rangeListL1 == NULL) + return NULL; // Failure case + return &_rangeListL1[EffectiveBitsForLevel(address, 1)]; + } + + RangeListMini* GetRangeListForAddress(void* address) + { +#ifdef TARGET_64BIT + RangeListMini**** _rangeListL3 = VolatileRead(&_rangeListL4[EffectiveBitsForLevel(address, 4)]); // Use a VolatileRead on the top level operation to ensure that the entire map is synchronized to a state that includes all data needed to examine currently active function pointers. + if (_rangeListL3 == NULL) + return NULL; + RangeListMini*** _rangeListL2 = _rangeListL3[EffectiveBitsForLevel(address, 3)]; + if (_rangeListL2 == NULL) + return NULL; + RangeListMini** _rangeListL1 = _rangeListL2[EffectiveBitsForLevel(address, 2)]; +#else + RangeListMini** _rangeListL1 = VolatileRead(&_rangeListL2[EffectiveBitsForLevel(address, 2)]); +#endif + if (_rangeListL2 == NULL) + return NULL; + + return _rangeListL1[EffectiveBitsForLevel(address, 1)]; + } + + uintptr_t RangeListMiniCount(RangeList *pRangeList) + { + uintptr_t rangeSize = reinterpret_cast(pRangeList->range.end) - reinterpret_cast(pRangeList->range.begin); + rangeSize /= bytesAtLastLevel; + rangeSize + 1; + + return rangeSize; + } + + void* IncrementAddressByMaxSizeOfMini(void* input) + { + uintptr_t inputAsInt = reinterpret_cast(input); + return reinterpret_cast(inputAsInt + bytesAtLastLevel); + } + +public: + RangeListMap() + { + } + + bool Init() + { + *GetTopLevelAddress() = AllocateLevel(); + if (*GetTopLevelAddress() == NULL) + return false; + + return true; + } + + bool AttachRangeListToMap(RangeList* pRangeList) + { + uintptr_t rangeListMiniCount = RangeListMiniCount(pRangeList); + RangeListMini* minis = (RangeListMini*)calloc(rangeListMiniCount, sizeof(RangeListMini)); + + if (minis == NULL) + { + return false; + } + + RangeListMini*** entriesInMapToUpdate = (RangeList***)calloc(rangeListMiniCount, sizeof(RangeList**)); + if (entriesInMapToUpdate == NULL) + { + free(minis); + return false; + } + + minis[0].isPrimaryRangeListMini = true; + + void* addressToPrepForUpdate = pRangeList->range.begin; + for (uintptr_t iMini = 0; iMini < rangeListMiniCount; iMini++) + { + minis[iMini].pRangeList = pRangeList; + minis[iMini].range = pRangeList->range; + RangeListMini** entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); + if (entryInMapToUpdate == NULL) + { + free(minis); + free(entriesInMapToUpdate); + return false; + } + + entriesInMapToUpdate[iMini] = entryInMapToUpdate; + addressToPrepForUpdate = IncrementAddressByMaxSizeOfMini(addressToPrepForUpdate); + } + + // At this point all the needed memory is allocated, and it is no longer possible to fail. + for (uintptr_t iMini = 0; iMini < rangeListMiniCount; iMini++) + { + RangeListMini* initialMiniInMap = VolatileRead(entriesInMapToUpdate[iMini]); + do + { + VolatileWrite(&minis[iMini].pRangeListMiniNext, initialMiniInMap); + RangeListMini* currentMiniInMap = (RangeListMini*)InterlockedCompareExchangePointer((volatile PVOID*)entriesInMapToUpdate[iMini], &(minis[iMini]), initialMiniInMap); + if (currentMiniInMap == initialMiniInMap) + { + break; + } + initialMiniInMap = currentMiniInMap; + } while (true); + } + + // entriesInMapToUpdate was just a temporary allocation + free(entriesInMapToUpdate); + + return true; + } + + RangeList* LookupRangeListByAddressForKnownValidAddressesOrUnderLock(void* address) + { + RangeListMini* mini = GetRangeListForAddress(address); + if (mini == NULL) + return NULL; + + while (mini != NULL && !mini->range.InRange(address)) + { + mini = VolatileRead(&mini->pRangeListMiniNext); + } + + if (mini != NULL) + { + return mini->pRangeList; + } + + return NULL; + } + + void RemoveRangeList(RangeList* pRangeList) + { + uintptr_t rangeListMiniCount = RangeListMiniCount(pRangeList); + void* addressToPrepForDelete = pRangeList->range.begin; + for (uintptr_t iMini = 0; iMini < rangeListMiniCount; iMini++) + { + // Implement as marking the rangelist minis as deleted + // Actual deletion from the RangeListMini list is done in CleanupWhileNoThreadMayLookupRangeLists + + addressToPrepForDelete = IncrementAddressByMaxSizeOfMini(addressToPrepForDelete); + } + } + void CleanupWhileNoThreadMayLookupRangeLists() + { + // Set a lock + } +}; + +int main() +{ + std::cout << "Hello World!\n"; +} + +// Run program: Ctrl + F5 or Debug > Start Without Debugging menu +// Debug program: F5 or Debug > Start Debugging menu + +// Tips for Getting Started: +// 1. Use the Solution Explorer window to add/manage files +// 2. Use the Team Explorer window to connect to source control +// 3. Use the Output window to see build output and other messages +// 4. Use the Error List window to view errors +// 5. Go to Project > Add New Item to create new code files, or Project > Add Existing Item to add existing code files to the project +// 6. In the future, to open this project again, go to File > Open > Project and select the .sln file diff --git a/temp/RangeListMap.vcxproj b/temp/RangeListMap.vcxproj new file mode 100644 index 0000000000000..317ff6e6bc245 --- /dev/null +++ b/temp/RangeListMap.vcxproj @@ -0,0 +1,135 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 16.0 + Win32Proj + {09366505-33bb-4a1b-80f0-8a1cd78d159b} + RangeListMap + 10.0 + + + + Application + true + v143 + Unicode + + + Application + false + v143 + true + Unicode + + + Application + true + v143 + Unicode + + + Application + false + v143 + true + Unicode + + + + + + + + + + + + + + + + + + + + + + Level3 + true + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + + + + + Level3 + true + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + true + true + + + + + Level3 + true + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + + + + + Level3 + true + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + true + true + + + + + + + + + \ No newline at end of file diff --git a/temp/RangeListMap.vcxproj.filters b/temp/RangeListMap.vcxproj.filters new file mode 100644 index 0000000000000..2c6b143d51646 --- /dev/null +++ b/temp/RangeListMap.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + \ No newline at end of file From 2fa62aae691af4635a6107180ba631af4d266d91 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 28 Oct 2022 15:19:48 -0700 Subject: [PATCH 03/25] More progress --- temp/RangeListMap.cpp | 159 ++++++++++++++++++++++++++++++++---------- 1 file changed, 122 insertions(+), 37 deletions(-) diff --git a/temp/RangeListMap.cpp b/temp/RangeListMap.cpp index cce008ebc3841..01fb724d4fb8e 100644 --- a/temp/RangeListMap.cpp +++ b/temp/RangeListMap.cpp @@ -2,6 +2,7 @@ // #include +#include #include #define TARGET_64BIT @@ -12,16 +13,17 @@ class Range public: void* begin; void* end; - bool InRange(void* address) - { - return address >= begin && address <= end; - } }; class RangeList { public: - Range range; + RangeList(Range range) : + _range(range) + {} + + Range _range; + RangeList* pRangeListNextForDelete = nullptr; // Used for adding to the cleanup list }; @@ -31,9 +33,9 @@ class RangeListMini { public: RangeListMini* pRangeListMiniNext; - RangeListMini* pRangeListMiniNextForFree; // Used for adding to the cleanup list - Range range; + Range _range; RangeList* pRangeList; + bool InRange(void* address) { return address >= _range.begin && address <= _range.end && pRangeList->pRangeListNextForDelete == NULL; } bool isPrimaryRangeListMini; // RangeListMini are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. }; @@ -61,7 +63,7 @@ class SingleElementSegmentMap template -void VolatileWrite(T* ptr, T val) +void VolatileStore(T* ptr, T val) { *ptr = val; } @@ -72,6 +74,12 @@ T VolatileRead(T* ptr) return *ptr; } +template +T VolatileLoadWithoutBarrier(T* ptr) +{ + return *ptr; +} + class RangeListMap { #ifdef TARGET_64BIT @@ -90,10 +98,11 @@ class RangeListMap RangeListMini** _rangeListL2; void** GetTopLevelAddress() { return reinterpret_cast(&_rangeListL2); } #endif - RangeListMini* pRangeListMinisReadyToDelete = nullptr; + int _lock = 0; // 0 indicates unlocked. -1 indicates in the process of cleanup, Positive numbers indicate read locks + RangeList* pCleanupList = nullptr; const uintptr_t bitsAtLastLevel = maxSetBit - (bitsPerLevel * mapLevels) + 1; - const uintptr_t bytesAtLastLevel = (1 << (bitsAtLastLevel - 1)); + const uintptr_t bytesAtLastLevel = (((uintptr_t)1) << (bitsAtLastLevel - 1)); void* AllocateLevel() { return calloc(entriesPerMapLevel, sizeof(void*)); } @@ -107,14 +116,14 @@ class RangeListMap } template - T EnsureLevel(T* outerLevel, uintptr_t level) + T EnsureLevel(void *address, T* outerLevel, uintptr_t level) { uintptr_t index = EffectiveBitsForLevel(address, level); T rangeListResult = outerLevel[index]; if (rangeListResult == NULL) { T rangeListNew = static_cast(AllocateLevel()); - T rangeListOld = InterlockedCompareExchangePointer((volatile PVOID*)&outerLevel[index], (PVOID)rangeListNew, NULL); + T rangeListOld = (T)InterlockedCompareExchangePointer((volatile PVOID*)&outerLevel[index], (PVOID)rangeListNew, NULL); if (rangeListOld != NULL) { @@ -133,17 +142,15 @@ class RangeListMap // Returns pointer to address in last level map that actually points at RangeList space. RangeListMini** EnsureMapsForAddress(void* address) { - uintptr_t index; - void* newRangeList; #ifdef TARGET_64BIT - RangeListMini**** _rangeListL3 = EnsureLevel(_rangeListL4, 4); + RangeListMini**** _rangeListL3 = EnsureLevel(address, _rangeListL4, 4); if (_rangeListL3 == NULL) return NULL; // Failure case - RangeListMini*** _rangeListL2 = EnsureLevel(_rangeListL3, 3); + RangeListMini*** _rangeListL2 = EnsureLevel(address, _rangeListL3, 3); if (_rangeListL2 == NULL) return NULL; // Failure case #endif - RangeListMini** _rangeListL1 = EnsureLevel(_rangeListL2, 2); + RangeListMini** _rangeListL1 = EnsureLevel(address, _rangeListL2, 2); if (_rangeListL1 == NULL) return NULL; // Failure case return &_rangeListL1[EffectiveBitsForLevel(address, 1)]; @@ -170,11 +177,9 @@ class RangeListMap uintptr_t RangeListMiniCount(RangeList *pRangeList) { - uintptr_t rangeSize = reinterpret_cast(pRangeList->range.end) - reinterpret_cast(pRangeList->range.begin); + uintptr_t rangeSize = reinterpret_cast(pRangeList->_range.end) - reinterpret_cast(pRangeList->_range.begin); rangeSize /= bytesAtLastLevel; - rangeSize + 1; - - return rangeSize; + return rangeSize + 1; } void* IncrementAddressByMaxSizeOfMini(void* input) @@ -207,7 +212,7 @@ class RangeListMap return false; } - RangeListMini*** entriesInMapToUpdate = (RangeList***)calloc(rangeListMiniCount, sizeof(RangeList**)); + RangeListMini*** entriesInMapToUpdate = (RangeListMini***)calloc(rangeListMiniCount, sizeof(RangeListMini**)); if (entriesInMapToUpdate == NULL) { free(minis); @@ -216,11 +221,11 @@ class RangeListMap minis[0].isPrimaryRangeListMini = true; - void* addressToPrepForUpdate = pRangeList->range.begin; + void* addressToPrepForUpdate = pRangeList->_range.begin; for (uintptr_t iMini = 0; iMini < rangeListMiniCount; iMini++) { minis[iMini].pRangeList = pRangeList; - minis[iMini].range = pRangeList->range; + minis[iMini]._range = pRangeList->_range; RangeListMini** entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); if (entryInMapToUpdate == NULL) { @@ -239,7 +244,7 @@ class RangeListMap RangeListMini* initialMiniInMap = VolatileRead(entriesInMapToUpdate[iMini]); do { - VolatileWrite(&minis[iMini].pRangeListMiniNext, initialMiniInMap); + VolatileStore(&minis[iMini].pRangeListMiniNext, initialMiniInMap); RangeListMini* currentMiniInMap = (RangeListMini*)InterlockedCompareExchangePointer((volatile PVOID*)entriesInMapToUpdate[iMini], &(minis[iMini]), initialMiniInMap); if (currentMiniInMap == initialMiniInMap) { @@ -255,15 +260,16 @@ class RangeListMap return true; } - RangeList* LookupRangeListByAddressForKnownValidAddressesOrUnderLock(void* address) +private: + RangeList* LookupRangeListByAddressForKnownValidAddressWhileCleanupCannotHappenOrUnderLock(void* address) { RangeListMini* mini = GetRangeListForAddress(address); if (mini == NULL) return NULL; - while (mini != NULL && !mini->range.InRange(address)) + while (mini != NULL && !mini->InRange(address)) { - mini = VolatileRead(&mini->pRangeListMiniNext); + mini = VolatileLoadWithoutBarrier(&mini->pRangeListMiniNext); } if (mini != NULL) @@ -274,21 +280,100 @@ class RangeListMap return NULL; } - void RemoveRangeList(RangeList* pRangeList) +public: + bool TryLookupRangeListByAddressForKnownValidAddress(void* address, RangeList** pRangeList) { - uintptr_t rangeListMiniCount = RangeListMiniCount(pRangeList); - void* addressToPrepForDelete = pRangeList->range.begin; - for (uintptr_t iMini = 0; iMini < rangeListMiniCount; iMini++) + *pRangeList = NULL; + + bool locked = false; + int lockVal; + + do { - // Implement as marking the rangelist minis as deleted - // Actual deletion from the RangeListMini list is done in CleanupWhileNoThreadMayLookupRangeLists + lockVal = VolatileRead(&_lock); - addressToPrepForDelete = IncrementAddressByMaxSizeOfMini(addressToPrepForDelete); - } + // Cleanup in process. Do not succeed in producing result + if (lockVal < 0) + return false; + + // Take reader lock + } while (InterlockedCompareExchange((volatile unsigned*)&_lock, (unsigned)lockVal + 1, (unsigned)lockVal) != lockVal); + + *pRangeList = LookupRangeListByAddressForKnownValidAddressWhileCleanupCannotHappenOrUnderLock(address); + + // Release lock + InterlockedDecrement((volatile unsigned*)&_lock); + } + + // Due to the thread safety semantics of removal, the address passed in here MUST be the address of a function on the stack, and therefore not eligible to be cleaned up due to some race. + RangeList* LookupRangeListCannotCallInParallelWithCleanup(void* address) + { + // Locked readers may be reading, but no cleanup can be happening + assert(_lock != -1); + return LookupRangeListByAddressForKnownValidAddressWhileCleanupCannotHappenOrUnderLock(address); + } + + void RemoveRangeListCannotCallInParallelWithCleanup(RangeList* pRangeList) + { + assert(pRangeList->pRangeListNextForDelete = nullptr); + assert(pRangeList == LookupRangeListCannotCallInParallelWithCleanup(pRangeList->_range.begin)); + + // Removal is implemented by placing onto the cleanup linked list. This is then processed later during cleanup + RangeList* pLatestRemovedRangeList; + do + { + pLatestRemovedRangeList = VolatileRead(&pCleanupList); + VolatileStore(&pRangeList->pRangeListNextForDelete, pLatestRemovedRangeList); + } while (InterlockedCompareExchangePointer((volatile PVOID *)&pCleanupList, pRangeList, pLatestRemovedRangeList) == pLatestRemovedRangeList); } + void CleanupWhileNoThreadMayLookupRangeLists() { - // Set a lock + // Take cleanup lock + if (InterlockedCompareExchange((volatile unsigned*)&_lock, (unsigned)(-1), 0) != 0) + { + // If a locked read is in progress. That's OK. We'll clean up some in a future call to cleanup. + return; + } + + RangeListMini *minisToFree = nullptr; + + while (this->pCleanupList != nullptr) + { + RangeList* pRangeListToCleanup = this->pCleanupList; + RangeListMini* pRangeListMiniToFree = nullptr; + this->pCleanupList = pRangeListToCleanup->pRangeListNextForDelete; + + uintptr_t rangeListMiniCount = RangeListMiniCount(pRangeListToCleanup); + + void* addressToPrepForCleanup = pRangeListToCleanup->_range.begin; + + for (uintptr_t iMini = 0; iMini < rangeListMiniCount; iMini++) + { + RangeListMini** entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); + assert(entryInMapToUpdate != NULL); + + while ((*entryInMapToUpdate)->pRangeList != pRangeListToCleanup) + { + entryInMapToUpdate = &(*entryInMapToUpdate)->pRangeListMiniNext; + } + + if (iMini == 0) + { + pRangeListMiniToFree = *entryInMapToUpdate; + assert(pRangeListMiniToFree->isPrimaryRangeListMini); + } + + *entryInMapToUpdate = (*entryInMapToUpdate)->pRangeListMiniNext; + + addressToPrepForCleanup = IncrementAddressByMaxSizeOfMini(addressToPrepForCleanup); + } + + free(pRangeListMiniToFree); + } + + // Release lock + VolatileStore(&_lock, 0); } }; From 55a5bb620c1e776da7b3c97ed705bc6f46a904d0 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Mon, 14 Nov 2022 15:13:25 -0800 Subject: [PATCH 04/25] Increase accuracy of comments around range section locking - This is improving docs on the old scheme - Also remove the ngen specific logic --- src/coreclr/vm/codeman.cpp | 42 +++++++--------------------------- src/coreclr/vm/codeman.h | 5 +--- src/coreclr/vm/genericdict.cpp | 5 +--- src/coreclr/vm/prestub.cpp | 6 ----- 4 files changed, 10 insertions(+), 48 deletions(-) diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index ddf189c013f6a..07855709672a7 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -793,14 +793,17 @@ values: m_CodeRangeList and hold it while walking the lists Uses ReaderLockHolder (allows multiple reeaders with no writers) ----------------------------------------- ExecutionManager::FindCodeRange -ExecutionManager::FindZapModule +ExecutionManager::FindReadyToRunModule ExecutionManager::EnumMemoryRegions +AND +ExecutionManager::IsManagedCode +ExecutionManager::IsManagedCodeWithLock +The IsManagedCode checks are notable as they protect not just access to the RangeSection walking, +but the actual RangeSection while determining if a given ip IsManagedCode. Uses WriterLockHolder (allows single writer and no readers) ----------------------------------------- -ExecutionManager::AddRangeHelper -ExecutionManager::DeleteRangeHelper - +ExecutionManager::DeleteRange */ //----------------------------------------------------------------------------- @@ -5024,36 +5027,6 @@ RangeSection* ExecutionManager::GetRangeSectionAndPrev(RangeSection *pHead, TADD return result; } -/* static */ -PTR_Module ExecutionManager::FindZapModule(TADDR currentData) -{ - CONTRACTL - { - NOTHROW; - GC_NOTRIGGER; - MODE_ANY; - STATIC_CONTRACT_HOST_CALLS; - SUPPORTS_DAC; - } - CONTRACTL_END; - - ReaderLockHolder rlh; - - RangeSection * pRS = GetRangeSection(currentData); - if (pRS == NULL) - return NULL; - - if (pRS->flags & RangeSection::RANGE_SECTION_CODEHEAP) - return NULL; - -#ifdef FEATURE_READYTORUN - if (pRS->flags & RangeSection::RANGE_SECTION_READYTORUN) - return NULL; -#endif - - return dac_cast(pRS->pHeapListOrZapModule); -} - /* static */ PTR_Module ExecutionManager::FindReadyToRunModule(TADDR currentData) { @@ -5080,6 +5053,7 @@ PTR_Module ExecutionManager::FindReadyToRunModule(TADDR currentData) if (pRS->flags & RangeSection::RANGE_SECTION_READYTORUN) return dac_cast(pRS->pHeapListOrZapModule);; + _ASSERTE(FALSE); ..= return NULL; #else return NULL; diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index f66b2f05784db..4657bca895e56 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -1300,10 +1300,9 @@ class ExecutionManager return (ICodeManager *)m_pDefaultCodeMan; } - static PTR_Module FindZapModule(TADDR currentData); static PTR_Module FindReadyToRunModule(TADDR currentData); - // FindZapModule flavor to be used during GC to find GCRefMap + // FindReadyToRunModule flavor to be used during GC to find GCRefMap static PTR_Module FindModuleForGCRefMap(TADDR currentData); static RangeSection* GetRangeSectionAndPrev(RangeSection *pRS, TADDR addr, RangeSection **ppPrev); @@ -1377,8 +1376,6 @@ class ExecutionManager IJitManager* pJit, RangeSection::RangeSectionFlags flags, TADDR pHeapListOrZapModule); - static void DeleteRangeHelper(RangeSection** ppRangeList, - TADDR StartRange); #ifndef DACCESS_COMPILE static PCODE getNextJumpStub(MethodDesc* pMD, diff --git a/src/coreclr/vm/genericdict.cpp b/src/coreclr/vm/genericdict.cpp index db18a4044f567..52645f0d92cf5 100644 --- a/src/coreclr/vm/genericdict.cpp +++ b/src/coreclr/vm/genericdict.cpp @@ -690,10 +690,7 @@ Dictionary::PopulateEntry( ptr = SigPointer((PCCOR_SIGNATURE)signature); IfFailThrow(ptr.GetData(&kind)); - Module * pContainingZapModule = ExecutionManager::FindZapModule(dac_cast(signature)); - - zapSigContext = ZapSig::Context(CoreLibBinder::GetModule(), (void *)pContainingZapModule, ZapSig::NormalTokens); - pZapSigContext = (pContainingZapModule != NULL) ? &zapSigContext : NULL; + pZapSigContext = NULL; } ModuleBase * pLookupModule = (isReadyToRunModule) ? pZapSigContext->pInfoModule : CoreLibBinder::GetModule(); diff --git a/src/coreclr/vm/prestub.cpp b/src/coreclr/vm/prestub.cpp index 7df9cf15540ed..076f907e45dbd 100644 --- a/src/coreclr/vm/prestub.cpp +++ b/src/coreclr/vm/prestub.cpp @@ -2390,12 +2390,6 @@ EXTERN_C PCODE STDCALL ExternalMethodFixupWorker(TransitionBlock * pTransitionBl } #endif - // FUTURE: Consider always passing in module and section index to avoid the lookups - if (pModule == NULL) - { - pModule = ExecutionManager::FindZapModule(pIndirection); - sectionIndex = (DWORD)-1; - } _ASSERTE(pModule != NULL); pEMFrame->SetCallSite(pModule, pIndirection); From f5357bbeb29582130e5edce549222a30a29f4be6 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Mon, 28 Nov 2022 16:11:10 -0800 Subject: [PATCH 05/25] Changes to RangeListMap to make it work - still with the wrong locking though... --- temp/RangeListMap.cpp | 293 ++++++++++++++++++++++++------------------ 1 file changed, 166 insertions(+), 127 deletions(-) diff --git a/temp/RangeListMap.cpp b/temp/RangeListMap.cpp index 01fb724d4fb8e..671cc1c56702f 100644 --- a/temp/RangeListMap.cpp +++ b/temp/RangeListMap.cpp @@ -7,6 +7,23 @@ #define TARGET_64BIT +template +void VolatileStore(T* ptr, T val) +{ + *ptr = val; +} + +template +T VolatileLoad(T* ptr) +{ + return *ptr; +} + +template +T VolatileLoadWithoutBarrier(T* ptr) +{ + return *ptr; +} class Range { @@ -15,31 +32,18 @@ class Range void* end; }; -class RangeList +class RangeSection { public: - RangeList(Range range) : + RangeSection(Range range) : _range(range) {} Range _range; - RangeList* pRangeListNextForDelete = nullptr; // Used for adding to the cleanup list + RangeSection* pRangeListNextForDelete = nullptr; // Used for adding to the cleanup list }; -// Unlike a RangeList, a RangeListMini cannot span multiple elements of the last level of the SegmentMap -// Always allocated via calloc -class RangeListMini -{ -public: - RangeListMini* pRangeListMiniNext; - Range _range; - RangeList* pRangeList; - bool InRange(void* address) { return address >= _range.begin && address <= _range.end && pRangeList->pRangeListNextForDelete == NULL; } - bool isPrimaryRangeListMini; // RangeListMini are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. -}; - - // For 64bit, we work with 8KB chunks of memory holding pointers to the next level. This provides 10 bits of address resolution per level. // For *reasons* the X64 hardware is limited to 57bits of addressable address space, and the minimum granularity that makes sense for range lists is 64KB (or every 2^16 bits) // Similarly the Arm64 specification requires addresses to use at most 52 bits. Thus we use the maximum addressable range of X64 to provide the real max range @@ -47,7 +51,7 @@ class RangeListMini // Then [46:37] -> L3 // [36:27] -> L2 // [26:17] -> L1 -// This leaves 17 bits of the address to be handled by the RangeList linked list +// This leaves 17 bits of the address to be handled by the RangeSection linked list // // For 32bit VA processes, use 1KB chunks holding pointers to the next level. This provides 8 bites of address resolution per level. [31:24] and [23:16]. @@ -56,50 +60,48 @@ class RangeListMini -class SingleElementSegmentMap -{ - -}; - - -template -void VolatileStore(T* ptr, T val) +class RangeListMap { - *ptr = val; -} + // Unlike a RangeSection, a RangeSectionFragment cannot span multiple elements of the last level of the RangeListMap + // Always allocated via calloc + class RangeSectionFragment + { + public: + RangeSectionFragment* pRangeListFragmentNext; + Range _range; + RangeSection* pRangeList; + bool InRange(void* address) { return address >= _range.begin && address <= _range.end && pRangeList->pRangeListNextForDelete == NULL; } + bool isPrimaryRangeListFragment; // RangeSectionFragment are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. + }; -template -T VolatileRead(T* ptr) -{ - return *ptr; -} +#ifdef TARGET_64BIT + static const uintptr_t entriesPerMapLevel = 1024; +#else + static const uintptr_t entriesPerMapLevel = 256; +#endif -template -T VolatileLoadWithoutBarrier(T* ptr) -{ - return *ptr; -} + typedef RangeSectionFragment* RangeSectionList; + typedef RangeSectionList RangeSectionL1[entriesPerMapLevel]; + typedef RangeSectionL1* RangeSectionL2[entriesPerMapLevel]; + typedef RangeSectionL2* RangeSectionL3[entriesPerMapLevel]; + typedef RangeSectionL3* RangeSectionL4[entriesPerMapLevel]; -class RangeListMap -{ #ifdef TARGET_64BIT - static const uintptr_t entriesPerMapLevel = 1024; + typedef RangeSectionL4 RangeSectionTopLevel; static const uintptr_t mapLevels = 4; static const uintptr_t maxSetBit = 56; // This is 0 indexed static const uintptr_t bitsPerLevel = 10; - RangeListMini***** _rangeListL4; - void** GetTopLevelAddress() { return reinterpret_cast(&_rangeListL4); } #else - static const uintptr_t entriesPerMapLevel = 256; + typedef RangeSectionL2 RangeSectionTopLevel; static const uintptr_t mapLevels = 2; static const uintptr_t maxSetBit = 31; // This is 0 indexed static const uintptr_t bitsPerLevel = 8; - - RangeListMini** _rangeListL2; - void** GetTopLevelAddress() { return reinterpret_cast(&_rangeListL2); } #endif + + RangeSectionTopLevel *_topLevel = nullptr; + int _lock = 0; // 0 indicates unlocked. -1 indicates in the process of cleanup, Positive numbers indicate read locks - RangeList* pCleanupList = nullptr; + RangeSection* pCleanupList = nullptr; const uintptr_t bitsAtLastLevel = maxSetBit - (bitsPerLevel * mapLevels) + 1; const uintptr_t bytesAtLastLevel = (((uintptr_t)1) << (bitsAtLastLevel - 1)); @@ -115,142 +117,151 @@ class RangeListMap return addressBitsUsedInLevel; } - template - T EnsureLevel(void *address, T* outerLevel, uintptr_t level) + template + auto EnsureLevel(void *address, T* outerLevel, uintptr_t level) -> decltype(&((**outerLevel)[0])) { uintptr_t index = EffectiveBitsForLevel(address, level); - T rangeListResult = outerLevel[index]; - if (rangeListResult == NULL) - { - T rangeListNew = static_cast(AllocateLevel()); - T rangeListOld = (T)InterlockedCompareExchangePointer((volatile PVOID*)&outerLevel[index], (PVOID)rangeListNew, NULL); + auto levelToGetPointerIn = VolatileLoadWithoutBarrier(outerLevel); - if (rangeListOld != NULL) + if (levelToGetPointerIn == NULL) + { + auto levelNew = static_cast(AllocateLevel()); + if (levelNew == NULL) + return NULL; + auto levelPreviouslyStored = (decltype(&(*outerLevel)[0]))InterlockedCompareExchangePointer((volatile PVOID*)outerLevel, (PVOID)levelNew, NULL); + if (levelPreviouslyStored != nullptr) { - rangeListResult = rangeListOld; - free(rangeListNew); + // Handle race where another thread grew the table + levelToGetPointerIn = levelPreviouslyStored; + free(levelNew); } else { - rangeListResult = rangeListNew; + levelToGetPointerIn = levelNew; } + assert(levelToGetPointerIn != nullptr); } - return rangeListResult; + return &((*levelToGetPointerIn)[index]); } - // Returns pointer to address in last level map that actually points at RangeList space. - RangeListMini** EnsureMapsForAddress(void* address) + // Returns pointer to address in last level map that actually points at RangeSection space. + RangeSectionFragment** EnsureMapsForAddress(void* address) { + uintptr_t level = mapLevels; #ifdef TARGET_64BIT - RangeListMini**** _rangeListL3 = EnsureLevel(address, _rangeListL4, 4); + auto _rangeListL3 = EnsureLevel(address, &_topLevel, level); if (_rangeListL3 == NULL) return NULL; // Failure case - RangeListMini*** _rangeListL2 = EnsureLevel(address, _rangeListL3, 3); + auto _rangeListL2 = EnsureLevel(address, _rangeListL3, --level); if (_rangeListL2 == NULL) return NULL; // Failure case +#else + auto _rangeListL2 = &topLevel; #endif - RangeListMini** _rangeListL1 = EnsureLevel(address, _rangeListL2, 2); + auto _rangeListL1 = EnsureLevel(address, _rangeListL2, --level); if (_rangeListL1 == NULL) return NULL; // Failure case - return &_rangeListL1[EffectiveBitsForLevel(address, 1)]; + + auto result = EnsureLevel(address, _rangeListL1, --level); + if (result == NULL) + return NULL; // Failure case + + return result; } - RangeListMini* GetRangeListForAddress(void* address) + RangeSectionFragment* GetRangeListForAddress(void* address) { #ifdef TARGET_64BIT - RangeListMini**** _rangeListL3 = VolatileRead(&_rangeListL4[EffectiveBitsForLevel(address, 4)]); // Use a VolatileRead on the top level operation to ensure that the entire map is synchronized to a state that includes all data needed to examine currently active function pointers. + auto _rangeListL4 = VolatileLoad(&_topLevel); + auto _rangeListL3 = (*_rangeListL4)[EffectiveBitsForLevel(address, 4)]; if (_rangeListL3 == NULL) return NULL; - RangeListMini*** _rangeListL2 = _rangeListL3[EffectiveBitsForLevel(address, 3)]; + auto _rangeListL2 = (*_rangeListL3)[EffectiveBitsForLevel(address, 3)]; if (_rangeListL2 == NULL) return NULL; - RangeListMini** _rangeListL1 = _rangeListL2[EffectiveBitsForLevel(address, 2)]; + auto _rangeListL1 = (*_rangeListL2)[EffectiveBitsForLevel(address, 2)]; #else - RangeListMini** _rangeListL1 = VolatileRead(&_rangeListL2[EffectiveBitsForLevel(address, 2)]); + auto _rangeListL1 = VolatileLoad(&_topLevel[EffectiveBitsForLevel(address, 2)]); // Use a VolatileLoad on the top level operation to ensure that the entire map is synchronized to a state that includes all data needed to examine currently active function pointers. #endif - if (_rangeListL2 == NULL) + if (_rangeListL1 == NULL) return NULL; - return _rangeListL1[EffectiveBitsForLevel(address, 1)]; + return (*_rangeListL1)[EffectiveBitsForLevel(address, 1)]; } - uintptr_t RangeListMiniCount(RangeList *pRangeList) + uintptr_t RangeListFragmentCount(RangeSection *pRangeList) { uintptr_t rangeSize = reinterpret_cast(pRangeList->_range.end) - reinterpret_cast(pRangeList->_range.begin); rangeSize /= bytesAtLastLevel; return rangeSize + 1; } - void* IncrementAddressByMaxSizeOfMini(void* input) + void* IncrementAddressByMaxSizeOfFragment(void* input) { uintptr_t inputAsInt = reinterpret_cast(input); return reinterpret_cast(inputAsInt + bytesAtLastLevel); } public: - RangeListMap() + RangeListMap() : _topLevel{0} { } bool Init() { - *GetTopLevelAddress() = AllocateLevel(); - if (*GetTopLevelAddress() == NULL) - return false; - return true; } - bool AttachRangeListToMap(RangeList* pRangeList) + bool AttachRangeListToMap(RangeSection* pRangeList) { - uintptr_t rangeListMiniCount = RangeListMiniCount(pRangeList); - RangeListMini* minis = (RangeListMini*)calloc(rangeListMiniCount, sizeof(RangeListMini)); + uintptr_t rangeListFragmentCount = RangeListFragmentCount(pRangeList); + RangeSectionFragment* fragments = (RangeSectionFragment*)calloc(rangeListFragmentCount, sizeof(RangeSectionFragment)); - if (minis == NULL) + if (fragments == NULL) { return false; } - RangeListMini*** entriesInMapToUpdate = (RangeListMini***)calloc(rangeListMiniCount, sizeof(RangeListMini**)); + RangeSectionFragment*** entriesInMapToUpdate = (RangeSectionFragment***)calloc(rangeListFragmentCount, sizeof(RangeSectionFragment**)); if (entriesInMapToUpdate == NULL) { - free(minis); + free(fragments); return false; } - minis[0].isPrimaryRangeListMini = true; + fragments[0].isPrimaryRangeListFragment = true; void* addressToPrepForUpdate = pRangeList->_range.begin; - for (uintptr_t iMini = 0; iMini < rangeListMiniCount; iMini++) + for (uintptr_t iFragment = 0; iFragment < rangeListFragmentCount; iFragment++) { - minis[iMini].pRangeList = pRangeList; - minis[iMini]._range = pRangeList->_range; - RangeListMini** entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); + fragments[iFragment].pRangeList = pRangeList; + fragments[iFragment]._range = pRangeList->_range; + RangeSectionFragment** entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); if (entryInMapToUpdate == NULL) { - free(minis); + free(fragments); free(entriesInMapToUpdate); return false; } - entriesInMapToUpdate[iMini] = entryInMapToUpdate; - addressToPrepForUpdate = IncrementAddressByMaxSizeOfMini(addressToPrepForUpdate); + entriesInMapToUpdate[iFragment] = entryInMapToUpdate; + addressToPrepForUpdate = IncrementAddressByMaxSizeOfFragment(addressToPrepForUpdate); } // At this point all the needed memory is allocated, and it is no longer possible to fail. - for (uintptr_t iMini = 0; iMini < rangeListMiniCount; iMini++) + for (uintptr_t iFragment = 0; iFragment < rangeListFragmentCount; iFragment++) { - RangeListMini* initialMiniInMap = VolatileRead(entriesInMapToUpdate[iMini]); + RangeSectionFragment* initialFragmentInMap = VolatileLoad(entriesInMapToUpdate[iFragment]); do { - VolatileStore(&minis[iMini].pRangeListMiniNext, initialMiniInMap); - RangeListMini* currentMiniInMap = (RangeListMini*)InterlockedCompareExchangePointer((volatile PVOID*)entriesInMapToUpdate[iMini], &(minis[iMini]), initialMiniInMap); - if (currentMiniInMap == initialMiniInMap) + VolatileStore(&fragments[iFragment].pRangeListFragmentNext, initialFragmentInMap); + RangeSectionFragment* currentFragmentInMap = (RangeSectionFragment*)InterlockedCompareExchangePointer((volatile PVOID*)entriesInMapToUpdate[iFragment], &(fragments[iFragment]), initialFragmentInMap); + if (currentFragmentInMap == initialFragmentInMap) { break; } - initialMiniInMap = currentMiniInMap; + initialFragmentInMap = currentFragmentInMap; } while (true); } @@ -261,27 +272,27 @@ class RangeListMap } private: - RangeList* LookupRangeListByAddressForKnownValidAddressWhileCleanupCannotHappenOrUnderLock(void* address) + RangeSection* LookupRangeListByAddressForKnownValidAddressWhileCleanupCannotHappenOrUnderLock(void* address) { - RangeListMini* mini = GetRangeListForAddress(address); - if (mini == NULL) + RangeSectionFragment* fragment = GetRangeListForAddress(address); + if (fragment == NULL) return NULL; - while (mini != NULL && !mini->InRange(address)) + while (fragment != NULL && !fragment->InRange(address)) { - mini = VolatileLoadWithoutBarrier(&mini->pRangeListMiniNext); + fragment = VolatileLoadWithoutBarrier(&fragment->pRangeListFragmentNext); } - if (mini != NULL) + if (fragment != NULL) { - return mini->pRangeList; + return fragment->pRangeList; } return NULL; } public: - bool TryLookupRangeListByAddressForKnownValidAddress(void* address, RangeList** pRangeList) + bool TryLookupRangeListByAddressForKnownValidAddress(void* address, RangeSection** pRangeList) { *pRangeList = NULL; @@ -290,7 +301,7 @@ class RangeListMap do { - lockVal = VolatileRead(&_lock); + lockVal = VolatileLoad(&_lock); // Cleanup in process. Do not succeed in producing result if (lockVal < 0) @@ -306,23 +317,23 @@ class RangeListMap } // Due to the thread safety semantics of removal, the address passed in here MUST be the address of a function on the stack, and therefore not eligible to be cleaned up due to some race. - RangeList* LookupRangeListCannotCallInParallelWithCleanup(void* address) + RangeSection* LookupRangeListCannotCallInParallelWithCleanup(void* address) { // Locked readers may be reading, but no cleanup can be happening assert(_lock != -1); return LookupRangeListByAddressForKnownValidAddressWhileCleanupCannotHappenOrUnderLock(address); } - void RemoveRangeListCannotCallInParallelWithCleanup(RangeList* pRangeList) + void RemoveRangeListCannotCallInParallelWithCleanup(RangeSection* pRangeList) { assert(pRangeList->pRangeListNextForDelete = nullptr); assert(pRangeList == LookupRangeListCannotCallInParallelWithCleanup(pRangeList->_range.begin)); // Removal is implemented by placing onto the cleanup linked list. This is then processed later during cleanup - RangeList* pLatestRemovedRangeList; + RangeSection* pLatestRemovedRangeList; do { - pLatestRemovedRangeList = VolatileRead(&pCleanupList); + pLatestRemovedRangeList = VolatileLoad(&pCleanupList); VolatileStore(&pRangeList->pRangeListNextForDelete, pLatestRemovedRangeList); } while (InterlockedCompareExchangePointer((volatile PVOID *)&pCleanupList, pRangeList, pLatestRemovedRangeList) == pLatestRemovedRangeList); } @@ -336,40 +347,41 @@ class RangeListMap return; } - RangeListMini *minisToFree = nullptr; - while (this->pCleanupList != nullptr) { - RangeList* pRangeListToCleanup = this->pCleanupList; - RangeListMini* pRangeListMiniToFree = nullptr; + RangeSection* pRangeListToCleanup = this->pCleanupList; + RangeSectionFragment* pRangeListFragmentToFree = nullptr; this->pCleanupList = pRangeListToCleanup->pRangeListNextForDelete; - uintptr_t rangeListMiniCount = RangeListMiniCount(pRangeListToCleanup); + uintptr_t rangeListFragmentCount = RangeListFragmentCount(pRangeListToCleanup); void* addressToPrepForCleanup = pRangeListToCleanup->_range.begin; - for (uintptr_t iMini = 0; iMini < rangeListMiniCount; iMini++) + // Remove fragments from each of the fragment linked lists + for (uintptr_t iFragment = 0; iFragment < rangeListFragmentCount; iFragment++) { - RangeListMini** entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); + RangeSectionFragment** entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); assert(entryInMapToUpdate != NULL); while ((*entryInMapToUpdate)->pRangeList != pRangeListToCleanup) { - entryInMapToUpdate = &(*entryInMapToUpdate)->pRangeListMiniNext; + entryInMapToUpdate = &(*entryInMapToUpdate)->pRangeListFragmentNext; } - if (iMini == 0) + // The fragment associated with the start of the range has the address that was allocated earlier + if (iFragment == 0) { - pRangeListMiniToFree = *entryInMapToUpdate; - assert(pRangeListMiniToFree->isPrimaryRangeListMini); + pRangeListFragmentToFree = *entryInMapToUpdate; + assert(pRangeListFragmentToFree->isPrimaryRangeListFragment); } - *entryInMapToUpdate = (*entryInMapToUpdate)->pRangeListMiniNext; + *entryInMapToUpdate = (*entryInMapToUpdate)->pRangeListFragmentNext; - addressToPrepForCleanup = IncrementAddressByMaxSizeOfMini(addressToPrepForCleanup); + addressToPrepForCleanup = IncrementAddressByMaxSizeOfFragment(addressToPrepForCleanup); } - free(pRangeListMiniToFree); + // Free the array of fragments + free(pRangeListFragmentToFree); } // Release lock @@ -379,7 +391,34 @@ class RangeListMap int main() { - std::cout << "Hello World!\n"; + RangeListMap map; + Range rFirst; + rFirst.begin = (void*)0x1111000; + rFirst.end = (void*)0x1111050; + Range rSecond; + rSecond.begin = (void*)0x1111051; + rSecond.end = (void*)0x1192050; + RangeSection rSectionFirst(rFirst); + RangeSection rSectionSecond(rSecond); + + + map.AttachRangeListToMap(&rSectionFirst); + map.AttachRangeListToMap(&rSectionSecond); + + RangeSection *result; + + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + assert(result == &rSectionFirst); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + assert(result == &rSectionFirst); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + assert(result == &rSectionSecond); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + assert(result == &rSectionSecond); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + assert(result == &rSectionSecond); + + std::cout << "Done\n"; } // Run program: Ctrl + F5 or Debug > Start Without Debugging menu From 1abc9c63a2d21cb4fad5c83e1abd3aed84940569 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Nov 2022 11:22:04 -0800 Subject: [PATCH 06/25] Add low bit check indirection --- temp/RangeListMap.cpp | 177 +++++++++++++++++++++++++++++++++++------- 1 file changed, 149 insertions(+), 28 deletions(-) diff --git a/temp/RangeListMap.cpp b/temp/RangeListMap.cpp index 671cc1c56702f..bc913a349a747 100644 --- a/temp/RangeListMap.cpp +++ b/temp/RangeListMap.cpp @@ -32,8 +32,11 @@ class Range void* end; }; +class RangeListMap; + class RangeSection { + friend class RangeListMap; public: RangeSection(Range range) : _range(range) @@ -58,20 +61,87 @@ class RangeSection // The memory safety model for segment maps is that the pointers held within the individual segments can never change other than to go from NULL to a meaningful pointer, // except for the final level, which is only permitted to change when CleanupWhileNoThreadMayLookupRangeLists is in use. - - class RangeListMap { + RangeSection ClearListMarker; + + class RangeSectionFragment; + class RangeSectionFragmentPointer + { + private: + uintptr_t _ptr; + + uintptr_t FragmentToPtr(RangeSectionFragment* fragment) + { + uintptr_t ptr = (uintptr_t)fragment; + if (ptr == 0) + return ptr; + + if (fragment->isCollectibleRangeListFragment) + { + ptr += 1; + } + + return ptr; + } + + RangeSectionFragmentPointer() { _ptr = 0; } + public: + + RangeSectionFragmentPointer(RangeSectionFragmentPointer &) = delete; + RangeSectionFragmentPointer(RangeSectionFragmentPointer &&) = delete; + RangeSectionFragmentPointer& operator=(const RangeSectionFragmentPointer&) = delete; + + bool PointerIsCollectible() + { + return ((_ptr & 1) == 1); + } + + bool IsNull() + { + return _ptr == 0; + } + + RangeSectionFragment* VolatileLoadWithoutBarrier() + { + uintptr_t ptr = ::VolatileLoadWithoutBarrier(&_ptr); + if ((ptr & 1) == 1) + { + return (RangeSectionFragment*)(ptr - 1); + } + else + { + return (RangeSectionFragment*)(ptr); + } + } + + void VolatileStore(RangeSectionFragment* fragment) + { + ::VolatileStore(&_ptr, FragmentToPtr(fragment)); + } + + bool AtomicReplace(RangeSectionFragment* newFragment, RangeSectionFragment* oldFragment) + { + uintptr_t oldPtr = FragmentToPtr(oldFragment); + uintptr_t newPtr = FragmentToPtr(newFragment); + + return oldPtr == (uintptr_t)InterlockedCompareExchangePointer((volatile PVOID*)&_ptr, (PVOID)newPtr, (PVOID)oldPtr); + } + }; + + + // Unlike a RangeSection, a RangeSectionFragment cannot span multiple elements of the last level of the RangeListMap // Always allocated via calloc class RangeSectionFragment { public: - RangeSectionFragment* pRangeListFragmentNext; + RangeSectionFragmentPointer pRangeListFragmentNext; Range _range; RangeSection* pRangeList; bool InRange(void* address) { return address >= _range.begin && address <= _range.end && pRangeList->pRangeListNextForDelete == NULL; } bool isPrimaryRangeListFragment; // RangeSectionFragment are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. + bool isCollectibleRangeListFragment; // RangeSectionFragments }; #ifdef TARGET_64BIT @@ -80,7 +150,7 @@ class RangeListMap static const uintptr_t entriesPerMapLevel = 256; #endif - typedef RangeSectionFragment* RangeSectionList; + typedef RangeSectionFragmentPointer RangeSectionList; typedef RangeSectionList RangeSectionL1[entriesPerMapLevel]; typedef RangeSectionL1* RangeSectionL2[entriesPerMapLevel]; typedef RangeSectionL2* RangeSectionL3[entriesPerMapLevel]; @@ -101,7 +171,7 @@ class RangeListMap RangeSectionTopLevel *_topLevel = nullptr; int _lock = 0; // 0 indicates unlocked. -1 indicates in the process of cleanup, Positive numbers indicate read locks - RangeSection* pCleanupList = nullptr; + RangeSection* pCleanupList; const uintptr_t bitsAtLastLevel = maxSetBit - (bitsPerLevel * mapLevels) + 1; const uintptr_t bytesAtLastLevel = (((uintptr_t)1) << (bitsAtLastLevel - 1)); @@ -146,7 +216,7 @@ class RangeListMap } // Returns pointer to address in last level map that actually points at RangeSection space. - RangeSectionFragment** EnsureMapsForAddress(void* address) + RangeSectionFragmentPointer* EnsureMapsForAddress(void* address) { uintptr_t level = mapLevels; #ifdef TARGET_64BIT @@ -187,7 +257,7 @@ class RangeListMap if (_rangeListL1 == NULL) return NULL; - return (*_rangeListL1)[EffectiveBitsForLevel(address, 1)]; + return ((*_rangeListL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(); } uintptr_t RangeListFragmentCount(RangeSection *pRangeList) @@ -204,7 +274,7 @@ class RangeListMap } public: - RangeListMap() : _topLevel{0} + RangeListMap() : _topLevel{0}, ClearListMarker(Range()), pCleanupList(&ClearListMarker) { } @@ -223,7 +293,7 @@ class RangeListMap return false; } - RangeSectionFragment*** entriesInMapToUpdate = (RangeSectionFragment***)calloc(rangeListFragmentCount, sizeof(RangeSectionFragment**)); + RangeSectionFragmentPointer** entriesInMapToUpdate = (RangeSectionFragmentPointer**)calloc(rangeListFragmentCount, sizeof(RangeSectionFragmentPointer*)); if (entriesInMapToUpdate == NULL) { free(fragments); @@ -237,7 +307,7 @@ class RangeListMap { fragments[iFragment].pRangeList = pRangeList; fragments[iFragment]._range = pRangeList->_range; - RangeSectionFragment** entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); + RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); if (entryInMapToUpdate == NULL) { free(fragments); @@ -252,16 +322,12 @@ class RangeListMap // At this point all the needed memory is allocated, and it is no longer possible to fail. for (uintptr_t iFragment = 0; iFragment < rangeListFragmentCount; iFragment++) { - RangeSectionFragment* initialFragmentInMap = VolatileLoad(entriesInMapToUpdate[iFragment]); do { - VolatileStore(&fragments[iFragment].pRangeListFragmentNext, initialFragmentInMap); - RangeSectionFragment* currentFragmentInMap = (RangeSectionFragment*)InterlockedCompareExchangePointer((volatile PVOID*)entriesInMapToUpdate[iFragment], &(fragments[iFragment]), initialFragmentInMap); - if (currentFragmentInMap == initialFragmentInMap) - { + RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(); + fragments[iFragment].pRangeListFragmentNext.VolatileStore(initialFragmentInMap); + if (entriesInMapToUpdate[iFragment]->AtomicReplace(&(fragments[iFragment]), initialFragmentInMap)) break; - } - initialFragmentInMap = currentFragmentInMap; } while (true); } @@ -278,13 +344,15 @@ class RangeListMap if (fragment == NULL) return NULL; - while (fragment != NULL && !fragment->InRange(address)) + while ((fragment != NULL) && !fragment->InRange(address)) { - fragment = VolatileLoadWithoutBarrier(&fragment->pRangeListFragmentNext); + fragment = fragment->pRangeListFragmentNext.VolatileLoadWithoutBarrier(); } if (fragment != NULL) { + if (fragment->pRangeList->pRangeListNextForDelete != NULL) + return NULL; return fragment->pRangeList; } @@ -326,7 +394,7 @@ class RangeListMap void RemoveRangeListCannotCallInParallelWithCleanup(RangeSection* pRangeList) { - assert(pRangeList->pRangeListNextForDelete = nullptr); + assert(pRangeList->pRangeListNextForDelete == nullptr); assert(pRangeList == LookupRangeListCannotCallInParallelWithCleanup(pRangeList->_range.begin)); // Removal is implemented by placing onto the cleanup linked list. This is then processed later during cleanup @@ -335,7 +403,7 @@ class RangeListMap { pLatestRemovedRangeList = VolatileLoad(&pCleanupList); VolatileStore(&pRangeList->pRangeListNextForDelete, pLatestRemovedRangeList); - } while (InterlockedCompareExchangePointer((volatile PVOID *)&pCleanupList, pRangeList, pLatestRemovedRangeList) == pLatestRemovedRangeList); + } while (InterlockedCompareExchangePointer((volatile PVOID *)&pCleanupList, pRangeList, pLatestRemovedRangeList) != pLatestRemovedRangeList); } void CleanupWhileNoThreadMayLookupRangeLists() @@ -347,7 +415,7 @@ class RangeListMap return; } - while (this->pCleanupList != nullptr) + while (this->pCleanupList != &ClearListMarker) { RangeSection* pRangeListToCleanup = this->pCleanupList; RangeSectionFragment* pRangeListFragmentToFree = nullptr; @@ -360,23 +428,24 @@ class RangeListMap // Remove fragments from each of the fragment linked lists for (uintptr_t iFragment = 0; iFragment < rangeListFragmentCount; iFragment++) { - RangeSectionFragment** entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); + RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); assert(entryInMapToUpdate != NULL); - while ((*entryInMapToUpdate)->pRangeList != pRangeListToCleanup) + while ((entryInMapToUpdate->VolatileLoadWithoutBarrier())->pRangeList != pRangeListToCleanup) { - entryInMapToUpdate = &(*entryInMapToUpdate)->pRangeListFragmentNext; + entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier())->pRangeListFragmentNext; } + RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(); + // The fragment associated with the start of the range has the address that was allocated earlier if (iFragment == 0) { - pRangeListFragmentToFree = *entryInMapToUpdate; + pRangeListFragmentToFree = fragment; assert(pRangeListFragmentToFree->isPrimaryRangeListFragment); } - *entryInMapToUpdate = (*entryInMapToUpdate)->pRangeListFragmentNext; - + entryInMapToUpdate->VolatileStore(fragment->pRangeListFragmentNext.VolatileLoadWithoutBarrier()); addressToPrepForCleanup = IncrementAddressByMaxSizeOfFragment(addressToPrepForCleanup); } @@ -418,6 +487,58 @@ int main() result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); assert(result == &rSectionSecond); + map.RemoveRangeListCannotCallInParallelWithCleanup(&rSectionFirst); + + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + assert(result == &rSectionSecond); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + assert(result == &rSectionSecond); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + assert(result == &rSectionSecond); + + map.CleanupWhileNoThreadMayLookupRangeLists(); + + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + assert(result == &rSectionSecond); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + assert(result == &rSectionSecond); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + assert(result == &rSectionSecond); + + map.RemoveRangeListCannotCallInParallelWithCleanup(&rSectionSecond); + + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + assert(result == NULL); + + map.CleanupWhileNoThreadMayLookupRangeLists(); + + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + assert(result == NULL); + result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + assert(result == NULL); + std::cout << "Done\n"; } From 5eee830227ad8c09e7148bcfceefdb56ab9f179b Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Nov 2022 13:44:33 -0800 Subject: [PATCH 07/25] Rework to passing down the current locking status --- temp/RangeListMap.cpp | 174 +++++++++++++++++++----------------------- 1 file changed, 78 insertions(+), 96 deletions(-) diff --git a/temp/RangeListMap.cpp b/temp/RangeListMap.cpp index bc913a349a747..d463ffa3ecac3 100644 --- a/temp/RangeListMap.cpp +++ b/temp/RangeListMap.cpp @@ -7,6 +7,8 @@ #define TARGET_64BIT +typedef uintptr_t TADDR; + template void VolatileStore(T* ptr, T val) { @@ -47,6 +49,14 @@ class RangeSection RangeSection* pRangeListNextForDelete = nullptr; // Used for adding to the cleanup list }; +enum class RangeListLockState +{ + None, + NeedsLock, + ReaderLocked, + WriteLocked, +}; + // For 64bit, we work with 8KB chunks of memory holding pointers to the next level. This provides 10 bits of address resolution per level. // For *reasons* the X64 hardware is limited to 57bits of addressable address space, and the minimum granularity that makes sense for range lists is 64KB (or every 2^16 bits) // Similarly the Arm64 specification requires addresses to use at most 52 bits. Thus we use the maximum addressable range of X64 to provide the real max range @@ -59,11 +69,11 @@ class RangeSection // For 32bit VA processes, use 1KB chunks holding pointers to the next level. This provides 8 bites of address resolution per level. [31:24] and [23:16]. // The memory safety model for segment maps is that the pointers held within the individual segments can never change other than to go from NULL to a meaningful pointer, -// except for the final level, which is only permitted to change when CleanupWhileNoThreadMayLookupRangeLists is in use. +// except for the final level, which is only permitted to change when CleanupRangeLists is in use. class RangeListMap { - RangeSection ClearListMarker; + RangeSection EndOfCleanupListMarker; class RangeSectionFragment; class RangeSectionFragmentPointer @@ -102,11 +112,16 @@ class RangeListMap return _ptr == 0; } - RangeSectionFragment* VolatileLoadWithoutBarrier() + RangeSectionFragment* VolatileLoadWithoutBarrier(RangeListLockState *pLockState) { uintptr_t ptr = ::VolatileLoadWithoutBarrier(&_ptr); if ((ptr & 1) == 1) { + if ((*pLockState == RangeListLockState::None) || (*pLockState == RangeListLockState::NeedsLock)) + { + *pLockState = RangeListLockState::NeedsLock; + return NULL; + } return (RangeSectionFragment*)(ptr - 1); } else @@ -129,8 +144,6 @@ class RangeListMap } }; - - // Unlike a RangeSection, a RangeSectionFragment cannot span multiple elements of the last level of the RangeListMap // Always allocated via calloc class RangeSectionFragment @@ -170,7 +183,6 @@ class RangeListMap RangeSectionTopLevel *_topLevel = nullptr; - int _lock = 0; // 0 indicates unlocked. -1 indicates in the process of cleanup, Positive numbers indicate read locks RangeSection* pCleanupList; const uintptr_t bitsAtLastLevel = maxSetBit - (bitsPerLevel * mapLevels) + 1; @@ -240,7 +252,7 @@ class RangeListMap return result; } - RangeSectionFragment* GetRangeListForAddress(void* address) + RangeSectionFragment* GetRangeListForAddress(void* address, RangeListLockState *pLockState) { #ifdef TARGET_64BIT auto _rangeListL4 = VolatileLoad(&_topLevel); @@ -257,7 +269,7 @@ class RangeListMap if (_rangeListL1 == NULL) return NULL; - return ((*_rangeListL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(); + return ((*_rangeListL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(pLockState); } uintptr_t RangeListFragmentCount(RangeSection *pRangeList) @@ -274,7 +286,7 @@ class RangeListMap } public: - RangeListMap() : _topLevel{0}, ClearListMarker(Range()), pCleanupList(&ClearListMarker) + RangeListMap() : _topLevel{0}, EndOfCleanupListMarker(Range()), pCleanupList(&EndOfCleanupListMarker) { } @@ -283,8 +295,10 @@ class RangeListMap return true; } - bool AttachRangeListToMap(RangeSection* pRangeList) + bool AttachRangeListToMap(RangeSection* pRangeList, RangeListLockState *pLockState) { + assert(*pLockState == RangeListLockState::ReaderLocked); // Must be locked so that the cannot fail case, can't fail. NOTE: This only needs the reader lock, as the attach process can happen in parallel to reads. + uintptr_t rangeListFragmentCount = RangeListFragmentCount(pRangeList); RangeSectionFragment* fragments = (RangeSectionFragment*)calloc(rangeListFragmentCount, sizeof(RangeSectionFragment)); @@ -324,7 +338,7 @@ class RangeListMap { do { - RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(); + RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(pLockState); fragments[iFragment].pRangeListFragmentNext.VolatileStore(initialFragmentInMap); if (entriesInMapToUpdate[iFragment]->AtomicReplace(&(fragments[iFragment]), initialFragmentInMap)) break; @@ -337,16 +351,15 @@ class RangeListMap return true; } -private: - RangeSection* LookupRangeListByAddressForKnownValidAddressWhileCleanupCannotHappenOrUnderLock(void* address) + RangeSection* LookupRangeList(void* address, RangeListLockState *pLockState) { - RangeSectionFragment* fragment = GetRangeListForAddress(address); + RangeSectionFragment* fragment = GetRangeListForAddress(address, pLockState); if (fragment == NULL) return NULL; while ((fragment != NULL) && !fragment->InRange(address)) { - fragment = fragment->pRangeListFragmentNext.VolatileLoadWithoutBarrier(); + fragment = fragment->pRangeListFragmentNext.VolatileLoadWithoutBarrier(pLockState); } if (fragment != NULL) @@ -359,43 +372,9 @@ class RangeListMap return NULL; } -public: - bool TryLookupRangeListByAddressForKnownValidAddress(void* address, RangeSection** pRangeList) - { - *pRangeList = NULL; - - bool locked = false; - int lockVal; - - do - { - lockVal = VolatileLoad(&_lock); - - // Cleanup in process. Do not succeed in producing result - if (lockVal < 0) - return false; - - // Take reader lock - } while (InterlockedCompareExchange((volatile unsigned*)&_lock, (unsigned)lockVal + 1, (unsigned)lockVal) != lockVal); - - *pRangeList = LookupRangeListByAddressForKnownValidAddressWhileCleanupCannotHappenOrUnderLock(address); - - // Release lock - InterlockedDecrement((volatile unsigned*)&_lock); - } - - // Due to the thread safety semantics of removal, the address passed in here MUST be the address of a function on the stack, and therefore not eligible to be cleaned up due to some race. - RangeSection* LookupRangeListCannotCallInParallelWithCleanup(void* address) - { - // Locked readers may be reading, but no cleanup can be happening - assert(_lock != -1); - return LookupRangeListByAddressForKnownValidAddressWhileCleanupCannotHappenOrUnderLock(address); - } - - void RemoveRangeListCannotCallInParallelWithCleanup(RangeSection* pRangeList) + void RemoveRangeList(RangeSection* pRangeList) { assert(pRangeList->pRangeListNextForDelete == nullptr); - assert(pRangeList == LookupRangeListCannotCallInParallelWithCleanup(pRangeList->_range.begin)); // Removal is implemented by placing onto the cleanup linked list. This is then processed later during cleanup RangeSection* pLatestRemovedRangeList; @@ -406,16 +385,11 @@ class RangeListMap } while (InterlockedCompareExchangePointer((volatile PVOID *)&pCleanupList, pRangeList, pLatestRemovedRangeList) != pLatestRemovedRangeList); } - void CleanupWhileNoThreadMayLookupRangeLists() + void CleanupRangeLists(RangeListLockState *pLockState) { - // Take cleanup lock - if (InterlockedCompareExchange((volatile unsigned*)&_lock, (unsigned)(-1), 0) != 0) - { - // If a locked read is in progress. That's OK. We'll clean up some in a future call to cleanup. - return; - } + assert(*pLockState == RangeListLockState::WriteLocked); - while (this->pCleanupList != &ClearListMarker) + while (this->pCleanupList != &EndOfCleanupListMarker) { RangeSection* pRangeListToCleanup = this->pCleanupList; RangeSectionFragment* pRangeListFragmentToFree = nullptr; @@ -431,12 +405,12 @@ class RangeListMap RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); assert(entryInMapToUpdate != NULL); - while ((entryInMapToUpdate->VolatileLoadWithoutBarrier())->pRangeList != pRangeListToCleanup) + while ((entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeList != pRangeListToCleanup) { - entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier())->pRangeListFragmentNext; + entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeListFragmentNext; } - RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(); + RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState); // The fragment associated with the start of the range has the address that was allocated earlier if (iFragment == 0) @@ -445,16 +419,13 @@ class RangeListMap assert(pRangeListFragmentToFree->isPrimaryRangeListFragment); } - entryInMapToUpdate->VolatileStore(fragment->pRangeListFragmentNext.VolatileLoadWithoutBarrier()); + entryInMapToUpdate->VolatileStore(fragment->pRangeListFragmentNext.VolatileLoadWithoutBarrier(pLockState)); addressToPrepForCleanup = IncrementAddressByMaxSizeOfFragment(addressToPrepForCleanup); } // Free the array of fragments free(pRangeListFragmentToFree); } - - // Release lock - VolatileStore(&_lock, 0); } }; @@ -471,72 +442,83 @@ int main() RangeSection rSectionSecond(rSecond); - map.AttachRangeListToMap(&rSectionFirst); - map.AttachRangeListToMap(&rSectionSecond); + RangeListLockState lockState = RangeListLockState::ReaderLocked; + map.AttachRangeListToMap(&rSectionFirst, &lockState); + map.AttachRangeListToMap(&rSectionSecond, &lockState); RangeSection *result; - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + lockState = RangeListLockState::None; + + result = map.LookupRangeList((void*)0x1111000, &lockState); assert(result == &rSectionFirst); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + result = map.LookupRangeList((void*)0x1111050, &lockState); assert(result == &rSectionFirst); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + result = map.LookupRangeList((void*)0x1111051, &lockState); assert(result == &rSectionSecond); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + result = map.LookupRangeList((void*)0x1151050, &lockState); assert(result == &rSectionSecond); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + result = map.LookupRangeList((void*)0x1192050, &lockState); assert(result == &rSectionSecond); - map.RemoveRangeListCannotCallInParallelWithCleanup(&rSectionFirst); + assert(lockState == RangeListLockState::None); + map.RemoveRangeList(&rSectionFirst); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + result = map.LookupRangeList((void*)0x1111000, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + result = map.LookupRangeList((void*)0x1111050, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + result = map.LookupRangeList((void*)0x1111051, &lockState); assert(result == &rSectionSecond); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + result = map.LookupRangeList((void*)0x1151050, &lockState); assert(result == &rSectionSecond); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + result = map.LookupRangeList((void*)0x1192050, &lockState); assert(result == &rSectionSecond); - map.CleanupWhileNoThreadMayLookupRangeLists(); + assert(lockState == RangeListLockState::None); + lockState = RangeListLockState::WriteLocked; + map.CleanupRangeLists(&lockState); + lockState = RangeListLockState::None; - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + result = map.LookupRangeList((void*)0x1111000, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + result = map.LookupRangeList((void*)0x1111050, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + result = map.LookupRangeList((void*)0x1111051, &lockState); assert(result == &rSectionSecond); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + result = map.LookupRangeList((void*)0x1151050, &lockState); assert(result == &rSectionSecond); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + result = map.LookupRangeList((void*)0x1192050, &lockState); assert(result == &rSectionSecond); - map.RemoveRangeListCannotCallInParallelWithCleanup(&rSectionSecond); + assert(lockState == RangeListLockState::None); + map.RemoveRangeList(&rSectionSecond); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + result = map.LookupRangeList((void*)0x1111000, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + result = map.LookupRangeList((void*)0x1111050, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + result = map.LookupRangeList((void*)0x1111051, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + result = map.LookupRangeList((void*)0x1151050, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + result = map.LookupRangeList((void*)0x1192050, &lockState); assert(result == NULL); - map.CleanupWhileNoThreadMayLookupRangeLists(); + assert(lockState == RangeListLockState::None); + lockState = RangeListLockState::WriteLocked; + map.CleanupRangeLists(&lockState); + lockState = RangeListLockState::None; - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111000); + result = map.LookupRangeList((void*)0x1111000, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111050); + result = map.LookupRangeList((void*)0x1111050, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1111051); + result = map.LookupRangeList((void*)0x1111051, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1151050); + result = map.LookupRangeList((void*)0x1151050, &lockState); assert(result == NULL); - result = map.LookupRangeListCannotCallInParallelWithCleanup((void*)0x1192050); + result = map.LookupRangeList((void*)0x1192050, &lockState); assert(result == NULL); std::cout << "Done\n"; From e4fb4dbb9acc1c1463468261ebb970a38710b248 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Nov 2022 14:45:12 -0800 Subject: [PATCH 08/25] Allocation of RangeSection and freeing added, as well as testing for the collectible handling characteristics --- temp/RangeListMap.cpp | 202 +++++++++++++++++++++++++++++------------- 1 file changed, 142 insertions(+), 60 deletions(-) diff --git a/temp/RangeListMap.cpp b/temp/RangeListMap.cpp index d463ffa3ecac3..bd724674e9c51 100644 --- a/temp/RangeListMap.cpp +++ b/temp/RangeListMap.cpp @@ -4,10 +4,20 @@ #include #include #include +using namespace std; +class IJitManager; +class Module; +class HeapList; +typedef Module* PTR_Module; +typedef HeapList* PTR_HeapList; +typedef uintptr_t TADDR; #define TARGET_64BIT -typedef uintptr_t TADDR; + + + + template void VolatileStore(T* ptr, T val) @@ -30,6 +40,7 @@ T VolatileLoadWithoutBarrier(T* ptr) class Range { public: + // [begin,end] (This is an inclusive range) void* begin; void* end; }; @@ -40,11 +51,39 @@ class RangeSection { friend class RangeListMap; public: - RangeSection(Range range) : - _range(range) + enum RangeSectionFlags + { + RANGE_SECTION_NONE = 0x0, + RANGE_SECTION_COLLECTIBLE = 0x1, + RANGE_SECTION_CODEHEAP = 0x2, + }; + +#ifdef FEATURE_READYTORUN + RangeSection(Range range, IJitManager* pJit, RangeSectionFlags flags, PTR_Module pR2RModule) : + _range(range), + _flags(flags), + _pjit(pJit), + _pR2RModule(pR2RModule), + _pHeapList(NULL) + { + assert(!(flags & RANGE_SECTION_COLLECTIBLE)); + assert(pR2RModule != NULL); + } +#endif + + RangeSection(Range range, IJitManager* pJit, RangeSectionFlags flags, PTR_HeapList pHeapList) : + _range(range), + _flags(flags), + _pjit(pJit), + _pR2RModule(NULL), + _pHeapList(pHeapList) {} - Range _range; + const Range _range; + const RangeSectionFlags _flags; + IJitManager *const _pjit; + const PTR_Module _pR2RModule; + const PTR_HeapList _pHeapList; RangeSection* pRangeListNextForDelete = nullptr; // Used for adding to the cleanup list }; @@ -73,7 +112,7 @@ enum class RangeListLockState class RangeListMap { - RangeSection EndOfCleanupListMarker; + RangeSection* EndOfCleanupListMarker() { return (RangeSection*)1; } class RangeSectionFragment; class RangeSectionFragmentPointer @@ -285,16 +324,6 @@ class RangeListMap return reinterpret_cast(inputAsInt + bytesAtLastLevel); } -public: - RangeListMap() : _topLevel{0}, EndOfCleanupListMarker(Range()), pCleanupList(&EndOfCleanupListMarker) - { - } - - bool Init() - { - return true; - } - bool AttachRangeListToMap(RangeSection* pRangeList, RangeListLockState *pLockState) { assert(*pLockState == RangeListLockState::ReaderLocked); // Must be locked so that the cannot fail case, can't fail. NOTE: This only needs the reader lock, as the attach process can happen in parallel to reads. @@ -321,6 +350,7 @@ class RangeListMap { fragments[iFragment].pRangeList = pRangeList; fragments[iFragment]._range = pRangeList->_range; + fragments[iFragment].isCollectibleRangeListFragment = !!(pRangeList->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); if (entryInMapToUpdate == NULL) { @@ -351,6 +381,48 @@ class RangeListMap return true; } +public: + RangeListMap() : _topLevel{0}, pCleanupList(EndOfCleanupListMarker()) + { + } + + bool Init() + { + return true; + } + +#ifdef FEATURE_READYTORUN + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_Module pR2RModule, RangeListLockState* pLockState) + { + RangeListLockState lockState = RangeListLockState::ReaderLocked; + RangeSection *pSection = new(nothrow)RangeSection(range, pJit, flags, pR2RModule); + if (pSection == NULL) + return NULL; + + if (!AttachRangeListToMap(pSection, pLockState)) + { + delete pSection; + return NULL; + } + return pSection; + } +#endif + + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_HeapList pHeapList, RangeListLockState* pLockState) + { + RangeListLockState lockState = RangeListLockState::ReaderLocked; + RangeSection *pSection = new(nothrow)RangeSection(range, pJit, flags, pHeapList); + if (pSection == NULL) + return NULL; + + if (!AttachRangeListToMap(pSection, pLockState)) + { + delete pSection; + return NULL; + } + return pSection; + } + RangeSection* LookupRangeList(void* address, RangeListLockState *pLockState) { RangeSectionFragment* fragment = GetRangeListForAddress(address, pLockState); @@ -375,6 +447,10 @@ class RangeListMap void RemoveRangeList(RangeSection* pRangeList) { assert(pRangeList->pRangeListNextForDelete == nullptr); + assert(pRangeList->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); +#ifdef FEATURE_READYTORUN + assert(pRangeList->pR2RModule == NULL); +#endif // Removal is implemented by placing onto the cleanup linked list. This is then processed later during cleanup RangeSection* pLatestRemovedRangeList; @@ -389,7 +465,7 @@ class RangeListMap { assert(*pLockState == RangeListLockState::WriteLocked); - while (this->pCleanupList != &EndOfCleanupListMarker) + while (this->pCleanupList != EndOfCleanupListMarker()) { RangeSection* pRangeListToCleanup = this->pCleanupList; RangeSectionFragment* pRangeListFragmentToFree = nullptr; @@ -424,6 +500,7 @@ class RangeListMap } // Free the array of fragments + delete pRangeListToCleanup; free(pRangeListFragmentToFree); } } @@ -438,72 +515,72 @@ int main() Range rSecond; rSecond.begin = (void*)0x1111051; rSecond.end = (void*)0x1192050; - RangeSection rSectionFirst(rFirst); - RangeSection rSectionSecond(rSecond); - RangeListLockState lockState = RangeListLockState::ReaderLocked; - map.AttachRangeListToMap(&rSectionFirst, &lockState); - map.AttachRangeListToMap(&rSectionSecond, &lockState); + RangeSection *rSectionFirst = map.AllocateRange(rFirst, NULL, RangeSection::RANGE_SECTION_COLLECTIBLE, (PTR_HeapList)NULL, &lockState); + RangeSection *rSectionSecond = map.AllocateRange(rSecond, NULL, RangeSection::RANGE_SECTION_NONE, (PTR_HeapList)NULL, &lockState); RangeSection *result; lockState = RangeListLockState::None; - - result = map.LookupRangeList((void*)0x1111000, &lockState); - assert(result == &rSectionFirst); - result = map.LookupRangeList((void*)0x1111050, &lockState); - assert(result == &rSectionFirst); - result = map.LookupRangeList((void*)0x1111051, &lockState); - assert(result == &rSectionSecond); - result = map.LookupRangeList((void*)0x1151050, &lockState); - assert(result == &rSectionSecond); - result = map.LookupRangeList((void*)0x1192050, &lockState); - assert(result == &rSectionSecond); - - assert(lockState == RangeListLockState::None); - map.RemoveRangeList(&rSectionFirst); - result = map.LookupRangeList((void*)0x1111000, &lockState); + assert(lockState == RangeListLockState::NeedsLock); assert(result == NULL); + lockState = RangeListLockState::ReaderLocked; + result = map.LookupRangeList((void*)0x1111000, &lockState); + assert(result == rSectionFirst); + assert(lockState == RangeListLockState::ReaderLocked); + + lockState = RangeListLockState::None; result = map.LookupRangeList((void*)0x1111050, &lockState); + assert(lockState == RangeListLockState::NeedsLock); assert(result == NULL); + lockState = RangeListLockState::ReaderLocked; + result = map.LookupRangeList((void*)0x1111050, &lockState); + assert(result == rSectionFirst); + assert(lockState == RangeListLockState::ReaderLocked); + lockState = RangeListLockState::None; + result = map.LookupRangeList((void*)0x1111051, &lockState); - assert(result == &rSectionSecond); + assert(lockState == RangeListLockState::None); + assert(result == rSectionSecond); result = map.LookupRangeList((void*)0x1151050, &lockState); - assert(result == &rSectionSecond); + assert(lockState == RangeListLockState::None); + assert(result == rSectionSecond); result = map.LookupRangeList((void*)0x1192050, &lockState); - assert(result == &rSectionSecond); - assert(lockState == RangeListLockState::None); - lockState = RangeListLockState::WriteLocked; - map.CleanupRangeLists(&lockState); - lockState = RangeListLockState::None; + assert(result == rSectionSecond); + + map.RemoveRangeList(rSectionFirst); + lockState = RangeListLockState::None; result = map.LookupRangeList((void*)0x1111000, &lockState); + assert(lockState == RangeListLockState::NeedsLock); assert(result == NULL); - result = map.LookupRangeList((void*)0x1111050, &lockState); + lockState = RangeListLockState::ReaderLocked; + result = map.LookupRangeList((void*)0x1111000, &lockState); assert(result == NULL); - result = map.LookupRangeList((void*)0x1111051, &lockState); - assert(result == &rSectionSecond); - result = map.LookupRangeList((void*)0x1151050, &lockState); - assert(result == &rSectionSecond); - result = map.LookupRangeList((void*)0x1192050, &lockState); - assert(result == &rSectionSecond); + assert(lockState == RangeListLockState::ReaderLocked); - assert(lockState == RangeListLockState::None); - map.RemoveRangeList(&rSectionSecond); - - result = map.LookupRangeList((void*)0x1111000, &lockState); + lockState = RangeListLockState::None; + result = map.LookupRangeList((void*)0x1111050, &lockState); + assert(lockState == RangeListLockState::NeedsLock); assert(result == NULL); + lockState = RangeListLockState::ReaderLocked; result = map.LookupRangeList((void*)0x1111050, &lockState); assert(result == NULL); + assert(lockState == RangeListLockState::ReaderLocked); + lockState = RangeListLockState::None; + result = map.LookupRangeList((void*)0x1111051, &lockState); - assert(result == NULL); + assert(result == rSectionSecond); + assert(lockState == RangeListLockState::None); result = map.LookupRangeList((void*)0x1151050, &lockState); - assert(result == NULL); + assert(result == rSectionSecond); + assert(lockState == RangeListLockState::None); result = map.LookupRangeList((void*)0x1192050, &lockState); - assert(result == NULL); + assert(result == rSectionSecond); + assert(lockState == RangeListLockState::None); assert(lockState == RangeListLockState::None); lockState = RangeListLockState::WriteLocked; @@ -512,14 +589,19 @@ int main() result = map.LookupRangeList((void*)0x1111000, &lockState); assert(result == NULL); + assert(lockState == RangeListLockState::None); result = map.LookupRangeList((void*)0x1111050, &lockState); assert(result == NULL); + assert(lockState == RangeListLockState::None); result = map.LookupRangeList((void*)0x1111051, &lockState); - assert(result == NULL); + assert(result == rSectionSecond); + assert(lockState == RangeListLockState::None); result = map.LookupRangeList((void*)0x1151050, &lockState); - assert(result == NULL); + assert(result == rSectionSecond); + assert(lockState == RangeListLockState::None); result = map.LookupRangeList((void*)0x1192050, &lockState); - assert(result == NULL); + assert(result == rSectionSecond); + assert(lockState == RangeListLockState::None); std::cout << "Done\n"; } From 6505eb5393daf5463bd16c9d1dc960cd518bbf48 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Nov 2022 14:52:20 -0800 Subject: [PATCH 09/25] Fix some naming --- temp/RangeListMap.cpp | 292 +++++++++++++++++++++--------------------- 1 file changed, 148 insertions(+), 144 deletions(-) diff --git a/temp/RangeListMap.cpp b/temp/RangeListMap.cpp index bd724674e9c51..ffe4db21b8267 100644 --- a/temp/RangeListMap.cpp +++ b/temp/RangeListMap.cpp @@ -1,4 +1,4 @@ -// RangeListMap.cpp : This file contains the 'main' function. Program execution begins and ends there. +// RangeSectionMap.cpp : This file contains the 'main' function. Program execution begins and ends there. // #include @@ -45,11 +45,11 @@ class Range void* end; }; -class RangeListMap; +class RangeSectionMap; class RangeSection { - friend class RangeListMap; + friend class RangeSectionMap; public: enum RangeSectionFlags { @@ -85,10 +85,14 @@ class RangeSection const PTR_Module _pR2RModule; const PTR_HeapList _pHeapList; - RangeSection* pRangeListNextForDelete = nullptr; // Used for adding to the cleanup list +#if defined(TARGET_AMD64) + PTR_UnwindInfoTable pUnwindInfoTable; // Points to unwind information for this memory range. +#endif // defined(TARGET_AMD64) + + RangeSection* _pRangeSectionNextForDelete = nullptr; // Used for adding to the cleanup list }; -enum class RangeListLockState +enum class RangeSectionLockState { None, NeedsLock, @@ -108,12 +112,10 @@ enum class RangeListLockState // For 32bit VA processes, use 1KB chunks holding pointers to the next level. This provides 8 bites of address resolution per level. [31:24] and [23:16]. // The memory safety model for segment maps is that the pointers held within the individual segments can never change other than to go from NULL to a meaningful pointer, -// except for the final level, which is only permitted to change when CleanupRangeLists is in use. +// except for the final level, which is only permitted to change when CleanupRangeSections is in use. -class RangeListMap +class RangeSectionMap { - RangeSection* EndOfCleanupListMarker() { return (RangeSection*)1; } - class RangeSectionFragment; class RangeSectionFragmentPointer { @@ -126,7 +128,7 @@ class RangeListMap if (ptr == 0) return ptr; - if (fragment->isCollectibleRangeListFragment) + if (fragment->isCollectibleRangeSectionFragment) { ptr += 1; } @@ -151,14 +153,14 @@ class RangeListMap return _ptr == 0; } - RangeSectionFragment* VolatileLoadWithoutBarrier(RangeListLockState *pLockState) + RangeSectionFragment* VolatileLoadWithoutBarrier(RangeSectionLockState *pLockState) { uintptr_t ptr = ::VolatileLoadWithoutBarrier(&_ptr); if ((ptr & 1) == 1) { - if ((*pLockState == RangeListLockState::None) || (*pLockState == RangeListLockState::NeedsLock)) + if ((*pLockState == RangeSectionLockState::None) || (*pLockState == RangeSectionLockState::NeedsLock)) { - *pLockState = RangeListLockState::NeedsLock; + *pLockState = RangeSectionLockState::NeedsLock; return NULL; } return (RangeSectionFragment*)(ptr - 1); @@ -183,17 +185,17 @@ class RangeListMap } }; - // Unlike a RangeSection, a RangeSectionFragment cannot span multiple elements of the last level of the RangeListMap + // Unlike a RangeSection, a RangeSectionFragment cannot span multiple elements of the last level of the RangeSectionMap // Always allocated via calloc class RangeSectionFragment { public: - RangeSectionFragmentPointer pRangeListFragmentNext; + RangeSectionFragmentPointer pRangeSectionFragmentNext; Range _range; - RangeSection* pRangeList; - bool InRange(void* address) { return address >= _range.begin && address <= _range.end && pRangeList->pRangeListNextForDelete == NULL; } - bool isPrimaryRangeListFragment; // RangeSectionFragment are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. - bool isCollectibleRangeListFragment; // RangeSectionFragments + RangeSection* pRangeSection; + bool InRange(void* address) { return address >= _range.begin && address <= _range.end && pRangeSection->_pRangeSectionNextForDelete == NULL; } + bool isPrimaryRangeSectionFragment; // RangeSectionFragment are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. + bool isCollectibleRangeSectionFragment; // RangeSectionFragments }; #ifdef TARGET_64BIT @@ -222,11 +224,13 @@ class RangeListMap RangeSectionTopLevel *_topLevel = nullptr; - RangeSection* pCleanupList; + RangeSection* _pCleanupList; const uintptr_t bitsAtLastLevel = maxSetBit - (bitsPerLevel * mapLevels) + 1; const uintptr_t bytesAtLastLevel = (((uintptr_t)1) << (bitsAtLastLevel - 1)); + RangeSection* EndOfCleanupListMarker() { return (RangeSection*)1; } + void* AllocateLevel() { return calloc(entriesPerMapLevel, sizeof(void*)); } uintptr_t EffectiveBitsForLevel(void* address, uintptr_t level) @@ -271,49 +275,49 @@ class RangeListMap { uintptr_t level = mapLevels; #ifdef TARGET_64BIT - auto _rangeListL3 = EnsureLevel(address, &_topLevel, level); - if (_rangeListL3 == NULL) + auto _RangeSectionL3 = EnsureLevel(address, &_topLevel, level); + if (_RangeSectionL3 == NULL) return NULL; // Failure case - auto _rangeListL2 = EnsureLevel(address, _rangeListL3, --level); - if (_rangeListL2 == NULL) + auto _RangeSectionL2 = EnsureLevel(address, _RangeSectionL3, --level); + if (_RangeSectionL2 == NULL) return NULL; // Failure case #else - auto _rangeListL2 = &topLevel; + auto _RangeSectionL2 = &topLevel; #endif - auto _rangeListL1 = EnsureLevel(address, _rangeListL2, --level); - if (_rangeListL1 == NULL) + auto _RangeSectionL1 = EnsureLevel(address, _RangeSectionL2, --level); + if (_RangeSectionL1 == NULL) return NULL; // Failure case - auto result = EnsureLevel(address, _rangeListL1, --level); + auto result = EnsureLevel(address, _RangeSectionL1, --level); if (result == NULL) return NULL; // Failure case return result; } - RangeSectionFragment* GetRangeListForAddress(void* address, RangeListLockState *pLockState) + RangeSectionFragment* GetRangeSectionForAddress(void* address, RangeSectionLockState *pLockState) { #ifdef TARGET_64BIT - auto _rangeListL4 = VolatileLoad(&_topLevel); - auto _rangeListL3 = (*_rangeListL4)[EffectiveBitsForLevel(address, 4)]; - if (_rangeListL3 == NULL) + auto _RangeSectionL4 = VolatileLoad(&_topLevel); + auto _RangeSectionL3 = (*_RangeSectionL4)[EffectiveBitsForLevel(address, 4)]; + if (_RangeSectionL3 == NULL) return NULL; - auto _rangeListL2 = (*_rangeListL3)[EffectiveBitsForLevel(address, 3)]; - if (_rangeListL2 == NULL) + auto _RangeSectionL2 = (*_RangeSectionL3)[EffectiveBitsForLevel(address, 3)]; + if (_RangeSectionL2 == NULL) return NULL; - auto _rangeListL1 = (*_rangeListL2)[EffectiveBitsForLevel(address, 2)]; + auto _RangeSectionL1 = (*_RangeSectionL2)[EffectiveBitsForLevel(address, 2)]; #else - auto _rangeListL1 = VolatileLoad(&_topLevel[EffectiveBitsForLevel(address, 2)]); // Use a VolatileLoad on the top level operation to ensure that the entire map is synchronized to a state that includes all data needed to examine currently active function pointers. + auto _RangeSectionL1 = VolatileLoad(&_topLevel[EffectiveBitsForLevel(address, 2)]); // Use a VolatileLoad on the top level operation to ensure that the entire map is synchronized to a state that includes all data needed to examine currently active function pointers. #endif - if (_rangeListL1 == NULL) + if (_RangeSectionL1 == NULL) return NULL; - return ((*_rangeListL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(pLockState); + return ((*_RangeSectionL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(pLockState); } - uintptr_t RangeListFragmentCount(RangeSection *pRangeList) + uintptr_t RangeSectionFragmentCount(RangeSection *pRangeSection) { - uintptr_t rangeSize = reinterpret_cast(pRangeList->_range.end) - reinterpret_cast(pRangeList->_range.begin); + uintptr_t rangeSize = reinterpret_cast(pRangeSection->_range.end) - reinterpret_cast(pRangeSection->_range.begin); rangeSize /= bytesAtLastLevel; return rangeSize + 1; } @@ -324,33 +328,33 @@ class RangeListMap return reinterpret_cast(inputAsInt + bytesAtLastLevel); } - bool AttachRangeListToMap(RangeSection* pRangeList, RangeListLockState *pLockState) + bool AttachRangeSectionToMap(RangeSection* pRangeSection, RangeSectionLockState *pLockState) { - assert(*pLockState == RangeListLockState::ReaderLocked); // Must be locked so that the cannot fail case, can't fail. NOTE: This only needs the reader lock, as the attach process can happen in parallel to reads. + assert(*pLockState == RangeSectionLockState::ReaderLocked); // Must be locked so that the cannot fail case, can't fail. NOTE: This only needs the reader lock, as the attach process can happen in parallel to reads. - uintptr_t rangeListFragmentCount = RangeListFragmentCount(pRangeList); - RangeSectionFragment* fragments = (RangeSectionFragment*)calloc(rangeListFragmentCount, sizeof(RangeSectionFragment)); + uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSection); + RangeSectionFragment* fragments = (RangeSectionFragment*)calloc(rangeSectionFragmentCount, sizeof(RangeSectionFragment)); if (fragments == NULL) { return false; } - RangeSectionFragmentPointer** entriesInMapToUpdate = (RangeSectionFragmentPointer**)calloc(rangeListFragmentCount, sizeof(RangeSectionFragmentPointer*)); + RangeSectionFragmentPointer** entriesInMapToUpdate = (RangeSectionFragmentPointer**)calloc(rangeSectionFragmentCount, sizeof(RangeSectionFragmentPointer*)); if (entriesInMapToUpdate == NULL) { free(fragments); return false; } - fragments[0].isPrimaryRangeListFragment = true; + fragments[0].isPrimaryRangeSectionFragment = true; - void* addressToPrepForUpdate = pRangeList->_range.begin; - for (uintptr_t iFragment = 0; iFragment < rangeListFragmentCount; iFragment++) + void* addressToPrepForUpdate = pRangeSection->_range.begin; + for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) { - fragments[iFragment].pRangeList = pRangeList; - fragments[iFragment]._range = pRangeList->_range; - fragments[iFragment].isCollectibleRangeListFragment = !!(pRangeList->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); + fragments[iFragment].pRangeSection = pRangeSection; + fragments[iFragment]._range = pRangeSection->_range; + fragments[iFragment].isCollectibleRangeSectionFragment = !!(pRangeSection->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); if (entryInMapToUpdate == NULL) { @@ -364,12 +368,12 @@ class RangeListMap } // At this point all the needed memory is allocated, and it is no longer possible to fail. - for (uintptr_t iFragment = 0; iFragment < rangeListFragmentCount; iFragment++) + for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) { do { RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(pLockState); - fragments[iFragment].pRangeListFragmentNext.VolatileStore(initialFragmentInMap); + fragments[iFragment].pRangeSectionFragmentNext.VolatileStore(initialFragmentInMap); if (entriesInMapToUpdate[iFragment]->AtomicReplace(&(fragments[iFragment]), initialFragmentInMap)) break; } while (true); @@ -382,7 +386,7 @@ class RangeListMap } public: - RangeListMap() : _topLevel{0}, pCleanupList(EndOfCleanupListMarker()) + RangeSectionMap() : _topLevel{0}, _pCleanupList(EndOfCleanupListMarker()) { } @@ -392,14 +396,14 @@ class RangeListMap } #ifdef FEATURE_READYTORUN - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_Module pR2RModule, RangeListLockState* pLockState) + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_Module pR2RModule, RangeSectionLockState* pLockState) { - RangeListLockState lockState = RangeListLockState::ReaderLocked; + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; RangeSection *pSection = new(nothrow)RangeSection(range, pJit, flags, pR2RModule); if (pSection == NULL) return NULL; - if (!AttachRangeListToMap(pSection, pLockState)) + if (!AttachRangeSectionToMap(pSection, pLockState)) { delete pSection; return NULL; @@ -408,14 +412,14 @@ class RangeListMap } #endif - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_HeapList pHeapList, RangeListLockState* pLockState) + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_HeapList pHeapList, RangeSectionLockState* pLockState) { - RangeListLockState lockState = RangeListLockState::ReaderLocked; + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; RangeSection *pSection = new(nothrow)RangeSection(range, pJit, flags, pHeapList); if (pSection == NULL) return NULL; - if (!AttachRangeListToMap(pSection, pLockState)) + if (!AttachRangeSectionToMap(pSection, pLockState)) { delete pSection; return NULL; @@ -423,67 +427,67 @@ class RangeListMap return pSection; } - RangeSection* LookupRangeList(void* address, RangeListLockState *pLockState) + RangeSection* LookupRangeSection(void* address, RangeSectionLockState *pLockState) { - RangeSectionFragment* fragment = GetRangeListForAddress(address, pLockState); + RangeSectionFragment* fragment = GetRangeSectionForAddress(address, pLockState); if (fragment == NULL) return NULL; while ((fragment != NULL) && !fragment->InRange(address)) { - fragment = fragment->pRangeListFragmentNext.VolatileLoadWithoutBarrier(pLockState); + fragment = fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState); } if (fragment != NULL) { - if (fragment->pRangeList->pRangeListNextForDelete != NULL) + if (fragment->pRangeSection->_pRangeSectionNextForDelete != NULL) return NULL; - return fragment->pRangeList; + return fragment->pRangeSection; } return NULL; } - void RemoveRangeList(RangeSection* pRangeList) + void RemoveRangeSection(RangeSection* pRangeSection) { - assert(pRangeList->pRangeListNextForDelete == nullptr); - assert(pRangeList->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); + assert(pRangeSection->_pRangeSectionNextForDelete == nullptr); + assert(pRangeSection->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); #ifdef FEATURE_READYTORUN - assert(pRangeList->pR2RModule == NULL); + assert(pRangeSection->pR2RModule == NULL); #endif // Removal is implemented by placing onto the cleanup linked list. This is then processed later during cleanup - RangeSection* pLatestRemovedRangeList; + RangeSection* pLatestRemovedRangeSection; do { - pLatestRemovedRangeList = VolatileLoad(&pCleanupList); - VolatileStore(&pRangeList->pRangeListNextForDelete, pLatestRemovedRangeList); - } while (InterlockedCompareExchangePointer((volatile PVOID *)&pCleanupList, pRangeList, pLatestRemovedRangeList) != pLatestRemovedRangeList); + pLatestRemovedRangeSection = VolatileLoad(&_pCleanupList); + VolatileStore(&pRangeSection->_pRangeSectionNextForDelete, pLatestRemovedRangeSection); + } while (InterlockedCompareExchangePointer((volatile PVOID *)&_pCleanupList, pRangeSection, pLatestRemovedRangeSection) != pLatestRemovedRangeSection); } - void CleanupRangeLists(RangeListLockState *pLockState) + void CleanupRangeSections(RangeSectionLockState *pLockState) { - assert(*pLockState == RangeListLockState::WriteLocked); + assert(*pLockState == RangeSectionLockState::WriteLocked); - while (this->pCleanupList != EndOfCleanupListMarker()) + while (this->_pCleanupList != EndOfCleanupListMarker()) { - RangeSection* pRangeListToCleanup = this->pCleanupList; - RangeSectionFragment* pRangeListFragmentToFree = nullptr; - this->pCleanupList = pRangeListToCleanup->pRangeListNextForDelete; + RangeSection* pRangeSectionToCleanup = this->_pCleanupList; + RangeSectionFragment* pRangeSectionFragmentToFree = nullptr; + this->_pCleanupList = pRangeSectionToCleanup->_pRangeSectionNextForDelete; - uintptr_t rangeListFragmentCount = RangeListFragmentCount(pRangeListToCleanup); + uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSectionToCleanup); - void* addressToPrepForCleanup = pRangeListToCleanup->_range.begin; + void* addressToPrepForCleanup = pRangeSectionToCleanup->_range.begin; // Remove fragments from each of the fragment linked lists - for (uintptr_t iFragment = 0; iFragment < rangeListFragmentCount; iFragment++) + for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) { RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); assert(entryInMapToUpdate != NULL); - while ((entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeList != pRangeListToCleanup) + while ((entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSection != pRangeSectionToCleanup) { - entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeListFragmentNext; + entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSectionFragmentNext; } RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState); @@ -491,24 +495,24 @@ class RangeListMap // The fragment associated with the start of the range has the address that was allocated earlier if (iFragment == 0) { - pRangeListFragmentToFree = fragment; - assert(pRangeListFragmentToFree->isPrimaryRangeListFragment); + pRangeSectionFragmentToFree = fragment; + assert(pRangeSectionFragmentToFree->isPrimaryRangeSectionFragment); } - entryInMapToUpdate->VolatileStore(fragment->pRangeListFragmentNext.VolatileLoadWithoutBarrier(pLockState)); + entryInMapToUpdate->VolatileStore(fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState)); addressToPrepForCleanup = IncrementAddressByMaxSizeOfFragment(addressToPrepForCleanup); } // Free the array of fragments - delete pRangeListToCleanup; - free(pRangeListFragmentToFree); + delete pRangeSectionToCleanup; + free(pRangeSectionFragmentToFree); } } }; int main() { - RangeListMap map; + RangeSectionMap map; Range rFirst; rFirst.begin = (void*)0x1111000; rFirst.end = (void*)0x1111050; @@ -516,92 +520,92 @@ int main() rSecond.begin = (void*)0x1111051; rSecond.end = (void*)0x1192050; - RangeListLockState lockState = RangeListLockState::ReaderLocked; + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; RangeSection *rSectionFirst = map.AllocateRange(rFirst, NULL, RangeSection::RANGE_SECTION_COLLECTIBLE, (PTR_HeapList)NULL, &lockState); RangeSection *rSectionSecond = map.AllocateRange(rSecond, NULL, RangeSection::RANGE_SECTION_NONE, (PTR_HeapList)NULL, &lockState); RangeSection *result; - lockState = RangeListLockState::None; - result = map.LookupRangeList((void*)0x1111000, &lockState); - assert(lockState == RangeListLockState::NeedsLock); + lockState = RangeSectionLockState::None; + result = map.LookupRangeSection((void*)0x1111000, &lockState); + assert(lockState == RangeSectionLockState::NeedsLock); assert(result == NULL); - lockState = RangeListLockState::ReaderLocked; - result = map.LookupRangeList((void*)0x1111000, &lockState); + lockState = RangeSectionLockState::ReaderLocked; + result = map.LookupRangeSection((void*)0x1111000, &lockState); assert(result == rSectionFirst); - assert(lockState == RangeListLockState::ReaderLocked); + assert(lockState == RangeSectionLockState::ReaderLocked); - lockState = RangeListLockState::None; - result = map.LookupRangeList((void*)0x1111050, &lockState); - assert(lockState == RangeListLockState::NeedsLock); + lockState = RangeSectionLockState::None; + result = map.LookupRangeSection((void*)0x1111050, &lockState); + assert(lockState == RangeSectionLockState::NeedsLock); assert(result == NULL); - lockState = RangeListLockState::ReaderLocked; - result = map.LookupRangeList((void*)0x1111050, &lockState); + lockState = RangeSectionLockState::ReaderLocked; + result = map.LookupRangeSection((void*)0x1111050, &lockState); assert(result == rSectionFirst); - assert(lockState == RangeListLockState::ReaderLocked); - lockState = RangeListLockState::None; + assert(lockState == RangeSectionLockState::ReaderLocked); + lockState = RangeSectionLockState::None; - result = map.LookupRangeList((void*)0x1111051, &lockState); - assert(lockState == RangeListLockState::None); + result = map.LookupRangeSection((void*)0x1111051, &lockState); + assert(lockState == RangeSectionLockState::None); assert(result == rSectionSecond); - result = map.LookupRangeList((void*)0x1151050, &lockState); - assert(lockState == RangeListLockState::None); + result = map.LookupRangeSection((void*)0x1151050, &lockState); + assert(lockState == RangeSectionLockState::None); assert(result == rSectionSecond); - result = map.LookupRangeList((void*)0x1192050, &lockState); - assert(lockState == RangeListLockState::None); + result = map.LookupRangeSection((void*)0x1192050, &lockState); + assert(lockState == RangeSectionLockState::None); assert(result == rSectionSecond); - map.RemoveRangeList(rSectionFirst); + map.RemoveRangeSection(rSectionFirst); - lockState = RangeListLockState::None; - result = map.LookupRangeList((void*)0x1111000, &lockState); - assert(lockState == RangeListLockState::NeedsLock); + lockState = RangeSectionLockState::None; + result = map.LookupRangeSection((void*)0x1111000, &lockState); + assert(lockState == RangeSectionLockState::NeedsLock); assert(result == NULL); - lockState = RangeListLockState::ReaderLocked; - result = map.LookupRangeList((void*)0x1111000, &lockState); + lockState = RangeSectionLockState::ReaderLocked; + result = map.LookupRangeSection((void*)0x1111000, &lockState); assert(result == NULL); - assert(lockState == RangeListLockState::ReaderLocked); + assert(lockState == RangeSectionLockState::ReaderLocked); - lockState = RangeListLockState::None; - result = map.LookupRangeList((void*)0x1111050, &lockState); - assert(lockState == RangeListLockState::NeedsLock); + lockState = RangeSectionLockState::None; + result = map.LookupRangeSection((void*)0x1111050, &lockState); + assert(lockState == RangeSectionLockState::NeedsLock); assert(result == NULL); - lockState = RangeListLockState::ReaderLocked; - result = map.LookupRangeList((void*)0x1111050, &lockState); + lockState = RangeSectionLockState::ReaderLocked; + result = map.LookupRangeSection((void*)0x1111050, &lockState); assert(result == NULL); - assert(lockState == RangeListLockState::ReaderLocked); - lockState = RangeListLockState::None; + assert(lockState == RangeSectionLockState::ReaderLocked); + lockState = RangeSectionLockState::None; - result = map.LookupRangeList((void*)0x1111051, &lockState); + result = map.LookupRangeSection((void*)0x1111051, &lockState); assert(result == rSectionSecond); - assert(lockState == RangeListLockState::None); - result = map.LookupRangeList((void*)0x1151050, &lockState); + assert(lockState == RangeSectionLockState::None); + result = map.LookupRangeSection((void*)0x1151050, &lockState); assert(result == rSectionSecond); - assert(lockState == RangeListLockState::None); - result = map.LookupRangeList((void*)0x1192050, &lockState); + assert(lockState == RangeSectionLockState::None); + result = map.LookupRangeSection((void*)0x1192050, &lockState); assert(result == rSectionSecond); - assert(lockState == RangeListLockState::None); + assert(lockState == RangeSectionLockState::None); - assert(lockState == RangeListLockState::None); - lockState = RangeListLockState::WriteLocked; - map.CleanupRangeLists(&lockState); - lockState = RangeListLockState::None; + assert(lockState == RangeSectionLockState::None); + lockState = RangeSectionLockState::WriteLocked; + map.CleanupRangeSections(&lockState); + lockState = RangeSectionLockState::None; - result = map.LookupRangeList((void*)0x1111000, &lockState); + result = map.LookupRangeSection((void*)0x1111000, &lockState); assert(result == NULL); - assert(lockState == RangeListLockState::None); - result = map.LookupRangeList((void*)0x1111050, &lockState); + assert(lockState == RangeSectionLockState::None); + result = map.LookupRangeSection((void*)0x1111050, &lockState); assert(result == NULL); - assert(lockState == RangeListLockState::None); - result = map.LookupRangeList((void*)0x1111051, &lockState); + assert(lockState == RangeSectionLockState::None); + result = map.LookupRangeSection((void*)0x1111051, &lockState); assert(result == rSectionSecond); - assert(lockState == RangeListLockState::None); - result = map.LookupRangeList((void*)0x1151050, &lockState); + assert(lockState == RangeSectionLockState::None); + result = map.LookupRangeSection((void*)0x1151050, &lockState); assert(result == rSectionSecond); - assert(lockState == RangeListLockState::None); - result = map.LookupRangeList((void*)0x1192050, &lockState); + assert(lockState == RangeSectionLockState::None); + result = map.LookupRangeSection((void*)0x1192050, &lockState); assert(result == rSectionSecond); - assert(lockState == RangeListLockState::None); + assert(lockState == RangeSectionLockState::None); std::cout << "Done\n"; } From 62b03763492e17c547c05095bcd136a9305576d0 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Nov 2022 18:11:33 -0800 Subject: [PATCH 10/25] Miraculously it builds! --- src/coreclr/inc/daccess.h | 4 +- src/coreclr/inc/dacvars.h | 2 +- src/coreclr/nativeaot/Runtime/inc/daccess.h | 4 +- src/coreclr/vm/ceeload.cpp | 4 +- src/coreclr/vm/codeman.cpp | 469 ++++++------------ src/coreclr/vm/codeman.h | 523 ++++++++++++++++++-- src/coreclr/vm/codeman.inl | 4 +- src/coreclr/vm/common.h | 1 + src/coreclr/vm/jitinterface.cpp | 4 +- src/coreclr/vm/method.cpp | 4 +- src/coreclr/vm/stubmgr.cpp | 2 +- 11 files changed, 637 insertions(+), 384 deletions(-) diff --git a/src/coreclr/inc/daccess.h b/src/coreclr/inc/daccess.h index a8056a5451561..0fe8d91411e37 100644 --- a/src/coreclr/inc/daccess.h +++ b/src/coreclr/inc/daccess.h @@ -97,7 +97,7 @@ // pRS=pRS->pright; // else // { -// return pRS->pjit; +// return pRS->_pjit; // } // } // @@ -108,7 +108,7 @@ // In the assignment statement the compiler will automatically use // the implicit conversion from PTR_RangeSection to RangeSection*, // causing a host instance to be created. Finally, if an appropriate -// section is found the use of pRS->pjit will cause an implicit +// section is found the use of pRS->_pjit will cause an implicit // conversion from PTR_IJitManager to IJitManager. The VPTR code // will look at target memory to determine the actual derived class // for the JitManager and instantiate the right class in the host so diff --git a/src/coreclr/inc/dacvars.h b/src/coreclr/inc/dacvars.h index 13642fe1ab6c9..6b6a9ef0223d9 100644 --- a/src/coreclr/inc/dacvars.h +++ b/src/coreclr/inc/dacvars.h @@ -75,7 +75,7 @@ #define UNKNOWN_POINTER_TYPE SIZE_T -DEFINE_DACVAR_VOLATILE(PTR_RangeSection, ExecutionManager__m_CodeRangeList, ExecutionManager::m_CodeRangeList) +DEFINE_DACVAR(PTR_RangeSectionMap, ExecutionManager__g_pCodeRangeMap, ExecutionManager::g_pCodeRangeMap) DEFINE_DACVAR(PTR_EECodeManager, ExecutionManager__m_pDefaultCodeMan, ExecutionManager::m_pDefaultCodeMan) DEFINE_DACVAR_VOLATILE(LONG, ExecutionManager__m_dwReaderCount, ExecutionManager::m_dwReaderCount) DEFINE_DACVAR_VOLATILE(LONG, ExecutionManager__m_dwWriterLock, ExecutionManager::m_dwWriterLock) diff --git a/src/coreclr/nativeaot/Runtime/inc/daccess.h b/src/coreclr/nativeaot/Runtime/inc/daccess.h index a7853555aca61..3fa4d8b0008bf 100644 --- a/src/coreclr/nativeaot/Runtime/inc/daccess.h +++ b/src/coreclr/nativeaot/Runtime/inc/daccess.h @@ -95,7 +95,7 @@ // pRS=pRS->pright; // else // { -// return pRS->pjit; +// return pRS->_pjit; // } // } // @@ -106,7 +106,7 @@ // In the assignment statement the compiler will automatically use // the implicit conversion from PTR_RangeSection to RangeSection*, // causing a host instance to be created. Finally, if an appropriate -// section is found the use of pRS->pjit will cause an implicit +// section is found the use of pRS->_pjit will cause an implicit // conversion from PTR_IJitManager to IJitManager. The VPTR code // will look at target memory to determine the actual derived class // for the JitManager and instantiate the right class in the host so diff --git a/src/coreclr/vm/ceeload.cpp b/src/coreclr/vm/ceeload.cpp index 73eb37933ab81..0fcffd5309021 100644 --- a/src/coreclr/vm/ceeload.cpp +++ b/src/coreclr/vm/ceeload.cpp @@ -4401,9 +4401,9 @@ void Module::RunEagerFixupsUnlocked() TADDR base = dac_cast(pNativeImage->GetBase()); ExecutionManager::AddCodeRange( - base, base + (TADDR)pNativeImage->GetVirtualSize(), + base, base + (TADDR)pNativeImage->GetVirtualSize() - 1, ExecutionManager::GetReadyToRunJitManager(), - RangeSection::RANGE_SECTION_READYTORUN, + RangeSection::RANGE_SECTION_NONE, this /* pHeapListOrZapModule */); } #endif // !DACCESS_COMPILE diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index 07855709672a7..acd53b2a03210 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -49,14 +49,13 @@ SPTR_IMPL(EEJitManager, ExecutionManager, m_pEEJitManager); SPTR_IMPL(ReadyToRunJitManager, ExecutionManager, m_pReadyToRunJitManager); #endif -VOLATILE_SPTR_IMPL_INIT(RangeSection, ExecutionManager, m_CodeRangeList, NULL); +SPTR_IMPL(RangeSectionMap, ExecutionManager, g_pCodeRangeMap); VOLATILE_SVAL_IMPL_INIT(LONG, ExecutionManager, m_dwReaderCount, 0); VOLATILE_SVAL_IMPL_INIT(LONG, ExecutionManager, m_dwWriterLock, 0); #ifndef DACCESS_COMPILE CrstStatic ExecutionManager::m_JumpStubCrst; -CrstStatic ExecutionManager::m_RangeCrst; unsigned ExecutionManager::m_normal_JumpStubLookup; unsigned ExecutionManager::m_normal_JumpStubUnique; @@ -391,7 +390,7 @@ void UnwindInfoTable::AddToUnwindInfoTable(UnwindInfoTable** unwindInfoPtr, PT_R if (pRS != NULL) { for(int i = 0; i < unwindInfoCount; i++) - AddToUnwindInfoTable(&pRS->pUnwindInfoTable, &unwindInfo[i], pRS->LowAddress, pRS->HighAddress); + AddToUnwindInfoTable(&pRS->pUnwindInfoTable, &unwindInfo[i], pRS->_range.begin, pRS->_range.end); } } @@ -409,14 +408,14 @@ void UnwindInfoTable::AddToUnwindInfoTable(UnwindInfoTable** unwindInfoPtr, PT_R _ASSERTE(pRS != NULL); if (pRS != NULL) { - _ASSERTE(pRS->pjit->GetCodeType() == (miManaged | miIL)); - if (pRS->pjit->GetCodeType() == (miManaged | miIL)) + _ASSERTE(pRS->_pjit->GetCodeType() == (miManaged | miIL)); + if (pRS->_pjit->GetCodeType() == (miManaged | miIL)) { // This cast is justified because only EEJitManager's have the code type above. - EEJitManager* pJitMgr = (EEJitManager*)(pRS->pjit); + EEJitManager* pJitMgr = (EEJitManager*)(pRS->_pjit); CodeHeader * pHeader = pJitMgr->GetCodeHeaderFromStartAddress(entryPoint); for(ULONG i = 0; i < pHeader->GetNumberOfUnwindInfos(); i++) - RemoveFromUnwindInfoTable(&pRS->pUnwindInfoTable, pRS->LowAddress, pRS->LowAddress + pHeader->GetUnwindInfo(i)->BeginAddress); + RemoveFromUnwindInfoTable(&pRS->pUnwindInfoTable, pRS->_range.begin, pRS->_range.begin + pHeader->GetUnwindInfo(i)->BeginAddress); } } } @@ -453,15 +452,15 @@ extern CrstStatic g_StubUnwindInfoHeapSegmentsCrst; PCODE methodEntry =(PCODE) heapIterator.GetMethodCode(); RangeSection * pRS = ExecutionManager::FindCodeRange(methodEntry, ExecutionManager::GetScanFlags()); _ASSERTE(pRS != NULL); - _ASSERTE(pRS->pjit->GetCodeType() == (miManaged | miIL)); - if (pRS != NULL && pRS->pjit->GetCodeType() == (miManaged | miIL)) + _ASSERTE(pRS->_pjit->GetCodeType() == (miManaged | miIL)); + if (pRS != NULL && pRS->_pjit->GetCodeType() == (miManaged | miIL)) { // This cast is justified because only EEJitManager's have the code type above. - EEJitManager* pJitMgr = (EEJitManager*)(pRS->pjit); + EEJitManager* pJitMgr = (EEJitManager*)(pRS->_pjit); CodeHeader * pHeader = pJitMgr->GetCodeHeaderFromStartAddress(methodEntry); int unwindInfoCount = pHeader->GetNumberOfUnwindInfos(); for(int i = 0; i < unwindInfoCount; i++) - AddToUnwindInfoTable(&pRS->pUnwindInfoTable, pHeader->GetUnwindInfo(i), pRS->LowAddress, pRS->HighAddress); + AddToUnwindInfoTable(&pRS->pUnwindInfoTable, pHeader->GetUnwindInfo(i), pRS->_range.begin, pRS->_range.end); } } } @@ -4181,7 +4180,7 @@ BOOL EEJitManager::JitCodeToMethodInfo( _ASSERTE(pRangeSection != NULL); - TADDR start = dac_cast(pRangeSection->pjit)->FindMethodCode(pRangeSection, currentPC); + TADDR start = dac_cast(pRangeSection->_pjit)->FindMethodCode(pRangeSection, currentPC); if (start == NULL) return FALSE; @@ -4221,7 +4220,7 @@ StubCodeBlockKind EEJitManager::GetStubCodeBlockKind(RangeSection * pRangeSectio SUPPORTS_DAC; } CONTRACTL_END; - TADDR start = dac_cast(pRangeSection->pjit)->FindMethodCode(pRangeSection, currentPC); + TADDR start = dac_cast(pRangeSection->_pjit)->FindMethodCode(pRangeSection, currentPC); if (start == NULL) return STUB_CODE_BLOCK_NOCODE; CodeHeader * pCHdr = PTR_CodeHeader(start - sizeof(CodeHeader)); @@ -4237,9 +4236,9 @@ TADDR EEJitManager::FindMethodCode(PCODE currentPC) } CONTRACTL_END; RangeSection * pRS = ExecutionManager::FindCodeRange(currentPC, ExecutionManager::GetScanFlags()); - if (pRS == NULL || (pRS->flags & RangeSection::RANGE_SECTION_CODEHEAP) == 0) + if (pRS == NULL || (pRS->_flags & RangeSection::RANGE_SECTION_CODEHEAP) == 0) return STUB_CODE_BLOCK_NOCODE; - return dac_cast(pRS->pjit)->FindMethodCode(pRS, currentPC); + return dac_cast(pRS->_pjit)->FindMethodCode(pRS, currentPC); } // Finds the header corresponding to the code at offset "delta". @@ -4251,7 +4250,7 @@ TADDR EEJitManager::FindMethodCode(RangeSection * pRangeSection, PCODE currentPC _ASSERTE(pRangeSection != NULL); - HeapList *pHp = dac_cast(pRangeSection->pHeapListOrZapModule); + HeapList *pHp = pRangeSection->_pHeapList; if ((currentPC < pHp->startAddress) || (currentPC > pHp->endAddress)) @@ -4665,12 +4664,13 @@ void ExecutionManager::Init() m_JumpStubCrst.Init(CrstJumpStubCache, CrstFlags(CRST_UNSAFE_ANYMODE|CRST_DEBUGGER_THREAD)); - m_RangeCrst.Init(CrstExecuteManRangeLock, CRST_UNSAFE_ANYMODE); + g_pCodeRangeMap = new RangeSectionMap(); m_pDefaultCodeMan = new EECodeManager(); m_pEEJitManager = new EEJitManager(); + #ifdef FEATURE_READYTORUN m_pReadyToRunJitManager = new ReadyToRunJitManager(); #endif @@ -4694,7 +4694,9 @@ ExecutionManager::FindCodeRange(PCODE currentPC, ScanFlag scanFlag) if (scanFlag == ScanReaderLock) return FindCodeRangeWithLock(currentPC); - return GetRangeSection(currentPC); + // Since ScanReaderLock is not set, then we should behave AS IF the ReaderLock is held + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; + return GetRangeSection(currentPC, &lockState); } //************************************************************************** @@ -4708,8 +4710,15 @@ ExecutionManager::FindCodeRangeWithLock(PCODE currentPC) SUPPORTS_DAC; } CONTRACTL_END; - ReaderLockHolder rlh; - return GetRangeSection(currentPC); + RangeSectionLockState lockState = RangeSectionLockState::None; + RangeSection *result = GetRangeSection(currentPC, &lockState); + if (lockState == RangeSectionLockState::NeedsLock) + { + ReaderLockHolder rlh; + lockState = RangeSectionLockState::ReaderLocked; + result = GetRangeSection(currentPC, &lockState); + } + return result; } @@ -4771,7 +4780,9 @@ BOOL ExecutionManager::IsManagedCode(PCODE currentPC) if (GetScanFlags() == ScanReaderLock) return IsManagedCodeWithLock(currentPC); - return IsManagedCodeWorker(currentPC); + // Since ScanReaderLock is not set, then we must assume that the ReaderLock is effectively taken. + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; + return IsManagedCodeWorker(currentPC, &lockState); } //************************************************************************** @@ -4783,8 +4794,17 @@ BOOL ExecutionManager::IsManagedCodeWithLock(PCODE currentPC) GC_NOTRIGGER; } CONTRACTL_END; - ReaderLockHolder rlh; - return IsManagedCodeWorker(currentPC); + RangeSectionLockState lockState = RangeSectionLockState::None; + BOOL result = IsManagedCodeWorker(currentPC, &lockState); + + if (lockState == RangeSectionLockState::NeedsLock) + { + ReaderLockHolder rlh; + lockState = RangeSectionLockState::ReaderLocked; + result = IsManagedCodeWorker(currentPC, &lockState); + } + + return result; } //************************************************************************** @@ -4811,14 +4831,15 @@ BOOL ExecutionManager::IsManagedCode(PCODE currentPC, HostCallPreference hostCal return FALSE; } - return IsManagedCodeWorker(currentPC); + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; + return IsManagedCodeWorker(currentPC, &lockState); #endif } //************************************************************************** // Assumes that the ExecutionManager reader/writer lock is taken or that // it is safe not to take it. -BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC) +BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC, RangeSectionLockState *pLockState) { CONTRACTL { NOTHROW; @@ -4829,16 +4850,16 @@ BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC) // taken over the call to JitCodeToMethodInfo too so that nobody pulls out // the range section from underneath us. - RangeSection * pRS = GetRangeSection(currentPC); + RangeSection * pRS = GetRangeSection(currentPC, pLockState); if (pRS == NULL) return FALSE; - if (pRS->flags & RangeSection::RANGE_SECTION_CODEHEAP) + if (pRS->_flags & RangeSection::RANGE_SECTION_CODEHEAP) { // Typically if we find a Jit Manager we are inside a managed method // but on we could also be in a stub, so we check for that // as well and we don't consider stub to be real managed code. - TADDR start = dac_cast(pRS->pjit)->FindMethodCode(pRS, currentPC); + TADDR start = dac_cast(pRS->_pjit)->FindMethodCode(pRS, currentPC); if (start == NULL) return FALSE; CodeHeader * pCHdr = PTR_CodeHeader(start - sizeof(CodeHeader)); @@ -4847,9 +4868,9 @@ BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC) } #ifdef FEATURE_READYTORUN else - if (pRS->flags & RangeSection::RANGE_SECTION_READYTORUN) + if (pRS->_pR2RModule != NULL) { - if (dac_cast(pRS->pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) + if (dac_cast(pRS->_pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) return TRUE; } #endif @@ -4871,10 +4892,11 @@ BOOL ExecutionManager::IsReadyToRunCode(PCODE currentPC) // the range section from underneath us. #ifdef FEATURE_READYTORUN - RangeSection * pRS = GetRangeSection(currentPC); - if (pRS != NULL && (pRS->flags & RangeSection::RANGE_SECTION_READYTORUN)) + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // TODO! Looking at users of this API I don't know that the lock structure here is safe. Needs checking. + RangeSection * pRS = GetRangeSection(currentPC, &lockState); + if (pRS != NULL && (pRS->_pR2RModule != NULL)) { - if (dac_cast(pRS->pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) + if (dac_cast(pRS->_pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) return TRUE; } #endif @@ -4904,7 +4926,7 @@ LPCWSTR ExecutionManager::GetJitName() } #endif // !FEATURE_MERGE_JIT_AND_ENGINE -RangeSection* ExecutionManager::GetRangeSection(TADDR addr) +RangeSection* ExecutionManager::GetRangeSection(TADDR addr, RangeSectionLockState *pLockState) { CONTRACTL { NOTHROW; @@ -4913,118 +4935,7 @@ RangeSection* ExecutionManager::GetRangeSection(TADDR addr) SUPPORTS_DAC; } CONTRACTL_END; - RangeSection * pHead = m_CodeRangeList; - - if (pHead == NULL) - { - return NULL; - } - - RangeSection *pCurr = pHead; - RangeSection *pLast = NULL; - -#ifndef DACCESS_COMPILE - RangeSection *pLastUsedRS = (pCurr != NULL) ? pCurr->pLastUsed : NULL; - - if (pLastUsedRS != NULL) - { - // positive case - if ((addr >= pLastUsedRS->LowAddress) && - (addr < pLastUsedRS->HighAddress) ) - { - return pLastUsedRS; - } - - RangeSection * pNextAfterLastUsedRS = pLastUsedRS->pnext; - - // negative case - if ((addr < pLastUsedRS->LowAddress) && - (pNextAfterLastUsedRS == NULL || addr >= pNextAfterLastUsedRS->HighAddress)) - { - return NULL; - } - } -#endif - - while (pCurr != NULL) - { - // See if addr is in [pCurr->LowAddress .. pCurr->HighAddress) - if (pCurr->LowAddress <= addr) - { - // Since we are sorted, once pCurr->HighAddress is less than addr - // then all subsequence ones will also be lower, so we are done. - if (addr >= pCurr->HighAddress) - { - // we'll return NULL and put pLast into pLastUsed - pCurr = NULL; - } - else - { - // addr must be in [pCurr->LowAddress .. pCurr->HighAddress) - _ASSERTE((pCurr->LowAddress <= addr) && (addr < pCurr->HighAddress)); - - // Found the matching RangeSection - // we'll return pCurr and put it into pLastUsed - pLast = pCurr; - } - - break; - } - pLast = pCurr; - pCurr = pCurr->pnext; - } - -#ifndef DACCESS_COMPILE - // Cache pCurr as pLastUsed in the head node - // Unless we are on an MP system with many cpus - // where this sort of caching actually diminishes scaling during server GC - // due to many processors writing to a common location - if (g_SystemInfo.dwNumberOfProcessors < 4 || !GCHeapUtilities::IsServerHeap() || !GCHeapUtilities::IsGCInProgress()) - pHead->pLastUsed = pLast; -#endif - - return pCurr; -} - -RangeSection* ExecutionManager::GetRangeSectionAndPrev(RangeSection *pHead, TADDR addr, RangeSection** ppPrev) -{ - WRAPPER_NO_CONTRACT; - - RangeSection *pCurr; - RangeSection *pPrev; - RangeSection *result = NULL; - - for (pPrev = NULL, pCurr = pHead; - pCurr != NULL; - pPrev = pCurr, pCurr = pCurr->pnext) - { - // See if addr is in [pCurr->LowAddress .. pCurr->HighAddress) - if (pCurr->LowAddress > addr) - continue; - - if (addr >= pCurr->HighAddress) - break; - - // addr must be in [pCurr->LowAddress .. pCurr->HighAddress) - _ASSERTE((pCurr->LowAddress <= addr) && (addr < pCurr->HighAddress)); - - // Found the matching RangeSection - result = pCurr; - - // Write back pPrev to ppPrev if it is non-null - if (ppPrev != NULL) - *ppPrev = pPrev; - - break; - } - - // If we failed to find a match write NULL to ppPrev if it is non-null - if ((ppPrev != NULL) && (result == NULL)) - { - *ppPrev = NULL; - } - - return result; + return g_pCodeRangeMap->LookupRangeSection(addr, pLockState); } /* static */ @@ -5041,20 +4952,24 @@ PTR_Module ExecutionManager::FindReadyToRunModule(TADDR currentData) CONTRACTL_END; #ifdef FEATURE_READYTORUN - ReaderLockHolder rlh; - - RangeSection * pRS = GetRangeSection(currentData); - if (pRS == NULL) - return NULL; - - if (pRS->flags & RangeSection::RANGE_SECTION_CODEHEAP) - return NULL; - - if (pRS->flags & RangeSection::RANGE_SECTION_READYTORUN) - return dac_cast(pRS->pHeapListOrZapModule);; + RangeSectionLockState lockState = RangeSectionLockState::None; + RangeSection * pRS = GetRangeSection(currentData, &lockState); + if (lockState != RangeSectionLockState::NeedsLock) + { + if (pRS == NULL) + return NULL; - _ASSERTE(FALSE); ..= - return NULL; + return pRS->_pR2RModule; + } + else + { + ReaderLockHolder rlh; + lockState = RangeSectionLockState::ReaderLocked; + pRS = GetRangeSection(currentData, &lockState); + if (pRS == NULL) + return NULL; + return pRS->_pR2RModule; + } #else return NULL; #endif @@ -5072,118 +4987,66 @@ PTR_Module ExecutionManager::FindModuleForGCRefMap(TADDR currentData) } CONTRACTL_END; +#ifndef FEATURE_READYTORUN + return NULL; +#else RangeSection * pRS = FindCodeRange(currentData, ExecutionManager::GetScanFlags()); if (pRS == NULL) return NULL; - if (pRS->flags & RangeSection::RANGE_SECTION_CODEHEAP) - return NULL; - -#ifdef FEATURE_READYTORUN - // RANGE_SECTION_READYTORUN is intentionally not filtered out here -#endif - - return dac_cast(pRS->pHeapListOrZapModule); + return pRS->_pR2RModule; +#endif // FEATURE_READYTORUN } #ifndef DACCESS_COMPILE -/* NGenMem depends on this entrypoint */ NOINLINE void ExecutionManager::AddCodeRange(TADDR pStartRange, TADDR pEndRange, IJitManager * pJit, RangeSection::RangeSectionFlags flags, - void * pHp) + PTR_Module pModule) { CONTRACTL { THROWS; GC_NOTRIGGER; + HOST_CALLS; + PRECONDITION(pStartRange < pEndRange); PRECONDITION(CheckPointer(pJit)); - PRECONDITION(CheckPointer(pHp)); + PRECONDITION(CheckPointer(pModule)); } CONTRACTL_END; - AddRangeHelper(pStartRange, - pEndRange, - pJit, - flags, - dac_cast(pHp)); + ReaderLockHolder rlh; + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // + + PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pModule, &lockState); + if (pRange == NULL) + ThrowOutOfMemory(); } -void ExecutionManager::AddRangeHelper(TADDR pStartRange, - TADDR pEndRange, - IJitManager * pJit, - RangeSection::RangeSectionFlags flags, - TADDR pHeapListOrZapModule) +NOINLINE +void ExecutionManager::AddCodeRange(TADDR pStartRange, + TADDR pEndRange, + IJitManager * pJit, + RangeSection::RangeSectionFlags flags, + PTR_HeapList pHp) { CONTRACTL { THROWS; GC_NOTRIGGER; HOST_CALLS; PRECONDITION(pStartRange < pEndRange); - PRECONDITION(pHeapListOrZapModule != NULL); + PRECONDITION(CheckPointer(pJit)); + PRECONDITION(CheckPointer(pHp)); } CONTRACTL_END; - RangeSection *pnewrange = new RangeSection; - - _ASSERTE(pEndRange > pStartRange); - - pnewrange->LowAddress = pStartRange; - pnewrange->HighAddress = pEndRange; - pnewrange->pjit = pJit; - pnewrange->pnext = NULL; - pnewrange->flags = flags; - pnewrange->pLastUsed = NULL; - pnewrange->pHeapListOrZapModule = pHeapListOrZapModule; -#if defined(TARGET_AMD64) - pnewrange->pUnwindInfoTable = NULL; -#endif // defined(TARGET_AMD64) - { - CrstHolder ch(&m_RangeCrst); // Acquire the Crst before linking in a new RangeList - - RangeSection * current = m_CodeRangeList; - RangeSection * previous = NULL; + ReaderLockHolder rlh; + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // - if (current != NULL) - { - while (true) - { - // Sort addresses top down so that more recently created ranges - // will populate the top of the list - if (pnewrange->LowAddress > current->LowAddress) - { - // Asserts if ranges are overlapping - _ASSERTE(pnewrange->LowAddress >= current->HighAddress); - pnewrange->pnext = current; + PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pHp, &lockState); - if (previous == NULL) // insert new head - { - m_CodeRangeList = pnewrange; - } - else - { // insert in the middle - previous->pnext = pnewrange; - } - break; - } - - RangeSection * next = current->pnext; - if (next == NULL) // insert at end of list - { - current->pnext = pnewrange; - break; - } - - // Continue walking the RangeSection list - previous = current; - current = next; - } - } - else - { - m_CodeRangeList = pnewrange; - } - } + if (pRange == NULL) + ThrowOutOfMemory(); } // Deletes a single range starting at pStartRange @@ -5194,114 +5057,64 @@ void ExecutionManager::DeleteRange(TADDR pStartRange) GC_NOTRIGGER; } CONTRACTL_END; - RangeSection *pCurr = NULL; - { - // Acquire the Crst before unlinking a RangeList. - // NOTE: The Crst must be acquired BEFORE we grab the writer lock, as the - // writer lock forces us into a forbid suspend thread region, and it's illegal - // to enter a Crst after the forbid suspend thread region is entered - CrstHolder ch(&m_RangeCrst); + RangeSection *pCurr = FindCodeRangeWithLock(pStartRange); + g_pCodeRangeMap->RemoveRangeSection(pCurr); + +#if defined(TARGET_AMD64) + PTR_UnwindInfoTable unwindTable = pCurr->pUnwindInfoTable; +#endif + + { // Acquire the WriterLock and prevent any readers from walking the RangeList. // This also forces us to enter a forbid suspend thread region, to prevent // hijacking profilers from grabbing this thread and walking it (the walk may // require the reader lock, which would cause a deadlock). WriterLockHolder wlh; - RangeSection *pPrev = NULL; - - pCurr = GetRangeSectionAndPrev(m_CodeRangeList, pStartRange, &pPrev); - - // pCurr points at the Range that needs to be unlinked from the RangeList - if (pCurr != NULL) - { - - // If pPrev is NULL the head of this list is to be deleted - if (pPrev == NULL) - { - m_CodeRangeList = pCurr->pnext; - } - else - { - _ASSERT(pPrev->pnext == pCurr); - - pPrev->pnext = pCurr->pnext; - } - - // Clear the cache pLastUsed in the head node (if any) - RangeSection * head = m_CodeRangeList; - if (head != NULL) - { - head->pLastUsed = NULL; - } - - // - // Cannot delete pCurr here because we own the WriterLock and if this is - // a hosted scenario then the hosting api callback cannot occur in a forbid - // suspend region, which the writer lock is. - // - } + RangeSectionLockState lockState = RangeSectionLockState::WriteLocked; + + g_pCodeRangeMap->CleanupRangeSections(&lockState); + // Unlike the previous implementation, we no longer attempt to avoid freeing + // the memory behind the RangeSection here, as we do not support the hosting + // api taking over memory allocation. } // - // Now delete the node + // Now delete the unwind info table // - if (pCurr != NULL) - { #if defined(TARGET_AMD64) - if (pCurr->pUnwindInfoTable != 0) - delete pCurr->pUnwindInfoTable; + if (unwindTable != 0) + delete unwindTable; #endif // defined(TARGET_AMD64) - delete pCurr; - } } #endif // #ifndef DACCESS_COMPILE #ifdef DACCESS_COMPILE -void ExecutionManager::EnumRangeList(RangeSection* list, - CLRDataEnumMemoryFlags flags) +void RangeSection::EnumMemoryRegions(CLRDataEnumMemoryFlags flags) { - while (list != NULL) - { - // If we can't read the target memory, stop immediately so we don't work - // with broken data. - if (!DacEnumMemoryRegion(dac_cast(list), sizeof(*list))) - break; - - if (list->pjit.IsValid()) - { - list->pjit->EnumMemoryRegions(flags); - } - - if (!(list->flags & RangeSection::RANGE_SECTION_CODEHEAP)) - { - PTR_Module pModule = dac_cast(list->pHeapListOrZapModule); + if (!DacEnumMemoryRegion(dac_cast(this), sizeof(*this))) + return; - if (pModule.IsValid()) - { - pModule->EnumMemoryRegions(flags, true); - } - } + if (_pjit.IsValid()) + { + _pjit->EnumMemoryRegions(flags); + } - list = list->pnext; -#if defined (_DEBUG) - // Test hook: when testing on debug builds, we want an easy way to test that the while - // correctly terminates in the face of ridiculous stuff from the target. - if (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_DumpGeneration_IntentionallyCorruptDataFromTarget) == 1) +#ifdef FEATURE_READYTORUN + if (_pR2RModule != NULL) + { + if (_pR2RModule.IsValid()) { - // Force us to struggle on with something bad. - if (list == NULL) - { - list = (RangeSection *)&flags; - } + _pR2RModule->EnumMemoryRegions(flags, true); } -#endif // (_DEBUG) - } +#endif // FEATURE_READYTORUN } + void ExecutionManager::EnumMemoryRegions(CLRDataEnumMemoryFlags flags) { STATIC_CONTRACT_HOST_CALLS; @@ -5312,16 +5125,16 @@ void ExecutionManager::EnumMemoryRegions(CLRDataEnumMemoryFlags flags) // Report the global data portions. // - m_CodeRangeList.EnumMem(); + g_pCodeRangeMap.EnumMem(); m_pDefaultCodeMan.EnumMem(); // // Walk structures and report. // - if (m_CodeRangeList.IsValid()) + if (g_pCodeRangeMap.IsValid()) { - EnumRangeList(m_CodeRangeList, flags); + g_pCodeRangeMap->EnumMemoryRegions(flags); } } #endif // #ifdef DACCESS_COMPILE @@ -5908,7 +5721,7 @@ ReadyToRunInfo * ReadyToRunJitManager::JitTokenToReadyToRunInfo(const METHODTOKE SUPPORTS_DAC; } CONTRACTL_END; - return dac_cast(MethodToken.m_pRangeSection->pHeapListOrZapModule)->GetReadyToRunInfo(); + return MethodToken.m_pRangeSection->_pR2RModule->GetReadyToRunInfo(); } UINT32 ReadyToRunJitManager::JitTokenToGCInfoVersion(const METHODTOKEN& MethodToken) @@ -6045,9 +5858,9 @@ StubCodeBlockKind ReadyToRunJitManager::GetStubCodeBlockKind(RangeSection * pRan } CONTRACTL_END; - DWORD rva = (DWORD)(currentPC - pRangeSection->LowAddress); + DWORD rva = (DWORD)(currentPC - pRangeSection->_range.begin); - PTR_ReadyToRunInfo pReadyToRunInfo = dac_cast(pRangeSection->pHeapListOrZapModule)->GetReadyToRunInfo(); + PTR_ReadyToRunInfo pReadyToRunInfo = pRangeSection->_pR2RModule->GetReadyToRunInfo(); PTR_IMAGE_DATA_DIRECTORY pDelayLoadMethodCallThunksDir = pReadyToRunInfo->GetDelayMethodCallThunksSection(); if (pDelayLoadMethodCallThunksDir != NULL) @@ -6198,11 +6011,11 @@ BOOL ReadyToRunJitManager::JitCodeToMethodInfo(RangeSection * pRangeSection, TADDR currentInstr = PCODEToPINSTR(currentPC); - TADDR ImageBase = pRangeSection->LowAddress; + TADDR ImageBase = pRangeSection->_range.begin; DWORD RelativePc = (DWORD)(currentInstr - ImageBase); - Module * pModule = dac_cast(pRangeSection->pHeapListOrZapModule); + Module * pModule = pRangeSection->_pR2RModule; ReadyToRunInfo * pInfo = pModule->GetReadyToRunInfo(); COUNT_T nRuntimeFunctions = pInfo->m_nRuntimeFunctions; diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 4657bca895e56..1d94396dd3798 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -609,43 +609,491 @@ class UnwindInfoTable { typedef DPTR(struct RangeSection) PTR_RangeSection; -struct RangeSection -{ - TADDR LowAddress; - TADDR HighAddress; - - PTR_IJitManager pjit; // The owner of this address range +class RangeSectionMap; -#ifndef DACCESS_COMPILE - // Volatile because of the list can be walked lock-free - Volatile pnext; // link rangesections in a sorted list -#else - PTR_RangeSection pnext; -#endif +class Range +{ +public: + Range(TADDR begin, TADDR end) : begin(begin), end(end) {} - PTR_RangeSection pLastUsed; // for the head node only: a link to rangesections that was used most recently + // [begin,end] (This is an inclusive range) + TADDR begin; + TADDR end; +}; +struct RangeSection +{ + friend class RangeSectionMap; enum RangeSectionFlags { RANGE_SECTION_NONE = 0x0, RANGE_SECTION_COLLECTIBLE = 0x1, RANGE_SECTION_CODEHEAP = 0x2, + }; + #ifdef FEATURE_READYTORUN - RANGE_SECTION_READYTORUN = 0x4, + RangeSection(Range range, IJitManager* pJit, RangeSectionFlags flags, PTR_Module pR2RModule) : + _range(range), + _flags(flags), + _pjit(pJit), + _pR2RModule(pR2RModule), + _pHeapList(dac_cast((TADDR)0)) + { + assert(!(flags & RANGE_SECTION_COLLECTIBLE)); + assert(pR2RModule != NULL); + } #endif - }; - DWORD flags; + RangeSection(Range range, IJitManager* pJit, RangeSectionFlags flags, PTR_HeapList pHeapList) : + _range(range), + _flags(flags), + _pjit(pJit), + _pR2RModule(dac_cast((TADDR)0)), + _pHeapList(pHeapList) + {} - // union - // { - // PTR_CodeHeap pCodeHeap; // valid if RANGE_SECTION_HEAP is set - // PTR_Module pZapModule; // valid if RANGE_SECTION_HEAP is not set - // }; - TADDR pHeapListOrZapModule; -#if defined(HOST_64BIT) +#ifdef DACCESS_COMPILE + void EnumMemoryRegions(CLRDataEnumMemoryFlags flags); +#endif + + const Range _range; + const RangeSectionFlags _flags; + const PTR_IJitManager _pjit; + const PTR_Module _pR2RModule; + const PTR_HeapList _pHeapList; + +#if defined(TARGET_AMD64) PTR_UnwindInfoTable pUnwindInfoTable; // Points to unwind information for this memory range. -#endif // defined(HOST_64BIT) +#endif // defined(TARGET_AMD64) + + + RangeSection* _pRangeSectionNextForDelete = nullptr; // Used for adding to the cleanup list +}; + +enum class RangeSectionLockState +{ + None, + NeedsLock, + ReaderLocked, + WriteLocked, +}; + +// For 64bit, we work with 8KB chunks of memory holding pointers to the next level. This provides 10 bits of address resolution per level. +// For *reasons* the X64 hardware is limited to 57bits of addressable address space, and the minimum granularity that makes sense for range lists is 64KB (or every 2^16 bits) +// Similarly the Arm64 specification requires addresses to use at most 52 bits. Thus we use the maximum addressable range of X64 to provide the real max range +// So the first level is bits [56:47] -> L4 +// Then [46:37] -> L3 +// [36:27] -> L2 +// [26:17] -> L1 +// This leaves 17 bits of the address to be handled by the RangeSection linked list +// +// For 32bit VA processes, use 1KB chunks holding pointers to the next level. This provides 8 bites of address resolution per level. [31:24] and [23:16]. + +// The memory safety model for segment maps is that the pointers held within the individual segments can never change other than to go from NULL to a meaningful pointer, +// except for the final level, which is only permitted to change when CleanupRangeSections is in use. + +class RangeSectionMap +{ + class RangeSectionFragment; + class RangeSectionFragmentPointer + { + private: + uintptr_t _ptr; + + uintptr_t FragmentToPtr(RangeSectionFragment* fragment) + { + uintptr_t ptr = (uintptr_t)fragment; + if (ptr == 0) + return ptr; + + if (fragment->isCollectibleRangeSectionFragment) + { + ptr += 1; + } + + return ptr; + } + + RangeSectionFragmentPointer() { _ptr = 0; } + public: + + RangeSectionFragmentPointer(RangeSectionFragmentPointer &) = delete; + RangeSectionFragmentPointer(RangeSectionFragmentPointer &&) = delete; + RangeSectionFragmentPointer& operator=(const RangeSectionFragmentPointer&) = delete; + + bool PointerIsCollectible() + { + return ((_ptr & 1) == 1); + } + + bool IsNull() + { + return _ptr == 0; + } + + RangeSectionFragment* VolatileLoadWithoutBarrier(RangeSectionLockState *pLockState) + { + uintptr_t ptr = ::VolatileLoadWithoutBarrier(&_ptr); + if ((ptr & 1) == 1) + { + if ((*pLockState == RangeSectionLockState::None) || (*pLockState == RangeSectionLockState::NeedsLock)) + { + *pLockState = RangeSectionLockState::NeedsLock; + return NULL; + } + return (RangeSectionFragment*)(ptr - 1); + } + else + { + return (RangeSectionFragment*)(ptr); + } + } + + void VolatileStore(RangeSectionFragment* fragment) + { + ::VolatileStore(&_ptr, FragmentToPtr(fragment)); + } + + bool AtomicReplace(RangeSectionFragment* newFragment, RangeSectionFragment* oldFragment) + { + uintptr_t oldPtr = FragmentToPtr(oldFragment); + uintptr_t newPtr = FragmentToPtr(newFragment); + + return oldPtr == InterlockedCompareExchangeT(&_ptr, newPtr, oldPtr); + } + }; + + // Unlike a RangeSection, a RangeSectionFragment cannot span multiple elements of the last level of the RangeSectionMap + // Always allocated via calloc + class RangeSectionFragment + { + public: + RangeSectionFragmentPointer pRangeSectionFragmentNext; + Range _range; + PTR_RangeSection pRangeSection; + bool InRange(TADDR address) { return address >= _range.begin && address <= _range.end && pRangeSection->_pRangeSectionNextForDelete == NULL; } + bool isPrimaryRangeSectionFragment; // RangeSectionFragment are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. + bool isCollectibleRangeSectionFragment; // RangeSectionFragments + }; + +#ifdef TARGET_64BIT + static const uintptr_t entriesPerMapLevel = 1024; +#else + static const uintptr_t entriesPerMapLevel = 256; +#endif + + typedef RangeSectionFragmentPointer RangeSectionList; + typedef RangeSectionList RangeSectionL1[entriesPerMapLevel]; + typedef RangeSectionL1* RangeSectionL2[entriesPerMapLevel]; + typedef RangeSectionL2* RangeSectionL3[entriesPerMapLevel]; + typedef RangeSectionL3* RangeSectionL4[entriesPerMapLevel]; + +#ifdef TARGET_64BIT + typedef RangeSectionL4 RangeSectionTopLevel; + static const uintptr_t mapLevels = 4; + static const uintptr_t maxSetBit = 56; // This is 0 indexed + static const uintptr_t bitsPerLevel = 10; +#else + typedef RangeSectionL2 RangeSectionTopLevel; + static const uintptr_t mapLevels = 2; + static const uintptr_t maxSetBit = 31; // This is 0 indexed + static const uintptr_t bitsPerLevel = 8; +#endif + + RangeSectionTopLevel *_topLevel = nullptr; + + RangeSection* _pCleanupList; + + const uintptr_t bitsAtLastLevel = maxSetBit - (bitsPerLevel * mapLevels) + 1; + const uintptr_t bytesAtLastLevel = (((uintptr_t)1) << (bitsAtLastLevel - 1)); + + RangeSection* EndOfCleanupListMarker() { return (RangeSection*)1; } + + void* AllocateLevel() { return calloc(entriesPerMapLevel, sizeof(void*)); } + + uintptr_t EffectiveBitsForLevel(TADDR address, uintptr_t level) + { + TADDR addressAsInt = address; + TADDR addressBitsUsedInMap = addressAsInt >> (maxSetBit - (mapLevels * bitsPerLevel)); + TADDR addressBitsShifted = addressBitsUsedInMap >> ((level - 1) * bitsPerLevel); + TADDR addressBitsUsedInLevel = (entriesPerMapLevel - 1) & addressBitsShifted; + return addressBitsUsedInLevel; + } + + template + auto EnsureLevel(TADDR address, T* outerLevel, uintptr_t level) -> decltype(&((**outerLevel)[0])) + { + uintptr_t index = EffectiveBitsForLevel(address, level); + auto levelToGetPointerIn = VolatileLoadWithoutBarrier(outerLevel); + + if (levelToGetPointerIn == NULL) + { + auto levelNew = static_cast(AllocateLevel()); + if (levelNew == NULL) + return NULL; + auto levelPreviouslyStored = InterlockedCompareExchangeT(outerLevel, levelNew, NULL); + if (levelPreviouslyStored != nullptr) + { + // Handle race where another thread grew the table + levelToGetPointerIn = levelPreviouslyStored; + free(levelNew); + } + else + { + levelToGetPointerIn = levelNew; + } + assert(levelToGetPointerIn != nullptr); + } + + return &((*levelToGetPointerIn)[index]); + } + + // Returns pointer to address in last level map that actually points at RangeSection space. + RangeSectionFragmentPointer* EnsureMapsForAddress(TADDR address) + { + uintptr_t level = mapLevels; +#ifdef TARGET_64BIT + auto _RangeSectionL3 = EnsureLevel(address, &_topLevel, level); + if (_RangeSectionL3 == NULL) + return NULL; // Failure case + auto _RangeSectionL2 = EnsureLevel(address, _RangeSectionL3, --level); + if (_RangeSectionL2 == NULL) + return NULL; // Failure case +#else + auto _RangeSectionL2 = &topLevel; +#endif + auto _RangeSectionL1 = EnsureLevel(address, _RangeSectionL2, --level); + if (_RangeSectionL1 == NULL) + return NULL; // Failure case + + auto result = EnsureLevel(address, _RangeSectionL1, --level); + if (result == NULL) + return NULL; // Failure case + + return result; + } + + RangeSectionFragment* GetRangeSectionForAddress(TADDR address, RangeSectionLockState *pLockState) + { +#ifdef TARGET_64BIT + auto _RangeSectionL4 = VolatileLoad(&_topLevel); + auto _RangeSectionL3 = (*_RangeSectionL4)[EffectiveBitsForLevel(address, 4)]; + if (_RangeSectionL3 == NULL) + return NULL; + auto _RangeSectionL2 = (*_RangeSectionL3)[EffectiveBitsForLevel(address, 3)]; + if (_RangeSectionL2 == NULL) + return NULL; + auto _RangeSectionL1 = (*_RangeSectionL2)[EffectiveBitsForLevel(address, 2)]; +#else + auto _RangeSectionL1 = VolatileLoad(&_topLevel[EffectiveBitsForLevel(address, 2)]); // Use a VolatileLoad on the top level operation to ensure that the entire map is synchronized to a state that includes all data needed to examine currently active function pointers. +#endif + if (_RangeSectionL1 == NULL) + return NULL; + + return ((*_RangeSectionL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(pLockState); + } + + uintptr_t RangeSectionFragmentCount(PTR_RangeSection pRangeSection) + { + uintptr_t rangeSize = (pRangeSection->_range.end - pRangeSection->_range.begin); + rangeSize /= bytesAtLastLevel; + return rangeSize + 1; + } + + TADDR IncrementAddressByMaxSizeOfFragment(TADDR input) + { + return input + bytesAtLastLevel; + } + + bool AttachRangeSectionToMap(PTR_RangeSection pRangeSection, RangeSectionLockState *pLockState) + { + assert(*pLockState == RangeSectionLockState::ReaderLocked); // Must be locked so that the cannot fail case, can't fail. NOTE: This only needs the reader lock, as the attach process can happen in parallel to reads. + + uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSection); + RangeSectionFragment* fragments = (RangeSectionFragment*)calloc(rangeSectionFragmentCount, sizeof(RangeSectionFragment)); + + if (fragments == NULL) + { + return false; + } + + RangeSectionFragmentPointer** entriesInMapToUpdate = (RangeSectionFragmentPointer**)calloc(rangeSectionFragmentCount, sizeof(RangeSectionFragmentPointer*)); + if (entriesInMapToUpdate == NULL) + { + free(fragments); + return false; + } + + fragments[0].isPrimaryRangeSectionFragment = true; + + TADDR addressToPrepForUpdate = pRangeSection->_range.begin; + for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) + { + fragments[iFragment].pRangeSection = pRangeSection; + fragments[iFragment]._range = pRangeSection->_range; + fragments[iFragment].isCollectibleRangeSectionFragment = !!(pRangeSection->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); + RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); + if (entryInMapToUpdate == NULL) + { + free(fragments); + free(entriesInMapToUpdate); + return false; + } + + entriesInMapToUpdate[iFragment] = entryInMapToUpdate; + addressToPrepForUpdate = IncrementAddressByMaxSizeOfFragment(addressToPrepForUpdate); + } + + // At this point all the needed memory is allocated, and it is no longer possible to fail. + for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) + { + do + { + RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(pLockState); + fragments[iFragment].pRangeSectionFragmentNext.VolatileStore(initialFragmentInMap); + if (entriesInMapToUpdate[iFragment]->AtomicReplace(&(fragments[iFragment]), initialFragmentInMap)) + break; + } while (true); + } + + // entriesInMapToUpdate was just a temporary allocation + free(entriesInMapToUpdate); + + return true; + } + +public: + RangeSectionMap() : _topLevel{0}, _pCleanupList(EndOfCleanupListMarker()) + { + } + + bool Init() + { + return true; + } + +#ifdef FEATURE_READYTORUN + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_Module pR2RModule, RangeSectionLockState* pLockState) + { + PTR_RangeSection pSection(new(nothrow)RangeSection(range, pJit, flags, pR2RModule)); + if (pSection == NULL) + return NULL; + + if (!AttachRangeSectionToMap(pSection, pLockState)) + { + delete pSection; + return NULL; + } + return pSection; + } +#endif + + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_HeapList pHeapList, RangeSectionLockState* pLockState) + { + PTR_RangeSection pSection(new(nothrow)RangeSection(range, pJit, flags, pHeapList)); + if (pSection == NULL) + return NULL; + + if (!AttachRangeSectionToMap(pSection, pLockState)) + { + delete pSection; + return NULL; + } + return pSection; + } + + PTR_RangeSection LookupRangeSection(TADDR address, RangeSectionLockState *pLockState) + { + RangeSectionFragment* fragment = GetRangeSectionForAddress(address, pLockState); + if (fragment == NULL) + return NULL; + + while ((fragment != NULL) && !fragment->InRange(address)) + { + fragment = fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState); + } + + if (fragment != NULL) + { + if (fragment->pRangeSection->_pRangeSectionNextForDelete != NULL) + return NULL; + return fragment->pRangeSection; + } + + return NULL; + } + + void RemoveRangeSection(RangeSection* pRangeSection) + { + assert(pRangeSection->_pRangeSectionNextForDelete == nullptr); + assert(pRangeSection->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); +#ifdef FEATURE_READYTORUN + assert(pRangeSection->_pR2RModule == NULL); +#endif + + // Removal is implemented by placing onto the cleanup linked list. This is then processed later during cleanup + RangeSection* pLatestRemovedRangeSection; + do + { + pLatestRemovedRangeSection = VolatileLoad(&_pCleanupList); + VolatileStore(&pRangeSection->_pRangeSectionNextForDelete, pLatestRemovedRangeSection); + } while (InterlockedCompareExchangeT(&_pCleanupList, pRangeSection, pLatestRemovedRangeSection) != pLatestRemovedRangeSection); + } + + void CleanupRangeSections(RangeSectionLockState *pLockState) + { + assert(*pLockState == RangeSectionLockState::WriteLocked); + + while (this->_pCleanupList != EndOfCleanupListMarker()) + { + PTR_RangeSection pRangeSectionToCleanup(this->_pCleanupList); + RangeSectionFragment* pRangeSectionFragmentToFree = nullptr; + this->_pCleanupList = pRangeSectionToCleanup->_pRangeSectionNextForDelete; + + uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSectionToCleanup); + + TADDR addressToPrepForCleanup = pRangeSectionToCleanup->_range.begin; + + // Remove fragments from each of the fragment linked lists + for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) + { + RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); + assert(entryInMapToUpdate != NULL); + + while ((entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSection != pRangeSectionToCleanup) + { + entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSectionFragmentNext; + } + + RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState); + + // The fragment associated with the start of the range has the address that was allocated earlier + if (iFragment == 0) + { + pRangeSectionFragmentToFree = fragment; + assert(pRangeSectionFragmentToFree->isPrimaryRangeSectionFragment); + } + + entryInMapToUpdate->VolatileStore(fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState)); + addressToPrepForCleanup = IncrementAddressByMaxSizeOfFragment(addressToPrepForCleanup); + } + + // Free the array of fragments + delete pRangeSectionToCleanup; + free(pRangeSectionFragmentToFree); + } + } + + #ifdef DACCESS_COMPILE + void EnumMemoryRegions(CLRDataEnumMemoryFlags flags) + { + if (!DacEnumMemoryRegion(dac_cast(this), sizeof(*this))) + return; + + _ASSERTE(FALSE); // Add an implementation here. + } + #endif// DACCESS_COMPILE + }; /*****************************************************************************/ @@ -1239,7 +1687,7 @@ class ExecutionManager } CONTRACTL_END; RangeSection * pRange = FindCodeRange(currentPC, GetScanFlags()); - return (pRange != NULL) ? pRange->pjit : NULL; + return (pRange != NULL) ? pRange->_pjit : NULL; } static RangeSection * FindCodeRange(PCODE currentPC, ScanFlag scanFlag); @@ -1284,11 +1732,12 @@ class ExecutionManager static void AddCodeRange(TADDR StartRange, TADDR EndRange, IJitManager* pJit, RangeSection::RangeSectionFlags flags, - void * pHp); + PTR_HeapList pHp); - static void AddNativeImageRange(TADDR StartRange, - SIZE_T Size, - Module * pModule); + static void AddCodeRange(TADDR StartRange, TADDR EndRange, + IJitManager* pJit, + RangeSection::RangeSectionFlags flags, + PTR_Module pModule); static void DeleteRange(TADDR StartRange); @@ -1305,11 +1754,7 @@ class ExecutionManager // FindReadyToRunModule flavor to be used during GC to find GCRefMap static PTR_Module FindModuleForGCRefMap(TADDR currentData); - static RangeSection* GetRangeSectionAndPrev(RangeSection *pRS, TADDR addr, RangeSection **ppPrev); - #ifdef DACCESS_COMPILE - static void EnumRangeList(RangeSection* list, - CLRDataEnumMemoryFlags flags); static void EnumMemoryRegions(CLRDataEnumMemoryFlags flags); #endif @@ -1326,9 +1771,9 @@ class ExecutionManager static RangeSection * FindCodeRangeWithLock(PCODE currentPC); static BOOL IsManagedCodeWithLock(PCODE currentPC); - static BOOL IsManagedCodeWorker(PCODE currentPC); + static BOOL IsManagedCodeWorker(PCODE currentPC, RangeSectionLockState *pLockState); - static RangeSection* GetRangeSection(TADDR addr); + static RangeSection* GetRangeSection(TADDR addr, RangeSectionLockState *pLockState); SPTR_DECL(EECodeManager, m_pDefaultCodeMan); @@ -1342,7 +1787,7 @@ class ExecutionManager // infrastructure to manage readers so we can lock them out and delete domain data // make ReaderCount volatile because we have order dependency in READER_INCREMENT - VOLATILE_SPTR_DECL(RangeSection, m_CodeRangeList); + SPTR_DECL(RangeSectionMap, g_pCodeRangeMap); VOLATILE_SVAL_DECL(LONG, m_dwReaderCount); VOLATILE_SVAL_DECL(LONG, m_dwWriterLock); @@ -1371,12 +1816,6 @@ class ExecutionManager } #endif // defined(_DEBUG) - static void AddRangeHelper(TADDR StartRange, - TADDR EndRange, - IJitManager* pJit, - RangeSection::RangeSectionFlags flags, - TADDR pHeapListOrZapModule); - #ifndef DACCESS_COMPILE static PCODE getNextJumpStub(MethodDesc* pMD, PCODE target, diff --git a/src/coreclr/vm/codeman.inl b/src/coreclr/vm/codeman.inl index da36c9fa14263..2237bd921e9fb 100644 --- a/src/coreclr/vm/codeman.inl +++ b/src/coreclr/vm/codeman.inl @@ -6,10 +6,10 @@ inline BOOL ExecutionManager::IsCollectibleMethod(const METHODTOKEN& MethodToken) { WRAPPER_NO_CONTRACT; - return MethodToken.m_pRangeSection->flags & RangeSection::RANGE_SECTION_COLLECTIBLE; + return MethodToken.m_pRangeSection->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE; } inline TADDR IJitManager::JitTokenToModuleBase(const METHODTOKEN& MethodToken) { - return MethodToken.m_pRangeSection->LowAddress; + return MethodToken.m_pRangeSection->_range.begin; } diff --git a/src/coreclr/vm/common.h b/src/coreclr/vm/common.h index 6430970b87b6e..c8cee620894bd 100644 --- a/src/coreclr/vm/common.h +++ b/src/coreclr/vm/common.h @@ -117,6 +117,7 @@ typedef DPTR(class EEClass) PTR_EEClass; typedef DPTR(class DelegateEEClass) PTR_DelegateEEClass; typedef DPTR(struct DomainLocalModule) PTR_DomainLocalModule; typedef VPTR(class EECodeManager) PTR_EECodeManager; +typedef DPTR(class RangeSectionMap) PTR_RangeSectionMap; typedef DPTR(class EEConfig) PTR_EEConfig; typedef VPTR(class EEDbgInterfaceImpl) PTR_EEDbgInterfaceImpl; typedef VPTR(class DebugInfoManager) PTR_DebugInfoManager; diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index ef511c1814fe0..c7638970a25fc 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -14458,10 +14458,10 @@ void EECodeInfo::Init(PCODE codeAddress, ExecutionManager::ScanFlag scanFlag) if (pRS == NULL) goto Invalid; - if (!pRS->pjit->JitCodeToMethodInfo(pRS, codeAddress, &m_pMD, this)) + if (!pRS->_pjit->JitCodeToMethodInfo(pRS, codeAddress, &m_pMD, this)) goto Invalid; - m_pJM = pRS->pjit; + m_pJM = pRS->_pjit; return; Invalid: diff --git a/src/coreclr/vm/method.cpp b/src/coreclr/vm/method.cpp index 311442060612b..458ace376489b 100644 --- a/src/coreclr/vm/method.cpp +++ b/src/coreclr/vm/method.cpp @@ -2145,10 +2145,10 @@ MethodDesc* NonVirtualEntry2MethodDesc(PCODE entryPoint) } MethodDesc* pMD; - if (pRS->pjit->JitCodeToMethodInfo(pRS, entryPoint, &pMD, NULL)) + if (pRS->_pjit->JitCodeToMethodInfo(pRS, entryPoint, &pMD, NULL)) return pMD; - if (pRS->pjit->GetStubCodeBlockKind(pRS, entryPoint) == STUB_CODE_BLOCK_PRECODE) + if (pRS->_pjit->GetStubCodeBlockKind(pRS, entryPoint) == STUB_CODE_BLOCK_PRECODE) return MethodDesc::GetMethodDescFromStubAddr(entryPoint); // We should never get here diff --git a/src/coreclr/vm/stubmgr.cpp b/src/coreclr/vm/stubmgr.cpp index 94bc431533055..f5f7246885299 100644 --- a/src/coreclr/vm/stubmgr.cpp +++ b/src/coreclr/vm/stubmgr.cpp @@ -1551,7 +1551,7 @@ RangeSectionStubManager::GetStubKind(PCODE stubStartAddress) if (pRS == NULL) return STUB_CODE_BLOCK_UNKNOWN; - return pRS->pjit->GetStubCodeBlockKind(pRS, stubStartAddress); + return pRS->_pjit->GetStubCodeBlockKind(pRS, stubStartAddress); } // From ebadf2ada4219273338366a7b18d7ea7a9a70484 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 29 Nov 2022 18:32:01 -0800 Subject: [PATCH 11/25] May actually work. DAC access known to be broken. --- src/coreclr/vm/codeman.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 1d94396dd3798..675f05667cc86 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -851,9 +851,9 @@ class RangeSectionMap // Returns pointer to address in last level map that actually points at RangeSection space. RangeSectionFragmentPointer* EnsureMapsForAddress(TADDR address) { - uintptr_t level = mapLevels; + uintptr_t level = mapLevels + 1; #ifdef TARGET_64BIT - auto _RangeSectionL3 = EnsureLevel(address, &_topLevel, level); + auto _RangeSectionL3 = EnsureLevel(address, &_topLevel, --level); if (_RangeSectionL3 == NULL) return NULL; // Failure case auto _RangeSectionL2 = EnsureLevel(address, _RangeSectionL3, --level); @@ -877,16 +877,18 @@ class RangeSectionMap { #ifdef TARGET_64BIT auto _RangeSectionL4 = VolatileLoad(&_topLevel); + if (_RangeSectionL4 == NULL) + return NULL; auto _RangeSectionL3 = (*_RangeSectionL4)[EffectiveBitsForLevel(address, 4)]; if (_RangeSectionL3 == NULL) return NULL; auto _RangeSectionL2 = (*_RangeSectionL3)[EffectiveBitsForLevel(address, 3)]; +#else + auto _RangeSectionL2 = VolatileLoad(&_topLevel); +#endif if (_RangeSectionL2 == NULL) return NULL; auto _RangeSectionL1 = (*_RangeSectionL2)[EffectiveBitsForLevel(address, 2)]; -#else - auto _RangeSectionL1 = VolatileLoad(&_topLevel[EffectiveBitsForLevel(address, 2)]); // Use a VolatileLoad on the top level operation to ensure that the entire map is synchronized to a state that includes all data needed to examine currently active function pointers. -#endif if (_RangeSectionL1 == NULL) return NULL; From 3ecfff820d46ee73f5f45703764805bebfc1a92a Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 30 Nov 2022 18:44:55 -0800 Subject: [PATCH 12/25] Fix handling of unwind table --- src/coreclr/vm/codeman.cpp | 8 ++++---- src/coreclr/vm/codeman.h | 12 +++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index 5ecfb273122fc..d31ea05bd41da 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -390,7 +390,7 @@ void UnwindInfoTable::AddToUnwindInfoTable(UnwindInfoTable** unwindInfoPtr, PT_R if (pRS != NULL) { for(int i = 0; i < unwindInfoCount; i++) - AddToUnwindInfoTable(&pRS->pUnwindInfoTable, &unwindInfo[i], pRS->_range.begin, pRS->_range.end); + AddToUnwindInfoTable(&pRS->_pUnwindInfoTable, &unwindInfo[i], pRS->_range.begin, pRS->_range.end); } } @@ -415,7 +415,7 @@ void UnwindInfoTable::AddToUnwindInfoTable(UnwindInfoTable** unwindInfoPtr, PT_R EEJitManager* pJitMgr = (EEJitManager*)(pRS->_pjit); CodeHeader * pHeader = pJitMgr->GetCodeHeaderFromStartAddress(entryPoint); for(ULONG i = 0; i < pHeader->GetNumberOfUnwindInfos(); i++) - RemoveFromUnwindInfoTable(&pRS->pUnwindInfoTable, pRS->_range.begin, pRS->_range.begin + pHeader->GetUnwindInfo(i)->BeginAddress); + RemoveFromUnwindInfoTable(&pRS->_pUnwindInfoTable, pRS->_range.begin, pRS->_range.begin + pHeader->GetUnwindInfo(i)->BeginAddress); } } } @@ -460,7 +460,7 @@ extern CrstStatic g_StubUnwindInfoHeapSegmentsCrst; CodeHeader * pHeader = pJitMgr->GetCodeHeaderFromStartAddress(methodEntry); int unwindInfoCount = pHeader->GetNumberOfUnwindInfos(); for(int i = 0; i < unwindInfoCount; i++) - AddToUnwindInfoTable(&pRS->pUnwindInfoTable, pHeader->GetUnwindInfo(i), pRS->_range.begin, pRS->_range.end); + AddToUnwindInfoTable(&pRS->_pUnwindInfoTable, pHeader->GetUnwindInfo(i), pRS->_range.begin, pRS->_range.end); } } } @@ -5059,7 +5059,7 @@ void ExecutionManager::DeleteRange(TADDR pStartRange) #if defined(TARGET_AMD64) - PTR_UnwindInfoTable unwindTable = pCurr->pUnwindInfoTable; + PTR_UnwindInfoTable unwindTable = pCurr->_pUnwindInfoTable; #endif { diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index c764d9b4944ff..97bd345be5bee 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -637,7 +637,10 @@ struct RangeSection _flags(flags), _pjit(pJit), _pR2RModule(pR2RModule), - _pHeapList(dac_cast((TADDR)0)) + _pHeapList(dac_cast((TADDR)0)), +#if defined(TARGET_AMD64) + _pUnwindInfoTable(dac_cast((TADDR)0)) +#endif { assert(!(flags & RANGE_SECTION_COLLECTIBLE)); assert(pR2RModule != NULL); @@ -649,7 +652,10 @@ struct RangeSection _flags(flags), _pjit(pJit), _pR2RModule(dac_cast((TADDR)0)), - _pHeapList(pHeapList) + _pHeapList(pHeapList), +#if defined(TARGET_AMD64) + _pUnwindInfoTable(dac_cast((TADDR)0)) +#endif {} #ifdef DACCESS_COMPILE @@ -663,7 +669,7 @@ struct RangeSection const PTR_HeapList _pHeapList; #if defined(TARGET_AMD64) - PTR_UnwindInfoTable pUnwindInfoTable; // Points to unwind information for this memory range. + PTR_UnwindInfoTable _pUnwindInfoTable; // Points to unwind information for this memory range. #endif // defined(TARGET_AMD64) From 209877b70f3bc9d376269ebfb9ed25e1ad8f3774 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 1 Dec 2022 17:05:46 -0800 Subject: [PATCH 13/25] Seems to work well on WinX64 Now that it is profileable, there are tweaks to be made --- src/coreclr/inc/vptr_list.h | 1 + src/coreclr/vm/assembly.cpp | 1 - src/coreclr/vm/assemblynative.cpp | 1 - src/coreclr/vm/codeman.cpp | 34 +++++++ src/coreclr/vm/codeman.h | 38 +++++++ src/coreclr/vm/gccover.cpp | 8 +- src/coreclr/vm/loaderallocator.cpp | 21 ++-- src/coreclr/vm/loaderallocator.hpp | 154 ++++++++++++++++++++++++++++- src/coreclr/vm/method.cpp | 22 ++--- src/coreclr/vm/stubmgr.cpp | 5 +- src/coreclr/vm/stubmgr.h | 22 ----- 11 files changed, 245 insertions(+), 62 deletions(-) diff --git a/src/coreclr/inc/vptr_list.h b/src/coreclr/inc/vptr_list.h index d8e6cd42bd7c3..4683dd86e6512 100644 --- a/src/coreclr/inc/vptr_list.h +++ b/src/coreclr/inc/vptr_list.h @@ -13,6 +13,7 @@ VPTR_CLASS(EECodeManager) VPTR_CLASS(RangeList) VPTR_CLASS(LockedRangeList) +VPTR_CLASS(CodeRangeMapRangeList) #ifdef EnC_SUPPORTED VPTR_CLASS(EditAndContinueModule) diff --git a/src/coreclr/vm/assembly.cpp b/src/coreclr/vm/assembly.cpp index 1c3408ec3a8c1..586d11854c763 100644 --- a/src/coreclr/vm/assembly.cpp +++ b/src/coreclr/vm/assembly.cpp @@ -465,7 +465,6 @@ Assembly *Assembly::CreateDynamic(AssemblyBinder* pBinder, NativeAssemblyNamePar if ((access & ASSEMBLY_ACCESS_COLLECT) != 0) { AssemblyLoaderAllocator *pCollectibleLoaderAllocator = new AssemblyLoaderAllocator(); - pCollectibleLoaderAllocator->SetCollectible(); pLoaderAllocator = pCollectibleLoaderAllocator; // Some of the initialization functions are not virtual. Call through the derived class diff --git a/src/coreclr/vm/assemblynative.cpp b/src/coreclr/vm/assemblynative.cpp index 5636de48aa23b..526dbe4f61703 100644 --- a/src/coreclr/vm/assemblynative.cpp +++ b/src/coreclr/vm/assemblynative.cpp @@ -1135,7 +1135,6 @@ extern "C" INT_PTR QCALLTYPE AssemblyNative_InitializeAssemblyLoadContext(INT_PT { // Create a new AssemblyLoaderAllocator for an AssemblyLoadContext loaderAllocator = new AssemblyLoaderAllocator(); - loaderAllocator->SetCollectible(); GCX_COOP(); LOADERALLOCATORREF pManagedLoaderAllocator = NULL; diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index d31ea05bd41da..eb6857db24c86 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -4180,6 +4180,9 @@ BOOL EEJitManager::JitCodeToMethodInfo( _ASSERTE(pRangeSection != NULL); + if (pRangeSection->_flags & RangeSection::RANGE_SECTION_RANGELIST) + return FALSE; + TADDR start = dac_cast(pRangeSection->_pjit)->FindMethodCode(pRangeSection, currentPC); if (start == NULL) return FALSE; @@ -4220,6 +4223,11 @@ StubCodeBlockKind EEJitManager::GetStubCodeBlockKind(RangeSection * pRangeSectio SUPPORTS_DAC; } CONTRACTL_END; + if (pRangeSection->_flags & RangeSection::RANGE_SECTION_RANGELIST) + { + return pRangeSection->_pRangeList->GetCodeBlockKind(); + } + TADDR start = dac_cast(pRangeSection->_pjit)->FindMethodCode(pRangeSection, currentPC); if (start == NULL) return STUB_CODE_BLOCK_NOCODE; @@ -4249,6 +4257,7 @@ TADDR EEJitManager::FindMethodCode(RangeSection * pRangeSection, PCODE currentPC LIMITED_METHOD_DAC_CONTRACT; _ASSERTE(pRangeSection != NULL); + _ASSERTE(pRangeSection->_flags & RangeSection::RANGE_SECTION_CODEHEAP); HeapList *pHp = pRangeSection->_pHeapList; @@ -5046,6 +5055,31 @@ void ExecutionManager::AddCodeRange(TADDR pStartRange, ThrowOutOfMemory(); } +NOINLINE +void ExecutionManager::AddCodeRange(TADDR pStartRange, + TADDR pEndRange, + IJitManager * pJit, + RangeSection::RangeSectionFlags flags, + PTR_CodeRangeMapRangeList pRangeList) +{ + CONTRACTL { + THROWS; + GC_NOTRIGGER; + HOST_CALLS; + PRECONDITION(pStartRange < pEndRange); + PRECONDITION(CheckPointer(pJit)); + PRECONDITION(CheckPointer(pRangeList)); + } CONTRACTL_END; + + ReaderLockHolder rlh; + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // + + PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pRangeList, &lockState); + + if (pRange == NULL) + ThrowOutOfMemory(); +} + // Deletes a single range starting at pStartRange void ExecutionManager::DeleteRange(TADDR pStartRange) { diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 97bd345be5bee..48e599f70f3fa 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -100,6 +100,8 @@ enum StubCodeBlockKind : int STUB_CODE_BLOCK_JUMPSTUB, STUB_CODE_BLOCK_PRECODE, STUB_CODE_BLOCK_DYNAMICHELPER, + STUB_CODE_BLOCK_STUBPRECODE, + STUB_CODE_BLOCK_FIXUPPRECODE, // Last valid value. Note that the definition is duplicated in debug\daccess\fntableaccess.cpp STUB_CODE_BLOCK_LAST = 0xF, // Placeholders returned by code:GetStubCodeBlockKind @@ -608,6 +610,7 @@ class UnwindInfoTable { // address range to track the code heaps. typedef DPTR(struct RangeSection) PTR_RangeSection; +typedef VPTR(class CodeRangeMapRangeList) PTR_CodeRangeMapRangeList; class RangeSectionMap; @@ -629,6 +632,7 @@ struct RangeSection RANGE_SECTION_NONE = 0x0, RANGE_SECTION_COLLECTIBLE = 0x1, RANGE_SECTION_CODEHEAP = 0x2, + RANGE_SECTION_RANGELIST = 0x4, }; #ifdef FEATURE_READYTORUN @@ -638,6 +642,7 @@ struct RangeSection _pjit(pJit), _pR2RModule(pR2RModule), _pHeapList(dac_cast((TADDR)0)), + _pRangeList(dac_cast((TADDR)0)), #if defined(TARGET_AMD64) _pUnwindInfoTable(dac_cast((TADDR)0)) #endif @@ -653,6 +658,19 @@ struct RangeSection _pjit(pJit), _pR2RModule(dac_cast((TADDR)0)), _pHeapList(pHeapList), + _pRangeList(dac_cast((TADDR)0)), +#if defined(TARGET_AMD64) + _pUnwindInfoTable(dac_cast((TADDR)0)) +#endif + {} + + RangeSection(Range range, IJitManager* pJit, RangeSectionFlags flags, PTR_CodeRangeMapRangeList pRangeList) : + _range(range), + _flags(flags), + _pjit(pJit), + _pR2RModule(dac_cast((TADDR)0)), + _pHeapList(dac_cast((TADDR)0)), + _pRangeList(pRangeList), #if defined(TARGET_AMD64) _pUnwindInfoTable(dac_cast((TADDR)0)) #endif @@ -667,6 +685,7 @@ struct RangeSection const PTR_IJitManager _pjit; const PTR_Module _pR2RModule; const PTR_HeapList _pHeapList; + const PTR_CodeRangeMapRangeList _pRangeList; #if defined(TARGET_AMD64) PTR_UnwindInfoTable _pUnwindInfoTable; // Points to unwind information for this memory range. @@ -1010,6 +1029,20 @@ class RangeSectionMap return pSection; } + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_CodeRangeMapRangeList pRangeList, RangeSectionLockState* pLockState) + { + PTR_RangeSection pSection(new(nothrow)RangeSection(range, pJit, flags, pRangeList)); + if (pSection == NULL) + return NULL; + + if (!AttachRangeSectionToMap(pSection, pLockState)) + { + delete pSection; + return NULL; + } + return pSection; + } + PTR_RangeSection LookupRangeSection(TADDR address, RangeSectionLockState *pLockState) { RangeSectionFragment* fragment = GetRangeSectionForAddress(address, pLockState); @@ -1737,6 +1770,11 @@ class ExecutionManager static void Unload(LoaderAllocator *pLoaderAllocator); + static void AddCodeRange(TADDR StartRange, TADDR EndRange, + IJitManager* pJit, + RangeSection::RangeSectionFlags flags, + PTR_CodeRangeMapRangeList pRangeList); + static void AddCodeRange(TADDR StartRange, TADDR EndRange, IJitManager* pJit, RangeSection::RangeSectionFlags flags, diff --git a/src/coreclr/vm/gccover.cpp b/src/coreclr/vm/gccover.cpp index 71a49b63be540..248c37350d167 100644 --- a/src/coreclr/vm/gccover.cpp +++ b/src/coreclr/vm/gccover.cpp @@ -61,18 +61,20 @@ static MethodDesc* getTargetMethodDesc(PCODE target) _ASSERTE(token.IsValid()); return VirtualCallStubManager::GetInterfaceMethodDescFromToken(token); } - if (RangeSectionStubManager::GetStubKind(target) == STUB_CODE_BLOCK_PRECODE) + + auto stubKind = RangeSectionStubManager::GetStubKind(target); + if (stubKind == STUB_CODE_BLOCK_PRECODE) { // The address looks like a value stub, try to get the method descriptor. return MethodDesc::GetMethodDescFromStubAddr(target, TRUE); } - if (PrecodeStubManager::g_pManager->GetStubPrecodeRangeList()->IsInRange(target)) + if (stubKind == STUB_CODE_BLOCK_STUBPRECODE) { return (MethodDesc*)((StubPrecode*)PCODEToPINSTR(target))->GetMethodDesc(); } - if (PrecodeStubManager::g_pManager->GetFixupPrecodeRangeList()->IsInRange(target)) + if (stubKind == STUB_CODE_BLOCK_FIXUPPRECODE) { if (!FixupPrecode::IsFixupPrecodeByASM(target)) { diff --git a/src/coreclr/vm/loaderallocator.cpp b/src/coreclr/vm/loaderallocator.cpp index f1ba447b34c95..b49a100d105f4 100644 --- a/src/coreclr/vm/loaderallocator.cpp +++ b/src/coreclr/vm/loaderallocator.cpp @@ -17,7 +17,9 @@ UINT64 LoaderAllocator::cLoaderAllocatorsCreated = 1; -LoaderAllocator::LoaderAllocator() +LoaderAllocator::LoaderAllocator(bool collectible) : + m_stubPrecodeRangeList(STUB_CODE_BLOCK_STUBPRECODE, collectible), + m_fixupPrecodeRangeList(STUB_CODE_BLOCK_FIXUPPRECODE, collectible) { LIMITED_METHOD_CONTRACT; @@ -66,7 +68,7 @@ LoaderAllocator::LoaderAllocator() m_pLastUsedCodeHeap = NULL; m_pLastUsedDynamicCodeHeap = NULL; m_pJumpStubCache = NULL; - m_IsCollectible = false; + m_IsCollectible = collectible; m_pMarshalingData = NULL; @@ -1194,7 +1196,7 @@ void LoaderAllocator::Init(BaseDomain *pDomain, BYTE *pExecutableHeapMemory) m_pNewStubPrecodeHeap = new (&m_NewStubPrecodeHeapInstance) LoaderHeap(2 * GetOsPageSize(), 2 * GetOsPageSize(), - PrecodeStubManager::g_pManager->GetStubPrecodeRangeList(), + &m_stubPrecodeRangeList, UnlockedLoaderHeap::HeapKind::Interleaved, false /* fUnlocked */, StubPrecode::GenerateCodePage, @@ -1202,7 +1204,7 @@ void LoaderAllocator::Init(BaseDomain *pDomain, BYTE *pExecutableHeapMemory) m_pFixupPrecodeHeap = new (&m_FixupPrecodeHeapInstance) LoaderHeap(2 * GetOsPageSize(), 2 * GetOsPageSize(), - PrecodeStubManager::g_pManager->GetFixupPrecodeRangeList(), + &m_fixupPrecodeRangeList, UnlockedLoaderHeap::HeapKind::Interleaved, false /* fUnlocked */, FixupPrecode::GenerateCodePage, @@ -1687,17 +1689,6 @@ void DomainAssemblyIterator::operator++() pNextAssembly = pCurrentAssembly ? pCurrentAssembly->GetNextDomainAssemblyInSameALC() : NULL; } -void AssemblyLoaderAllocator::SetCollectible() -{ - CONTRACTL - { - NOTHROW; - } - CONTRACTL_END; - - m_IsCollectible = true; -} - #ifndef DACCESS_COMPILE void AssemblyLoaderAllocator::Init(AppDomain* pAppDomain) diff --git a/src/coreclr/vm/loaderallocator.hpp b/src/coreclr/vm/loaderallocator.hpp index b943ea37ad4dd..c56b774aa3585 100644 --- a/src/coreclr/vm/loaderallocator.hpp +++ b/src/coreclr/vm/loaderallocator.hpp @@ -39,6 +39,149 @@ typedef SHash> LoaderAllocatorSet; class CustomAssemblyBinder; + +// This implements the Add/Remove rangelist api on top of the CodeRangeMap in the code manager +// It does not implement the IsInRange api, and destruction isn't safe either. Systems which need +// to check IsInRange must use the CodeRangeMap directly. +class CodeRangeMapRangeList : public RangeList +{ +public: + VPTR_VTABLE_CLASS(CodeRangeMapRangeList, RangeList) + +#ifdef DACCESS_COMPILE + CodeRangeMapRangeList() : + _RangeListRWLock(COOPERATIVE_OR_PREEMPTIVE, LOCK_TYPE_DEFAULT), + _rangeListType(STUB_CODE_BLOCK_UNKNOWN), + _id(NULL), + _collectible(true) + {} +#endif + + CodeRangeMapRangeList(StubCodeBlockKind rangeListType, bool collectible) : + _RangeListRWLock(COOPERATIVE_OR_PREEMPTIVE, LOCK_TYPE_DEFAULT), + _rangeListType(rangeListType), + _id(NULL), + _collectible(collectible) + { + LIMITED_METHOD_CONTRACT; + } + + ~CodeRangeMapRangeList() + { + LIMITED_METHOD_CONTRACT; + RemoveRangesWorker(_id, NULL, NULL); + } + + StubCodeBlockKind GetCodeBlockKind() + { + LIMITED_METHOD_CONTRACT; + return _rangeListType; + } + +private: +#ifndef DACCESS_COMPILE + void AddRangeWorkerHelper(TADDR start, TADDR end, void* id) + { + SimpleWriteLockHolder lh(&_RangeListRWLock); + + _ASSERTE(id == _id || _id == NULL); + _id = id; + // Grow the array first, so that a failure cannot break the + + RangeSection::RangeSectionFlags flags = RangeSection::RANGE_SECTION_RANGELIST; + if (_collectible) + { + _starts.Preallocate(_starts.GetCount() + 1); + flags = (RangeSection::RangeSectionFlags)(flags | RangeSection::RANGE_SECTION_COLLECTIBLE); + } + + ExecutionManager::AddCodeRange(start, end, ExecutionManager::GetEEJitManager(), flags, this); + + if (_collectible) + { + // This cannot fail as the array was Preallocated above. + _starts.Append(start); + } + } +#endif + +protected: + virtual BOOL AddRangeWorker(const BYTE *start, const BYTE *end, void *id) + { + CONTRACTL + { + NOTHROW; + GC_NOTRIGGER; + } + CONTRACTL_END; + +#ifndef DACCESS_COMPILE + BOOL result = FALSE; + EX_TRY + { + AddRangeWorkerHelper((TADDR)start, (TADDR)end, id); + result = TRUE; + } + EX_CATCH + { + } + EX_END_CATCH(SwallowAllExceptions) + + return result; +#else + return FALSE; +#endif // DACCESS_COMPILE + } + + virtual void RemoveRangesWorker(void *id, const BYTE *start, const BYTE *end) + { + CONTRACTL + { + NOTHROW; + GC_NOTRIGGER; + } + CONTRACTL_END; + +#ifndef DACCESS_COMPILE + // This implementation only works for the case where the RangeList is used in a single LoaderHeap + _ASSERTE(start == NULL); + _ASSERTE(end == NULL); + + SimpleWriteLockHolder lh(&_RangeListRWLock); + _ASSERTE(id == _id || (_id == NULL && _starts.IsEmpty())); + + // Iterate backwards to improve efficiency of removals + // as any linked lists in the RangeSectionMap code are in reverse order of insertion. + auto iter = _starts.End(); + while (iter != _starts.Begin()) + { + --iter; + ExecutionManager::DeleteRange(*iter); + } + _starts.Clear(); +#endif // DACCESS_COMPILE + } + + virtual BOOL IsInRangeWorker(TADDR address, TADDR *pID = NULL) + { + WRAPPER_NO_CONTRACT; + RangeSection *pRS = ExecutionManager::FindCodeRange(address, ExecutionManager::ScanReaderLock); + if (pRS == NULL) + return FALSE; + if ((pRS->_flags & RangeSection::RANGE_SECTION_RANGELIST) == 0) + return FALSE; + + return (pRS->_pRangeList == this); + } + +private: + SimpleRWLock _RangeListRWLock; + StubCodeBlockKind _rangeListType; + SArray _starts; + void* _id; + bool _collectible; +}; + // Iterator over a DomainAssembly in the same ALC class DomainAssemblyIterator { @@ -197,6 +340,9 @@ class LoaderAllocator // IL stub cache with fabricated MethodTable parented by a random module in this LoaderAllocator. ILStubCache m_ILStubCache; + CodeRangeMapRangeList m_stubPrecodeRangeList; + CodeRangeMapRangeList m_fixupPrecodeRangeList; + #ifdef FEATURE_PGO // PgoManager to hold pgo data associated with this LoaderAllocator Volatile m_pgoManager; @@ -555,7 +701,7 @@ class LoaderAllocator OBJECTREF GetHandleValue(LOADERHANDLE handle); - LoaderAllocator(); + LoaderAllocator(bool collectible); virtual ~LoaderAllocator(); BaseDomain *GetDomain() { LIMITED_METHOD_CONTRACT; return m_pDomain; } virtual BOOL CanUnload() = 0; @@ -702,7 +848,7 @@ class GlobalLoaderAllocator : public LoaderAllocator public: void Init(BaseDomain *pDomain); - GlobalLoaderAllocator() : m_Id(LAT_Global, (void*)1) { LIMITED_METHOD_CONTRACT;}; + GlobalLoaderAllocator() : LoaderAllocator(false), m_Id(LAT_Global, (void*)1) { LIMITED_METHOD_CONTRACT;}; virtual LoaderAllocatorID* Id(); virtual BOOL CanUnload(); }; @@ -722,7 +868,7 @@ class AssemblyLoaderAllocator : public LoaderAllocator ShuffleThunkCache* m_pShuffleThunkCache; public: virtual LoaderAllocatorID* Id(); - AssemblyLoaderAllocator() : m_Id(LAT_Assembly), m_pShuffleThunkCache(NULL) + AssemblyLoaderAllocator() : LoaderAllocator(true), m_Id(LAT_Assembly), m_pShuffleThunkCache(NULL) #if !defined(DACCESS_COMPILE) , m_binderToRelease(NULL) #endif @@ -730,8 +876,6 @@ class AssemblyLoaderAllocator : public LoaderAllocator void Init(AppDomain *pAppDomain); virtual BOOL CanUnload(); - void SetCollectible(); - void AddDomainAssembly(DomainAssembly *pDomainAssembly) { WRAPPER_NO_CONTRACT; diff --git a/src/coreclr/vm/method.cpp b/src/coreclr/vm/method.cpp index 8f9a09ec265fe..d2f9c3b73d0cd 100644 --- a/src/coreclr/vm/method.cpp +++ b/src/coreclr/vm/method.cpp @@ -2109,17 +2109,6 @@ MethodDesc* NonVirtualEntry2MethodDesc(PCODE entryPoint) RangeSection* pRS = ExecutionManager::FindCodeRange(entryPoint, ExecutionManager::GetScanFlags()); if (pRS == NULL) { - TADDR pInstr = PCODEToPINSTR(entryPoint); - if (PrecodeStubManager::g_pManager->GetStubPrecodeRangeList()->IsInRange(entryPoint)) - { - return (MethodDesc*)((StubPrecode*)pInstr)->GetMethodDesc(); - } - - if (PrecodeStubManager::g_pManager->GetFixupPrecodeRangeList()->IsInRange(entryPoint)) - { - return (MethodDesc*)((FixupPrecode*)pInstr)->GetMethodDesc(); - } - // Is it an FCALL? MethodDesc* pFCallMD = ECall::MapTargetBackToMethod(entryPoint); if (pFCallMD != NULL) @@ -2134,8 +2123,17 @@ MethodDesc* NonVirtualEntry2MethodDesc(PCODE entryPoint) if (pRS->_pjit->JitCodeToMethodInfo(pRS, entryPoint, &pMD, NULL)) return pMD; - if (pRS->_pjit->GetStubCodeBlockKind(pRS, entryPoint) == STUB_CODE_BLOCK_PRECODE) + auto stubCodeBlockKind = pRS->_pjit->GetStubCodeBlockKind(pRS, entryPoint); + + switch(stubCodeBlockKind) + { + case STUB_CODE_BLOCK_PRECODE: return MethodDesc::GetMethodDescFromStubAddr(entryPoint); + case STUB_CODE_BLOCK_FIXUPPRECODE: + return (MethodDesc*)((FixupPrecode*)PCODEToPINSTR(entryPoint))->GetMethodDesc(); + case STUB_CODE_BLOCK_STUBPRECODE: + return (MethodDesc*)((StubPrecode*)PCODEToPINSTR(entryPoint))->GetMethodDesc(); + } // We should never get here _ASSERTE(!"NonVirtualEntry2MethodDesc failed for RangeSection"); diff --git a/src/coreclr/vm/stubmgr.cpp b/src/coreclr/vm/stubmgr.cpp index c31917df824d9..1083add024ced 100644 --- a/src/coreclr/vm/stubmgr.cpp +++ b/src/coreclr/vm/stubmgr.cpp @@ -1003,7 +1003,8 @@ BOOL PrecodeStubManager::CheckIsStub_Internal(PCODE stubStartAddress) } CONTRACTL_END; - return GetStubPrecodeRangeList()->IsInRange(stubStartAddress) || GetFixupPrecodeRangeList()->IsInRange(stubStartAddress); + auto stubKind = RangeSectionStubManager::GetStubKind(stubStartAddress); + return (stubKind == STUB_CODE_BLOCK_FIXUPPRECODE) || (stubKind == STUB_CODE_BLOCK_STUBPRECODE); } BOOL PrecodeStubManager::DoTraceStub(PCODE stubStartAddress, @@ -2380,8 +2381,6 @@ PrecodeStubManager::DoEnumMemoryRegions(CLRDataEnumMemoryFlags flags) WRAPPER_NO_CONTRACT; DAC_ENUM_VTHIS(); EMEM_OUT(("MEM: %p PrecodeStubManager\n", dac_cast(this))); - GetStubPrecodeRangeList()->EnumMemoryRegions(flags); - GetFixupPrecodeRangeList()->EnumMemoryRegions(flags); } void diff --git a/src/coreclr/vm/stubmgr.h b/src/coreclr/vm/stubmgr.h index 49e2e83770476..c4c80d0b17db6 100644 --- a/src/coreclr/vm/stubmgr.h +++ b/src/coreclr/vm/stubmgr.h @@ -399,28 +399,6 @@ class PrecodeStubManager : public StubManager ~PrecodeStubManager() {WRAPPER_NO_CONTRACT;} #endif - protected: - LockedRangeList m_stubPrecodeRangeList; - LockedRangeList m_fixupPrecodeRangeList; - - public: - // Get dac-ized pointer to rangelist. - PTR_RangeList GetStubPrecodeRangeList() - { - SUPPORTS_DAC; - - TADDR addr = PTR_HOST_MEMBER_TADDR(PrecodeStubManager, this, m_stubPrecodeRangeList); - return PTR_RangeList(addr); - } - - PTR_RangeList GetFixupPrecodeRangeList() - { - SUPPORTS_DAC; - - TADDR addr = PTR_HOST_MEMBER_TADDR(PrecodeStubManager, this, m_fixupPrecodeRangeList); - return PTR_RangeList(addr); - } - public: virtual BOOL CheckIsStub_Internal(PCODE stubStartAddress); From 50712b7277dce484c142939e672a02e51aef7099 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 2 Dec 2022 14:49:00 -0800 Subject: [PATCH 14/25] Fix performance of the DelegateConstruct method - Implement a fast path for determining that an instruction pointer is a FixupPrecode/StubPrecode. It turns out those are more likely, than actual jitted code (as we typically have wrapper stubs of some form at this point). - Implement a fast path for computing the number of arguments to a method instead of using MetaSig. - Remove fancy locking scheme from RangeSectionMap. The existing fancy locking scheme works fine. - Fix bug in ExecutionManager::IsReadyToRun and ExecutionManager::FindReadyToRunModule where locks were not used --- src/coreclr/vm/codeman.cpp | 87 ++++++++++++++-------------------- src/coreclr/vm/codeman.h | 81 +++++++++---------------------- src/coreclr/vm/comdelegate.cpp | 27 +++++++++-- src/coreclr/vm/method.cpp | 13 +++++ 4 files changed, 94 insertions(+), 114 deletions(-) diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index eb6857db24c86..7ab3fdd2cf5f6 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -4700,9 +4700,7 @@ ExecutionManager::FindCodeRange(PCODE currentPC, ScanFlag scanFlag) if (scanFlag == ScanReaderLock) return FindCodeRangeWithLock(currentPC); - // Since ScanReaderLock is not set, then we should behave AS IF the ReaderLock is held - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; - return GetRangeSection(currentPC, &lockState); + return GetRangeSection(currentPC); } //************************************************************************** @@ -4716,15 +4714,8 @@ ExecutionManager::FindCodeRangeWithLock(PCODE currentPC) SUPPORTS_DAC; } CONTRACTL_END; - RangeSectionLockState lockState = RangeSectionLockState::None; - RangeSection *result = GetRangeSection(currentPC, &lockState); - if (lockState == RangeSectionLockState::NeedsLock) - { - ReaderLockHolder rlh; - lockState = RangeSectionLockState::ReaderLocked; - result = GetRangeSection(currentPC, &lockState); - } - return result; + ReaderLockHolder rlh; + return GetRangeSection(currentPC); } @@ -4786,9 +4777,7 @@ BOOL ExecutionManager::IsManagedCode(PCODE currentPC) if (GetScanFlags() == ScanReaderLock) return IsManagedCodeWithLock(currentPC); - // Since ScanReaderLock is not set, then we must assume that the ReaderLock is effectively taken. - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; - return IsManagedCodeWorker(currentPC, &lockState); + return IsManagedCodeWorker(currentPC); } //************************************************************************** @@ -4800,17 +4789,8 @@ BOOL ExecutionManager::IsManagedCodeWithLock(PCODE currentPC) GC_NOTRIGGER; } CONTRACTL_END; - RangeSectionLockState lockState = RangeSectionLockState::None; - BOOL result = IsManagedCodeWorker(currentPC, &lockState); - - if (lockState == RangeSectionLockState::NeedsLock) - { - ReaderLockHolder rlh; - lockState = RangeSectionLockState::ReaderLocked; - result = IsManagedCodeWorker(currentPC, &lockState); - } - - return result; + ReaderLockHolder rlh; + return IsManagedCodeWorker(currentPC); } //************************************************************************** @@ -4837,15 +4817,14 @@ BOOL ExecutionManager::IsManagedCode(PCODE currentPC, HostCallPreference hostCal return FALSE; } - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; - return IsManagedCodeWorker(currentPC, &lockState); + return IsManagedCodeWorker(currentPC); #endif } //************************************************************************** // Assumes that the ExecutionManager reader/writer lock is taken or that // it is safe not to take it. -BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC, RangeSectionLockState *pLockState) +BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC) { CONTRACTL { NOTHROW; @@ -4856,7 +4835,7 @@ BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC, RangeSectionLockStat // taken over the call to JitCodeToMethodInfo too so that nobody pulls out // the range section from underneath us. - RangeSection * pRS = GetRangeSection(currentPC, pLockState); + RangeSection * pRS = GetRangeSection(currentPC); if (pRS == NULL) return FALSE; @@ -4898,12 +4877,24 @@ BOOL ExecutionManager::IsReadyToRunCode(PCODE currentPC) // the range section from underneath us. #ifdef FEATURE_READYTORUN - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // TODO! Looking at users of this API I don't know that the lock structure here is safe. Needs checking. - RangeSection * pRS = GetRangeSection(currentPC, &lockState); - if (pRS != NULL && (pRS->_pR2RModule != NULL)) + if (ExecutionManager::GetScanFlags() == ScanNoReaderLock) { - if (dac_cast(pRS->_pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) - return TRUE; + RangeSection * pRS = GetRangeSection(currentPC); + if (pRS != NULL && (pRS->_pR2RModule != NULL)) + { + if (dac_cast(pRS->_pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) + return TRUE; + } + } + else + { + ReaderLockHolder rlh; + RangeSection * pRS = GetRangeSection(currentPC); + if (pRS != NULL && (pRS->_pR2RModule != NULL)) + { + if (dac_cast(pRS->_pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) + return TRUE; + } } #endif @@ -4932,7 +4923,7 @@ LPCWSTR ExecutionManager::GetJitName() } #endif // !FEATURE_MERGE_JIT_AND_ENGINE -RangeSection* ExecutionManager::GetRangeSection(TADDR addr, RangeSectionLockState *pLockState) +RangeSection* ExecutionManager::GetRangeSection(TADDR addr) { CONTRACTL { NOTHROW; @@ -4941,7 +4932,7 @@ RangeSection* ExecutionManager::GetRangeSection(TADDR addr, RangeSectionLockStat SUPPORTS_DAC; } CONTRACTL_END; - return g_pCodeRangeMap->LookupRangeSection(addr, pLockState); + return g_pCodeRangeMap->LookupRangeSection(addr); } /* static */ @@ -4958,10 +4949,9 @@ PTR_Module ExecutionManager::FindReadyToRunModule(TADDR currentData) CONTRACTL_END; #ifdef FEATURE_READYTORUN - RangeSectionLockState lockState = RangeSectionLockState::None; - RangeSection * pRS = GetRangeSection(currentData, &lockState); - if (lockState != RangeSectionLockState::NeedsLock) + if (ExecutionManager::GetScanFlags() == ScanNoReaderLock) { + RangeSection * pRS = GetRangeSection(currentData); if (pRS == NULL) return NULL; @@ -4970,10 +4960,10 @@ PTR_Module ExecutionManager::FindReadyToRunModule(TADDR currentData) else { ReaderLockHolder rlh; - lockState = RangeSectionLockState::ReaderLocked; - pRS = GetRangeSection(currentData, &lockState); + RangeSection * pRS = GetRangeSection(currentData); if (pRS == NULL) return NULL; + return pRS->_pR2RModule; } #else @@ -5023,9 +5013,8 @@ void ExecutionManager::AddCodeRange(TADDR pStartRange, } CONTRACTL_END; ReaderLockHolder rlh; - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // - PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pModule, &lockState); + PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pModule); if (pRange == NULL) ThrowOutOfMemory(); } @@ -5047,9 +5036,8 @@ void ExecutionManager::AddCodeRange(TADDR pStartRange, } CONTRACTL_END; ReaderLockHolder rlh; - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // - PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pHp, &lockState); + PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pHp); if (pRange == NULL) ThrowOutOfMemory(); @@ -5072,9 +5060,8 @@ void ExecutionManager::AddCodeRange(TADDR pStartRange, } CONTRACTL_END; ReaderLockHolder rlh; - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // - PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pRangeList, &lockState); + PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pRangeList); if (pRange == NULL) ThrowOutOfMemory(); @@ -5103,9 +5090,7 @@ void ExecutionManager::DeleteRange(TADDR pStartRange) // require the reader lock, which would cause a deadlock). WriterLockHolder wlh; - RangeSectionLockState lockState = RangeSectionLockState::WriteLocked; - - g_pCodeRangeMap->CleanupRangeSections(&lockState); + g_pCodeRangeMap->CleanupRangeSections(); // Unlike the previous implementation, we no longer attempt to avoid freeing // the memory behind the RangeSection here, as we do not support the hosting // api taking over memory allocation. diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 48e599f70f3fa..37b27a1997f3d 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -695,14 +695,6 @@ struct RangeSection RangeSection* _pRangeSectionNextForDelete = nullptr; // Used for adding to the cleanup list }; -enum class RangeSectionLockState -{ - None, - NeedsLock, - ReaderLocked, - WriteLocked, -}; - // For 64bit, we work with 8KB chunks of memory holding pointers to the next level. This provides 10 bits of address resolution per level. // For *reasons* the X64 hardware is limited to 57bits of addressable address space, and the minimum granularity that makes sense for range lists is 64KB (or every 2^16 bits) // Similarly the Arm64 specification requires addresses to use at most 52 bits. Thus we use the maximum addressable range of X64 to provide the real max range @@ -728,14 +720,6 @@ class RangeSectionMap uintptr_t FragmentToPtr(RangeSectionFragment* fragment) { uintptr_t ptr = (uintptr_t)fragment; - if (ptr == 0) - return ptr; - - if (fragment->isCollectibleRangeSectionFragment) - { - ptr += 1; - } - return ptr; } @@ -746,32 +730,15 @@ class RangeSectionMap RangeSectionFragmentPointer(RangeSectionFragmentPointer &&) = delete; RangeSectionFragmentPointer& operator=(const RangeSectionFragmentPointer&) = delete; - bool PointerIsCollectible() - { - return ((_ptr & 1) == 1); - } - bool IsNull() { return _ptr == 0; } - RangeSectionFragment* VolatileLoadWithoutBarrier(RangeSectionLockState *pLockState) + RangeSectionFragment* VolatileLoadWithoutBarrier() { uintptr_t ptr = ::VolatileLoadWithoutBarrier(&_ptr); - if ((ptr & 1) == 1) - { - if ((*pLockState == RangeSectionLockState::None) || (*pLockState == RangeSectionLockState::NeedsLock)) - { - *pLockState = RangeSectionLockState::NeedsLock; - return NULL; - } - return (RangeSectionFragment*)(ptr - 1); - } - else - { - return (RangeSectionFragment*)(ptr); - } + return (RangeSectionFragment*)(ptr); } void VolatileStore(RangeSectionFragment* fragment) @@ -898,7 +865,7 @@ class RangeSectionMap return result; } - RangeSectionFragment* GetRangeSectionForAddress(TADDR address, RangeSectionLockState *pLockState) + RangeSectionFragment* GetRangeSectionForAddress(TADDR address) { #ifdef TARGET_64BIT auto _RangeSectionL4 = VolatileLoad(&_topLevel); @@ -917,7 +884,7 @@ class RangeSectionMap if (_RangeSectionL1 == NULL) return NULL; - return ((*_RangeSectionL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(pLockState); + return ((*_RangeSectionL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(); } uintptr_t RangeSectionFragmentCount(PTR_RangeSection pRangeSection) @@ -932,10 +899,8 @@ class RangeSectionMap return input + bytesAtLastLevel; } - bool AttachRangeSectionToMap(PTR_RangeSection pRangeSection, RangeSectionLockState *pLockState) + bool AttachRangeSectionToMap(PTR_RangeSection pRangeSection) { - assert(*pLockState == RangeSectionLockState::ReaderLocked); // Must be locked so that the cannot fail case, can't fail. NOTE: This only needs the reader lock, as the attach process can happen in parallel to reads. - uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSection); RangeSectionFragment* fragments = (RangeSectionFragment*)calloc(rangeSectionFragmentCount, sizeof(RangeSectionFragment)); @@ -976,7 +941,7 @@ class RangeSectionMap { do { - RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(pLockState); + RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(); fragments[iFragment].pRangeSectionFragmentNext.VolatileStore(initialFragmentInMap); if (entriesInMapToUpdate[iFragment]->AtomicReplace(&(fragments[iFragment]), initialFragmentInMap)) break; @@ -1000,13 +965,13 @@ class RangeSectionMap } #ifdef FEATURE_READYTORUN - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_Module pR2RModule, RangeSectionLockState* pLockState) + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_Module pR2RModule) { PTR_RangeSection pSection(new(nothrow)RangeSection(range, pJit, flags, pR2RModule)); if (pSection == NULL) return NULL; - if (!AttachRangeSectionToMap(pSection, pLockState)) + if (!AttachRangeSectionToMap(pSection)) { delete pSection; return NULL; @@ -1015,13 +980,13 @@ class RangeSectionMap } #endif - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_HeapList pHeapList, RangeSectionLockState* pLockState) + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_HeapList pHeapList) { PTR_RangeSection pSection(new(nothrow)RangeSection(range, pJit, flags, pHeapList)); if (pSection == NULL) return NULL; - if (!AttachRangeSectionToMap(pSection, pLockState)) + if (!AttachRangeSectionToMap(pSection)) { delete pSection; return NULL; @@ -1029,13 +994,13 @@ class RangeSectionMap return pSection; } - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_CodeRangeMapRangeList pRangeList, RangeSectionLockState* pLockState) + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_CodeRangeMapRangeList pRangeList) { PTR_RangeSection pSection(new(nothrow)RangeSection(range, pJit, flags, pRangeList)); if (pSection == NULL) return NULL; - if (!AttachRangeSectionToMap(pSection, pLockState)) + if (!AttachRangeSectionToMap(pSection)) { delete pSection; return NULL; @@ -1043,15 +1008,15 @@ class RangeSectionMap return pSection; } - PTR_RangeSection LookupRangeSection(TADDR address, RangeSectionLockState *pLockState) + PTR_RangeSection LookupRangeSection(TADDR address) { - RangeSectionFragment* fragment = GetRangeSectionForAddress(address, pLockState); + RangeSectionFragment* fragment = GetRangeSectionForAddress(address); if (fragment == NULL) return NULL; while ((fragment != NULL) && !fragment->InRange(address)) { - fragment = fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState); + fragment = fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(); } if (fragment != NULL) @@ -1081,10 +1046,8 @@ class RangeSectionMap } while (InterlockedCompareExchangeT(&_pCleanupList, pRangeSection, pLatestRemovedRangeSection) != pLatestRemovedRangeSection); } - void CleanupRangeSections(RangeSectionLockState *pLockState) + void CleanupRangeSections() { - assert(*pLockState == RangeSectionLockState::WriteLocked); - while (this->_pCleanupList != EndOfCleanupListMarker()) { PTR_RangeSection pRangeSectionToCleanup(this->_pCleanupList); @@ -1101,12 +1064,12 @@ class RangeSectionMap RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); assert(entryInMapToUpdate != NULL); - while ((entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSection != pRangeSectionToCleanup) + while ((entryInMapToUpdate->VolatileLoadWithoutBarrier())->pRangeSection != pRangeSectionToCleanup) { - entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSectionFragmentNext; + entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier())->pRangeSectionFragmentNext; } - RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState); + RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(); // The fragment associated with the start of the range has the address that was allocated earlier if (iFragment == 0) @@ -1115,7 +1078,7 @@ class RangeSectionMap assert(pRangeSectionFragmentToFree->isPrimaryRangeSectionFragment); } - entryInMapToUpdate->VolatileStore(fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState)); + entryInMapToUpdate->VolatileStore(fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier()); addressToPrepForCleanup = IncrementAddressByMaxSizeOfFragment(addressToPrepForCleanup); } @@ -1817,9 +1780,9 @@ class ExecutionManager static RangeSection * FindCodeRangeWithLock(PCODE currentPC); static BOOL IsManagedCodeWithLock(PCODE currentPC); - static BOOL IsManagedCodeWorker(PCODE currentPC, RangeSectionLockState *pLockState); + static BOOL IsManagedCodeWorker(PCODE currentPC); - static RangeSection* GetRangeSection(TADDR addr, RangeSectionLockState *pLockState); + static RangeSection* GetRangeSection(TADDR addr); SPTR_DECL(EECodeManager, m_pDefaultCodeMan); diff --git a/src/coreclr/vm/comdelegate.cpp b/src/coreclr/vm/comdelegate.cpp index aec64968e6370..d2a347e0c9d10 100644 --- a/src/coreclr/vm/comdelegate.cpp +++ b/src/coreclr/vm/comdelegate.cpp @@ -1568,6 +1568,27 @@ FCIMPLEND extern "C" void * _ReturnAddress(void); #endif // _MSC_VER && !TARGET_UNIX +uint32_t MethodDescToNumFixedArgs(MethodDesc *pMD) +{ + WRAPPER_NO_CONTRACT; + + PCCOR_SIGNATURE pSig; + DWORD cbSigSize; + pMD->GetSig(&pSig, &cbSigSize); + + // Since the signature is known to be valid if we've loaded the Method, we can use the + // non-error checking parser here. + uint32_t data = CorSigUncompressCallingConv(pSig); + if (data & IMAGE_CEE_CS_CALLCONV_GENERIC) + { + // Skip over generic argument count + CorSigUncompressData(pSig); + } + + // Return argument count + return CorSigUncompressData(pSig); +} + // This is the single constructor for all Delegates. The compiler // doesn't provide an implementation of the Delegate constructor. We // provide that implementation through an ECall call to this method. @@ -1635,10 +1656,8 @@ FCIMPL3(void, COMDelegate::DelegateConstruct, Object* refThisUNSAFE, Object* tar DelegateEEClass *pDelCls = (DelegateEEClass*)pDelMT->GetClass(); MethodDesc *pDelegateInvoke = COMDelegate::FindDelegateInvokeMethod(pDelMT); - MetaSig invokeSig(pDelegateInvoke); - MetaSig methodSig(pMeth); - UINT invokeArgCount = invokeSig.NumFixedArgs(); - UINT methodArgCount = methodSig.NumFixedArgs(); + UINT invokeArgCount = MethodDescToNumFixedArgs(pDelegateInvoke); + UINT methodArgCount = MethodDescToNumFixedArgs(pMeth); BOOL isStatic = pMeth->IsStatic(); if (!isStatic) { diff --git a/src/coreclr/vm/method.cpp b/src/coreclr/vm/method.cpp index d2f9c3b73d0cd..936107520b145 100644 --- a/src/coreclr/vm/method.cpp +++ b/src/coreclr/vm/method.cpp @@ -2119,6 +2119,19 @@ MethodDesc* NonVirtualEntry2MethodDesc(PCODE entryPoint) return NULL; } + // Inlined fast path for fixup precode and stub precode from RangeList implementation + if (pRS->_flags == RangeSection::RANGE_SECTION_RANGELIST) + { + if (pRS->_pRangeList->GetCodeBlockKind() == STUB_CODE_BLOCK_FIXUPPRECODE) + { + return (MethodDesc*)((FixupPrecode*)PCODEToPINSTR(entryPoint))->GetMethodDesc(); + } + if (pRS->_pRangeList->GetCodeBlockKind() == STUB_CODE_BLOCK_STUBPRECODE) + { + return (MethodDesc*)((StubPrecode*)PCODEToPINSTR(entryPoint))->GetMethodDesc(); + } + } + MethodDesc* pMD; if (pRS->_pjit->JitCodeToMethodInfo(pRS, entryPoint, &pMD, NULL)) return pMD; From 38acd5cf9e05247564522bc4bae418b7ff3f7255 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 2 Dec 2022 15:12:41 -0800 Subject: [PATCH 15/25] It should build everywhere now --- src/coreclr/vm/codeman.h | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 37b27a1997f3d..22cfbf5fa428e 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -642,9 +642,9 @@ struct RangeSection _pjit(pJit), _pR2RModule(pR2RModule), _pHeapList(dac_cast((TADDR)0)), - _pRangeList(dac_cast((TADDR)0)), + _pRangeList(dac_cast((TADDR)0)) #if defined(TARGET_AMD64) - _pUnwindInfoTable(dac_cast((TADDR)0)) + , _pUnwindInfoTable(dac_cast((TADDR)0)) #endif { assert(!(flags & RANGE_SECTION_COLLECTIBLE)); @@ -658,9 +658,9 @@ struct RangeSection _pjit(pJit), _pR2RModule(dac_cast((TADDR)0)), _pHeapList(pHeapList), - _pRangeList(dac_cast((TADDR)0)), + _pRangeList(dac_cast((TADDR)0)) #if defined(TARGET_AMD64) - _pUnwindInfoTable(dac_cast((TADDR)0)) + , _pUnwindInfoTable(dac_cast((TADDR)0)) #endif {} @@ -670,9 +670,9 @@ struct RangeSection _pjit(pJit), _pR2RModule(dac_cast((TADDR)0)), _pHeapList(dac_cast((TADDR)0)), - _pRangeList(pRangeList), + _pRangeList(pRangeList) #if defined(TARGET_AMD64) - _pUnwindInfoTable(dac_cast((TADDR)0)) + , _pUnwindInfoTable(dac_cast((TADDR)0)) #endif {} @@ -712,14 +712,17 @@ struct RangeSection class RangeSectionMap { class RangeSectionFragment; + + // Helper structure which forces all access to the various pointers to be handled via volatile/interlocked operations + // The copy/move constructors are all deleted to forbid accidental reads into temporaries, etc. class RangeSectionFragmentPointer { private: - uintptr_t _ptr; + TADDR _ptr; - uintptr_t FragmentToPtr(RangeSectionFragment* fragment) + TADDR FragmentToPtr(RangeSectionFragment* fragment) { - uintptr_t ptr = (uintptr_t)fragment; + TADDR ptr = (TADDR)fragment; return ptr; } @@ -756,7 +759,7 @@ class RangeSectionMap }; // Unlike a RangeSection, a RangeSectionFragment cannot span multiple elements of the last level of the RangeSectionMap - // Always allocated via calloc + // Always allocated via memset/free class RangeSectionFragment { public: @@ -801,7 +804,13 @@ class RangeSectionMap RangeSection* EndOfCleanupListMarker() { return (RangeSection*)1; } - void* AllocateLevel() { return calloc(entriesPerMapLevel, sizeof(void*)); } + void* AllocateLevel() + { + size_t size = entriesPerMapLevel * sizeof(void*); + void *buf = malloc(size); + memset(buf, 0, size); + return buf; + } uintptr_t EffectiveBitsForLevel(TADDR address, uintptr_t level) { @@ -852,7 +861,7 @@ class RangeSectionMap if (_RangeSectionL2 == NULL) return NULL; // Failure case #else - auto _RangeSectionL2 = &topLevel; + auto _RangeSectionL2 = &_topLevel; #endif auto _RangeSectionL1 = EnsureLevel(address, _RangeSectionL2, --level); if (_RangeSectionL1 == NULL) From 9148e31b9a1617de568112592ed3993333d89590 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 2 Dec 2022 15:50:45 -0800 Subject: [PATCH 16/25] Delete temporary bits --- temp/RangeListMap.cpp | 622 ------------------------------ temp/RangeListMap.vcxproj | 135 ------- temp/RangeListMap.vcxproj.filters | 22 -- 3 files changed, 779 deletions(-) delete mode 100644 temp/RangeListMap.cpp delete mode 100644 temp/RangeListMap.vcxproj delete mode 100644 temp/RangeListMap.vcxproj.filters diff --git a/temp/RangeListMap.cpp b/temp/RangeListMap.cpp deleted file mode 100644 index ffe4db21b8267..0000000000000 --- a/temp/RangeListMap.cpp +++ /dev/null @@ -1,622 +0,0 @@ -// RangeSectionMap.cpp : This file contains the 'main' function. Program execution begins and ends there. -// - -#include -#include -#include -using namespace std; -class IJitManager; -class Module; -class HeapList; - -typedef Module* PTR_Module; -typedef HeapList* PTR_HeapList; -typedef uintptr_t TADDR; -#define TARGET_64BIT - - - - - - -template -void VolatileStore(T* ptr, T val) -{ - *ptr = val; -} - -template -T VolatileLoad(T* ptr) -{ - return *ptr; -} - -template -T VolatileLoadWithoutBarrier(T* ptr) -{ - return *ptr; -} - -class Range -{ -public: - // [begin,end] (This is an inclusive range) - void* begin; - void* end; -}; - -class RangeSectionMap; - -class RangeSection -{ - friend class RangeSectionMap; -public: - enum RangeSectionFlags - { - RANGE_SECTION_NONE = 0x0, - RANGE_SECTION_COLLECTIBLE = 0x1, - RANGE_SECTION_CODEHEAP = 0x2, - }; - -#ifdef FEATURE_READYTORUN - RangeSection(Range range, IJitManager* pJit, RangeSectionFlags flags, PTR_Module pR2RModule) : - _range(range), - _flags(flags), - _pjit(pJit), - _pR2RModule(pR2RModule), - _pHeapList(NULL) - { - assert(!(flags & RANGE_SECTION_COLLECTIBLE)); - assert(pR2RModule != NULL); - } -#endif - - RangeSection(Range range, IJitManager* pJit, RangeSectionFlags flags, PTR_HeapList pHeapList) : - _range(range), - _flags(flags), - _pjit(pJit), - _pR2RModule(NULL), - _pHeapList(pHeapList) - {} - - const Range _range; - const RangeSectionFlags _flags; - IJitManager *const _pjit; - const PTR_Module _pR2RModule; - const PTR_HeapList _pHeapList; - -#if defined(TARGET_AMD64) - PTR_UnwindInfoTable pUnwindInfoTable; // Points to unwind information for this memory range. -#endif // defined(TARGET_AMD64) - - RangeSection* _pRangeSectionNextForDelete = nullptr; // Used for adding to the cleanup list -}; - -enum class RangeSectionLockState -{ - None, - NeedsLock, - ReaderLocked, - WriteLocked, -}; - -// For 64bit, we work with 8KB chunks of memory holding pointers to the next level. This provides 10 bits of address resolution per level. -// For *reasons* the X64 hardware is limited to 57bits of addressable address space, and the minimum granularity that makes sense for range lists is 64KB (or every 2^16 bits) -// Similarly the Arm64 specification requires addresses to use at most 52 bits. Thus we use the maximum addressable range of X64 to provide the real max range -// So the first level is bits [56:47] -> L4 -// Then [46:37] -> L3 -// [36:27] -> L2 -// [26:17] -> L1 -// This leaves 17 bits of the address to be handled by the RangeSection linked list -// -// For 32bit VA processes, use 1KB chunks holding pointers to the next level. This provides 8 bites of address resolution per level. [31:24] and [23:16]. - -// The memory safety model for segment maps is that the pointers held within the individual segments can never change other than to go from NULL to a meaningful pointer, -// except for the final level, which is only permitted to change when CleanupRangeSections is in use. - -class RangeSectionMap -{ - class RangeSectionFragment; - class RangeSectionFragmentPointer - { - private: - uintptr_t _ptr; - - uintptr_t FragmentToPtr(RangeSectionFragment* fragment) - { - uintptr_t ptr = (uintptr_t)fragment; - if (ptr == 0) - return ptr; - - if (fragment->isCollectibleRangeSectionFragment) - { - ptr += 1; - } - - return ptr; - } - - RangeSectionFragmentPointer() { _ptr = 0; } - public: - - RangeSectionFragmentPointer(RangeSectionFragmentPointer &) = delete; - RangeSectionFragmentPointer(RangeSectionFragmentPointer &&) = delete; - RangeSectionFragmentPointer& operator=(const RangeSectionFragmentPointer&) = delete; - - bool PointerIsCollectible() - { - return ((_ptr & 1) == 1); - } - - bool IsNull() - { - return _ptr == 0; - } - - RangeSectionFragment* VolatileLoadWithoutBarrier(RangeSectionLockState *pLockState) - { - uintptr_t ptr = ::VolatileLoadWithoutBarrier(&_ptr); - if ((ptr & 1) == 1) - { - if ((*pLockState == RangeSectionLockState::None) || (*pLockState == RangeSectionLockState::NeedsLock)) - { - *pLockState = RangeSectionLockState::NeedsLock; - return NULL; - } - return (RangeSectionFragment*)(ptr - 1); - } - else - { - return (RangeSectionFragment*)(ptr); - } - } - - void VolatileStore(RangeSectionFragment* fragment) - { - ::VolatileStore(&_ptr, FragmentToPtr(fragment)); - } - - bool AtomicReplace(RangeSectionFragment* newFragment, RangeSectionFragment* oldFragment) - { - uintptr_t oldPtr = FragmentToPtr(oldFragment); - uintptr_t newPtr = FragmentToPtr(newFragment); - - return oldPtr == (uintptr_t)InterlockedCompareExchangePointer((volatile PVOID*)&_ptr, (PVOID)newPtr, (PVOID)oldPtr); - } - }; - - // Unlike a RangeSection, a RangeSectionFragment cannot span multiple elements of the last level of the RangeSectionMap - // Always allocated via calloc - class RangeSectionFragment - { - public: - RangeSectionFragmentPointer pRangeSectionFragmentNext; - Range _range; - RangeSection* pRangeSection; - bool InRange(void* address) { return address >= _range.begin && address <= _range.end && pRangeSection->_pRangeSectionNextForDelete == NULL; } - bool isPrimaryRangeSectionFragment; // RangeSectionFragment are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. - bool isCollectibleRangeSectionFragment; // RangeSectionFragments - }; - -#ifdef TARGET_64BIT - static const uintptr_t entriesPerMapLevel = 1024; -#else - static const uintptr_t entriesPerMapLevel = 256; -#endif - - typedef RangeSectionFragmentPointer RangeSectionList; - typedef RangeSectionList RangeSectionL1[entriesPerMapLevel]; - typedef RangeSectionL1* RangeSectionL2[entriesPerMapLevel]; - typedef RangeSectionL2* RangeSectionL3[entriesPerMapLevel]; - typedef RangeSectionL3* RangeSectionL4[entriesPerMapLevel]; - -#ifdef TARGET_64BIT - typedef RangeSectionL4 RangeSectionTopLevel; - static const uintptr_t mapLevels = 4; - static const uintptr_t maxSetBit = 56; // This is 0 indexed - static const uintptr_t bitsPerLevel = 10; -#else - typedef RangeSectionL2 RangeSectionTopLevel; - static const uintptr_t mapLevels = 2; - static const uintptr_t maxSetBit = 31; // This is 0 indexed - static const uintptr_t bitsPerLevel = 8; -#endif - - RangeSectionTopLevel *_topLevel = nullptr; - - RangeSection* _pCleanupList; - - const uintptr_t bitsAtLastLevel = maxSetBit - (bitsPerLevel * mapLevels) + 1; - const uintptr_t bytesAtLastLevel = (((uintptr_t)1) << (bitsAtLastLevel - 1)); - - RangeSection* EndOfCleanupListMarker() { return (RangeSection*)1; } - - void* AllocateLevel() { return calloc(entriesPerMapLevel, sizeof(void*)); } - - uintptr_t EffectiveBitsForLevel(void* address, uintptr_t level) - { - uintptr_t addressAsInt = (uintptr_t)address; - uintptr_t addressBitsUsedInMap = addressAsInt >> (maxSetBit - (mapLevels * bitsPerLevel)); - uintptr_t addressBitsShifted = addressBitsUsedInMap >> ((level - 1) * bitsPerLevel); - uintptr_t addressBitsUsedInLevel = (entriesPerMapLevel - 1) & addressBitsShifted; - return addressBitsUsedInLevel; - } - - template - auto EnsureLevel(void *address, T* outerLevel, uintptr_t level) -> decltype(&((**outerLevel)[0])) - { - uintptr_t index = EffectiveBitsForLevel(address, level); - auto levelToGetPointerIn = VolatileLoadWithoutBarrier(outerLevel); - - if (levelToGetPointerIn == NULL) - { - auto levelNew = static_cast(AllocateLevel()); - if (levelNew == NULL) - return NULL; - auto levelPreviouslyStored = (decltype(&(*outerLevel)[0]))InterlockedCompareExchangePointer((volatile PVOID*)outerLevel, (PVOID)levelNew, NULL); - if (levelPreviouslyStored != nullptr) - { - // Handle race where another thread grew the table - levelToGetPointerIn = levelPreviouslyStored; - free(levelNew); - } - else - { - levelToGetPointerIn = levelNew; - } - assert(levelToGetPointerIn != nullptr); - } - - return &((*levelToGetPointerIn)[index]); - } - - // Returns pointer to address in last level map that actually points at RangeSection space. - RangeSectionFragmentPointer* EnsureMapsForAddress(void* address) - { - uintptr_t level = mapLevels; -#ifdef TARGET_64BIT - auto _RangeSectionL3 = EnsureLevel(address, &_topLevel, level); - if (_RangeSectionL3 == NULL) - return NULL; // Failure case - auto _RangeSectionL2 = EnsureLevel(address, _RangeSectionL3, --level); - if (_RangeSectionL2 == NULL) - return NULL; // Failure case -#else - auto _RangeSectionL2 = &topLevel; -#endif - auto _RangeSectionL1 = EnsureLevel(address, _RangeSectionL2, --level); - if (_RangeSectionL1 == NULL) - return NULL; // Failure case - - auto result = EnsureLevel(address, _RangeSectionL1, --level); - if (result == NULL) - return NULL; // Failure case - - return result; - } - - RangeSectionFragment* GetRangeSectionForAddress(void* address, RangeSectionLockState *pLockState) - { -#ifdef TARGET_64BIT - auto _RangeSectionL4 = VolatileLoad(&_topLevel); - auto _RangeSectionL3 = (*_RangeSectionL4)[EffectiveBitsForLevel(address, 4)]; - if (_RangeSectionL3 == NULL) - return NULL; - auto _RangeSectionL2 = (*_RangeSectionL3)[EffectiveBitsForLevel(address, 3)]; - if (_RangeSectionL2 == NULL) - return NULL; - auto _RangeSectionL1 = (*_RangeSectionL2)[EffectiveBitsForLevel(address, 2)]; -#else - auto _RangeSectionL1 = VolatileLoad(&_topLevel[EffectiveBitsForLevel(address, 2)]); // Use a VolatileLoad on the top level operation to ensure that the entire map is synchronized to a state that includes all data needed to examine currently active function pointers. -#endif - if (_RangeSectionL1 == NULL) - return NULL; - - return ((*_RangeSectionL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(pLockState); - } - - uintptr_t RangeSectionFragmentCount(RangeSection *pRangeSection) - { - uintptr_t rangeSize = reinterpret_cast(pRangeSection->_range.end) - reinterpret_cast(pRangeSection->_range.begin); - rangeSize /= bytesAtLastLevel; - return rangeSize + 1; - } - - void* IncrementAddressByMaxSizeOfFragment(void* input) - { - uintptr_t inputAsInt = reinterpret_cast(input); - return reinterpret_cast(inputAsInt + bytesAtLastLevel); - } - - bool AttachRangeSectionToMap(RangeSection* pRangeSection, RangeSectionLockState *pLockState) - { - assert(*pLockState == RangeSectionLockState::ReaderLocked); // Must be locked so that the cannot fail case, can't fail. NOTE: This only needs the reader lock, as the attach process can happen in parallel to reads. - - uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSection); - RangeSectionFragment* fragments = (RangeSectionFragment*)calloc(rangeSectionFragmentCount, sizeof(RangeSectionFragment)); - - if (fragments == NULL) - { - return false; - } - - RangeSectionFragmentPointer** entriesInMapToUpdate = (RangeSectionFragmentPointer**)calloc(rangeSectionFragmentCount, sizeof(RangeSectionFragmentPointer*)); - if (entriesInMapToUpdate == NULL) - { - free(fragments); - return false; - } - - fragments[0].isPrimaryRangeSectionFragment = true; - - void* addressToPrepForUpdate = pRangeSection->_range.begin; - for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) - { - fragments[iFragment].pRangeSection = pRangeSection; - fragments[iFragment]._range = pRangeSection->_range; - fragments[iFragment].isCollectibleRangeSectionFragment = !!(pRangeSection->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); - RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForUpdate); - if (entryInMapToUpdate == NULL) - { - free(fragments); - free(entriesInMapToUpdate); - return false; - } - - entriesInMapToUpdate[iFragment] = entryInMapToUpdate; - addressToPrepForUpdate = IncrementAddressByMaxSizeOfFragment(addressToPrepForUpdate); - } - - // At this point all the needed memory is allocated, and it is no longer possible to fail. - for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) - { - do - { - RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(pLockState); - fragments[iFragment].pRangeSectionFragmentNext.VolatileStore(initialFragmentInMap); - if (entriesInMapToUpdate[iFragment]->AtomicReplace(&(fragments[iFragment]), initialFragmentInMap)) - break; - } while (true); - } - - // entriesInMapToUpdate was just a temporary allocation - free(entriesInMapToUpdate); - - return true; - } - -public: - RangeSectionMap() : _topLevel{0}, _pCleanupList(EndOfCleanupListMarker()) - { - } - - bool Init() - { - return true; - } - -#ifdef FEATURE_READYTORUN - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_Module pR2RModule, RangeSectionLockState* pLockState) - { - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; - RangeSection *pSection = new(nothrow)RangeSection(range, pJit, flags, pR2RModule); - if (pSection == NULL) - return NULL; - - if (!AttachRangeSectionToMap(pSection, pLockState)) - { - delete pSection; - return NULL; - } - return pSection; - } -#endif - - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_HeapList pHeapList, RangeSectionLockState* pLockState) - { - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; - RangeSection *pSection = new(nothrow)RangeSection(range, pJit, flags, pHeapList); - if (pSection == NULL) - return NULL; - - if (!AttachRangeSectionToMap(pSection, pLockState)) - { - delete pSection; - return NULL; - } - return pSection; - } - - RangeSection* LookupRangeSection(void* address, RangeSectionLockState *pLockState) - { - RangeSectionFragment* fragment = GetRangeSectionForAddress(address, pLockState); - if (fragment == NULL) - return NULL; - - while ((fragment != NULL) && !fragment->InRange(address)) - { - fragment = fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState); - } - - if (fragment != NULL) - { - if (fragment->pRangeSection->_pRangeSectionNextForDelete != NULL) - return NULL; - return fragment->pRangeSection; - } - - return NULL; - } - - void RemoveRangeSection(RangeSection* pRangeSection) - { - assert(pRangeSection->_pRangeSectionNextForDelete == nullptr); - assert(pRangeSection->_flags & RangeSection::RANGE_SECTION_COLLECTIBLE); -#ifdef FEATURE_READYTORUN - assert(pRangeSection->pR2RModule == NULL); -#endif - - // Removal is implemented by placing onto the cleanup linked list. This is then processed later during cleanup - RangeSection* pLatestRemovedRangeSection; - do - { - pLatestRemovedRangeSection = VolatileLoad(&_pCleanupList); - VolatileStore(&pRangeSection->_pRangeSectionNextForDelete, pLatestRemovedRangeSection); - } while (InterlockedCompareExchangePointer((volatile PVOID *)&_pCleanupList, pRangeSection, pLatestRemovedRangeSection) != pLatestRemovedRangeSection); - } - - void CleanupRangeSections(RangeSectionLockState *pLockState) - { - assert(*pLockState == RangeSectionLockState::WriteLocked); - - while (this->_pCleanupList != EndOfCleanupListMarker()) - { - RangeSection* pRangeSectionToCleanup = this->_pCleanupList; - RangeSectionFragment* pRangeSectionFragmentToFree = nullptr; - this->_pCleanupList = pRangeSectionToCleanup->_pRangeSectionNextForDelete; - - uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSectionToCleanup); - - void* addressToPrepForCleanup = pRangeSectionToCleanup->_range.begin; - - // Remove fragments from each of the fragment linked lists - for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) - { - RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); - assert(entryInMapToUpdate != NULL); - - while ((entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSection != pRangeSectionToCleanup) - { - entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSectionFragmentNext; - } - - RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState); - - // The fragment associated with the start of the range has the address that was allocated earlier - if (iFragment == 0) - { - pRangeSectionFragmentToFree = fragment; - assert(pRangeSectionFragmentToFree->isPrimaryRangeSectionFragment); - } - - entryInMapToUpdate->VolatileStore(fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState)); - addressToPrepForCleanup = IncrementAddressByMaxSizeOfFragment(addressToPrepForCleanup); - } - - // Free the array of fragments - delete pRangeSectionToCleanup; - free(pRangeSectionFragmentToFree); - } - } -}; - -int main() -{ - RangeSectionMap map; - Range rFirst; - rFirst.begin = (void*)0x1111000; - rFirst.end = (void*)0x1111050; - Range rSecond; - rSecond.begin = (void*)0x1111051; - rSecond.end = (void*)0x1192050; - - RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; - RangeSection *rSectionFirst = map.AllocateRange(rFirst, NULL, RangeSection::RANGE_SECTION_COLLECTIBLE, (PTR_HeapList)NULL, &lockState); - RangeSection *rSectionSecond = map.AllocateRange(rSecond, NULL, RangeSection::RANGE_SECTION_NONE, (PTR_HeapList)NULL, &lockState); - - RangeSection *result; - - lockState = RangeSectionLockState::None; - result = map.LookupRangeSection((void*)0x1111000, &lockState); - assert(lockState == RangeSectionLockState::NeedsLock); - assert(result == NULL); - lockState = RangeSectionLockState::ReaderLocked; - result = map.LookupRangeSection((void*)0x1111000, &lockState); - assert(result == rSectionFirst); - assert(lockState == RangeSectionLockState::ReaderLocked); - - lockState = RangeSectionLockState::None; - result = map.LookupRangeSection((void*)0x1111050, &lockState); - assert(lockState == RangeSectionLockState::NeedsLock); - assert(result == NULL); - lockState = RangeSectionLockState::ReaderLocked; - result = map.LookupRangeSection((void*)0x1111050, &lockState); - assert(result == rSectionFirst); - assert(lockState == RangeSectionLockState::ReaderLocked); - lockState = RangeSectionLockState::None; - - result = map.LookupRangeSection((void*)0x1111051, &lockState); - assert(lockState == RangeSectionLockState::None); - assert(result == rSectionSecond); - result = map.LookupRangeSection((void*)0x1151050, &lockState); - assert(lockState == RangeSectionLockState::None); - assert(result == rSectionSecond); - result = map.LookupRangeSection((void*)0x1192050, &lockState); - assert(lockState == RangeSectionLockState::None); - assert(result == rSectionSecond); - - map.RemoveRangeSection(rSectionFirst); - - lockState = RangeSectionLockState::None; - result = map.LookupRangeSection((void*)0x1111000, &lockState); - assert(lockState == RangeSectionLockState::NeedsLock); - assert(result == NULL); - lockState = RangeSectionLockState::ReaderLocked; - result = map.LookupRangeSection((void*)0x1111000, &lockState); - assert(result == NULL); - assert(lockState == RangeSectionLockState::ReaderLocked); - - lockState = RangeSectionLockState::None; - result = map.LookupRangeSection((void*)0x1111050, &lockState); - assert(lockState == RangeSectionLockState::NeedsLock); - assert(result == NULL); - lockState = RangeSectionLockState::ReaderLocked; - result = map.LookupRangeSection((void*)0x1111050, &lockState); - assert(result == NULL); - assert(lockState == RangeSectionLockState::ReaderLocked); - lockState = RangeSectionLockState::None; - - result = map.LookupRangeSection((void*)0x1111051, &lockState); - assert(result == rSectionSecond); - assert(lockState == RangeSectionLockState::None); - result = map.LookupRangeSection((void*)0x1151050, &lockState); - assert(result == rSectionSecond); - assert(lockState == RangeSectionLockState::None); - result = map.LookupRangeSection((void*)0x1192050, &lockState); - assert(result == rSectionSecond); - assert(lockState == RangeSectionLockState::None); - - assert(lockState == RangeSectionLockState::None); - lockState = RangeSectionLockState::WriteLocked; - map.CleanupRangeSections(&lockState); - lockState = RangeSectionLockState::None; - - result = map.LookupRangeSection((void*)0x1111000, &lockState); - assert(result == NULL); - assert(lockState == RangeSectionLockState::None); - result = map.LookupRangeSection((void*)0x1111050, &lockState); - assert(result == NULL); - assert(lockState == RangeSectionLockState::None); - result = map.LookupRangeSection((void*)0x1111051, &lockState); - assert(result == rSectionSecond); - assert(lockState == RangeSectionLockState::None); - result = map.LookupRangeSection((void*)0x1151050, &lockState); - assert(result == rSectionSecond); - assert(lockState == RangeSectionLockState::None); - result = map.LookupRangeSection((void*)0x1192050, &lockState); - assert(result == rSectionSecond); - assert(lockState == RangeSectionLockState::None); - - std::cout << "Done\n"; -} - -// Run program: Ctrl + F5 or Debug > Start Without Debugging menu -// Debug program: F5 or Debug > Start Debugging menu - -// Tips for Getting Started: -// 1. Use the Solution Explorer window to add/manage files -// 2. Use the Team Explorer window to connect to source control -// 3. Use the Output window to see build output and other messages -// 4. Use the Error List window to view errors -// 5. Go to Project > Add New Item to create new code files, or Project > Add Existing Item to add existing code files to the project -// 6. In the future, to open this project again, go to File > Open > Project and select the .sln file diff --git a/temp/RangeListMap.vcxproj b/temp/RangeListMap.vcxproj deleted file mode 100644 index 317ff6e6bc245..0000000000000 --- a/temp/RangeListMap.vcxproj +++ /dev/null @@ -1,135 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 16.0 - Win32Proj - {09366505-33bb-4a1b-80f0-8a1cd78d159b} - RangeListMap - 10.0 - - - - Application - true - v143 - Unicode - - - Application - false - v143 - true - Unicode - - - Application - true - v143 - Unicode - - - Application - false - v143 - true - Unicode - - - - - - - - - - - - - - - - - - - - - - Level3 - true - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - - - Console - true - - - - - Level3 - true - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - - - Console - true - true - true - - - - - Level3 - true - _DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - - - Console - true - - - - - Level3 - true - true - true - NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - - - Console - true - true - true - - - - - - - - - \ No newline at end of file diff --git a/temp/RangeListMap.vcxproj.filters b/temp/RangeListMap.vcxproj.filters deleted file mode 100644 index 2c6b143d51646..0000000000000 --- a/temp/RangeListMap.vcxproj.filters +++ /dev/null @@ -1,22 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - \ No newline at end of file From dce4bb1ca5701d93fded22756cc37959bbde1c5f Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 6 Dec 2022 14:09:06 -0800 Subject: [PATCH 17/25] Fix Windows builds --- src/coreclr/vm/codeman.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 22cfbf5fa428e..2b481c41f71ea 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -751,8 +751,8 @@ class RangeSectionMap bool AtomicReplace(RangeSectionFragment* newFragment, RangeSectionFragment* oldFragment) { - uintptr_t oldPtr = FragmentToPtr(oldFragment); - uintptr_t newPtr = FragmentToPtr(newFragment); + TADDR oldPtr = FragmentToPtr(oldFragment); + TADDR newPtr = FragmentToPtr(newFragment); return oldPtr == InterlockedCompareExchangeT(&_ptr, newPtr, oldPtr); } From 518011c9aa95c0a7bf695621dd58d756f0334e4f Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 6 Dec 2022 14:38:02 -0800 Subject: [PATCH 18/25] Fix contract issues --- src/coreclr/vm/loaderallocator.hpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/coreclr/vm/loaderallocator.hpp b/src/coreclr/vm/loaderallocator.hpp index c56b774aa3585..8f5ab1adc1c7b 100644 --- a/src/coreclr/vm/loaderallocator.hpp +++ b/src/coreclr/vm/loaderallocator.hpp @@ -152,13 +152,16 @@ class CodeRangeMapRangeList : public RangeList // Iterate backwards to improve efficiency of removals // as any linked lists in the RangeSectionMap code are in reverse order of insertion. - auto iter = _starts.End(); - while (iter != _starts.Begin()) + for (auto i = _starts.GetCount(); i > 0;) { - --iter; - ExecutionManager::DeleteRange(*iter); + --i; + if (_starts[i] != 0) + { + ExecutionManager::DeleteRange(_starts[i]); + _starts[i] = 0; + } + } - _starts.Clear(); #endif // DACCESS_COMPILE } From 6db0e7087a51c7cd00bc60ff369600665cabca08 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 6 Dec 2022 15:01:23 -0800 Subject: [PATCH 19/25] Fix CodeRangeMapRangeList to handle interval correctly --- src/coreclr/vm/loaderallocator.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/coreclr/vm/loaderallocator.hpp b/src/coreclr/vm/loaderallocator.hpp index 8f5ab1adc1c7b..377f9d632cb21 100644 --- a/src/coreclr/vm/loaderallocator.hpp +++ b/src/coreclr/vm/loaderallocator.hpp @@ -117,6 +117,9 @@ class CodeRangeMapRangeList : public RangeList #ifndef DACCESS_COMPILE BOOL result = FALSE; + + --end; // The RangeList apis surface works with ranges which are represented as open intervals, + // But the RangeSectionMap uses a closed interval EX_TRY { AddRangeWorkerHelper((TADDR)start, (TADDR)end, id); @@ -160,7 +163,6 @@ class CodeRangeMapRangeList : public RangeList ExecutionManager::DeleteRange(_starts[i]); _starts[i] = 0; } - } #endif // DACCESS_COMPILE } From c8a74e2113eacf6b4de1ab01a72a68578a3459f6 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 6 Dec 2022 15:34:44 -0800 Subject: [PATCH 20/25] It builds on Linux! --- src/coreclr/vm/codeman.h | 8 ++++++-- src/coreclr/vm/loaderallocator.hpp | 2 +- src/coreclr/vm/method.cpp | 10 +++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 2b481c41f71ea..4d935742d2a99 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -911,14 +911,18 @@ class RangeSectionMap bool AttachRangeSectionToMap(PTR_RangeSection pRangeSection) { uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSection); - RangeSectionFragment* fragments = (RangeSectionFragment*)calloc(rangeSectionFragmentCount, sizeof(RangeSectionFragment)); + size_t fragmentsSize = rangeSectionFragmentCount * sizeof(RangeSectionFragment); + RangeSectionFragment* fragments = (RangeSectionFragment*)malloc(fragmentsSize); + memset(fragments, 0, fragmentsSize); if (fragments == NULL) { return false; } - RangeSectionFragmentPointer** entriesInMapToUpdate = (RangeSectionFragmentPointer**)calloc(rangeSectionFragmentCount, sizeof(RangeSectionFragmentPointer*)); + size_t entryUpdateSize = rangeSectionFragmentCount * sizeof(RangeSectionFragmentPointer*); + RangeSectionFragmentPointer** entriesInMapToUpdate = (RangeSectionFragmentPointer**)malloc(entryUpdateSize); + memset(entriesInMapToUpdate, 0, entryUpdateSize); if (entriesInMapToUpdate == NULL) { free(fragments); diff --git a/src/coreclr/vm/loaderallocator.hpp b/src/coreclr/vm/loaderallocator.hpp index 377f9d632cb21..626b41c3fa102 100644 --- a/src/coreclr/vm/loaderallocator.hpp +++ b/src/coreclr/vm/loaderallocator.hpp @@ -48,7 +48,7 @@ class CodeRangeMapRangeList : public RangeList public: VPTR_VTABLE_CLASS(CodeRangeMapRangeList, RangeList) -#ifdef DACCESS_COMPILE +#if defined(DACCESS_COMPILE) || !defined(TARGET_WINDOWS) CodeRangeMapRangeList() : _RangeListRWLock(COOPERATIVE_OR_PREEMPTIVE, LOCK_TYPE_DEFAULT), _rangeListType(STUB_CODE_BLOCK_UNKNOWN), diff --git a/src/coreclr/vm/method.cpp b/src/coreclr/vm/method.cpp index 936107520b145..3827d864895d6 100644 --- a/src/coreclr/vm/method.cpp +++ b/src/coreclr/vm/method.cpp @@ -2146,11 +2146,11 @@ MethodDesc* NonVirtualEntry2MethodDesc(PCODE entryPoint) return (MethodDesc*)((FixupPrecode*)PCODEToPINSTR(entryPoint))->GetMethodDesc(); case STUB_CODE_BLOCK_STUBPRECODE: return (MethodDesc*)((StubPrecode*)PCODEToPINSTR(entryPoint))->GetMethodDesc(); - } - - // We should never get here - _ASSERTE(!"NonVirtualEntry2MethodDesc failed for RangeSection"); - return NULL; + default: + // We should never get here + _ASSERTE(!"NonVirtualEntry2MethodDesc failed for RangeSection"); + return NULL; + } } //******************************************************************************* From 2a012b27c2bd8799dfa564dc0c674e2ca4fe7914 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Tue, 6 Dec 2022 15:38:43 -0800 Subject: [PATCH 21/25] Make it even more lock free --- src/coreclr/vm/codeman.cpp | 87 ++++++++++++++++++++++---------------- src/coreclr/vm/codeman.h | 81 +++++++++++++++++++++++++---------- 2 files changed, 110 insertions(+), 58 deletions(-) diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index 7ab3fdd2cf5f6..eb6857db24c86 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -4700,7 +4700,9 @@ ExecutionManager::FindCodeRange(PCODE currentPC, ScanFlag scanFlag) if (scanFlag == ScanReaderLock) return FindCodeRangeWithLock(currentPC); - return GetRangeSection(currentPC); + // Since ScanReaderLock is not set, then we should behave AS IF the ReaderLock is held + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; + return GetRangeSection(currentPC, &lockState); } //************************************************************************** @@ -4714,8 +4716,15 @@ ExecutionManager::FindCodeRangeWithLock(PCODE currentPC) SUPPORTS_DAC; } CONTRACTL_END; - ReaderLockHolder rlh; - return GetRangeSection(currentPC); + RangeSectionLockState lockState = RangeSectionLockState::None; + RangeSection *result = GetRangeSection(currentPC, &lockState); + if (lockState == RangeSectionLockState::NeedsLock) + { + ReaderLockHolder rlh; + lockState = RangeSectionLockState::ReaderLocked; + result = GetRangeSection(currentPC, &lockState); + } + return result; } @@ -4777,7 +4786,9 @@ BOOL ExecutionManager::IsManagedCode(PCODE currentPC) if (GetScanFlags() == ScanReaderLock) return IsManagedCodeWithLock(currentPC); - return IsManagedCodeWorker(currentPC); + // Since ScanReaderLock is not set, then we must assume that the ReaderLock is effectively taken. + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; + return IsManagedCodeWorker(currentPC, &lockState); } //************************************************************************** @@ -4789,8 +4800,17 @@ BOOL ExecutionManager::IsManagedCodeWithLock(PCODE currentPC) GC_NOTRIGGER; } CONTRACTL_END; - ReaderLockHolder rlh; - return IsManagedCodeWorker(currentPC); + RangeSectionLockState lockState = RangeSectionLockState::None; + BOOL result = IsManagedCodeWorker(currentPC, &lockState); + + if (lockState == RangeSectionLockState::NeedsLock) + { + ReaderLockHolder rlh; + lockState = RangeSectionLockState::ReaderLocked; + result = IsManagedCodeWorker(currentPC, &lockState); + } + + return result; } //************************************************************************** @@ -4817,14 +4837,15 @@ BOOL ExecutionManager::IsManagedCode(PCODE currentPC, HostCallPreference hostCal return FALSE; } - return IsManagedCodeWorker(currentPC); + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; + return IsManagedCodeWorker(currentPC, &lockState); #endif } //************************************************************************** // Assumes that the ExecutionManager reader/writer lock is taken or that // it is safe not to take it. -BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC) +BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC, RangeSectionLockState *pLockState) { CONTRACTL { NOTHROW; @@ -4835,7 +4856,7 @@ BOOL ExecutionManager::IsManagedCodeWorker(PCODE currentPC) // taken over the call to JitCodeToMethodInfo too so that nobody pulls out // the range section from underneath us. - RangeSection * pRS = GetRangeSection(currentPC); + RangeSection * pRS = GetRangeSection(currentPC, pLockState); if (pRS == NULL) return FALSE; @@ -4877,24 +4898,12 @@ BOOL ExecutionManager::IsReadyToRunCode(PCODE currentPC) // the range section from underneath us. #ifdef FEATURE_READYTORUN - if (ExecutionManager::GetScanFlags() == ScanNoReaderLock) + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // TODO! Looking at users of this API I don't know that the lock structure here is safe. Needs checking. + RangeSection * pRS = GetRangeSection(currentPC, &lockState); + if (pRS != NULL && (pRS->_pR2RModule != NULL)) { - RangeSection * pRS = GetRangeSection(currentPC); - if (pRS != NULL && (pRS->_pR2RModule != NULL)) - { - if (dac_cast(pRS->_pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) - return TRUE; - } - } - else - { - ReaderLockHolder rlh; - RangeSection * pRS = GetRangeSection(currentPC); - if (pRS != NULL && (pRS->_pR2RModule != NULL)) - { - if (dac_cast(pRS->_pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) - return TRUE; - } + if (dac_cast(pRS->_pjit)->JitCodeToMethodInfo(pRS, currentPC, NULL, NULL)) + return TRUE; } #endif @@ -4923,7 +4932,7 @@ LPCWSTR ExecutionManager::GetJitName() } #endif // !FEATURE_MERGE_JIT_AND_ENGINE -RangeSection* ExecutionManager::GetRangeSection(TADDR addr) +RangeSection* ExecutionManager::GetRangeSection(TADDR addr, RangeSectionLockState *pLockState) { CONTRACTL { NOTHROW; @@ -4932,7 +4941,7 @@ RangeSection* ExecutionManager::GetRangeSection(TADDR addr) SUPPORTS_DAC; } CONTRACTL_END; - return g_pCodeRangeMap->LookupRangeSection(addr); + return g_pCodeRangeMap->LookupRangeSection(addr, pLockState); } /* static */ @@ -4949,9 +4958,10 @@ PTR_Module ExecutionManager::FindReadyToRunModule(TADDR currentData) CONTRACTL_END; #ifdef FEATURE_READYTORUN - if (ExecutionManager::GetScanFlags() == ScanNoReaderLock) + RangeSectionLockState lockState = RangeSectionLockState::None; + RangeSection * pRS = GetRangeSection(currentData, &lockState); + if (lockState != RangeSectionLockState::NeedsLock) { - RangeSection * pRS = GetRangeSection(currentData); if (pRS == NULL) return NULL; @@ -4960,10 +4970,10 @@ PTR_Module ExecutionManager::FindReadyToRunModule(TADDR currentData) else { ReaderLockHolder rlh; - RangeSection * pRS = GetRangeSection(currentData); + lockState = RangeSectionLockState::ReaderLocked; + pRS = GetRangeSection(currentData, &lockState); if (pRS == NULL) return NULL; - return pRS->_pR2RModule; } #else @@ -5013,8 +5023,9 @@ void ExecutionManager::AddCodeRange(TADDR pStartRange, } CONTRACTL_END; ReaderLockHolder rlh; + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // - PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pModule); + PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pModule, &lockState); if (pRange == NULL) ThrowOutOfMemory(); } @@ -5036,8 +5047,9 @@ void ExecutionManager::AddCodeRange(TADDR pStartRange, } CONTRACTL_END; ReaderLockHolder rlh; + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // - PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pHp); + PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pHp, &lockState); if (pRange == NULL) ThrowOutOfMemory(); @@ -5060,8 +5072,9 @@ void ExecutionManager::AddCodeRange(TADDR pStartRange, } CONTRACTL_END; ReaderLockHolder rlh; + RangeSectionLockState lockState = RangeSectionLockState::ReaderLocked; // - PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pRangeList); + PTR_RangeSection pRange = g_pCodeRangeMap->AllocateRange(Range(pStartRange, pEndRange), pJit, flags, pRangeList, &lockState); if (pRange == NULL) ThrowOutOfMemory(); @@ -5090,7 +5103,9 @@ void ExecutionManager::DeleteRange(TADDR pStartRange) // require the reader lock, which would cause a deadlock). WriterLockHolder wlh; - g_pCodeRangeMap->CleanupRangeSections(); + RangeSectionLockState lockState = RangeSectionLockState::WriteLocked; + + g_pCodeRangeMap->CleanupRangeSections(&lockState); // Unlike the previous implementation, we no longer attempt to avoid freeing // the memory behind the RangeSection here, as we do not support the hosting // api taking over memory allocation. diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 4d935742d2a99..6756d7b4e713a 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -695,6 +695,14 @@ struct RangeSection RangeSection* _pRangeSectionNextForDelete = nullptr; // Used for adding to the cleanup list }; +enum class RangeSectionLockState +{ + None, + NeedsLock, + ReaderLocked, + WriteLocked, +}; + // For 64bit, we work with 8KB chunks of memory holding pointers to the next level. This provides 10 bits of address resolution per level. // For *reasons* the X64 hardware is limited to 57bits of addressable address space, and the minimum granularity that makes sense for range lists is 64KB (or every 2^16 bits) // Similarly the Arm64 specification requires addresses to use at most 52 bits. Thus we use the maximum addressable range of X64 to provide the real max range @@ -723,6 +731,14 @@ class RangeSectionMap TADDR FragmentToPtr(RangeSectionFragment* fragment) { TADDR ptr = (TADDR)fragment; + if (ptr == 0) + return ptr; + + if (fragment->isCollectibleRangeSectionFragment) + { + ptr += 1; + } + return ptr; } @@ -733,15 +749,32 @@ class RangeSectionMap RangeSectionFragmentPointer(RangeSectionFragmentPointer &&) = delete; RangeSectionFragmentPointer& operator=(const RangeSectionFragmentPointer&) = delete; + bool PointerIsCollectible() + { + return ((_ptr & 1) == 1); + } + bool IsNull() { return _ptr == 0; } - RangeSectionFragment* VolatileLoadWithoutBarrier() + RangeSectionFragment* VolatileLoadWithoutBarrier(RangeSectionLockState *pLockState) { uintptr_t ptr = ::VolatileLoadWithoutBarrier(&_ptr); - return (RangeSectionFragment*)(ptr); + if ((ptr & 1) == 1) + { + if ((*pLockState == RangeSectionLockState::None) || (*pLockState == RangeSectionLockState::NeedsLock)) + { + *pLockState = RangeSectionLockState::NeedsLock; + return NULL; + } + return (RangeSectionFragment*)(ptr - 1); + } + else + { + return (RangeSectionFragment*)(ptr); + } } void VolatileStore(RangeSectionFragment* fragment) @@ -874,7 +907,7 @@ class RangeSectionMap return result; } - RangeSectionFragment* GetRangeSectionForAddress(TADDR address) + RangeSectionFragment* GetRangeSectionForAddress(TADDR address, RangeSectionLockState *pLockState) { #ifdef TARGET_64BIT auto _RangeSectionL4 = VolatileLoad(&_topLevel); @@ -893,7 +926,7 @@ class RangeSectionMap if (_RangeSectionL1 == NULL) return NULL; - return ((*_RangeSectionL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(); + return ((*_RangeSectionL1)[EffectiveBitsForLevel(address, 1)]).VolatileLoadWithoutBarrier(pLockState); } uintptr_t RangeSectionFragmentCount(PTR_RangeSection pRangeSection) @@ -908,8 +941,10 @@ class RangeSectionMap return input + bytesAtLastLevel; } - bool AttachRangeSectionToMap(PTR_RangeSection pRangeSection) + bool AttachRangeSectionToMap(PTR_RangeSection pRangeSection, RangeSectionLockState *pLockState) { + assert(*pLockState == RangeSectionLockState::ReaderLocked); // Must be locked so that the cannot fail case, can't fail. NOTE: This only needs the reader lock, as the attach process can happen in parallel to reads. + uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSection); size_t fragmentsSize = rangeSectionFragmentCount * sizeof(RangeSectionFragment); RangeSectionFragment* fragments = (RangeSectionFragment*)malloc(fragmentsSize); @@ -954,7 +989,7 @@ class RangeSectionMap { do { - RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(); + RangeSectionFragment* initialFragmentInMap = entriesInMapToUpdate[iFragment]->VolatileLoadWithoutBarrier(pLockState); fragments[iFragment].pRangeSectionFragmentNext.VolatileStore(initialFragmentInMap); if (entriesInMapToUpdate[iFragment]->AtomicReplace(&(fragments[iFragment]), initialFragmentInMap)) break; @@ -978,13 +1013,13 @@ class RangeSectionMap } #ifdef FEATURE_READYTORUN - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_Module pR2RModule) + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_Module pR2RModule, RangeSectionLockState* pLockState) { PTR_RangeSection pSection(new(nothrow)RangeSection(range, pJit, flags, pR2RModule)); if (pSection == NULL) return NULL; - if (!AttachRangeSectionToMap(pSection)) + if (!AttachRangeSectionToMap(pSection, pLockState)) { delete pSection; return NULL; @@ -993,13 +1028,13 @@ class RangeSectionMap } #endif - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_HeapList pHeapList) + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_HeapList pHeapList, RangeSectionLockState* pLockState) { PTR_RangeSection pSection(new(nothrow)RangeSection(range, pJit, flags, pHeapList)); if (pSection == NULL) return NULL; - if (!AttachRangeSectionToMap(pSection)) + if (!AttachRangeSectionToMap(pSection, pLockState)) { delete pSection; return NULL; @@ -1007,13 +1042,13 @@ class RangeSectionMap return pSection; } - RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_CodeRangeMapRangeList pRangeList) + RangeSection *AllocateRange(Range range, IJitManager* pJit, RangeSection::RangeSectionFlags flags, PTR_CodeRangeMapRangeList pRangeList, RangeSectionLockState* pLockState) { PTR_RangeSection pSection(new(nothrow)RangeSection(range, pJit, flags, pRangeList)); if (pSection == NULL) return NULL; - if (!AttachRangeSectionToMap(pSection)) + if (!AttachRangeSectionToMap(pSection, pLockState)) { delete pSection; return NULL; @@ -1021,15 +1056,15 @@ class RangeSectionMap return pSection; } - PTR_RangeSection LookupRangeSection(TADDR address) + PTR_RangeSection LookupRangeSection(TADDR address, RangeSectionLockState *pLockState) { - RangeSectionFragment* fragment = GetRangeSectionForAddress(address); + RangeSectionFragment* fragment = GetRangeSectionForAddress(address, pLockState); if (fragment == NULL) return NULL; while ((fragment != NULL) && !fragment->InRange(address)) { - fragment = fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(); + fragment = fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState); } if (fragment != NULL) @@ -1059,8 +1094,10 @@ class RangeSectionMap } while (InterlockedCompareExchangeT(&_pCleanupList, pRangeSection, pLatestRemovedRangeSection) != pLatestRemovedRangeSection); } - void CleanupRangeSections() + void CleanupRangeSections(RangeSectionLockState *pLockState) { + assert(*pLockState == RangeSectionLockState::WriteLocked); + while (this->_pCleanupList != EndOfCleanupListMarker()) { PTR_RangeSection pRangeSectionToCleanup(this->_pCleanupList); @@ -1077,12 +1114,12 @@ class RangeSectionMap RangeSectionFragmentPointer* entryInMapToUpdate = EnsureMapsForAddress(addressToPrepForCleanup); assert(entryInMapToUpdate != NULL); - while ((entryInMapToUpdate->VolatileLoadWithoutBarrier())->pRangeSection != pRangeSectionToCleanup) + while ((entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSection != pRangeSectionToCleanup) { - entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier())->pRangeSectionFragmentNext; + entryInMapToUpdate = &(entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState))->pRangeSectionFragmentNext; } - RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(); + RangeSectionFragment* fragment = entryInMapToUpdate->VolatileLoadWithoutBarrier(pLockState); // The fragment associated with the start of the range has the address that was allocated earlier if (iFragment == 0) @@ -1091,7 +1128,7 @@ class RangeSectionMap assert(pRangeSectionFragmentToFree->isPrimaryRangeSectionFragment); } - entryInMapToUpdate->VolatileStore(fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier()); + entryInMapToUpdate->VolatileStore(fragment->pRangeSectionFragmentNext.VolatileLoadWithoutBarrier(pLockState)); addressToPrepForCleanup = IncrementAddressByMaxSizeOfFragment(addressToPrepForCleanup); } @@ -1793,9 +1830,9 @@ class ExecutionManager static RangeSection * FindCodeRangeWithLock(PCODE currentPC); static BOOL IsManagedCodeWithLock(PCODE currentPC); - static BOOL IsManagedCodeWorker(PCODE currentPC); + static BOOL IsManagedCodeWorker(PCODE currentPC, RangeSectionLockState *pLockState); - static RangeSection* GetRangeSection(TADDR addr); + static RangeSection* GetRangeSection(TADDR addr, RangeSectionLockState *pLockState); SPTR_DECL(EECodeManager, m_pDefaultCodeMan); From de97c1764915ad7505d0283cdfa0b2382330c040 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 8 Dec 2022 09:01:42 -0800 Subject: [PATCH 22/25] Remove confusion around Range interval specification Some uses were relying on [,) semantics and some were [,] --- src/coreclr/vm/ceeload.cpp | 2 +- src/coreclr/vm/codeman.cpp | 10 ++-- src/coreclr/vm/codeman.h | 81 ++++++++++++++++++++++++++---- src/coreclr/vm/codeman.inl | 2 +- src/coreclr/vm/loaderallocator.hpp | 4 -- 5 files changed, 78 insertions(+), 21 deletions(-) diff --git a/src/coreclr/vm/ceeload.cpp b/src/coreclr/vm/ceeload.cpp index f9fc611ec7f95..8cd2c26ba70c4 100644 --- a/src/coreclr/vm/ceeload.cpp +++ b/src/coreclr/vm/ceeload.cpp @@ -4395,7 +4395,7 @@ void Module::RunEagerFixupsUnlocked() TADDR base = dac_cast(pNativeImage->GetBase()); ExecutionManager::AddCodeRange( - base, base + (TADDR)pNativeImage->GetVirtualSize() - 1, + base, base + (TADDR)pNativeImage->GetVirtualSize(), ExecutionManager::GetReadyToRunJitManager(), RangeSection::RANGE_SECTION_NONE, this /* pHeapListOrZapModule */); diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index eb6857db24c86..3f9a0b981ca91 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -390,7 +390,7 @@ void UnwindInfoTable::AddToUnwindInfoTable(UnwindInfoTable** unwindInfoPtr, PT_R if (pRS != NULL) { for(int i = 0; i < unwindInfoCount; i++) - AddToUnwindInfoTable(&pRS->_pUnwindInfoTable, &unwindInfo[i], pRS->_range.begin, pRS->_range.end); + AddToUnwindInfoTable(&pRS->_pUnwindInfoTable, &unwindInfo[i], pRS->_range.RangeStart(), pRS->_range.RangeEndOpen()); } } @@ -415,7 +415,7 @@ void UnwindInfoTable::AddToUnwindInfoTable(UnwindInfoTable** unwindInfoPtr, PT_R EEJitManager* pJitMgr = (EEJitManager*)(pRS->_pjit); CodeHeader * pHeader = pJitMgr->GetCodeHeaderFromStartAddress(entryPoint); for(ULONG i = 0; i < pHeader->GetNumberOfUnwindInfos(); i++) - RemoveFromUnwindInfoTable(&pRS->_pUnwindInfoTable, pRS->_range.begin, pRS->_range.begin + pHeader->GetUnwindInfo(i)->BeginAddress); + RemoveFromUnwindInfoTable(&pRS->_pUnwindInfoTable, pRS->_range.RangeStart(), pRS->_range.RangeStart() + pHeader->GetUnwindInfo(i)->BeginAddress); } } } @@ -460,7 +460,7 @@ extern CrstStatic g_StubUnwindInfoHeapSegmentsCrst; CodeHeader * pHeader = pJitMgr->GetCodeHeaderFromStartAddress(methodEntry); int unwindInfoCount = pHeader->GetNumberOfUnwindInfos(); for(int i = 0; i < unwindInfoCount; i++) - AddToUnwindInfoTable(&pRS->_pUnwindInfoTable, pHeader->GetUnwindInfo(i), pRS->_range.begin, pRS->_range.end); + AddToUnwindInfoTable(&pRS->_pUnwindInfoTable, pHeader->GetUnwindInfo(i), pRS->_range.RangeStart(), pRS->_range.RangeEndOpen()); } } } @@ -5969,7 +5969,7 @@ StubCodeBlockKind ReadyToRunJitManager::GetStubCodeBlockKind(RangeSection * pRan } CONTRACTL_END; - DWORD rva = (DWORD)(currentPC - pRangeSection->_range.begin); + DWORD rva = (DWORD)(currentPC - pRangeSection->_range.RangeStart()); PTR_ReadyToRunInfo pReadyToRunInfo = pRangeSection->_pR2RModule->GetReadyToRunInfo(); @@ -6120,7 +6120,7 @@ BOOL ReadyToRunJitManager::JitCodeToMethodInfo(RangeSection * pRangeSection, TADDR currentInstr = PCODEToPINSTR(currentPC); - TADDR ImageBase = pRangeSection->_range.begin; + TADDR ImageBase = pRangeSection->_range.RangeStart(); DWORD RelativePc = (DWORD)(currentInstr - ImageBase); diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 6756d7b4e713a..76edf9056fbe7 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -616,12 +616,41 @@ class RangeSectionMap; class Range { -public: - Range(TADDR begin, TADDR end) : begin(begin), end(end) {} - - // [begin,end] (This is an inclusive range) + // [begin,end) (This is an inclusive range) TADDR begin; TADDR end; + +public: + Range(TADDR begin, TADDR end) : begin(begin), end(end) + { + assert(end >= begin); + } + + bool IsInRange(TADDR address) const + { + return address >= begin && address < end; + } + + TADDR RangeSize() const + { + return end - begin; + } + + TADDR RangeStart() const + { + return begin; + } + + TADDR RangeEnd() const + { + assert(RangeSize() > 0); + return end - 1; + } + + TADDR RangeEndOpen() const + { + return end; + } }; struct RangeSection @@ -799,7 +828,7 @@ class RangeSectionMap RangeSectionFragmentPointer pRangeSectionFragmentNext; Range _range; PTR_RangeSection pRangeSection; - bool InRange(TADDR address) { return address >= _range.begin && address <= _range.end && pRangeSection->_pRangeSectionNextForDelete == NULL; } + bool InRange(TADDR address) { return _range.IsInRange(address) && pRangeSection->_pRangeSectionNextForDelete == NULL; } bool isPrimaryRangeSectionFragment; // RangeSectionFragment are allocated in arrays, but we only need to free the first allocated one. It will be marked with this flag. bool isCollectibleRangeSectionFragment; // RangeSectionFragments }; @@ -931,9 +960,12 @@ class RangeSectionMap uintptr_t RangeSectionFragmentCount(PTR_RangeSection pRangeSection) { - uintptr_t rangeSize = (pRangeSection->_range.end - pRangeSection->_range.begin); - rangeSize /= bytesAtLastLevel; - return rangeSize + 1; + uintptr_t rangeSize = pRangeSection->_range.RangeSize(); + if (rangeSize == 0) + return 0; + + uintptr_t fragmentCount = ((rangeSize - 1) / bytesAtLastLevel) + 1; + return fragmentCount; } TADDR IncrementAddressByMaxSizeOfFragment(TADDR input) @@ -945,6 +977,11 @@ class RangeSectionMap { assert(*pLockState == RangeSectionLockState::ReaderLocked); // Must be locked so that the cannot fail case, can't fail. NOTE: This only needs the reader lock, as the attach process can happen in parallel to reads. + // Currently all use of the RangeSection should be with aligned addresses, so validate that the start and end are at aligned boundaries + assert((pRangeSection->_range.RangeStart() & 0xF) == 0); + assert((pRangeSection->_range.RangeEnd() & 0xF) == 0xF); + assert((pRangeSection->_range.RangeEndOpen() & 0xF) == 0); + uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSection); size_t fragmentsSize = rangeSectionFragmentCount * sizeof(RangeSectionFragment); RangeSectionFragment* fragments = (RangeSectionFragment*)malloc(fragmentsSize); @@ -966,7 +1003,16 @@ class RangeSectionMap fragments[0].isPrimaryRangeSectionFragment = true; - TADDR addressToPrepForUpdate = pRangeSection->_range.begin; + TADDR addressToPrepForUpdate = pRangeSection->_range.RangeStart(); + + // Assert that range is not already mapped in any way + assert(LookupRangeSection(addressToPrepForUpdate, pLockState) == NULL); + assert(LookupRangeSection(pRangeSection->_range.RangeEnd(), pLockState) == NULL); + for (TADDR fragmentAddress = addressToPrepForUpdate; pRangeSection->_range.IsInRange(fragmentAddress); fragmentAddress = IncrementAddressByMaxSizeOfFragment(fragmentAddress)) + { + assert(LookupRangeSection(fragmentAddress, pLockState) == NULL); + } + for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) { fragments[iFragment].pRangeSection = pRangeSection; @@ -996,6 +1042,14 @@ class RangeSectionMap } while (true); } + // Assert that range is now found via lookup + assert(LookupRangeSection(pRangeSection->_range.RangeStart(), pLockState) == pRangeSection); + assert(LookupRangeSection(pRangeSection->_range.RangeEnd(), pLockState) == pRangeSection); + for (TADDR fragmentAddress = pRangeSection->_range.RangeStart(); pRangeSection->_range.IsInRange(fragmentAddress); fragmentAddress = IncrementAddressByMaxSizeOfFragment(fragmentAddress)) + { + assert(LookupRangeSection(fragmentAddress, pLockState) == pRangeSection); + } + // entriesInMapToUpdate was just a temporary allocation free(entriesInMapToUpdate); @@ -1106,7 +1160,14 @@ class RangeSectionMap uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSectionToCleanup); - TADDR addressToPrepForCleanup = pRangeSectionToCleanup->_range.begin; + TADDR addressToPrepForCleanup = pRangeSectionToCleanup->_range.RangeStart(); + + assert(LookupRangeSection(addressToPrepForCleanup, pLockState) == NULL); + assert(LookupRangeSection(pRangeSectionToCleanup->_range.RangeEnd(), pLockState) == NULL); + for (TADDR fragmentAddress = addressToPrepForCleanup; fragmentAddress < pRangeSectionToCleanup->_range.RangeEnd(); fragmentAddress = IncrementAddressByMaxSizeOfFragment(fragmentAddress)) + { + assert(LookupRangeSection(fragmentAddress, pLockState) == NULL); + } // Remove fragments from each of the fragment linked lists for (uintptr_t iFragment = 0; iFragment < rangeSectionFragmentCount; iFragment++) diff --git a/src/coreclr/vm/codeman.inl b/src/coreclr/vm/codeman.inl index 2237bd921e9fb..8af0fc0e48bfb 100644 --- a/src/coreclr/vm/codeman.inl +++ b/src/coreclr/vm/codeman.inl @@ -11,5 +11,5 @@ inline BOOL ExecutionManager::IsCollectibleMethod(const METHODTOKEN& MethodToken inline TADDR IJitManager::JitTokenToModuleBase(const METHODTOKEN& MethodToken) { - return MethodToken.m_pRangeSection->_range.begin; + return MethodToken.m_pRangeSection->_range.RangeStart(); } diff --git a/src/coreclr/vm/loaderallocator.hpp b/src/coreclr/vm/loaderallocator.hpp index 626b41c3fa102..da7349efc4354 100644 --- a/src/coreclr/vm/loaderallocator.hpp +++ b/src/coreclr/vm/loaderallocator.hpp @@ -41,8 +41,6 @@ class CustomAssemblyBinder; // This implements the Add/Remove rangelist api on top of the CodeRangeMap in the code manager -// It does not implement the IsInRange api, and destruction isn't safe either. Systems which need -// to check IsInRange must use the CodeRangeMap directly. class CodeRangeMapRangeList : public RangeList { public: @@ -118,8 +116,6 @@ class CodeRangeMapRangeList : public RangeList #ifndef DACCESS_COMPILE BOOL result = FALSE; - --end; // The RangeList apis surface works with ranges which are represented as open intervals, - // But the RangeSectionMap uses a closed interval EX_TRY { AddRangeWorkerHelper((TADDR)start, (TADDR)end, id); From a493cde062315e6e7071d272ab3c0635a01d598c Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 8 Dec 2022 09:07:37 -0800 Subject: [PATCH 23/25] Attempt to appease the GCC build --- src/coreclr/vm/codeman.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 76edf9056fbe7..1108b9b592636 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -984,8 +984,10 @@ class RangeSectionMap uintptr_t rangeSectionFragmentCount = RangeSectionFragmentCount(pRangeSection); size_t fragmentsSize = rangeSectionFragmentCount * sizeof(RangeSectionFragment); - RangeSectionFragment* fragments = (RangeSectionFragment*)malloc(fragmentsSize); - memset(fragments, 0, fragmentsSize); + void* fragmentsMemory = (RangeSectionFragment*)malloc(fragmentsSize); + memset(fragmentsMemory, 0, fragmentsSize); + + RangeSectionFragment* fragments = (RangeSectionFragment*)fragmentsMemory; if (fragments == NULL) { From 15072072f1a0087f6c4d2ce23af863629903712a Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 8 Dec 2022 16:22:27 -0800 Subject: [PATCH 24/25] Fix the math around fragment handling and index calculations --- src/coreclr/vm/codeman.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 1108b9b592636..4bda5d970d231 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -862,7 +862,7 @@ class RangeSectionMap RangeSection* _pCleanupList; const uintptr_t bitsAtLastLevel = maxSetBit - (bitsPerLevel * mapLevels) + 1; - const uintptr_t bytesAtLastLevel = (((uintptr_t)1) << (bitsAtLastLevel - 1)); + const uintptr_t bytesAtLastLevel = (((uintptr_t)1) << bitsAtLastLevel); RangeSection* EndOfCleanupListMarker() { return (RangeSection*)1; } @@ -877,7 +877,7 @@ class RangeSectionMap uintptr_t EffectiveBitsForLevel(TADDR address, uintptr_t level) { TADDR addressAsInt = address; - TADDR addressBitsUsedInMap = addressAsInt >> (maxSetBit - (mapLevels * bitsPerLevel)); + TADDR addressBitsUsedInMap = addressAsInt >> (maxSetBit + 1 - (mapLevels * bitsPerLevel)); TADDR addressBitsShifted = addressBitsUsedInMap >> ((level - 1) * bitsPerLevel); TADDR addressBitsUsedInLevel = (entriesPerMapLevel - 1) & addressBitsShifted; return addressBitsUsedInLevel; @@ -963,6 +963,9 @@ class RangeSectionMap uintptr_t rangeSize = pRangeSection->_range.RangeSize(); if (rangeSize == 0) return 0; + + // Account for the range not starting at the beginning of a last level fragment + rangeSize += pRangeSection->_range.RangeStart() & (this->bytesAtLastLevel - 1); uintptr_t fragmentCount = ((rangeSize - 1) / bytesAtLastLevel) + 1; return fragmentCount; @@ -1166,7 +1169,7 @@ class RangeSectionMap assert(LookupRangeSection(addressToPrepForCleanup, pLockState) == NULL); assert(LookupRangeSection(pRangeSectionToCleanup->_range.RangeEnd(), pLockState) == NULL); - for (TADDR fragmentAddress = addressToPrepForCleanup; fragmentAddress < pRangeSectionToCleanup->_range.RangeEnd(); fragmentAddress = IncrementAddressByMaxSizeOfFragment(fragmentAddress)) + for (TADDR fragmentAddress = addressToPrepForCleanup; pRangeSectionToCleanup->_range.IsInRange(fragmentAddress); fragmentAddress = IncrementAddressByMaxSizeOfFragment(fragmentAddress)) { assert(LookupRangeSection(fragmentAddress, pLockState) == NULL); } From 8d0f9bf4269f9b1459ac95d49c9aba1a8d1182ab Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 9 Dec 2022 12:02:57 -0800 Subject: [PATCH 25/25] Remove use of native EH from the IL_Throw exception path, in combination with the other work in this branch, produces much more scalable EH performance --- src/coreclr/pal/inc/pal.h | 12 ++ src/coreclr/pal/src/exception/seh-unwind.cpp | 71 +++++++ src/coreclr/vm/excep.cpp | 200 ++++++++++++------- src/coreclr/vm/exceptmacros.h | 6 +- src/coreclr/vm/fcall.h | 3 +- src/coreclr/vm/jithelpers.cpp | 16 +- 6 files changed, 229 insertions(+), 79 deletions(-) diff --git a/src/coreclr/pal/inc/pal.h b/src/coreclr/pal/inc/pal.h index b6879228acb58..3f379ee802e11 100644 --- a/src/coreclr/pal/inc/pal.h +++ b/src/coreclr/pal/inc/pal.h @@ -3235,6 +3235,18 @@ RaiseException( IN DWORD nNumberOfArguments, IN CONST ULONG_PTR *lpArguments); +struct PAL_SEHException; + +PALIMPORT +VOID +PALAPI +RaiseExceptionProducePALExceptionOnly( + IN DWORD dwExceptionCode, + IN DWORD dwExceptionFlags, + IN DWORD nNumberOfArguments, + IN CONST ULONG_PTR *lpArguments, + PAL_SEHException *pPalException); + PALIMPORT VOID PALAPI diff --git a/src/coreclr/pal/src/exception/seh-unwind.cpp b/src/coreclr/pal/src/exception/seh-unwind.cpp index e94922cea0451..210c7c8838d74 100644 --- a/src/coreclr/pal/src/exception/seh-unwind.cpp +++ b/src/coreclr/pal/src/exception/seh-unwind.cpp @@ -951,4 +951,75 @@ RaiseException(IN DWORD dwExceptionCode, LOGEXIT("RaiseException returns\n"); } +/*++ +Function: + RaiseException + +See MSDN doc. +--*/ +// no PAL_NORETURN, as callers must assume this can return for continuable exceptions. +__attribute__((noinline)) +VOID +PALAPI +RaiseExceptionProducePALExceptionOnly(IN DWORD dwExceptionCode, + IN DWORD dwExceptionFlags, + IN DWORD nNumberOfArguments, + IN CONST ULONG_PTR *lpArguments, + PAL_SEHException *pPalException) +{ + // PERF_ENTRY_ONLY is used here because RaiseException may or may not + // return. We can not get latency data without PERF_EXIT. For this reason, + // PERF_ENTRY_ONLY is used to profile frequency only. + PERF_ENTRY(RaiseExceptionProducePALExceptionOnly); + ENTRY("RaiseExceptionProducePALExceptionOnly(dwCode=%#x, dwFlags=%#x, nArgs=%u, lpArguments=%p)\n", + dwExceptionCode, dwExceptionFlags, nNumberOfArguments, lpArguments); + + /* Validate parameters */ + if (dwExceptionCode & RESERVED_SEH_BIT) + { + WARN("Exception code %08x has bit 28 set; clearing it.\n", dwExceptionCode); + dwExceptionCode ^= RESERVED_SEH_BIT; + } + + if (nNumberOfArguments > EXCEPTION_MAXIMUM_PARAMETERS) + { + WARN("Number of arguments (%d) exceeds the limit " + "EXCEPTION_MAXIMUM_PARAMETERS (%d); ignoring extra parameters.\n", + nNumberOfArguments, EXCEPTION_MAXIMUM_PARAMETERS); + nNumberOfArguments = EXCEPTION_MAXIMUM_PARAMETERS; + } + + CONTEXT *contextRecord; + EXCEPTION_RECORD *exceptionRecord; + AllocateExceptionRecords(&exceptionRecord, &contextRecord); + + ZeroMemory(exceptionRecord, sizeof(EXCEPTION_RECORD)); + + exceptionRecord->ExceptionCode = dwExceptionCode; + exceptionRecord->ExceptionFlags = dwExceptionFlags; + exceptionRecord->ExceptionRecord = NULL; + exceptionRecord->ExceptionAddress = NULL; // will be set by RtlpRaiseException + exceptionRecord->NumberParameters = nNumberOfArguments; + if (nNumberOfArguments) + { + CopyMemory(exceptionRecord->ExceptionInformation, lpArguments, + nNumberOfArguments * sizeof(ULONG_PTR)); + } + + // Capture the context of RaiseException. + ZeroMemory(contextRecord, sizeof(CONTEXT)); + contextRecord->ContextFlags = CONTEXT_FULL; + CONTEXT_CaptureContext(contextRecord); + + // We have to unwind one level to get the actual context user code could be resumed at. + PAL_VirtualUnwind(contextRecord, NULL); + + exceptionRecord->ExceptionAddress = (void *)CONTEXTGetPC(contextRecord); + + *pPalException = PAL_SEHException(exceptionRecord, contextRecord); + + LOGEXIT("RaiseExceptionProducePALExceptionOnly returns\n"); + PERF_EXIT(RaiseExceptionProducePALExceptionOnly); +} + #endif // !HOST_WINDOWS diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index efa7f786844b3..aee0e6b4fe293 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -2658,7 +2658,103 @@ HRESULT GetHRFromThrowable(OBJECTREF throwable) return hr; } -VOID DECLSPEC_NORETURN RaiseTheExceptionInternalOnly(OBJECTREF throwable, BOOL rethrow, BOOL fForStackOverflow) +struct Param : RaiseExceptionFilterParam +{ + OBJECTREF throwable; + BOOL fForStackOverflow; + ULONG_PTR exceptionArgs[INSTANCE_TAGGED_SEH_PARAM_ARRAY_SIZE]; + Thread *pThread; + ThreadExceptionState* pExState; +#ifndef TARGET_WINDOWS + PAL_SEHException *pPalException; +#endif + +}; + +FORCEINLINE void RaiseTheExceptionInternalOnlyTryBody(Param *pParam) +{ + //_ASSERTE(! pParam->isRethrown || pParam->pExState->m_pExceptionRecord); + ULONG_PTR *args = NULL; + ULONG argCount = 0; + ULONG flags = 0; + ULONG code = 0; + + // Always save the current object in the handle so on rethrow we can reuse it. This is important as it + // contains stack trace info. + // + // Note: we use SafeSetLastThrownObject, which will try to set the throwable and if there are any problems, + // it will set the throwable to something appropriate (like OOM exception) and return the new + // exception. Thus, the user's exception object can be replaced here. + pParam->throwable = pParam->pThread->SafeSetLastThrownObject(pParam->throwable); + + if (!pParam->isRethrown || +#ifdef FEATURE_INTERPRETER + !pParam->pExState->IsExceptionInProgress() || +#endif // FEATURE_INTERPRETER + pParam->pExState->IsComPlusException() || + (pParam->pExState->GetExceptionCode() == STATUS_STACK_OVERFLOW)) + { + ULONG_PTR hr = GetHRFromThrowable(pParam->throwable); + + args = pParam->exceptionArgs; + argCount = MarkAsThrownByUs(args, hr); + flags = EXCEPTION_NONCONTINUABLE; + code = EXCEPTION_COMPLUS; + } + else + { + // Exception code should be consistent. + _ASSERTE((DWORD)(pParam->pExState->GetExceptionRecord()->ExceptionCode) == pParam->pExState->GetExceptionCode()); + + args = pParam->pExState->GetExceptionRecord()->ExceptionInformation; + argCount = pParam->pExState->GetExceptionRecord()->NumberParameters; + flags = pParam->pExState->GetExceptionRecord()->ExceptionFlags; + code = pParam->pExState->GetExceptionRecord()->ExceptionCode; + } + + if (pParam->pThread->IsAbortInitiated () && IsExceptionOfType(kThreadAbortException,&pParam->throwable)) + { + pParam->pThread->ResetPreparingAbort(); + + if (pParam->pThread->GetFrame() == FRAME_TOP) + { + // There is no more managed code on stack. + pParam->pThread->ResetAbort(); + } + } + + // Can't access the exception object when are in pre-emptive, so find out before + // if its an SO. + BOOL fIsStackOverflow = IsExceptionOfType(kStackOverflowException, &pParam->throwable); + + if (fIsStackOverflow || pParam->fForStackOverflow) + { + // Don't probe if we're already handling an SO. Just throw the exception. + RaiseException(code, flags, argCount, args); + } + + // This needs to be both here and inside the handler below + // enable preemptive mode before call into OS + GCX_PREEMP_NO_DTOR(); + +#ifndef TARGET_WINDOWS + if (pParam->pPalException != NULL) + { + RaiseExceptionProducePALExceptionOnly(code, flags, argCount, args, pParam->pPalException); + } + else +#endif + { + // In non-debug, we can just raise the exception once we've probed. + RaiseException(code, flags, argCount, args); + } +} + +VOID RaiseTheExceptionInternalOnlyCore(OBJECTREF throwable, BOOL rethrow, BOOL fForStackOverflow +#ifndef TARGET_WINDOWS +, PAL_SEHException *pPalException +#endif +) { STATIC_CONTRACT_THROWS; STATIC_CONTRACT_GC_TRIGGERS; @@ -2688,18 +2784,14 @@ VOID DECLSPEC_NORETURN RaiseTheExceptionInternalOnly(OBJECTREF throwable, BOOL r } #endif - struct Param : RaiseExceptionFilterParam - { - OBJECTREF throwable; - BOOL fForStackOverflow; - ULONG_PTR exceptionArgs[INSTANCE_TAGGED_SEH_PARAM_ARRAY_SIZE]; - Thread *pThread; - ThreadExceptionState* pExState; - } param; + Param param; param.isRethrown = rethrow ? 1 : 0; // normalize because we use it as a count in RaiseExceptionFilter param.throwable = throwable; param.fForStackOverflow = fForStackOverflow; param.pThread = GetThread(); +#ifndef TARGET_WINDOWS + param.pPalException = pPalException; +#endif _ASSERTE(param.pThread); param.pExState = param.pThread->GetExceptionState(); @@ -2727,80 +2819,37 @@ VOID DECLSPEC_NORETURN RaiseTheExceptionInternalOnly(OBJECTREF throwable, BOOL r #endif #endif - // raise - PAL_TRY(Param *, pParam, ¶m) + if (rethrow +#ifndef TARGET_WINDOWS + && (pPalException == NULL) +#endif + ) { - //_ASSERTE(! pParam->isRethrown || pParam->pExState->m_pExceptionRecord); - ULONG_PTR *args = NULL; - ULONG argCount = 0; - ULONG flags = 0; - ULONG code = 0; - - // Always save the current object in the handle so on rethrow we can reuse it. This is important as it - // contains stack trace info. - // - // Note: we use SafeSetLastThrownObject, which will try to set the throwable and if there are any problems, - // it will set the throwable to something appropriate (like OOM exception) and return the new - // exception. Thus, the user's exception object can be replaced here. - pParam->throwable = pParam->pThread->SafeSetLastThrownObject(pParam->throwable); - - if (!pParam->isRethrown || -#ifdef FEATURE_INTERPRETER - !pParam->pExState->IsExceptionInProgress() || -#endif // FEATURE_INTERPRETER - pParam->pExState->IsComPlusException() || - (pParam->pExState->GetExceptionCode() == STATUS_STACK_OVERFLOW)) + // raise + PAL_TRY(Param *, pParam, ¶m) { - ULONG_PTR hr = GetHRFromThrowable(pParam->throwable); - - args = pParam->exceptionArgs; - argCount = MarkAsThrownByUs(args, hr); - flags = EXCEPTION_NONCONTINUABLE; - code = EXCEPTION_COMPLUS; + RaiseTheExceptionInternalOnlyTryBody(pParam); } - else + PAL_EXCEPT_FILTER (RaiseExceptionFilter) { - // Exception code should be consistent. - _ASSERTE((DWORD)(pParam->pExState->GetExceptionRecord()->ExceptionCode) == pParam->pExState->GetExceptionCode()); - - args = pParam->pExState->GetExceptionRecord()->ExceptionInformation; - argCount = pParam->pExState->GetExceptionRecord()->NumberParameters; - flags = pParam->pExState->GetExceptionRecord()->ExceptionFlags; - code = pParam->pExState->GetExceptionRecord()->ExceptionCode; } - - if (pParam->pThread->IsAbortInitiated () && IsExceptionOfType(kThreadAbortException,&pParam->throwable)) - { - pParam->pThread->ResetPreparingAbort(); - - if (pParam->pThread->GetFrame() == FRAME_TOP) - { - // There is no more managed code on stack. - pParam->pThread->ResetAbort(); - } - } - - // Can't access the exception object when are in pre-emptive, so find out before - // if its an SO. - BOOL fIsStackOverflow = IsExceptionOfType(kStackOverflowException, &pParam->throwable); - - if (fIsStackOverflow || pParam->fForStackOverflow) - { - // Don't probe if we're already handling an SO. Just throw the exception. - RaiseException(code, flags, argCount, args); - } - - // This needs to be both here and inside the handler below - // enable preemptive mode before call into OS - GCX_PREEMP_NO_DTOR(); - - // In non-debug, we can just raise the exception once we've probed. - RaiseException(code, flags, argCount, args); + PAL_ENDTRY } - PAL_EXCEPT_FILTER (RaiseExceptionFilter) + else { + RaiseTheExceptionInternalOnlyTryBody(¶m); +#ifndef TARGET_WINDOWS + if (rethrow) + { + RaiseExceptionFilter(&pPalException->ExceptionPointers, ¶m); + } +#endif } - PAL_ENDTRY +} + +VOID DECLSPEC_NORETURN RaiseTheExceptionInternalOnly(OBJECTREF throwable, BOOL rethrow, BOOL fForStackOverflow) +{ + RaiseTheExceptionInternalOnlyCore(throwable, rethrow, fForStackOverflow); _ASSERTE(!"Cannot continue after COM+ exception"); // Debugger can bring you here. // For example, // Debugger breaks in due to second chance exception (unhandled) @@ -2810,7 +2859,6 @@ VOID DECLSPEC_NORETURN RaiseTheExceptionInternalOnly(OBJECTREF throwable, BOOL r UNREACHABLE(); } - // INSTALL_COMPLUS_EXCEPTION_HANDLER has a filter, so must put the call in a separate fcn static VOID DECLSPEC_NORETURN RealCOMPlusThrowWorker(OBJECTREF throwable, BOOL rethrow) { diff --git a/src/coreclr/vm/exceptmacros.h b/src/coreclr/vm/exceptmacros.h index 1627e3d3d65cc..fb3e6ce45be20 100644 --- a/src/coreclr/vm/exceptmacros.h +++ b/src/coreclr/vm/exceptmacros.h @@ -255,7 +255,11 @@ VEH_ACTION CLRVectoredExceptionHandler(PEXCEPTION_POINTERS pExceptionInfo); extern LONG InternalUnhandledExceptionFilter_Worker(PEXCEPTION_POINTERS pExceptionInfo); VOID DECLSPEC_NORETURN RaiseTheExceptionInternalOnly(OBJECTREF throwable, BOOL rethrow, BOOL fForStackOverflow = FALSE); - +VOID RaiseTheExceptionInternalOnlyCore(OBJECTREF throwable, BOOL rethrow, BOOL fForStackOverflow +#ifndef TARGET_WINDOWS +, PAL_SEHException* pPalException = NULL +#endif +); #if defined(DACCESS_COMPILE) #define INSTALL_UNWIND_AND_CONTINUE_HANDLER diff --git a/src/coreclr/vm/fcall.h b/src/coreclr/vm/fcall.h index c2816c992ecb9..7425691c45c9a 100644 --- a/src/coreclr/vm/fcall.h +++ b/src/coreclr/vm/fcall.h @@ -584,6 +584,7 @@ LPVOID __FCThrowArgument(LPVOID me, enum RuntimeExceptionKind reKind, LPCWSTR ar /* gcpoll; */ \ INSTALL_MANAGED_EXCEPTION_DISPATCHER; \ __helperframe.Push(); \ + bool __popHelperMethodFrame=true; \ MAKE_CURRENT_THREAD_AVAILABLE_EX(__helperframe.GetThread()); \ INSTALL_UNWIND_AND_CONTINUE_HANDLER_FOR_HMF(&__helperframe); @@ -616,7 +617,7 @@ LPVOID __FCThrowArgument(LPVOID me, enum RuntimeExceptionKind reKind, LPCWSTR ar #define HELPER_METHOD_FRAME_END_EX(gcpoll,allowGC) \ UNINSTALL_UNWIND_AND_CONTINUE_HANDLER; \ - __helperframe.Pop(); \ + if (__popHelperMethodFrame) __helperframe.Pop(); \ UNINSTALL_MANAGED_EXCEPTION_DISPATCHER; \ HELPER_METHOD_FRAME_END_EX_BODY(gcpoll,allowGC); diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 2628f560ab436..f5a2a1eb4746f 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -4026,7 +4026,14 @@ HCIMPL1(void, IL_Throw, Object* obj) } } - RaiseTheExceptionInternalOnly(oref, FALSE); +#ifndef TARGET_WINDOWS + // See definition of UNINSTALL_MANAGED_EXCEPTION_DISPATCHER for details around where exCopy and hasCaughtException come from + RaiseTheExceptionInternalOnlyCore(oref, FALSE, FALSE, &exCopy); + hasCaughtException = true; + __popHelperMethodFrame = false; // Disable the helper method frame pop +#else + RaiseTheExceptionInternal(oref, FALSE, FALSE); +#endif HELPER_METHOD_FRAME_END(); } @@ -4045,7 +4052,14 @@ HCIMPL0(void, IL_Rethrow) OBJECTREF throwable = GetThread()->GetThrowable(); if (throwable != NULL) { +#ifndef TARGET_WINDOWS + // See definition of UNINSTALL_MANAGED_EXCEPTION_DISPATCHER for details around where exCopy and hasCaughtException come from + RaiseTheExceptionInternalOnlyCore(throwable, TRUE, FALSE, &exCopy); + hasCaughtException = true; + __popHelperMethodFrame = false; // Disable the helper method frame pop +#else RaiseTheExceptionInternalOnly(throwable, TRUE); +#endif } else {