diff --git a/src/coreclr/nativeaot/Runtime/AsmOffsets.h b/src/coreclr/nativeaot/Runtime/AsmOffsets.h index e28386e1d3b5b..203fee38bf15f 100644 --- a/src/coreclr/nativeaot/Runtime/AsmOffsets.h +++ b/src/coreclr/nativeaot/Runtime/AsmOffsets.h @@ -58,8 +58,6 @@ ASM_OFFSET( 0, 78, Thread, m_uHijackedReturnValueFlags) ASM_OFFSET( 48, 80, Thread, m_pExInfoStackHead) ASM_OFFSET( 4c, 88, Thread, m_threadAbortException) -ASM_OFFSET( 50, 90, Thread, m_pThreadLocalModuleStatics) - ASM_SIZEOF( 14, 20, EHEnum) ASM_OFFSET( 0, 0, gc_alloc_context, alloc_ptr) diff --git a/src/coreclr/nativeaot/Runtime/RuntimeInstance.cpp b/src/coreclr/nativeaot/Runtime/RuntimeInstance.cpp index 581c27cbb163d..146c15aeff192 100644 --- a/src/coreclr/nativeaot/Runtime/RuntimeInstance.cpp +++ b/src/coreclr/nativeaot/Runtime/RuntimeInstance.cpp @@ -275,6 +275,25 @@ RuntimeInstance::TypeManagerList& RuntimeInstance::GetTypeManagerList() return m_TypeManagerList; } +TypeManager* RuntimeInstance::GetSingleTypeManager() +{ + auto head = m_TypeManagerList.GetHead(); + if (head != NULL && head->m_pNext == NULL) + { + return head->m_pTypeManager; + } + + return NULL; +} + +COOP_PINVOKE_HELPER(TypeManagerHandle, RhGetSingleTypeManager, ()) +{ + TypeManager* typeManager = GetRuntimeInstance()->GetSingleTypeManager(); + ASSERT(typeManager != NULL); + + return TypeManagerHandle::Create(typeManager); +} + // static bool RuntimeInstance::Initialize(HANDLE hPalInstance) { diff --git a/src/coreclr/nativeaot/Runtime/RuntimeInstance.h b/src/coreclr/nativeaot/Runtime/RuntimeInstance.h index 2de7c220f0912..06db50839eeba 100644 --- a/src/coreclr/nativeaot/Runtime/RuntimeInstance.h +++ b/src/coreclr/nativeaot/Runtime/RuntimeInstance.h @@ -99,6 +99,7 @@ class RuntimeInstance bool RegisterTypeManager(TypeManager * pTypeManager); TypeManagerList& GetTypeManagerList(); + TypeManager* GetSingleTypeManager(); OsModuleList* GetOsModuleList(); bool RegisterUnboxingStubs(PTR_VOID pvStartRange, uint32_t cbRange); diff --git a/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc b/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc index 55ca399b4e405..5c2376955c70c 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc +++ b/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc @@ -235,11 +235,11 @@ Name dq offset AddressToExport _tls_array equ 58h ;; offsetof(TEB, ThreadLocalStoragePointer) ;; -;; __declspec(thread) version +;; __declspec(thread) variable ;; -INLINE_GETTHREAD macro destReg, trashReg +INLINE_GET_TLS_VAR macro destReg, trashReg, variable EXTERN _tls_index : DWORD - EXTERN tls_CurrentThread:DWORD + EXTERN variable:DWORD ;; ;; construct 'eax' from 'rax' so that the register size and data size match @@ -255,11 +255,18 @@ endif mov destRegDWORD, [_tls_index] mov trashReg, gs:[_tls_array] mov trashReg, [trashReg + destReg * 8] - mov destRegDWORD, SECTIONREL tls_CurrentThread + mov destRegDWORD, SECTIONREL variable add destReg, trashReg endm +;; +;; __declspec(thread) tls_CurrentThread +;; +INLINE_GETTHREAD macro destReg, trashReg + INLINE_GET_TLS_VAR destReg, trashReg, tls_CurrentThread +endm + INLINE_THREAD_UNHIJACK macro threadReg, trashReg1, trashReg2 ;; ;; Thread::Unhijack() diff --git a/src/coreclr/nativeaot/Runtime/amd64/MiscStubs.S b/src/coreclr/nativeaot/Runtime/amd64/MiscStubs.S index e2d0d91fce4bf..7faf58c75c45e 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/MiscStubs.S +++ b/src/coreclr/nativeaot/Runtime/amd64/MiscStubs.S @@ -45,57 +45,20 @@ LOCAL_LABEL(ProbeLoop): ret NESTED_END RhpStackProbe, _TEXT -NESTED_ENTRY RhpGetThreadStaticBaseForType, _TEXT, NoHandler - // On entry: - // rdi - TypeManagerSlot* - // rsi - type index +NESTED_ENTRY RhpGetInlinedThreadStaticBase, _TEXT, NoHandler // On exit: // rax - the thread static base for the given type - push_nonvol_reg rbx - push_nonvol_reg r12 - - mov rbx, rdi // Save TypeManagerSlot* - mov r12, rsi // Save type index - - // rax = GetThread() - INLINE_GETTHREAD - - mov r8d, [rbx + 8] // Get ModuleIndex out of the TypeManagerSlot + // rdi = &tls_InlinedThreadStatics + INLINE_GET_TLS_VAR tls_InlinedThreadStatics + mov rdi, rax // get per-thread storage - mov rax, [rax + OFFSETOF__Thread__m_pThreadLocalModuleStatics] - - // get per-module storage + mov rax, [rdi] test rax, rax - jz LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath) - cmp r8d, [rax + OFFSETOF__Array__m_Length] - jae LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath) - mov rax, [rax + r8 * 8 + 0x10] + jz C_FUNC(RhpGetInlinedThreadStaticBaseSlow) // rdi contains the storage ref - // get the actual per-type storage - test rax, rax - jz LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath) - cmp r12d, [rax + OFFSETOF__Array__m_Length] - jae LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath) - mov rax, [rax + r12 * 8 + 0x10] - - // if have storage, return it - test rax, rax - jz LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath) - - .cfi_remember_state - pop_nonvol_reg r12 - pop_nonvol_reg rbx + // return it ret +NESTED_END RhpGetInlinedThreadStaticBase, _TEXT - .cfi_restore_state - .cfi_def_cfa_offset 24 // workaround cfi_restore_state bug -LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath): - mov rdi, rbx // restore TypeManagerSlot* - mov rsi, r12 // restore type index - - pop_nonvol_reg r12 - pop_nonvol_reg rbx - jmp C_FUNC(RhpGetThreadStaticBaseForTypeSlow) -NESTED_END RhpGetThreadStaticBaseForType, _TEXT diff --git a/src/coreclr/nativeaot/Runtime/amd64/MiscStubs.asm b/src/coreclr/nativeaot/Runtime/amd64/MiscStubs.asm index 9f11b73c4444d..c4f39f5134558 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/MiscStubs.asm +++ b/src/coreclr/nativeaot/Runtime/amd64/MiscStubs.asm @@ -3,7 +3,7 @@ include AsmMacros.inc -EXTERN RhpGetThreadStaticBaseForTypeSlow : PROC +EXTERN RhpGetInlinedThreadStaticBaseSlow : PROC ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; The following helper will access ("probe") a word on each page of the stack @@ -39,41 +39,20 @@ ProbeLoop: LEAF_END RhpStackProbe, _TEXT -LEAF_ENTRY RhpGetThreadStaticBaseForType, _TEXT - ; On entry and thorough the procedure: - ; rcx - TypeManagerSlot* - ; rdx - type index +LEAF_ENTRY RhpGetInlinedThreadStaticBase, _TEXT ; On exit: ; rax - the thread static base for the given type - ;; rax = GetThread(), TRASHES r8 - INLINE_GETTHREAD rax, r8 - - mov r8d, [rcx + 8] ; Get ModuleIndex out of the TypeManagerSlot + ;; rcx = &tls_InlinedThreadStatics, TRASHES r8 + INLINE_GET_TLS_VAR rcx, r8, tls_InlinedThreadStatics ;; get per-thread storage - mov rax, [rax + OFFSETOF__Thread__m_pThreadLocalModuleStatics] - - ;; get per-module storage - test rax, rax - jz RhpGetThreadStaticBaseForTypeSlow - cmp r8d, [rax + OFFSETOF__Array__m_Length] - jae RhpGetThreadStaticBaseForTypeSlow - mov rax, [rax + r8 * 8 + 10h] - - ;; get the actual per-type storage + mov rax, [rcx] test rax, rax - jz RhpGetThreadStaticBaseForTypeSlow - cmp edx, [rax + OFFSETOF__Array__m_Length] - jae RhpGetThreadStaticBaseForTypeSlow - mov rax, [rax + rdx * 8 + 10h] - - ;; if have storage, return it - test rax, rax - jz RhpGetThreadStaticBaseForTypeSlow + jz RhpGetInlinedThreadStaticBaseSlow ;; rcx contains the storage ref + ;; return it ret - -LEAF_END RhpGetThreadStaticBaseForType, _TEXT +LEAF_END RhpGetInlinedThreadStaticBase, _TEXT end diff --git a/src/coreclr/nativeaot/Runtime/arm64/MiscStubs.S b/src/coreclr/nativeaot/Runtime/arm64/MiscStubs.S index fc1f6d465ca2d..6c70614f38136 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/MiscStubs.S +++ b/src/coreclr/nativeaot/Runtime/arm64/MiscStubs.S @@ -4,3 +4,21 @@ #include #include "AsmOffsets.inc" +NESTED_ENTRY RhpGetInlinedThreadStaticBase, _TEXT, NoHandler + // On exit: + // x0 - the thread static base for the given type + + // x1 = GetThread() + INLINE_GET_TLS_VAR x1, C_FUNC(tls_InlinedThreadStatics) + + // get per-thread storage + ldr x0, [x1] + cbnz x0, HaveValue + mov x0, x1 + b C_FUNC(RhpGetInlinedThreadStaticBaseSlow) + +HaveValue: + // return it + ret + +NESTED_END RhpGetInlinedThreadStaticBase, _TEXT diff --git a/src/coreclr/nativeaot/Runtime/arm64/MiscStubs.asm b/src/coreclr/nativeaot/Runtime/arm64/MiscStubs.asm index 49baea4977259..cdb076b3d56b2 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/MiscStubs.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/MiscStubs.asm @@ -3,6 +3,25 @@ #include "AsmMacros.h" + EXTERN RhpGetInlinedThreadStaticBaseSlow + TEXTAREA +;; On exit: +;; x0 - the thread static base for the given type + LEAF_ENTRY RhpGetInlinedThreadStaticBase + ;; x1 = &tls_InlinedThreadStatics, TRASHES x2 + INLINE_GET_TLS_VAR x1, x2, tls_InlinedThreadStatics + + ;; get per-thread storage + ldr x0, [x1] + cbnz x0, HaveValue + mov x0, x1 + b RhpGetInlinedThreadStaticBaseSlow + +HaveValue + ;; return it + ret + LEAF_END RhpGetInlinedThreadStaticBase + end diff --git a/src/coreclr/nativeaot/Runtime/gcrhscan.cpp b/src/coreclr/nativeaot/Runtime/gcrhscan.cpp index 641e55ec98371..e6c815094d29b 100644 --- a/src/coreclr/nativeaot/Runtime/gcrhscan.cpp +++ b/src/coreclr/nativeaot/Runtime/gcrhscan.cpp @@ -54,6 +54,14 @@ void GCToEEInterface::GcScanRoots(EnumGcRefCallbackFunc * fn, int condemned, in else #endif { + InlinedThreadStaticRoot* pRoot = pThread->GetInlinedThreadStaticList(); + while (pRoot != NULL) + { + STRESS_LOG2(LF_GC | LF_GCROOTS, LL_INFO100, "{ Scanning Thread's %p inline thread statics root %p. \n", pThread, pRoot); + GcEnumObject(&pRoot->m_threadStaticsBase, 0 /*flags*/, fn, sc); + pRoot = pRoot->m_next; + } + STRESS_LOG1(LF_GC | LF_GCROOTS, LL_INFO100, "{ Scanning Thread's %p thread statics root. \n", pThread); GcEnumObject(pThread->GetThreadStaticStorage(), 0 /*flags*/, fn, sc); diff --git a/src/coreclr/nativeaot/Runtime/thread.cpp b/src/coreclr/nativeaot/Runtime/thread.cpp index 6f599b97c9eb2..4d8c2c29d1a78 100644 --- a/src/coreclr/nativeaot/Runtime/thread.cpp +++ b/src/coreclr/nativeaot/Runtime/thread.cpp @@ -284,7 +284,8 @@ void Thread::Construct() // Everything else should be initialized to 0 via the static initialization of tls_CurrentThread. - ASSERT(m_pThreadLocalModuleStatics == NULL); + ASSERT(m_pThreadLocalStatics == NULL); + ASSERT(m_pInlinedThreadLocalStatics == NULL); ASSERT(m_pGCFrameRegistrations == NULL); @@ -1266,15 +1267,33 @@ COOP_PINVOKE_HELPER(Object *, RhpGetThreadAbortException, ()) Object** Thread::GetThreadStaticStorage() { - return &m_pThreadLocalModuleStatics; + return &m_pThreadLocalStatics; } COOP_PINVOKE_HELPER(Object**, RhGetThreadStaticStorage, ()) { - Thread * pCurrentThread = ThreadStore::RawGetCurrentThread(); + Thread* pCurrentThread = ThreadStore::RawGetCurrentThread(); return pCurrentThread->GetThreadStaticStorage(); } +InlinedThreadStaticRoot* Thread::GetInlinedThreadStaticList() +{ + return m_pInlinedThreadLocalStatics; +} + +void Thread::RegisterInlinedThreadStaticRoot(InlinedThreadStaticRoot* newRoot) +{ + ASSERT(newRoot->m_next == NULL); + newRoot->m_next = m_pInlinedThreadLocalStatics; + m_pInlinedThreadLocalStatics = newRoot; +} + +COOP_PINVOKE_HELPER(void, RhRegisterInlinedThreadStaticRoot, (Object** root)) +{ + Thread* pCurrentThread = ThreadStore::RawGetCurrentThread(); + pCurrentThread->RegisterInlinedThreadStaticRoot((InlinedThreadStaticRoot*)root); +} + // This is function is used to quickly query a value that can uniquely identify a thread COOP_PINVOKE_HELPER(uint8_t*, RhCurrentNativeThreadId, ()) { diff --git a/src/coreclr/nativeaot/Runtime/thread.h b/src/coreclr/nativeaot/Runtime/thread.h index 0dd855cbe968d..39310955e388c 100644 --- a/src/coreclr/nativeaot/Runtime/thread.h +++ b/src/coreclr/nativeaot/Runtime/thread.h @@ -74,6 +74,12 @@ struct GCFrameRegistration int m_MaybeInterior; }; +struct InlinedThreadStaticRoot +{ + Object* m_threadStaticsBase; + InlinedThreadStaticRoot* m_next; +}; + struct ThreadBuffer { uint8_t m_rgbAllocContextBuffer[SIZEOF_ALLOC_CONTEXT]; @@ -88,7 +94,8 @@ struct ThreadBuffer uintptr_t m_uHijackedReturnValueFlags; PTR_ExInfo m_pExInfoStackHead; Object* m_threadAbortException; // ThreadAbortException instance -set only during thread abort - Object* m_pThreadLocalModuleStatics; + Object* m_pThreadLocalStatics; + InlinedThreadStaticRoot* m_pInlinedThreadLocalStatics; GCFrameRegistration* m_pGCFrameRegistrations; PTR_VOID m_pStackLow; PTR_VOID m_pStackHigh; @@ -288,6 +295,9 @@ class Thread : private ThreadBuffer Object** GetThreadStaticStorage(); + InlinedThreadStaticRoot* GetInlinedThreadStaticList(); + void RegisterInlinedThreadStaticRoot(InlinedThreadStaticRoot* newRoot); + NATIVE_CONTEXT* GetInterruptedContext(); void PushGCFrameRegistration(GCFrameRegistration* pRegistration); diff --git a/src/coreclr/nativeaot/Runtime/threadstore.cpp b/src/coreclr/nativeaot/Runtime/threadstore.cpp index 12cdea592ce49..c65d957709566 100644 --- a/src/coreclr/nativeaot/Runtime/threadstore.cpp +++ b/src/coreclr/nativeaot/Runtime/threadstore.cpp @@ -430,6 +430,11 @@ C_ASSERT(sizeof(Thread) == sizeof(ThreadBuffer)); #ifndef _MSC_VER __thread ThreadBuffer tls_CurrentThread; + +// the root of inlined threadstatics storage +// there is only one now, +// eventually this will be emitted by ILC and we may have more than one such variable +__thread InlinedThreadStaticRoot tls_InlinedThreadStatics; #endif EXTERN_C ThreadBuffer* RhpGetThread() @@ -437,6 +442,11 @@ EXTERN_C ThreadBuffer* RhpGetThread() return &tls_CurrentThread; } +COOP_PINVOKE_HELPER(Object**, RhGetInlinedThreadStaticStorage, ()) +{ + return &tls_InlinedThreadStatics.m_threadStaticsBase; +} + #endif // !DACCESS_COMPILE #ifdef _WIN32 @@ -505,4 +515,4 @@ void ThreadStore::SaveCurrentThreadOffsetForDAC() { } -#endif // _WIN32 \ No newline at end of file +#endif // _WIN32 diff --git a/src/coreclr/nativeaot/Runtime/threadstore.inl b/src/coreclr/nativeaot/Runtime/threadstore.inl index 29495046a9827..6fe750f4b01c0 100644 --- a/src/coreclr/nativeaot/Runtime/threadstore.inl +++ b/src/coreclr/nativeaot/Runtime/threadstore.inl @@ -4,8 +4,14 @@ #ifdef _MSC_VER // a workaround to prevent tls_CurrentThread from becoming dynamically checked/initialized. EXTERN_C __declspec(selectany) __declspec(thread) ThreadBuffer tls_CurrentThread; + +// the root of inlined threadstatics storage +// there is only one now, +// eventually this will be emitted by ILC and we may have more than one such variable +EXTERN_C __declspec(selectany) __declspec(thread) InlinedThreadStaticRoot tls_InlinedThreadStatics; #else EXTERN_C __thread ThreadBuffer tls_CurrentThread; +EXTERN_C __thread InlinedThreadStaticRoot tls_InlinedThreadStatics; #endif // static diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/ThreadStatics.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/ThreadStatics.cs index b8dd4c0971e41..9734e2a0a2632 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/ThreadStatics.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/ThreadStatics.cs @@ -4,7 +4,7 @@ using System; using System.Runtime; using System.Runtime.CompilerServices; - +using System.Runtime.InteropServices; using Internal.Runtime.CompilerHelpers; using Debug = System.Diagnostics.Debug; @@ -22,6 +22,34 @@ internal static class ThreadStatics /// static storage for the given type. /// internal static unsafe object GetThreadStaticBaseForType(TypeManagerSlot* pModuleData, int typeTlsIndex) + { + if (typeTlsIndex >= 0) + return GetUninlinedThreadStaticBaseForType(pModuleData, typeTlsIndex); + + ref object? threadStorage = ref RuntimeImports.RhGetInlinedThreadStaticStorage(); + if (threadStorage != null) + return threadStorage; + + return GetInlinedThreadStaticBaseSlow(ref threadStorage); + } + + [RuntimeExport("RhpGetInlinedThreadStaticBaseSlow")] + internal static unsafe object GetInlinedThreadStaticBaseSlow(ref object? threadStorage) + { + Debug.Assert(threadStorage == null); + // Allocate an object that will represent a memory block for all thread static fields + TypeManagerHandle typeManager = RuntimeImports.RhGetSingleTypeManager(); + object threadStaticBase = AllocateThreadStaticStorageForType(typeManager, 0); + + // register the storage location with the thread for GC reporting. + RuntimeImports.RhRegisterInlinedThreadStaticRoot(ref threadStorage); + + // assign the storage block to the storage variable and return + threadStorage = threadStaticBase; + return threadStaticBase; + } + + internal static unsafe object GetUninlinedThreadStaticBaseForType(TypeManagerSlot* pModuleData, int typeTlsIndex) { Debug.Assert(typeTlsIndex >= 0); int moduleIndex = pModuleData->ModuleIndex; @@ -41,16 +69,15 @@ internal static unsafe object GetThreadStaticBaseForType(TypeManagerSlot* pModul } } - return GetThreadStaticBaseForTypeSlow(pModuleData, typeTlsIndex); + return GetUninlinedThreadStaticBaseForTypeSlow(pModuleData, typeTlsIndex); } - [RuntimeExport("RhpGetThreadStaticBaseForTypeSlow")] [MethodImpl(MethodImplOptions.NoInlining)] - internal static unsafe object GetThreadStaticBaseForTypeSlow(TypeManagerSlot* pModuleData, int typeTlsIndex) + internal static unsafe object GetUninlinedThreadStaticBaseForTypeSlow(TypeManagerSlot* pModuleData, int typeTlsIndex) { Debug.Assert(typeTlsIndex >= 0); int moduleIndex = pModuleData->ModuleIndex; - Debug.Assert(typeTlsIndex >= 0); + Debug.Assert(moduleIndex >= 0); // Get the array that holds thread statics for the current thread, if none present // allocate a new one big enough to hold the current module data diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs index f3b4cca51b052..9fe023ee52f88 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -557,6 +557,14 @@ internal static IntPtr RhGetModuleSection(TypeManagerHandle module, ReadyToRunSe [RuntimeImport(RuntimeLibrary, "RhGetThreadStaticStorage")] internal static extern ref object[][] RhGetThreadStaticStorage(); + [MethodImplAttribute(MethodImplOptions.InternalCall)] + [RuntimeImport(RuntimeLibrary, "RhGetInlinedThreadStaticStorage")] + internal static extern ref object? RhGetInlinedThreadStaticStorage(); + + [MethodImplAttribute(MethodImplOptions.InternalCall)] + [RuntimeImport(RuntimeLibrary, "RhRegisterInlinedThreadStaticRoot")] + internal static extern void RhRegisterInlinedThreadStaticRoot(ref object? root); + [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "RhCurrentNativeThreadId")] internal static extern unsafe IntPtr RhCurrentNativeThreadId(); @@ -581,6 +589,10 @@ internal static IntPtr RhGetModuleSection(TypeManagerHandle module, ReadyToRunSe [RuntimeImport(RuntimeLibrary, "RhGetTargetOfUnboxingAndInstantiatingStub")] public static extern IntPtr RhGetTargetOfUnboxingAndInstantiatingStub(IntPtr pCode); + [MethodImplAttribute(MethodImplOptions.InternalCall)] + [RuntimeImport(RuntimeLibrary, "RhGetSingleTypeManager")] + public static extern TypeManagerHandle RhGetSingleTypeManager(); + // // EH helpers // diff --git a/src/coreclr/tools/Common/Compiler/DependencyAnalysis/Target_ARM64/ARM64Emitter.cs b/src/coreclr/tools/Common/Compiler/DependencyAnalysis/Target_ARM64/ARM64Emitter.cs index 84f67052d691d..45b5018b58fa8 100644 --- a/src/coreclr/tools/Common/Compiler/DependencyAnalysis/Target_ARM64/ARM64Emitter.cs +++ b/src/coreclr/tools/Common/Compiler/DependencyAnalysis/Target_ARM64/ARM64Emitter.cs @@ -31,6 +31,13 @@ public void EmitMOV(Register regDst, ushort imm16) Builder.EmitUInt(instruction); } + public void EmitMVN(Register regDst, ushort imm16) + { + Debug.Assert((uint)regDst <= 0x1f); + uint instruction = 0x92800000u | ((uint)imm16 << 5) | (uint)regDst; + Builder.EmitUInt(instruction); + } + public void EmitMOV(Register regDst, ISymbolNode symbol) { // ADRP regDst, [symbol (21bit ADRP thing)] diff --git a/src/coreclr/tools/Common/TypeSystem/Common/Utilities/GCPointerMap.Algorithm.cs b/src/coreclr/tools/Common/TypeSystem/Common/Utilities/GCPointerMap.Algorithm.cs index a03addd6632b3..fd21b72a066a8 100644 --- a/src/coreclr/tools/Common/TypeSystem/Common/Utilities/GCPointerMap.Algorithm.cs +++ b/src/coreclr/tools/Common/TypeSystem/Common/Utilities/GCPointerMap.Algorithm.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using Debug = System.Diagnostics.Debug; namespace Internal.TypeSystem @@ -100,13 +101,8 @@ public static GCPointerMap FromStaticLayout(MetadataType type) return builder.ToGCMap(); } - /// - /// Computes the GC pointer map of the thread static region of the type. - /// - public static GCPointerMap FromThreadStaticLayout(MetadataType type) + private static void MapThreadStaticsForType(GCPointerMapBuilder builder, MetadataType type, int baseOffset) { - GCPointerMapBuilder builder = new GCPointerMapBuilder(type.ThreadGcStaticFieldSize.AsInt, type.Context.Target.PointerSize); - foreach (FieldDesc field in type.GetFields()) { if (!field.IsStatic || field.HasRva || field.IsLiteral || !field.IsThreadStatic || !field.HasGCStaticBase) @@ -115,7 +111,7 @@ public static GCPointerMap FromThreadStaticLayout(MetadataType type) TypeDesc fieldType = field.FieldType; if (fieldType.IsGCPointer) { - builder.MarkGCPointer(field.Offset.AsInt); + builder.MarkGCPointer(field.Offset.AsInt + baseOffset); } else if (fieldType.IsValueType) { @@ -123,14 +119,39 @@ public static GCPointerMap FromThreadStaticLayout(MetadataType type) if (fieldDefType.ContainsGCPointers) { GCPointerMapBuilder innerBuilder = - builder.GetInnerBuilder(field.Offset.AsInt, fieldDefType.InstanceByteCount.AsInt); + builder.GetInnerBuilder(field.Offset.AsInt + baseOffset, fieldDefType.InstanceByteCount.AsInt); FromInstanceLayoutHelper(ref innerBuilder, fieldDefType); } } } + } + + /// + /// Computes the GC pointer map of the thread static region of the type. + /// + public static GCPointerMap FromThreadStaticLayout(MetadataType type) + { + GCPointerMapBuilder builder = new GCPointerMapBuilder(type.ThreadGcStaticFieldSize.AsInt, type.Context.Target.PointerSize); + + MapThreadStaticsForType(builder, type, baseOffset: 0); Debug.Assert(builder.ToGCMap().Size * type.Context.Target.PointerSize >= type.ThreadGcStaticFieldSize.AsInt); return builder.ToGCMap(); } + + public static GCPointerMap FromInlinedThreadStatics( + List types, + Dictionary offsets, + int threadStaticSize, + int pointerSize) + { + GCPointerMapBuilder builder = new GCPointerMapBuilder(threadStaticSize, pointerSize); + foreach (var type in types) + { + MapThreadStaticsForType(builder, type, offsets[type]); + } + + return builder.ToGCMap(); + } } } diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilationBuilder.Aot.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilationBuilder.Aot.cs index 48d66ff85b26b..0f28af6440bf7 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilationBuilder.Aot.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilationBuilder.Aot.cs @@ -17,6 +17,7 @@ public partial class CompilationBuilder protected DictionaryLayoutProvider _dictionaryLayoutProvider = new LazyDictionaryLayoutProvider(); protected DebugInformationProvider _debugInformationProvider = new DebugInformationProvider(); protected DevirtualizationManager _devirtualizationManager = new DevirtualizationManager(); + protected InlinedThreadStatics _inlinedThreadStatics = new InlinedThreadStatics(); protected MethodImportationErrorProvider _methodImportationErrorProvider = new MethodImportationErrorProvider(); protected IInliningPolicy _inliningPolicy; protected bool _methodBodyFolding; @@ -109,6 +110,12 @@ public CompilationBuilder UseMethodImportationErrorProvider(MethodImportationErr return this; } + public CompilationBuilder UseInlinedThreadStatics(InlinedThreadStatics inlinedThreadStatics) + { + _inlinedThreadStatics = inlinedThreadStatics; + return this; + } + public CompilationBuilder UseDwarf5(bool value) { _useDwarf5 = value; diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ILScanNodeFactory.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ILScanNodeFactory.cs index 90e29d20bcc2d..738f76f27f11d 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ILScanNodeFactory.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ILScanNodeFactory.cs @@ -13,7 +13,7 @@ namespace ILCompiler.DependencyAnalysis public sealed class ILScanNodeFactory : NodeFactory { public ILScanNodeFactory(CompilerTypeSystemContext context, CompilationModuleGroup compilationModuleGroup, MetadataManager metadataManager, InteropStubManager interopStubManager, NameMangler nameMangler, PreinitializationManager preinitManager) - : base(context, compilationModuleGroup, metadataManager, interopStubManager, nameMangler, new LazyGenericsDisabledPolicy(), new LazyVTableSliceProvider(), new LazyDictionaryLayoutProvider(), new ExternSymbolsImportedNodeProvider(), preinitManager) + : base(context, compilationModuleGroup, metadataManager, interopStubManager, nameMangler, new LazyGenericsDisabledPolicy(), new LazyVTableSliceProvider(), new LazyDictionaryLayoutProvider(), new InlinedThreadStatics(), new ExternSymbolsImportedNodeProvider(), preinitManager) { } diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/NodeFactory.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/NodeFactory.cs index c726a9dc0a66d..53b44d32b142c 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/NodeFactory.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/NodeFactory.cs @@ -24,6 +24,7 @@ public abstract partial class NodeFactory private CompilationModuleGroup _compilationModuleGroup; private VTableSliceProvider _vtableSliceProvider; private DictionaryLayoutProvider _dictionaryLayoutProvider; + private InlinedThreadStatics _inlinedThreadStatics; protected readonly ImportedNodeProvider _importedNodeProvider; private bool _markingComplete; @@ -36,6 +37,7 @@ public NodeFactory( LazyGenericsPolicy lazyGenericsPolicy, VTableSliceProvider vtableSliceProvider, DictionaryLayoutProvider dictionaryLayoutProvider, + InlinedThreadStatics inlinedThreadStatics, ImportedNodeProvider importedNodeProvider, PreinitializationManager preinitializationManager) { @@ -44,6 +46,7 @@ public NodeFactory( _compilationModuleGroup = compilationModuleGroup; _vtableSliceProvider = vtableSliceProvider; _dictionaryLayoutProvider = dictionaryLayoutProvider; + _inlinedThreadStatics = inlinedThreadStatics; NameMangler = nameMangler; InteropStubManager = interoptStubManager; CreateNodeCaches(); @@ -207,8 +210,21 @@ private void CreateNodeCaches() _threadStatics = new NodeCache(CreateThreadStaticsNode); + TypeThreadStaticIndexNode inlinedThreadStatiscIndexNode = null; + if (_inlinedThreadStatics.IsComputed()) + { + _inlinedThreadStatiscNode = new ThreadStaticsNode(_inlinedThreadStatics, this); + inlinedThreadStatiscIndexNode = new TypeThreadStaticIndexNode(_inlinedThreadStatiscNode); + } + _typeThreadStaticIndices = new NodeCache(type => { + if (inlinedThreadStatiscIndexNode != null && + _inlinedThreadStatics.GetOffsets().ContainsKey(type)) + { + return inlinedThreadStatiscIndexNode; + } + return new TypeThreadStaticIndexNode(type); }); @@ -646,6 +662,7 @@ public EmbeddedObjectNode GCStaticIndirection(MetadataType type) } private NodeCache _threadStatics; + private ThreadStaticsNode _inlinedThreadStatiscNode; public ISymbolDefinitionNode TypeThreadStaticsSymbol(MetadataType type) { @@ -844,6 +861,17 @@ public IMethodNode StringAllocator(MethodDesc stringConstructor) return _stringAllocators.GetOrAdd(stringConstructor); } + public uint ThreadStaticBaseOffset(MetadataType type) + { + if (_inlinedThreadStatics.IsComputed() && + _inlinedThreadStatics.GetOffsets().TryGetValue(type, out var offset)) + { + return (uint)offset; + } + + return 0; + } + private sealed class MethodEntrypointHashtable : LockFreeReaderHashtable { private readonly NodeFactory _factory; @@ -1263,6 +1291,11 @@ public virtual void AttachToDependencyGraph(DependencyAnalyzerBase graph.AddRoot(InterfaceDispatchCellSection, "Interface dispatch cell section is always generated"); graph.AddRoot(ModuleInitializerList, "Module initializer list is always generated"); + if (_inlinedThreadStatics.IsComputed()) + { + graph.AddRoot(_inlinedThreadStatiscNode, "Inlined threadstatics are used if present"); + } + ReadyToRunHeader.Add(ReadyToRunSectionType.GCStaticRegion, GCStaticsRegion, GCStaticsRegion.StartSymbol, GCStaticsRegion.EndSymbol); ReadyToRunHeader.Add(ReadyToRunSectionType.ThreadStaticRegion, ThreadStaticsRegion, ThreadStaticsRegion.StartSymbol, ThreadStaticsRegion.EndSymbol); ReadyToRunHeader.Add(ReadyToRunSectionType.EagerCctor, EagerCctorTable, EagerCctorTable.StartSymbol, EagerCctorTable.EndSymbol); diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ReflectionFieldMapNode.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ReflectionFieldMapNode.cs index ec2c6ed59d151..3044fd4d979d0 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ReflectionFieldMapNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ReflectionFieldMapNode.cs @@ -128,9 +128,15 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) case FieldTableFlags.GCStatic: case FieldTableFlags.NonGCStatic: { + uint fieldOffset = (uint)field.Offset.AsInt; + if (field.IsThreadStatic && field.OwningType is MetadataType mt) + { + fieldOffset += factory.ThreadStaticBaseOffset(mt); + } + if (field.OwningType.HasInstantiation) { - vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant((uint)(field.Offset.AsInt))); + vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant(fieldOffset)); } else { @@ -138,22 +144,28 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) ISymbolNode staticsNode; if (field.IsThreadStatic) + { staticsNode = factory.TypeThreadStaticIndex(metadataType); + } else if (field.HasGCStaticBase) + { staticsNode = factory.TypeGCStaticsSymbol(metadataType); + } else + { staticsNode = factory.TypeNonGCStaticsSymbol(metadataType); + } if (!field.IsThreadStatic && !field.HasGCStaticBase) { - uint index = _externalReferences.GetIndex(staticsNode, field.Offset.AsInt); + uint index = _externalReferences.GetIndex(staticsNode, (int)fieldOffset); vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant(index)); } else { uint index = _externalReferences.GetIndex(staticsNode); vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant(index)); - vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant((uint)(field.Offset.AsInt))); + vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant(fieldOffset)); } } } diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_ARM64/ARM64ReadyToRunHelperNode.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_ARM64/ARM64ReadyToRunHelperNode.cs index c61a3ebf7b9ea..8e43f126e0cc7 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_ARM64/ARM64ReadyToRunHelperNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_ARM64/ARM64ReadyToRunHelperNode.cs @@ -71,30 +71,59 @@ protected override void EmitCode(NodeFactory factory, ref ARM64Emitter encoder, case ReadyToRunHelperId.GetThreadStaticBase: { MetadataType target = (MetadataType)Target; - encoder.EmitMOV(encoder.TargetRegister.Arg2, factory.TypeThreadStaticIndex(target)); - - // First arg: address of the TypeManager slot that provides the helper with - // information about module index and the type manager instance (which is used - // for initialization on first access). - encoder.EmitLDR(encoder.TargetRegister.Arg0, encoder.TargetRegister.Arg2); - - // Second arg: index of the type in the ThreadStatic section of the modules - encoder.EmitLDR(encoder.TargetRegister.Arg1, encoder.TargetRegister.Arg2, factory.Target.PointerSize); - - if (!factory.PreinitializationManager.HasLazyStaticConstructor(target)) + ISortableSymbolNode index = factory.TypeThreadStaticIndex(target); + if (index is TypeThreadStaticIndexNode ti && ti.Type == null) { - encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType)); + ISymbolNode helper = factory.ExternSymbol("RhpGetInlinedThreadStaticBase"); + + if (!factory.PreinitializationManager.HasLazyStaticConstructor(target)) + { + encoder.EmitJMP(helper); + } + else + { + encoder.EmitMOV(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target)); + encoder.EmitSUB(encoder.TargetRegister.Arg2, NonGCStaticsNode.GetClassConstructorContextSize(factory.Target)); + + encoder.EmitLDR(encoder.TargetRegister.Arg3, encoder.TargetRegister.Arg2); + encoder.EmitCMP(encoder.TargetRegister.Arg3, 0); + encoder.EmitJE(helper); + + // First arg: unused address of the TypeManager + encoder.EmitMOV(encoder.TargetRegister.Arg0, (ushort)0); + // Second arg: ~0 (index of inlined storage) + encoder.EmitMVN(encoder.TargetRegister.Arg1, 0); + encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase)); + } } else { - encoder.EmitMOV(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target)); - encoder.EmitSUB(encoder.TargetRegister.Arg2, NonGCStaticsNode.GetClassConstructorContextSize(factory.Target)); - - encoder.EmitLDR(encoder.TargetRegister.Arg3, encoder.TargetRegister.Arg2); - encoder.EmitCMP(encoder.TargetRegister.Arg3, 0); - encoder.EmitJE(factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType)); - - encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase)); + encoder.EmitMOV(encoder.TargetRegister.Arg2, index); + + // First arg: address of the TypeManager slot that provides the helper with + // information about module index and the type manager instance (which is used + // for initialization on first access). + encoder.EmitLDR(encoder.TargetRegister.Arg0, encoder.TargetRegister.Arg2); + + // Second arg: index of the type in the ThreadStatic section of the modules + encoder.EmitLDR(encoder.TargetRegister.Arg1, encoder.TargetRegister.Arg2, factory.Target.PointerSize); + + ISymbolNode helper = factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType); + if (!factory.PreinitializationManager.HasLazyStaticConstructor(target)) + { + encoder.EmitJMP(helper); + } + else + { + encoder.EmitMOV(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target)); + encoder.EmitSUB(encoder.TargetRegister.Arg2, NonGCStaticsNode.GetClassConstructorContextSize(factory.Target)); + + encoder.EmitLDR(encoder.TargetRegister.Arg3, encoder.TargetRegister.Arg2); + encoder.EmitCMP(encoder.TargetRegister.Arg3, 0); + encoder.EmitJE(helper); + + encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase)); + } } } break; diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X64/X64ReadyToRunHelperNode.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X64/X64ReadyToRunHelperNode.cs index a091fdfc94364..15c0a98e3dfeb 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X64/X64ReadyToRunHelperNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X64/X64ReadyToRunHelperNode.cs @@ -71,32 +71,59 @@ protected override void EmitCode(NodeFactory factory, ref X64Emitter encoder, bo case ReadyToRunHelperId.GetThreadStaticBase: { MetadataType target = (MetadataType)Target; - - encoder.EmitLEAQ(encoder.TargetRegister.Arg2, factory.TypeThreadStaticIndex(target)); - - // First arg: address of the TypeManager slot that provides the helper with - // information about module index and the type manager instance (which is used - // for initialization on first access). - AddrMode loadFromArg2 = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64); - encoder.EmitMOV(encoder.TargetRegister.Arg0, ref loadFromArg2); - - // Second arg: index of the type in the ThreadStatic section of the modules - AddrMode loadFromArg2AndDelta = new AddrMode(encoder.TargetRegister.Arg2, null, factory.Target.PointerSize, 0, AddrModeSize.Int32); - encoder.EmitMOV(encoder.TargetRegister.Arg1, ref loadFromArg2AndDelta); - - if (!factory.PreinitializationManager.HasLazyStaticConstructor(target)) + ISortableSymbolNode index = factory.TypeThreadStaticIndex(target); + if (index is TypeThreadStaticIndexNode ti && ti.Type == null) { - encoder.EmitJMP(factory.ExternSymbol("RhpGetThreadStaticBaseForType")); + ISymbolNode helper = factory.ExternSymbol("RhpGetInlinedThreadStaticBase"); + + if (!factory.PreinitializationManager.HasLazyStaticConstructor(target)) + { + encoder.EmitJMP(helper); + } + else + { + encoder.EmitLEAQ(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target), -NonGCStaticsNode.GetClassConstructorContextSize(factory.Target)); + + AddrMode initialized = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64); + encoder.EmitCMP(ref initialized, 0); + encoder.EmitJE(helper); + + // First arg: unused address of the TypeManager + encoder.EmitMOV(encoder.TargetRegister.Arg0, 0); + // Second arg: -1 (index of inlined storage) + encoder.EmitMOV(encoder.TargetRegister.Arg1, -1); + encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase)); + } } else { - encoder.EmitLEAQ(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target), - NonGCStaticsNode.GetClassConstructorContextSize(factory.Target)); - - AddrMode initialized = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64); - encoder.EmitCMP(ref initialized, 0); - encoder.EmitJE(factory.ExternSymbol("RhpGetThreadStaticBaseForType")); - - encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase)); + encoder.EmitLEAQ(encoder.TargetRegister.Arg2, index); + + // First arg: address of the TypeManager slot that provides the helper with + // information about module index and the type manager instance (which is used + // for initialization on first access). + AddrMode loadFromArg2 = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64); + encoder.EmitMOV(encoder.TargetRegister.Arg0, ref loadFromArg2); + + // Second arg: index of the type in the ThreadStatic section of the modules + AddrMode loadFromArg2AndDelta = new AddrMode(encoder.TargetRegister.Arg2, null, factory.Target.PointerSize, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Arg1, ref loadFromArg2AndDelta); + + ISymbolNode helper = factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType); + if (!factory.PreinitializationManager.HasLazyStaticConstructor(target)) + { + encoder.EmitJMP(helper); + } + else + { + encoder.EmitLEAQ(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target), -NonGCStaticsNode.GetClassConstructorContextSize(factory.Target)); + + AddrMode initialized = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64); + encoder.EmitCMP(ref initialized, 0); + encoder.EmitJE(helper); + + encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase)); + } } } break; diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ThreadStaticsNode.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ThreadStaticsNode.cs index 4c48b63430a50..6fa2a7a70cc11 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ThreadStaticsNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ThreadStaticsNode.cs @@ -16,6 +16,7 @@ namespace ILCompiler.DependencyAnalysis public class ThreadStaticsNode : EmbeddedObjectNode, ISymbolDefinitionNode { private MetadataType _type; + private InlinedThreadStatics _inlined; public ThreadStaticsNode(MetadataType type, NodeFactory factory) { @@ -24,6 +25,11 @@ public ThreadStaticsNode(MetadataType type, NodeFactory factory) _type = type; } + public ThreadStaticsNode(InlinedThreadStatics inlined, NodeFactory factory) + { + _inlined = inlined; + } + protected override string GetName(NodeFactory factory) => this.GetMangledName(factory.NameMangler); protected override void OnMarked(NodeFactory factory) @@ -42,12 +48,20 @@ public static string GetMangledName(TypeDesc type, NameMangler nameMangler) public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb) { - sb.Append(GetMangledName(_type, nameMangler)); + string mangledName = _type == null ? "_inlinedThreadStatics" : GetMangledName(_type, nameMangler); + sb.Append(mangledName); } private ISymbolNode GetGCStaticEETypeNode(NodeFactory factory) { - GCPointerMap map = GCPointerMap.FromThreadStaticLayout(_type); + GCPointerMap map = _type != null ? + GCPointerMap.FromThreadStaticLayout(_type) : + GCPointerMap.FromInlinedThreadStatics( + _inlined.GetTypes(), + _inlined.GetOffsets(), + _inlined.GetSize(), + factory.Target.PointerSize); + return factory.GCStaticEEType(map); } @@ -57,20 +71,41 @@ public override IEnumerable GetStaticDependencies(NodeFacto result.Add(new DependencyListEntry(GetGCStaticEETypeNode(factory), "ThreadStatic MethodTable")); - if (factory.PreinitializationManager.HasEagerStaticConstructor(_type)) + if (_type != null) { - result.Add(new DependencyListEntry(factory.EagerCctorIndirection(_type.GetStaticConstructor()), "Eager .cctor")); - } - ModuleUseBasedDependencyAlgorithm.AddDependenciesDueToModuleUse(ref result, factory, _type.Module); + if (factory.PreinitializationManager.HasEagerStaticConstructor(_type)) + { + result.Add(new DependencyListEntry(factory.EagerCctorIndirection(_type.GetStaticConstructor()), "Eager .cctor")); + } + + ModuleUseBasedDependencyAlgorithm.AddDependenciesDueToModuleUse(ref result, factory, _type.Module); + } + else + { + foreach (var type in _inlined.GetTypes()) + { + if (factory.PreinitializationManager.HasEagerStaticConstructor(type)) + { + result.Add(new DependencyListEntry(factory.EagerCctorIndirection(type.GetStaticConstructor()), "Eager .cctor")); + } + + ModuleUseBasedDependencyAlgorithm.AddDependenciesDueToModuleUse(ref result, factory, type.Module); + } + } return result; } - public override bool HasConditionalStaticDependencies => _type.ConvertToCanonForm(CanonicalFormKind.Specific) != _type; + public override bool HasConditionalStaticDependencies => + _type != null ? + _type.ConvertToCanonForm(CanonicalFormKind.Specific) != _type: + false; public override IEnumerable GetConditionalStaticDependencies(NodeFactory factory) { + Debug.Assert(_type != null); + // If we have a type loader template for this type, we need to keep track of the generated // bases in the type info hashtable. The type symbol node does such accounting. return new CombinedDependencyListEntry[] @@ -91,10 +126,21 @@ public override void EncodeData(ref ObjectDataBuilder builder, NodeFactory facto builder.EmitPointerReloc(GetGCStaticEETypeNode(factory)); } + public MetadataType Type => _type; + public override int ClassCode => 2091208431; public override int CompareToImpl(ISortableNode other, CompilerComparer comparer) { + // force the type of the storage block for inlined threadstatics to be "less" + // than other storage blocks, - to ensure it is serialized as the item #0 + if (_type == null) + { + // there should only be at most one inlined storage type. + Debug.Assert(other != null); + return -1; + } + return comparer.Compare(_type, ((ThreadStaticsNode)other)._type); } } diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/TypeThreadStaticIndexNode.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/TypeThreadStaticIndexNode.cs index f9200bef525b9..772970bf145ca 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/TypeThreadStaticIndexNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/TypeThreadStaticIndexNode.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics; using Internal.Text; using Internal.TypeSystem; @@ -12,18 +13,26 @@ namespace ILCompiler.DependencyAnalysis public class TypeThreadStaticIndexNode : DehydratableObjectNode, ISymbolDefinitionNode, ISortableSymbolNode { private MetadataType _type; + private ThreadStaticsNode _inlinedThreadStatics; public TypeThreadStaticIndexNode(MetadataType type) { _type = type; } + public TypeThreadStaticIndexNode(ThreadStaticsNode inlinedThreadStatics) + { + _inlinedThreadStatics = inlinedThreadStatics; + } + public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb) { - sb.Append(nameMangler.NodeMangler.ThreadStaticsIndex(_type)); + sb.Append(_type != null ? nameMangler.NodeMangler.ThreadStaticsIndex(_type) : "_inlinedThreadStaticsIndex"); } + public int Offset => 0; protected override string GetName(NodeFactory factory) => this.GetMangledName(factory.NameMangler); + protected override ObjectNodeSection GetDehydratedSection(NodeFactory factory) { if (factory.Target.IsWindows) @@ -31,14 +40,19 @@ protected override ObjectNodeSection GetDehydratedSection(NodeFactory factory) else return ObjectNodeSection.DataSection; } + public override bool IsShareable => true; public override bool StaticDependenciesAreComputed => true; protected override DependencyList ComputeNonRelocationBasedDependencies(NodeFactory factory) { + ISymbolDefinitionNode node = _type != null ? + factory.TypeThreadStaticsSymbol(_type) : + _inlinedThreadStatics; + return new DependencyList { - new DependencyListEntry(factory.TypeThreadStaticsSymbol(_type), "Thread static storage") + new DependencyListEntry(node, "Thread static storage") }; } @@ -52,8 +66,21 @@ protected override ObjectData GetDehydratableData(NodeFactory factory, bool relo int typeTlsIndex = 0; if (!relocsOnly) { - var node = factory.TypeThreadStaticsSymbol(_type); - typeTlsIndex = ((ThreadStaticsNode)node).IndexFromBeginningOfArray; + if (_type != null) + { + ISymbolDefinitionNode node = factory.TypeThreadStaticsSymbol(_type); + typeTlsIndex = ((ThreadStaticsNode)node).IndexFromBeginningOfArray; + } + else + { + // we use -1 to specify the index of inlined threadstatics, + // which are stored separately from uninlined ones. + typeTlsIndex = -1; + + // the type of the storage block for inlined threadstatics, if present, + // is serialized as the item #0 among other storage block types. + Debug.Assert(_inlinedThreadStatics.IndexFromBeginningOfArray == 0); + } } objData.EmitPointerReloc(factory.TypeManagerIndirection); diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScanner.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScanner.cs index 8f1bcca2e545d..953c059649ebb 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScanner.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScanner.cs @@ -252,6 +252,11 @@ public TypePreinit.TypePreinitializationPolicy GetPreinitializationPolicy() return new ScannedPreinitializationPolicy(_factory.PreinitializationManager, MarkedNodes); } + public InlinedThreadStatics GetInlinedThreadStatics() + { + return new ScannedInlinedThreadStatics(_factory, MarkedNodes); + } + private sealed class ScannedVTableProvider : VTableSliceProvider { private Dictionary> _vtableSlices = new Dictionary>(); @@ -674,6 +679,75 @@ public override TypeSystemException GetCompilationError(MethodDesc method) => _importationErrors.TryGetValue(method, out var exception) ? exception : null; } + private sealed class ScannedInlinedThreadStatics : InlinedThreadStatics + { + private readonly List _types; + private readonly Dictionary _offsets; + private readonly int _size; + + internal override bool IsComputed() => true; + internal override List GetTypes() => _types; + internal override Dictionary GetOffsets() => _offsets; + internal override int GetSize() => _size; + + public ScannedInlinedThreadStatics(NodeFactory factory, ImmutableArray> markedNodes) + { + List threadStaticNodes = new List(); + foreach (var markedNode in markedNodes) + { + if (markedNode is ThreadStaticsNode threadStaticNode) + { + threadStaticNodes.Add(threadStaticNode); + } + } + + // skip MT pointer + int nextDataOffset = factory.Target.PointerSize; + + List types = new List(); + Dictionary offsets = new Dictionary(); + + if (threadStaticNodes.Count > 0) + { + threadStaticNodes.Sort(CompilerComparer.Instance); + for (int i = 0; i < threadStaticNodes.Count; i++) + { + ThreadStaticsNode threadStaticNode = threadStaticNodes[i]; + MetadataType t = threadStaticNode.Type; + + // do not inline storage for shared generics + if (t.ConvertToCanonForm(CanonicalFormKind.Specific) != t) + continue; + +#if DEBUG + // do not inline storage for some types in debug - for test coverage + if (i % 8 == 0) + continue; +#endif + + types.Add(t); + + // N.B. for ARM32, we would need to deal with > PointerSize alignments. + // GCStaticEEType does not currently set RequiresAlign8Flag + Debug.Assert(t.ThreadGcStaticFieldAlignment.AsInt <= factory.Target.PointerSize); + nextDataOffset = nextDataOffset.AlignUp(t.ThreadGcStaticFieldAlignment.AsInt); + + // reported offset is from the MT pointer, adjust for that + offsets.Add(t, nextDataOffset - factory.Target.PointerSize); + + // ThreadGcStaticFieldSize includes MT pointer, we will not need space for it + int dataSize = t.ThreadGcStaticFieldSize.AsInt - factory.Target.PointerSize; + nextDataOffset += dataSize; + } + } + + _types = types; + _offsets = offsets; + // the size is at least MIN_OBJECT_SIZE + _size = Math.Max(nextDataOffset, factory.Target.PointerSize * 3); + } + } + private sealed class ScannedPreinitializationPolicy : TypePreinit.TypePreinitializationPolicy { private readonly HashSet _canonFormsWithCctorChecks = new HashSet(); diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/InlinedThreadStatics.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/InlinedThreadStatics.cs new file mode 100644 index 0000000000000..d746962ba00fc --- /dev/null +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/InlinedThreadStatics.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Internal.TypeSystem; + +namespace ILCompiler +{ + public class InlinedThreadStatics + { + internal virtual bool IsComputed() => false; + + internal virtual int GetSize() => throw new InvalidOperationException(); + internal virtual List GetTypes() => throw new InvalidOperationException(); + internal virtual Dictionary GetOffsets() => throw new InvalidOperationException(); + } +} diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj b/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj index fc41ba97ec0f0..ad9980639e259 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj @@ -453,6 +453,7 @@ + diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/DependencyAnalysis/RyuJitNodeFactory.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/DependencyAnalysis/RyuJitNodeFactory.cs index a51508005f536..abae360824375 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/DependencyAnalysis/RyuJitNodeFactory.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/DependencyAnalysis/RyuJitNodeFactory.cs @@ -10,8 +10,8 @@ namespace ILCompiler.DependencyAnalysis public sealed class RyuJitNodeFactory : NodeFactory { public RyuJitNodeFactory(CompilerTypeSystemContext context, CompilationModuleGroup compilationModuleGroup, MetadataManager metadataManager, - InteropStubManager interopStubManager, NameMangler nameMangler, VTableSliceProvider vtableSliceProvider, DictionaryLayoutProvider dictionaryLayoutProvider, PreinitializationManager preinitializationManager) - : base(context, compilationModuleGroup, metadataManager, interopStubManager, nameMangler, new LazyGenericsDisabledPolicy(), vtableSliceProvider, dictionaryLayoutProvider, new ExternSymbolsImportedNodeProvider(), preinitializationManager) + InteropStubManager interopStubManager, NameMangler nameMangler, VTableSliceProvider vtableSliceProvider, DictionaryLayoutProvider dictionaryLayoutProvider, InlinedThreadStatics inlinedThreadStatics, PreinitializationManager preinitializationManager) + : base(context, compilationModuleGroup, metadataManager, interopStubManager, nameMangler, new LazyGenericsDisabledPolicy(), vtableSliceProvider, dictionaryLayoutProvider, inlinedThreadStatics, new ExternSymbolsImportedNodeProvider(), preinitializationManager) { } diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/RyuJitCompilationBuilder.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/RyuJitCompilationBuilder.cs index 3c20598ffb099..ea29f7ff342c9 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/RyuJitCompilationBuilder.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/RyuJitCompilationBuilder.cs @@ -123,7 +123,7 @@ public override ICompilation ToCompilation() if (_resilient) options |= RyuJitCompilationOptions.UseResilience; - var factory = new RyuJitNodeFactory(_context, _compilationGroup, _metadataManager, _interopStubManager, _nameMangler, _vtableSliceProvider, _dictionaryLayoutProvider, GetPreinitializationManager()); + var factory = new RyuJitNodeFactory(_context, _compilationGroup, _metadataManager, _interopStubManager, _nameMangler, _vtableSliceProvider, _dictionaryLayoutProvider, _inlinedThreadStatics, GetPreinitializationManager()); JitConfigProvider.Initialize(_context.Target, jitFlagBuilder.ToArray(), _ryujitOptions, _jitPath); DependencyAnalyzerBase graph = CreateDependencyGraph(factory, new ObjectNode.ObjectNodeComparer(CompilerComparer.Instance)); diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs index b8a56bb708cb0..cdbf7fe492e27 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs @@ -13,7 +13,6 @@ using ILCompiler; using ILCompiler.DependencyAnalysis; -using Internal.TypeSystem.Ecma; #if SUPPORT_JIT using MethodCodeNode = Internal.Runtime.JitSupport.JitMethodCodeNode; @@ -2057,6 +2056,11 @@ private void getFieldInfo(ref CORINFO_RESOLVED_TOKEN pResolvedToken, CORINFO_MET CORINFO_FIELD_FLAGS fieldFlags = (CORINFO_FIELD_FLAGS)0; uint fieldOffset = (field.IsStatic && field.HasRva ? 0xBAADF00D : (uint)field.Offset.AsInt); + if (field.IsThreadStatic && field.OwningType is MetadataType mt) + { + fieldOffset += _compilation.NodeFactory.ThreadStaticBaseOffset(mt); + } + if (field.IsStatic) { fieldFlags |= CORINFO_FIELD_FLAGS.CORINFO_FLG_FIELD_STATIC; diff --git a/src/coreclr/tools/aot/ILCompiler/Program.cs b/src/coreclr/tools/aot/ILCompiler/Program.cs index 56689af570bf0..2a9aff5514198 100644 --- a/src/coreclr/tools/aot/ILCompiler/Program.cs +++ b/src/coreclr/tools/aot/ILCompiler/Program.cs @@ -485,6 +485,15 @@ void RunScanner() preinitManager = new PreinitializationManager(typeSystemContext, compilationGroup, ilProvider, scanResults.GetPreinitializationPolicy()); builder.UsePreinitializationManager(preinitManager); } + + // If we have a scanner, we can inline threadstatics storage using the information + // we collected at scanning time. + // Inlined storage implies a single type manager, thus we do not do it in multifile case. + // This could be a command line switch if we really wanted to. + if (!multiFile) + { + builder.UseInlinedThreadStatics(scanResults.GetInlinedThreadStatics()); + } } string ilDump = Get(_command.IlDump); diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index a6b7612c4895a..f14de76be7644 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -51,7 +51,7 @@ public static int Main(string[] args) // Just count the number of warnings and errors. There are so many right now that it's not worth enumerating the list #if DEBUG - const int MinWarnings = 16500; + const int MinWarnings = 12000; const int MaxWarnings = 20000; #else const int MinWarnings = 12000;