Add support for delegate GDV and method-based vtable GDV

src/coreclr/inc/corinfo.h

@@ -646,6 +646,9 @@ enum CorInfoHelpFunc
     CORINFO_HELP_VALIDATE_INDIRECT_CALL,    // CFG: Validate function pointer
     CORINFO_HELP_DISPATCH_INDIRECT_CALL,    // CFG: Validate and dispatch to pointer
 
+    CORINFO_HELP_METHODPROFILE32,           // Update 32-bit method profile for a call site
+    CORINFO_HELP_METHODPROFILE64,           // Update 64-bit method profile for a call site
+
     CORINFO_HELP_COUNT,
 };
 

src/coreclr/inc/corjit.h

@@ -330,7 +330,8 @@ class ICorJitInfo : public ICorDynamicInfo
 
     // Data structure for a single class probe using 32-bit count.
     //
-    // CLASS_FLAG and INTERFACE_FLAG are placed into the Other field in the schema
+    // CLASS_FLAG, INTERFACE_FLAG and DELEGATE_FLAG are placed into the Other field in the schema.
+    // If CLASS_FLAG is set the handle table consists of type handles, and otherwise method handles.
     //
     // Count is the number of times a call was made at that call site.
     //
@@ -339,24 +340,25 @@ class ICorJitInfo : public ICorDynamicInfo
     // SAMPLE_INTERVAL must be >= SIZE. SAMPLE_INTERVAL / SIZE
     // gives the average number of calls between table updates.
     // 
-    struct ClassProfile32
+    struct HandleHistogram32
     {
         enum { 
             SIZE = 8, 
             SAMPLE_INTERVAL = 32, 
             CLASS_FLAG     = 0x80000000, 
             INTERFACE_FLAG = 0x40000000,
-            OFFSET_MASK    = 0x3FFFFFFF
+            DELEGATE_FLAG  = 0x20000000,
+            OFFSET_MASK    = 0x0FFFFFFF
         };
 
         uint32_t Count;
-        CORINFO_CLASS_HANDLE ClassTable[SIZE];
+        void* HandleTable[SIZE];
     };
 
-    struct ClassProfile64
+    struct HandleHistogram64
     {
         uint64_t Count;
-        CORINFO_CLASS_HANDLE ClassTable[ClassProfile32::SIZE];
+        void* HandleTable[HandleHistogram32::SIZE];
     };
 
     enum class PgoInstrumentationKind
@@ -368,6 +370,7 @@ class ICorJitInfo : public ICorDynamicInfo
         FourByte = 1,
         EightByte = 2,
         TypeHandle = 3,
+        MethodHandle = 4,
 
         // Mask of all schema data types
         MarshalMask = 0xF,
@@ -385,14 +388,16 @@ class ICorJitInfo : public ICorDynamicInfo
         Done = None, // All instrumentation schemas must end with a record which is "Done"
         BasicBlockIntCount = (DescriptorMin * 1) | FourByte, // basic block counter using unsigned 4 byte int
         BasicBlockLongCount = (DescriptorMin * 1) | EightByte, // basic block counter using unsigned 8 byte int
-        TypeHandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment.
-        TypeHandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram
-        TypeHandleHistogramTypeHandle = (DescriptorMin * 3) | TypeHandle, // TypeHandle that is part of a type histogram
+        HandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match HandleHistogram32's alignment.
+        HandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram
+        HandleHistogramTypes = (DescriptorMin * 3) | TypeHandle, // Histogram of type handles
+        HandleHistogramMethods = (DescriptorMin * 3) | MethodHandle, // Histogram of method handles
         Version = (DescriptorMin * 4) | None, // Version is encoded in the Other field of the schema
         NumRuns = (DescriptorMin * 5) | None, // Number of runs is encoded in the Other field of the schema
         EdgeIntCount = (DescriptorMin * 6) | FourByte, // edge counter using unsigned 4 byte int
         EdgeLongCount = (DescriptorMin * 6) | EightByte, // edge counter using unsigned 8 byte int
         GetLikelyClass = (DescriptorMin * 7) | TypeHandle, // Compressed get likely class data
+        GetLikelyMethod = (DescriptorMin * 7) | MethodHandle, // Compressed get likely method data
     };
 
     struct PgoInstrumentationSchema
@@ -415,13 +420,13 @@ class ICorJitInfo : public ICorDynamicInfo
         Sampling= 6,    // PGO data derived from sampling
     };
 
-#define DEFAULT_UNKNOWN_TYPEHANDLE 1
-#define UNKNOWN_TYPEHANDLE_MIN 1
-#define UNKNOWN_TYPEHANDLE_MAX 33
+#define DEFAULT_UNKNOWN_HANDLE 1
+#define UNKNOWN_HANDLE_MIN 1
+#define UNKNOWN_HANDLE_MAX 33
 
-    static inline bool IsUnknownTypeHandle(intptr_t typeHandle)
+    static inline bool IsUnknownHandle(intptr_t handle)
     {
-        return ((typeHandle >= UNKNOWN_TYPEHANDLE_MIN) && (typeHandle <= UNKNOWN_TYPEHANDLE_MAX));
+        return ((handle >= UNKNOWN_HANDLE_MIN) && (handle <= UNKNOWN_HANDLE_MAX));
     }
 
     // get profile information to be used for optimizing a current method.  The format

src/coreclr/inc/eventtracebase.h

@@ -939,7 +939,7 @@ namespace ETW
         static VOID MethodRestored(MethodDesc * pMethodDesc);
         static VOID MethodTableRestored(MethodTable * pMethodTable);
         static VOID DynamicMethodDestroyed(MethodDesc *pMethodDesc);
-        static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t typeHandles);
+        static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t numTypeHandles, MethodDesc** pMethods, uint32_t numMethods);
 #else // FEATURE_EVENT_TRACE
     public:
         static VOID GetR2RGetEntryPointStart(MethodDesc *pMethodDesc) {};
@@ -951,7 +951,7 @@ namespace ETW
         static VOID MethodRestored(MethodDesc * pMethodDesc) {};
         static VOID MethodTableRestored(MethodTable * pMethodTable) {};
         static VOID DynamicMethodDestroyed(MethodDesc *pMethodDesc) {};
-        static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t typeHandles) {};
+        static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t numTypeHandles, MethodDesc** pMethods, uint32_t numMethods) {};
 #endif // FEATURE_EVENT_TRACE
     };
 

src/coreclr/inc/jiteeversionguid.h

@@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
 #define GUID_DEFINED
 #endif // !GUID_DEFINED
 
-constexpr GUID JITEEVersionIdentifier = { /* b0719856-6fe6-407c-bf40-7a57e22b2382 */
-    0xb0719856,
-    0x6fe6,
-    0x407c,
-    {0xbf, 0x40, 0x7a, 0x57, 0xe2, 0x2b, 0x23, 0x82}
+constexpr GUID JITEEVersionIdentifier = { /* 7503fe09-4852-40f6-829a-ff91402c9604 */
+    0x7503fe09,
+    0x4852,
+    0x40f6,
+    {0x82, 0x9a, 0xff, 0x91, 0x40, 0x2c, 0x96, 0x04}
   };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////

src/coreclr/inc/jithelpers.h

@@ -339,6 +339,9 @@
     JITHELPER(CORINFO_HELP_DISPATCH_INDIRECT_CALL, NULL, CORINFO_HELP_SIG_REG_ONLY)
 #endif
 
+    JITHELPER(CORINFO_HELP_METHODPROFILE32, JIT_MethodProfile32, CORINFO_HELP_SIG_8_STACK)
+    JITHELPER(CORINFO_HELP_METHODPROFILE64, JIT_MethodProfile64, CORINFO_HELP_SIG_8_STACK)
+
 #undef JITHELPER
 #undef DYNAMICJITHELPER
 #undef JITHELPER

src/coreclr/inc/pgo_formatprocessing.h

@@ -8,15 +8,7 @@
 
 #ifdef FEATURE_PGO
 
-inline bool AddTypeHandleToUnknownTypeHandleMask(INT_PTR typeHandle, uint32_t *unknownTypeHandleMask)
-{
-    uint32_t bitMask = (uint32_t)(1 << (typeHandle - UNKNOWN_TYPEHANDLE_MIN));
-    bool result = (bitMask & *unknownTypeHandleMask) == 0;
-    *unknownTypeHandleMask |= bitMask;
-    return result;
-}
-
-inline INT_PTR HashToPgoUnknownTypeHandle(uint32_t hash)
+inline INT_PTR HashToPgoUnknownHandle(uint32_t hash)
 {
     // Map from a 32bit hash to the 32 different unknown type handle values
     return (hash & 0x1F) + 1;
@@ -53,6 +45,7 @@ inline uint32_t InstrumentationKindToSize(ICorJitInfo::PgoInstrumentationKind ki
         case ICorJitInfo::PgoInstrumentationKind::EightByte:
             return 8;
         case ICorJitInfo::PgoInstrumentationKind::TypeHandle:
+        case ICorJitInfo::PgoInstrumentationKind::MethodHandle:
             return TARGET_POINTER_SIZE;
         default:
             _ASSERTE(FALSE);
@@ -242,6 +235,7 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa
     bool done = false;
     int64_t lastDataValue = 0;
     int64_t lastTypeDataValue = 0;
+    int64_t lastMethodDataValue = 0;
     int32_t dataCountToRead = 0;
 
     ReadCompressedInts(pByte, cbDataMax, [&](int64_t curValue)
@@ -267,8 +261,16 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa
                     return false;
                 }
                 break;
+            case ICorJitInfo::PgoInstrumentationKind::MethodHandle:
+                lastMethodDataValue += curValue;
+
+                if (!handler(schemaHandler.GetSchema(), lastMethodDataValue, schemaHandler.GetSchema().Count - dataCountToRead))
+                {
+                    return false;
+                }
+                break;
             default:
-                assert(false);
+                assert(!"Unexpected PGO instrumentation data type");
                 break;
             }
             dataCountToRead--;
@@ -516,6 +518,7 @@ class SchemaAndDataWriter
     ICorJitInfo::PgoInstrumentationSchema prevSchema = {};
     int64_t lastIntDataWritten = 0;
     int64_t lastTypeDataWritten = 0;
+    int64_t lastMethodDataWritten = 0;
 
 public:
     SchemaAndDataWriter(const ByteWriter& byteWriter, uint8_t* pInstrumentationData) :
@@ -532,8 +535,8 @@ class SchemaAndDataWriter
         return true;
     }
 
-    template<class TypeHandleProcessor>
-    bool AppendDataFromLastSchema(TypeHandleProcessor& thProcessor)
+    template<class TypeHandleProcessor, class MethodHandleProcessor>
+    bool AppendDataFromLastSchema(TypeHandleProcessor& thProcessor, MethodHandleProcessor& mhProcessor)
     {
         uint8_t *pData = (pInstrumentationData + prevSchema.Offset);
         for (int32_t iDataElem = 0; iDataElem < prevSchema.Count; iDataElem++)
@@ -577,6 +580,20 @@ class SchemaAndDataWriter
                     pData += sizeof(intptr_t);
                     break;
                 }
+                case ICorJitInfo::PgoInstrumentationKind::MethodHandle:
+                {
+                    logicalDataToWrite = *(volatile intptr_t*)pData;
+
+                    // As there could be tearing otherwise, inform the caller of exactly what value was written.
+                    mhProcessor(logicalDataToWrite);
+
+                    bool returnValue = WriteCompressedIntToBytes(logicalDataToWrite - lastMethodDataWritten, byteWriter);
+                    lastMethodDataWritten = logicalDataToWrite;
+                    if (!returnValue)
+                        return false;
+                    pData += sizeof(intptr_t);
+                    break;
+                }
                 default:
                     _ASSERTE(!"Unexpected type");
                     return false;

src/coreclr/inc/readytorun.h

@@ -16,7 +16,7 @@
 
 // Keep these in sync with src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs
 #define READYTORUN_MAJOR_VERSION 0x0006
-#define READYTORUN_MINOR_VERSION 0x0000
+#define READYTORUN_MINOR_VERSION 0x0001
 
 #define MINIMUM_READYTORUN_MAJOR_VERSION 0x006
 

src/coreclr/jit/block.h

@@ -546,7 +546,7 @@ enum BasicBlockFlags : unsigned __int64
     BBF_BACKWARD_JUMP_TARGET = MAKE_BBFLAG(35), // Block is a target of a backward jump
 
     BBF_PATCHPOINT           = MAKE_BBFLAG(36), // Block is a patchpoint
-    BBF_HAS_CLASS_PROFILE    = MAKE_BBFLAG(37), // BB contains a call needing a class profile
+    BBF_HAS_HISTOGRAM_PROFILE    = MAKE_BBFLAG(37), // BB contains a call needing a class profile
     BBF_PARTIAL_COMPILATION_PATCHPOINT  = MAKE_BBFLAG(38), // Block is a partial compilation patchpoint
     BBF_HAS_ALIGN            = MAKE_BBFLAG(39), // BB ends with 'align' instruction
     BBF_TAILCALL_SUCCESSOR   = MAKE_BBFLAG(40), // BB has pred that has potential tail call
@@ -582,7 +582,7 @@ enum BasicBlockFlags : unsigned __int64
     // TODO: Should BBF_RUN_RARELY be added to BBF_SPLIT_GAINED ?
 
     BBF_SPLIT_GAINED = BBF_DONT_REMOVE | BBF_HAS_JMP | BBF_BACKWARD_JUMP | BBF_HAS_IDX_LEN | BBF_HAS_NEWARRAY | BBF_PROF_WEIGHT | \
-                       BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK | BBF_HAS_CLASS_PROFILE,
+                       BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK | BBF_HAS_HISTOGRAM_PROFILE,
 };
 
 inline constexpr BasicBlockFlags operator ~(BasicBlockFlags a)
@@ -918,8 +918,8 @@ struct BasicBlock : private LIR::Range
     };
 
     union {
-        unsigned bbStkTempsOut;      // base# for output stack temps
-        int      bbClassSchemaIndex; // schema index for class instrumentation
+        unsigned bbStkTempsOut;          // base# for output stack temps
+        int      bbHistogramSchemaIndex; // schema index for histogram instrumentation
     };
 
 #define MAX_XCPTN_INDEX (USHRT_MAX - 1)

src/coreclr/jit/compiler.cpp

@@ -6232,10 +6232,10 @@ int Compiler::compCompileHelper(CORINFO_MODULE_HANDLE classPtr,
     compHndBBtabCount      = 0;
     compHndBBtabAllocCount = 0;
 
-    info.compNativeCodeSize    = 0;
-    info.compTotalHotCodeSize  = 0;
-    info.compTotalColdCodeSize = 0;
-    info.compClassProbeCount   = 0;
+    info.compNativeCodeSize            = 0;
+    info.compTotalHotCodeSize          = 0;
+    info.compTotalColdCodeSize         = 0;
+    info.compHandleHistogramProbeCount = 0;
 
     compHasBackwardJump          = false;
     compHasBackwardJumpInHandler = false;

src/coreclr/jit/compiler.h

@@ -1814,6 +1814,7 @@ class Compiler
     friend class MorphInitBlockHelper;
     friend class MorphCopyBlockHelper;
     friend class CallArgs;
+    friend class IndirectCallTransformer;
 
 #ifdef FEATURE_HW_INTRINSICS
     friend struct HWIntrinsicInfo;
@@ -2317,8 +2318,12 @@ class Compiler
 
     GenTreeCall* gtNewIndCallNode(GenTree* addr, var_types type, const DebugInfo& di = DebugInfo());
 
-    GenTreeCall* gtNewHelperCallNode(
-        unsigned helper, var_types type, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr, GenTree* arg3 = nullptr);
+    GenTreeCall* gtNewHelperCallNode(unsigned  helper,
+                                     var_types type,
+                                     GenTree*  arg1 = nullptr,
+                                     GenTree*  arg2 = nullptr,
+                                     GenTree*  arg3 = nullptr,
+                                     GenTree*  arg4 = nullptr);
 
     GenTreeCall* gtNewRuntimeLookupHelperCallNode(CORINFO_RUNTIME_LOOKUP* pRuntimeLookup,
                                                   GenTree*                ctxTree,
@@ -3518,6 +3523,18 @@ class Compiler
                              bool                    isExplicitTailCall,
                              IL_OFFSET               ilOffset = BAD_IL_OFFSET);
 
+    void impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset);
+
+    enum class GDVProbeType
+    {
+        None,
+        ClassProfile,
+        MethodProfile,
+        MethodAndClassProfile,
+    };
+
+    GDVProbeType compClassifyGDVProbeType(GenTreeCall* call);
+
     //=========================================================================
     //                          PROTECTED
     //=========================================================================
@@ -5415,7 +5432,7 @@ class Compiler
     bool fgGetProfileWeightForBasicBlock(IL_OFFSET offset, weight_t* weight);
 
     Instrumentor* fgCountInstrumentor;
-    Instrumentor* fgClassInstrumentor;
+    Instrumentor* fgHistogramInstrumentor;
 
     PhaseStatus fgPrepareToInstrumentMethod();
     PhaseStatus fgInstrumentMethod();
@@ -5441,6 +5458,7 @@ class Compiler
     UINT32                                 fgPgoBlockCounts;
     UINT32                                 fgPgoEdgeCounts;
     UINT32                                 fgPgoClassProfiles;
+    UINT32                                 fgPgoMethodProfiles;
     unsigned                               fgPgoInlineePgo;
     unsigned                               fgPgoInlineeNoPgo;
     unsigned                               fgPgoInlineeNoPgoSingleBlock;
@@ -5658,6 +5676,7 @@ class Compiler
                                                      Statement*       paramAssignmentInsertionPoint);
     GenTree* fgMorphCall(GenTreeCall* call);
     GenTree* fgExpandVirtualVtableCallTarget(GenTreeCall* call);
+
     void fgMorphCallInline(GenTreeCall* call, InlineResult* result);
     void fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
 #if DEBUG
@@ -6781,13 +6800,18 @@ class Compiler
         optMethodFlags &= ~OMF_HAS_GUARDEDDEVIRT;
     }
 
+    void pickGDV(GenTreeCall*           call,
+                 IL_OFFSET              ilOffset,
+                 bool                   isInterface,
+                 CORINFO_CLASS_HANDLE*  classGuess,
+                 CORINFO_METHOD_HANDLE* methodGuess,
+                 unsigned*              likelihood);
     void considerGuardedDevirtualization(GenTreeCall*            call,
                                          IL_OFFSET               ilOffset,
                                          bool                    isInterface,
                                          CORINFO_METHOD_HANDLE   baseMethod,
                                          CORINFO_CLASS_HANDLE    baseClass,
-                                         CORINFO_CONTEXT_HANDLE* pContextHandle DEBUGARG(CORINFO_CLASS_HANDLE objClass)
-                                             DEBUGARG(const char* objClassName));
+                                         CORINFO_CONTEXT_HANDLE* pContextHandle);
 
     void addGuardedDevirtualizationCandidate(GenTreeCall*          call,
                                              CORINFO_METHOD_HANDLE methodHandle,
@@ -9488,7 +9512,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
         unsigned genCPU; // What CPU are we running on
 
         // Number of class profile probes in this method
-        unsigned compClassProbeCount;
+        unsigned compHandleHistogramProbeCount;
 
     } info;