[MERGE #5081 @sigatrev] inline callbacks

Merge pull request #5081 from sigatrev:inlineCallbacks

This commit adds tracking of up to 1 function passed as an argument to any given callsite, and provides the neccessary data to the JIT to inline it. The inliner is able to follow the single-def chain up multiple functions to find the approriate data, allowing inling to occur at megamorphic in cases where the function is a callback. This will allow more efficient javascript implementations of built-ins like filter and sort.

lib/Backend/FunctionCodeGenJitTimeData.cpp

@@ -102,6 +102,14 @@ namespace Js
         return ldFldInlinees ? ldFldInlinees[inlineCacheIndex] : nullptr;
     }
 
+    const FunctionCodeGenJitTimeData * FunctionCodeGenJitTimeData::GetCallbackInlinee(const ProfileId profiledCallSiteId) const
+    {
+        Assert(GetFunctionBody());
+        Assert(profiledCallSiteId < GetFunctionBody()->GetProfiledCallSiteCount());
+
+        return callbackInlinees ? callbackInlinees[profiledCallSiteId] : nullptr;
+    }
+
     FunctionCodeGenJitTimeData *FunctionCodeGenJitTimeData::AddInlinee(
         Recycler *const recycler,
         const ProfileId profiledCallSiteId,
@@ -165,6 +173,30 @@ namespace Js
         return inlineeData;
     }
 
+    FunctionCodeGenJitTimeData * FunctionCodeGenJitTimeData::AddCallbackInlinee(
+        Recycler *const recycler,
+        const ProfileId profiledCallSiteId,
+        FunctionInfo *const inlinee)
+    {
+        Assert(recycler != nullptr);
+        FunctionBody * functionBody = GetFunctionBody();
+        Assert(functionBody != nullptr);
+        Assert(profiledCallSiteId < functionBody->GetProfiledCallSiteCount());
+        Assert(inlinee != nullptr);
+
+        if (!callbackInlinees)
+        {
+            callbackInlinees = RecyclerNewArrayZ(recycler, Field(FunctionCodeGenJitTimeData *), functionBody->GetProfiledCallSiteCount());
+        }
+
+        // Polymorphic arguments are not inlined.
+        Assert(callbackInlinees[profiledCallSiteId] == nullptr);
+
+        FunctionCodeGenJitTimeData * inlineeData = FunctionCodeGenJitTimeData::New(recycler, inlinee, nullptr /* entryPoint */, true /*isInlined*/);
+        callbackInlinees[profiledCallSiteId] = inlineeData;
+        return inlineeData;
+    }
+
     uint FunctionCodeGenJitTimeData::InlineeCount() const
     {
         return inlineeCount;

lib/Backend/FunctionCodeGenJitTimeData.h

@@ -31,6 +31,7 @@ namespace Js
         // There will be a non-null entry for each profiled call site where a function is to be inlined
         Field(Field(FunctionCodeGenJitTimeData*)*) inlinees;
         Field(Field(FunctionCodeGenJitTimeData*)*) ldFldInlinees;
+        Field(Field(FunctionCodeGenJitTimeData*)*) callbackInlinees;
         Field(RecyclerWeakReference<FunctionBody>*) weakFuncRef;
 
         Field(PolymorphicInlineCacheInfoIDL*) inlineeInfo;
@@ -90,6 +91,7 @@ namespace Js
     public:
         const FunctionCodeGenJitTimeData *GetInlinee(const ProfileId profiledCallSiteId) const;
         const FunctionCodeGenJitTimeData *GetLdFldInlinee(const InlineCacheIndex inlineCacheIndex) const;
+        const FunctionCodeGenJitTimeData * GetCallbackInlinee(const ProfileId profiledCallSiteId) const;
         FunctionCodeGenJitTimeData *AddInlinee(
             Recycler *const recycler,
             const ProfileId profiledCallSiteId,
@@ -118,6 +120,11 @@ namespace Js
             const InlineCacheIndex inlineCacheIndex,
             FunctionInfo *const inlinee);
 
+        FunctionCodeGenJitTimeData * AddCallbackInlinee(
+            Recycler *const recycler,
+            const ProfileId profiledCallSiteId,
+            FunctionInfo *const inlinee);
+
         bool IsPolymorphicCallSite(const ProfileId profiledCallSiteId) const;
         // This function walks all the chained jittimedata and returns the one which match the functionInfo.
         // This can return null, if the functionInfo doesn't match.

lib/Backend/FunctionJITTimeInfo.cpp

@@ -92,6 +92,24 @@ FunctionJITTimeInfo::BuildJITTimeData(
                 BuildJITTimeData(alloc, inlineeJITData, inlineeRuntimeData, jitData->inlinees[i], true, isForegroundJIT);
             }
         }
+
+        jitData->callbackInlineeCount = jitData->bodyData->profiledCallSiteCount;
+        jitData->callbackInlinees = AnewArrayZ(alloc, FunctionJITTimeDataIDL*, jitData->bodyData->profiledCallSiteCount);
+
+        for (Js::ProfileId i = 0; i < jitData->bodyData->profiledCallSiteCount; ++i)
+        {
+            const Js::FunctionCodeGenJitTimeData * inlineeJITData = codeGenData->GetCallbackInlinee(i);
+            if (inlineeJITData != nullptr)
+            {
+                const Js::FunctionCodeGenRuntimeData * inlineeRuntimeData = nullptr;
+                if (inlineeJITData->GetFunctionInfo()->HasBody())
+                {
+                    inlineeRuntimeData = isInlinee ? targetRuntimeData->GetCallbackInlinee(i) : functionBody->GetCallbackInlineeCodeGenRuntimeData(i);
+                }
+                jitData->callbackInlinees[i] = AnewStructZ(alloc, FunctionJITTimeDataIDL);
+                BuildJITTimeData(alloc, inlineeJITData, inlineeRuntimeData, jitData->callbackInlinees[i], true, isForegroundJIT);
+            }
+        }
     }
     jitData->profiledRuntimeData = AnewStructZ(alloc, FunctionJITRuntimeIDL);
     if (isInlinee && targetRuntimeData->ClonedInlineCaches()->HasInlineCaches())
@@ -250,6 +268,24 @@ FunctionJITTimeInfo::GetLdFldInlineeRuntimeData(const Js::InlineCacheIndex inlin
     return GetLdFldInlinee(inlineCacheIndex) ? GetLdFldInlinee(inlineCacheIndex)->GetRuntimeInfo() : nullptr;
 }
 
+const FunctionJITRuntimeInfo *
+FunctionJITTimeInfo::GetCallbackInlineeRuntimeData(const Js::ProfileId profiledCallSiteId) const
+{
+    return GetCallbackInlinee(profiledCallSiteId) ? GetCallbackInlinee(profiledCallSiteId)->GetRuntimeInfo() : nullptr;
+}
+
+const FunctionJITRuntimeInfo *
+FunctionJITTimeInfo::GetInlineeForCallbackInlineeRuntimeData(const Js::ProfileId profiledCallSiteId, intptr_t inlineeFuncBodyAddr) const
+{
+    const FunctionJITTimeInfo *inlineeData = GetCallbackInlinee(profiledCallSiteId);
+    while (inlineeData && inlineeData->GetBody()->GetAddr() != inlineeFuncBodyAddr)
+    {
+        inlineeData = inlineeData->GetNext();
+    }
+    __analysis_assume(inlineeData != nullptr);
+    return inlineeData->GetRuntimeInfo();
+}
+
 const FunctionJITRuntimeInfo *
 FunctionJITTimeInfo::GetRuntimeInfo() const
 {
@@ -290,6 +326,19 @@ FunctionJITTimeInfo::GetSourceContextId() const
     return GetBody()->GetSourceContextId();
 }
 
+const FunctionJITTimeInfo *
+FunctionJITTimeInfo::GetCallbackInlinee(Js::ProfileId profileId) const
+{
+    Assert(profileId < m_data.bodyData->profiledCallSiteCount);
+    if (!m_data.callbackInlinees)
+    {
+        return nullptr;
+    }
+    AssertOrFailFast(profileId < m_data.callbackInlineeCount);
+
+    return reinterpret_cast<const FunctionJITTimeInfo *>(m_data.callbackInlinees[profileId]);
+}
+
 const FunctionJITTimeInfo *
 FunctionJITTimeInfo::GetLdFldInlinee(Js::InlineCacheIndex inlineCacheIndex) const
 {

lib/Backend/FunctionJITTimeInfo.h

@@ -20,6 +20,7 @@ class FunctionJITTimeInfo
     uint GetInlineeCount() const;
     bool IsLdFldInlineePresent() const;
 
+    const FunctionJITTimeInfo * GetCallbackInlinee(Js::ProfileId profileId) const;
     const FunctionJITTimeInfo * GetLdFldInlinee(Js::InlineCacheIndex inlineCacheIndex) const;
     const FunctionJITTimeInfo * GetInlinee(Js::ProfileId profileId) const;
     const FunctionJITTimeInfo * GetNext() const;
@@ -41,6 +42,8 @@ class FunctionJITTimeInfo
     const FunctionJITRuntimeInfo * GetInlineeForTargetInlineeRuntimeData(const Js::ProfileId profiledCallSiteId, intptr_t inlineeFuncBodyAddr) const;
     const FunctionJITRuntimeInfo *GetInlineeRuntimeData(const Js::ProfileId profiledCallSiteId) const;
     const FunctionJITRuntimeInfo *GetLdFldInlineeRuntimeData(const Js::InlineCacheIndex inlineCacheIndex) const;
+    const FunctionJITRuntimeInfo * GetCallbackInlineeRuntimeData(const Js::ProfileId profiledCallSiteId) const;
+    const FunctionJITRuntimeInfo * GetInlineeForCallbackInlineeRuntimeData(const Js::ProfileId profiledCallSiteId, intptr_t inlineeFuncBodyAddr) const;
     bool ForceJITLoopBody() const;
     bool HasSharedPropertyGuards() const;
     bool HasSharedPropertyGuard(Js::PropertyId id) const;