Inlining in jitted loop bodies

lib/Backend/BackwardPass.cpp

@@ -6792,7 +6792,7 @@ BackwardPass::ProcessInlineeStart(IR::Instr* inlineeStart)
     inlineeStart->IterateMetaArgs([&](IR::Instr* metaArg)
     {
         if (i == Js::Constants::InlineeMetaArgIndex_ArgumentsObject &&
-            inlineeStart->m_func->GetHasArgumentObject())
+            inlineeStart->m_func->GetJnFunction()->GetUsesArgumentsObject())
         {
             Assert(!inlineeStart->m_func->GetHasUnoptimizedArgumentsAcccess());
             // Do not remove arguments object meta arg if there is a reference to arguments object

lib/Backend/Encoder.cpp

@@ -595,7 +595,7 @@ void Encoder::RecordInlineeFrame(Func* inlinee, uint32 currentOffset)
 {
     // The only restriction for not supporting loop bodies is that inlinee frame map is created on FunctionEntryPointInfo & not
     // the base class EntryPointInfo.
-    if (!this->m_func->IsLoopBody() && !this->m_func->IsSimpleJit())
+    if (!(this->m_func->IsLoopBody() && PHASE_OFF(Js::InlineInJitLoopBodyPhase, this->m_func)) && !this->m_func->IsSimpleJit())
     {
         InlineeFrameRecord* record = nullptr;
         if (inlinee->frameInfo && inlinee->m_hasInlineArgsOpt)

lib/Backend/Func.cpp

@@ -77,7 +77,6 @@ Func::Func(JitArenaAllocator *alloc, CodeGenWorkItem* workItem, const Js::Functi
     thisOrParentInlinerHasArguments(false),
     hasStackArgs(false),
     hasNonSimpleParams(false),
-    hasArgumentObject(false),
     hasUnoptimizedArgumentsAcccess(false),
     hasApplyTargetInlining(false),
     hasImplicitCalls(false),
@@ -151,7 +150,8 @@ Func::Func(JitArenaAllocator *alloc, CodeGenWorkItem* workItem, const Js::Functi
             // as determined by the bytecode generator.
             SetHasStackArgs(true);
         }
-        if (doStackNestedFunc && m_jnFunction->GetNestedCount() != 0)
+        if (doStackNestedFunc && m_jnFunction->GetNestedCount() != 0 &&
+            this->GetTopFunc()->m_workItem->Type() != JsLoopBodyWorkItemType) // make sure none of the functions inlined in a jitted loop body allocate nested functions on the stack
         {
             Assert(!(this->IsJitInDebugMode() && !m_jnFunction->GetUtf8SourceInfo()->GetIsLibraryCode()));
             stackNestedFunc = true;
@@ -379,7 +379,7 @@ Func::Codegen()
 
         BEGIN_CODEGEN_PHASE(this, Js::InlinePhase);
 
-        InliningHeuristics heuristics(this->GetJnFunction());
+        InliningHeuristics heuristics(this->GetJnFunction(), this->IsLoopBody());
         Inline inliner(this, heuristics);
         inliner.Optimize();
 

lib/Backend/Func.h

@@ -546,7 +546,6 @@ static const unsigned __int64 c_debugFillPattern8 = 0xcececececececece;
     bool                hasBailout: 1;
     bool                hasBailoutInEHRegion : 1;
     bool                hasStackArgs: 1;
-    bool                hasArgumentObject : 1;
     bool                hasUnoptimizedArgumentsAcccess : 1; // True if there are any arguments access beyond the simple case of this.apply pattern
     bool                m_canDoInlineArgsOpt : 1;
     bool                hasApplyTargetInlining:1;
@@ -621,7 +620,7 @@ static const unsigned __int64 c_debugFillPattern8 = 0xcececececececece;
     bool                IsStackArgsEnabled()
     {
                         Func* curFunc = this;
-                        bool isStackArgsEnabled = this->hasArgumentObject && curFunc->GetHasStackArgs();
+                        bool isStackArgsEnabled = this->m_jnFunction->GetUsesArgumentsObject() && curFunc->GetHasStackArgs();
                         Func * topFunc = curFunc->GetTopFunc();
                         if (topFunc != nullptr)
                         {
@@ -630,9 +629,6 @@ static const unsigned __int64 c_debugFillPattern8 = 0xcececececececece;
                         return isStackArgsEnabled;
     }
 
-    bool                GetHasArgumentObject() const { return this->hasArgumentObject;}
-    void                SetHasArgumentObject() { this->hasArgumentObject = true;}
-
     bool                GetHasUnoptimizedArgumentsAcccess() const { return this->hasUnoptimizedArgumentsAcccess; }
     void                SetHasUnoptimizedArgumentsAccess(bool args)
     {

lib/Backend/IRBuilder.cpp

@@ -1508,7 +1508,6 @@ IRBuilder::BuildReg1(Js::OpCode newOpcode, uint32 offset, Js::RegSlot R0)
             dstSym->m_isSafeThis = true;
             dstSym->m_isNotInt = true;
         }
-        this->m_func->SetHasArgumentObject();
         return;
     }
     case Js::OpCode::LdLetHeapArgsCached:
@@ -1517,7 +1516,6 @@ IRBuilder::BuildReg1(Js::OpCode newOpcode, uint32 offset, Js::RegSlot R0)
         //Fallthrough to next case block!
     }
     case Js::OpCode::LdHeapArgsCached:
-        this->m_func->SetHasArgumentObject();
         if (!m_func->GetJnFunction()->HasScopeObject())
         {
             Js::Throw::FatalInternalError();
@@ -6017,7 +6015,8 @@ IRBuilder::BuildProfiledCallI(Js::OpCode opcode, uint32 offset, Js::RegSlot retu
                 if(this->m_func->m_jitTimeData)
                 {
                     const Js::FunctionCodeGenJitTimeData *inlinerData = this->m_func->m_jitTimeData;
-                    if(!this->IsLoopBody() && inlinerData->inlineesBv && (!inlinerData->inlineesBv->Test(profileId)
+                    if(!(this->IsLoopBody() && PHASE_OFF(Js::InlineInJitLoopBodyPhase, this->m_func)) && 
+                        inlinerData && inlinerData->inlineesBv && (!inlinerData->inlineesBv->Test(profileId)
 #if DBG
                         || (PHASE_STRESS(Js::BailOnNoProfilePhase, this->m_func->GetTopFunc()) &&
                             (CONFIG_FLAG(SkipFuncCountForBailOnNoProfile) < 0 ||

lib/Backend/Inline.cpp

@@ -1907,7 +1907,7 @@ Inline::InlineBuiltInFunction(IR::Instr *callInstr, Js::FunctionInfo *funcInfo,
 
 #if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
     InliningDecider::TraceInlining(inlinerData->GetFunctionBody(), Js::JavascriptLibrary::GetNameForBuiltIn(builtInId),
-        nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, profileId, builtInId);
+        nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, profileId, callInstr->m_func->GetTopFunc()->IsLoopBody(), builtInId);
 #endif
 
     // From now on we are committed to inlining.
@@ -2255,7 +2255,7 @@ IR::Instr* Inline::InlineApply(IR::Instr *callInstr, Js::FunctionInfo *funcInfo,
 
 #if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
     InliningDecider::TraceInlining(inlinerData->GetFunctionBody(), Js::JavascriptLibrary::GetNameForBuiltIn(builtInId),
-        nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, callSiteId, builtInId);
+        nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, callSiteId, callInstr->m_func->GetTopFunc()->IsLoopBody(), builtInId);
     char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
 #endif
 
@@ -2718,7 +2718,7 @@ Inline::InlineCall(IR::Instr *callInstr, Js::FunctionInfo *funcInfo, const Js::F
 
 #if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
     InliningDecider::TraceInlining(inlinerData->GetFunctionBody(), Js::JavascriptLibrary::GetNameForBuiltIn(builtInId),
-        nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, callSiteId, builtInId);
+        nullptr, 0, this->topFunc->m_workItem->GetFunctionBody(), 0, nullptr, callSiteId, callInstr->m_func->GetTopFunc()->IsLoopBody(), builtInId);
 #endif
 
     uint actualCount = 0;
@@ -3618,6 +3618,16 @@ Inline::InlineScriptFunction(IR::Instr *callInstr, const Js::FunctionCodeGenJitT
     Js::FunctionBody *funcCaller = callInstr->m_func->GetJnFunction();
     Js::FunctionBody *funcBody = inlineeData->GetFunctionBody();
 
+    // We don't do stack args optimization in jitted loop body (because of lack of information about the code before and after the loop) 
+    // and we turn off stack arg optimization for the whole inline chain if we can't do it for one of the functionss.
+    // Inlining a function that uses arguments object could potentially hurt perf because we'll have to create arguments object on the 
+    // heap for that function (versus otherwise the function will be jitted and have its arguments object creation optimized).
+    // TODO: Allow arguments object creation to be optimized on a function level instead of an all-or-nothing approach.
+    if (callInstr->m_func->IsLoopBody() && funcBody->GetUsesArgumentsObject())
+    {
+        return instrNext;
+    }
+
     if (callInstr->GetSrc2() &&
         callInstr->GetSrc2()->IsSymOpnd() &&
         callInstr->GetSrc2()->AsSymOpnd()->m_sym->AsStackSym()->GetArgSlotNum() > Js::InlineeCallInfo::MaxInlineeArgoutCount)
@@ -5054,7 +5064,7 @@ Inline::HasArgumentsAccess(IR::Instr * instr, SymID argumentsSymId)
 bool
 Inline::GetInlineeHasArgumentObject(Func * inlinee)
 {
-    if (!inlinee->GetHasArgumentObject())
+    if (!inlinee->GetJnFunction()->GetUsesArgumentsObject())
     {
         // If inlinee has no arguments access return false
         return false;

lib/Backend/InliningDecider.cpp

@@ -5,7 +5,7 @@
 #include "Backend.h"
 
 InliningDecider::InliningDecider(Js::FunctionBody *const topFunc, bool isLoopBody, bool isInDebugMode, const ExecutionMode jitMode)
-    : topFunc(topFunc), isLoopBody(isLoopBody), isInDebugMode(isInDebugMode), jitMode(jitMode), bytecodeInlinedCount(0), numberOfInlineesWithLoop (0), inliningHeuristics(topFunc)
+    : topFunc(topFunc), isLoopBody(isLoopBody), isInDebugMode(isInDebugMode), jitMode(jitMode), bytecodeInlinedCount(0), numberOfInlineesWithLoop (0), inliningHeuristics(topFunc, isLoopBody)
 {
     Assert(topFunc);
 }
@@ -190,7 +190,7 @@ Js::FunctionInfo *InliningDecider::Inline(Js::FunctionBody *const inliner, Js::F
     Js::FunctionProxy * proxy = functionInfo->GetFunctionProxy();
     if (proxy && proxy->IsFunctionBody())
     {
-        if (isLoopBody && !PHASE_ON(Js::InlineInJitLoopBodyPhase, this->topFunc))
+        if (isLoopBody && PHASE_OFF(Js::InlineInJitLoopBodyPhase, this->topFunc))
         {
             INLINE_TESTTRACE_VERBOSE(_u("INLINING: Skip Inline: Jit loop body: %s (%s)\n"), this->topFunc->GetDisplayName(),
                 this->topFunc->GetDebugNumberSet(debugStringBuffer));
@@ -268,7 +268,7 @@ Js::FunctionInfo *InliningDecider::Inline(Js::FunctionBody *const inliner, Js::F
         }
 
 #if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
-        TraceInlining(inliner, inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inlinee->GetByteCodeCount(), this->topFunc, this->bytecodeInlinedCount, inlinee, callSiteId);
+        TraceInlining(inliner, inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inlinee->GetByteCodeCount(), this->topFunc, this->bytecodeInlinedCount, inlinee, callSiteId, this->isLoopBody);
 #endif
 
         this->bytecodeInlinedCount += inlinee->GetByteCodeCount();
@@ -608,32 +608,32 @@ bool InliningDecider::GetBuiltInInfo(
 #if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
 // static
 void InliningDecider::TraceInlining(Js::FunctionBody *const inliner, const char16* inlineeName, const char16* inlineeFunctionIdandNumberString, uint inlineeByteCodeCount,
-    Js::FunctionBody* topFunc, uint inlinedByteCodeCount, Js::FunctionBody *const inlinee, uint callSiteId, uint builtIn)
+    Js::FunctionBody* topFunc, uint inlinedByteCodeCount, Js::FunctionBody *const inlinee, uint callSiteId, bool inLoopBody, uint builtIn)
 {
     char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
     char16 debugStringBuffer2[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
     char16 debugStringBuffer3[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
     if (inlineeName == nullptr)
     {
-
         int len = swprintf_s(debugStringBuffer3, MAX_FUNCTION_BODY_DEBUG_STRING_SIZE, _u("built In Id: %u"), builtIn);
         Assert(len > 14);
         inlineeName = debugStringBuffer3;
     }
-    INLINE_TESTTRACE(_u("INLINING: Inlinee: %s (%s)\tSize: %d\tCaller: %s (%s)\tSize: %d\tInlineCount: %d\tRoot: %s (%s)\tSize: %d\tCallSiteId: %d\n"),
+    INLINE_TESTTRACE(_u("INLINING %s: Inlinee: %s (%s)\tSize: %d\tCaller: %s (%s)\tSize: %d\tInlineCount: %d\tRoot: %s (%s)\tSize: %d\tCallSiteId: %d\n"),
+        inLoopBody ? _u("IN LOOP BODY") : _u(""),
         inlineeName, inlineeFunctionIdandNumberString, inlineeByteCodeCount,
-        inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer), inliner->GetByteCodeCount(), inlinedByteCodeCount,
-        topFunc->GetDisplayName(),
-        topFunc->GetDebugNumberSet(debugStringBuffer2), topFunc->GetByteCodeCount(),
+        inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer), inliner->GetByteCodeCount(),
+        inlinedByteCodeCount,
+        topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer2), topFunc->GetByteCodeCount(),
         callSiteId
         );
 
-    INLINE_TRACE(_u("INLINING:\n\tInlinee: size: %4d  %s\n\tCaller: size: %4d  %-25s (%s)  InlineCount: %d\tRoot:  size: %4d  %s  (%s) CallSiteId %d\n"),
-        inlineeByteCodeCount, inlineeName,
-        inliner->GetByteCodeCount(), inliner->GetDisplayName(),
-        inliner->GetDebugNumberSet(debugStringBuffer), inlinedByteCodeCount,
-        topFunc->GetByteCodeCount(),
-        topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer2),
+    INLINE_TRACE(_u("INLINING %s: Inlinee: %s(%s)\tSize : %d\tCaller : %s(%s)\tSize : %d\tInlineCount : %d\tRoot : %s(%s)\tSize : %d\tCallSiteId : %d\n"),
+        inLoopBody ? _u("IN LOOP BODY") : _u(""),
+        inlineeName, inlineeFunctionIdandNumberString, inlineeByteCodeCount,
+        inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer), inliner->GetByteCodeCount(),
+        inlinedByteCodeCount,
+        topFunc->GetByteCodeCount(), topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer2),
         callSiteId
         );
 

lib/Backend/InliningDecider.h

@@ -51,8 +51,8 @@ class InliningDecider
         bytecodeInlinedCount = 0;
         numberOfInlineesWithLoop = 0;
     }
-    uint32 getNumberOfInlineesWithLoop() { return numberOfInlineesWithLoop; }
-    void incrementNumberOfInlineesWithLoop() { numberOfInlineesWithLoop++; }
+    uint32 GetNumberOfInlineesWithLoop() { return numberOfInlineesWithLoop; }
+    void IncrementNumberOfInlineesWithLoop() { numberOfInlineesWithLoop++; }
 
 
     static bool GetBuiltInInfo(
@@ -63,7 +63,7 @@ class InliningDecider
 
 #if defined(ENABLE_DEBUG_CONFIG_OPTIONS)
     static void TraceInlining(Js::FunctionBody *const inliner, const char16* inlineeName, const char16* inlineeFunctionIdandNumberString, uint inlineeByteCodeCount,
-        Js::FunctionBody* topFunc, uint inlinedByteCodeCount, Js::FunctionBody *const inlinee, uint callSiteId, uint builtIn = -1);
+        Js::FunctionBody* topFunc, uint inlinedByteCodeCount, Js::FunctionBody *const inlinee, uint callSiteId, bool isLoopBody, uint builtIn = -1);
 #endif
 
     PREVENT_COPY(InliningDecider)