diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 164102758a88f..e54d3fd834a27 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -300,7 +300,8 @@ function finish_cfg_inline!(state::CFGInliningState) end end -function ir_inline_linetable!(debuginfo::DebugInfoStream, inlinee_debuginfo::DebugInfo, inlinee::MethodInstance) +# TODO append `inlinee_debuginfo` to inner linetable when `inlined_at[2] ≠ 0` +function ir_inline_linetable!(debuginfo::DebugInfoStream, inlinee_debuginfo::DebugInfo, inlined_at::NTuple{3,Int32}) # Append the linetable of the inlined function to our edges table linetable_offset = 1 while true @@ -312,16 +313,14 @@ function ir_inline_linetable!(debuginfo::DebugInfoStream, inlinee_debuginfo::Deb end linetable_offset += 1 end - return Int32(linetable_offset) + return (inlined_at[1], Int32(linetable_offset), Int32(0)) end function ir_prepare_inlining!(insert_node!::Inserter, inline_target::Union{IRCode, IncrementalCompact}, - ir::IRCode, di::DebugInfo, mi::MethodInstance, inlined_at::Int32, argexprs::Vector{Any}) + ir::IRCode, di::DebugInfo, mi::MethodInstance, inlined_at::NTuple{3,Int32}, argexprs::Vector{Any}) def = mi.def::Method debuginfo = inline_target isa IRCode ? inline_target.debuginfo : inline_target.ir.debuginfo - - linetable_offset = ir_inline_linetable!(debuginfo, di, mi) - topline = (inlined_at, linetable_offset, Int32(0)) + topline = new_inlined_at = ir_inline_linetable!(debuginfo, di, inlined_at) if should_insert_coverage(def.module, di) insert_node!(NewInstruction(Expr(:code_coverage_effect), Nothing, topline)) end @@ -343,7 +342,7 @@ function ir_prepare_inlining!(insert_node!::Inserter, inline_target::Union{IRCod NewInstruction(Expr(:call, GlobalRef(Core, :getfield), argexprs[1], QuoteNode(:captures)), ir.argtypes[1], topline)) end - return SSASubstitute(mi, argexprs, spvals_ssa, (inlined_at, linetable_offset)) + return SSASubstitute(mi, argexprs, spvals_ssa, new_inlined_at) end function adjust_boundscheck!(inline_compact::IncrementalCompact, idx′::Int, stmt::Expr, boundscheck::Symbol) @@ -359,8 +358,7 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector item::InliningTodo, boundscheck::Symbol, todo_bbs::Vector{Tuple{Int, Int}}) # Ok, do the inlining here inlined_at = compact.result[idx][:line] - @assert inlined_at[2] == 0 "already inlined this instruction" - ssa_substitute = ir_prepare_inlining!(InsertHere(compact), compact, item.ir, item.di, item.mi, inlined_at[1], argexprs) + ssa_substitute = ir_prepare_inlining!(InsertHere(compact), compact, item.ir, item.di, item.mi, inlined_at, argexprs) boundscheck = has_flag(compact.result[idx], IR_FLAG_INBOUNDS) ? :off : boundscheck # If the iterator already moved on to the next basic block, @@ -1771,7 +1769,7 @@ struct SSASubstitute mi::MethodInstance arg_replacements::Vector{Any} spvals_ssa::Union{Nothing,SSAValue} - inlined_at::NTuple{2,Int32} # TODO: add a map also, so that ssaidx doesn't need to equal inlined_idx? + inlined_at::NTuple{3,Int32} # TODO: add a map also, so that ssaidx doesn't need to equal inlined_idx? end function insert_spval!(insert_node!::Inserter, spvals_ssa::SSAValue, spidx::Int, do_isdefined::Bool) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 946ba9633d366..328f7164a7b93 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1516,7 +1516,7 @@ function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int, add_inlining_backedge!(et, mi) # TODO: Should there be a special line number node for inlined finalizers? - inline_at = ir[SSAValue(idx)][:line][1] + inline_at = ir[SSAValue(idx)][:line] ssa_substitute = ir_prepare_inlining!(InsertBefore(ir, SSAValue(idx)), ir, src, di, mi, inline_at, argexprs) # TODO: Use the actual inliner here rather than open coding this special purpose inliner. diff --git a/multi-inlining.jl b/multi-inlining.jl new file mode 100644 index 0000000000000..ff5e79250fb2a --- /dev/null +++ b/multi-inlining.jl @@ -0,0 +1,23 @@ +include("test/compiler/irutils.jl") + +_mulfunc(a, b) = @noinline a * b +mulfunc(a, b) = _mulfunc(a, b) +function parentfunc(a::Int, b::Int) + v = mulfunc(a, b) + return v +end +# check if callsite `@noinline` annotation works +interp = Core.Compiler.NativeInterpreter() +# ir, = Base.code_ircode(; optimize_until="inlining", interp) do a::Int, b::Int +# mulfunc(a, b) +# end |> only +ir, = only(Base.code_ircode(parentfunc, (Int,Int); optimize_until="inlining", interp)) + +i = findfirst(isinvoke(:*), ir.stmts.stmt) +@test i !== nothing +# now delete the callsite flag, and see the second inlining pass can inline the call +@eval Core.Compiler $ir.stmts[$i][:flag] &= ~IR_FLAG_NOINLINE +inlining = Core.Compiler.InliningState(interp) +ir2 = Core.Compiler.ssa_inlining_pass!(CC.copy(ir), inlining, false) +@test count(isinvoke(:*), ir2.stmts.stmt) == 0 +@test count(iscall((ir2, Core.Intrinsics.mul_int)), ir2.stmts.stmt) == 1 diff --git a/test/compiler/inline.jl b/test/compiler/inline.jl index 2f916c375218a..3c408b06a87ae 100644 --- a/test/compiler/inline.jl +++ b/test/compiler/inline.jl @@ -1818,23 +1818,55 @@ let src = code_typed1((AtomicMemoryRef{Int},)) do a @test count(isinvokemodify(:+), src.code) == 1 end - - # apply `ssa_inlining_pass` multiple times -let interp = Core.Compiler.NativeInterpreter() +func_mul_int(a::Int, b::Int) = Core.Intrinsics.mul_int(a, b) +multi_inlining1(a::Int, b::Int) = @noinline func_mul_int(a, b) +let i::Int, continue_::Bool + interp = Core.Compiler.NativeInterpreter() # check if callsite `@noinline` annotation works - ir, = Base.code_ircode((Int,Int); optimize_until="inlining", interp) do a, b - @noinline a*b - end |> only - i = findfirst(isinvoke(:*), ir.stmts.stmt) + ir, = only(Base.code_ircode(multi_inlining1, (Int,Int); optimize_until="inlining", interp)) + i = findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) @test i !== nothing - - # ok, now delete the callsite flag, and see the second inlining pass can inline the call + # now delete the callsite flag, and see the second inlining pass can inline the call @eval Core.Compiler $ir.stmts[$i][:flag] &= ~IR_FLAG_NOINLINE inlining = Core.Compiler.InliningState(interp) ir = Core.Compiler.ssa_inlining_pass!(ir, inlining, false) - @test count(isinvoke(:*), ir.stmts.stmt) == 0 - @test count(iscall((ir, Core.Intrinsics.mul_int)), ir.stmts.stmt) == 1 + @test findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) === nothing + @test (i = findfirst(iscall((ir, Core.Intrinsics.mul_int)), ir.stmts.stmt)) !== nothing + lins = Base.IRShow.buildLineInfoNode(ir.debuginfo, nothing, i) + @test (continue_ = length(lins) == 2) # :multi_inlining1 -> :func_mul_int + if continue_ + def1 = lins[1].method + @test def1 isa Core.MethodInstance && def1.def.name === :multi_inlining1 + def2 = lins[2].method + @test def2 isa Core.MethodInstance && def2.def.name === :func_mul_int + end +end + +call_func_mul_int(a::Int, b::Int) = @noinline func_mul_int(a, b) +multi_inlining2(a::Int, b::Int) = call_func_mul_int(a, b) +let i::Int, continue_::Bool + interp = Core.Compiler.NativeInterpreter() + # check if callsite `@noinline` annotation works + ir, = only(Base.code_ircode(multi_inlining2, (Int,Int); optimize_until="inlining", interp)) + i = findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) + @test i !== nothing + # now delete the callsite flag, and see the second inlining pass can inline the call + @eval Core.Compiler $ir.stmts[$i][:flag] &= ~IR_FLAG_NOINLINE + inlining = Core.Compiler.InliningState(interp) + ir = Core.Compiler.ssa_inlining_pass!(ir, inlining, false) + @test findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) === nothing + @test (i = findfirst(iscall((ir, Core.Intrinsics.mul_int)), ir.stmts.stmt)) !== nothing + lins = Base.IRShow.buildLineInfoNode(ir.debuginfo, nothing, i) + @test_broken (continue_ = length(lins) == 3) # see TODO in `ir_inline_linetable!` + if continue_ + def1 = lins[1].method + @test def1 isa Core.MethodInstance && def1.def.name === :multi_inlining2 + def2 = lins[2].method + @test def2 isa Core.MethodInstance && def2.def.name === :call_func_mul_int + def3 = lins[3].method + @test def3 isa Core.MethodInstance && def3.def.name === :call_func_mul_int + end end # Test special purpose inliner for Core.ifelse