Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

optimizer: allow multiple inlining again #53911

Merged
merged 1 commit into from
Apr 1, 2024
Merged

optimizer: allow multiple inlining again #53911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 8 additions & 10 deletions base/compiler/ssair/inlining.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,8 @@ function finish_cfg_inline!(state::CFGInliningState)
end
end

function ir_inline_linetable!(debuginfo::DebugInfoStream, inlinee_debuginfo::DebugInfo, inlinee::MethodInstance)
# TODO append `inlinee_debuginfo` to inner linetable when `inlined_at[2] ≠ 0`
function ir_inline_linetable!(debuginfo::DebugInfoStream, inlinee_debuginfo::DebugInfo, inlined_at::NTuple{3,Int32})
# Append the linetable of the inlined function to our edges table
linetable_offset = 1
while true
Expand All @@ -312,16 +313,14 @@ function ir_inline_linetable!(debuginfo::DebugInfoStream, inlinee_debuginfo::Deb
end
linetable_offset += 1
end
return Int32(linetable_offset)
return (inlined_at[1], Int32(linetable_offset), Int32(0))
end

function ir_prepare_inlining!(insert_node!::Inserter, inline_target::Union{IRCode, IncrementalCompact},
ir::IRCode, di::DebugInfo, mi::MethodInstance, inlined_at::Int32, argexprs::Vector{Any})
ir::IRCode, di::DebugInfo, mi::MethodInstance, inlined_at::NTuple{3,Int32}, argexprs::Vector{Any})
def = mi.def::Method
debuginfo = inline_target isa IRCode ? inline_target.debuginfo : inline_target.ir.debuginfo

linetable_offset = ir_inline_linetable!(debuginfo, di, mi)
topline = (inlined_at, linetable_offset, Int32(0))
topline = new_inlined_at = ir_inline_linetable!(debuginfo, di, inlined_at)
if should_insert_coverage(def.module, di)
insert_node!(NewInstruction(Expr(:code_coverage_effect), Nothing, topline))
end
Expand All @@ -343,7 +342,7 @@ function ir_prepare_inlining!(insert_node!::Inserter, inline_target::Union{IRCod
NewInstruction(Expr(:call, GlobalRef(Core, :getfield), argexprs[1], QuoteNode(:captures)),
ir.argtypes[1], topline))
end
return SSASubstitute(mi, argexprs, spvals_ssa, (inlined_at, linetable_offset))
return SSASubstitute(mi, argexprs, spvals_ssa, new_inlined_at)
end

function adjust_boundscheck!(inline_compact::IncrementalCompact, idx′::Int, stmt::Expr, boundscheck::Symbol)
Expand All @@ -359,8 +358,7 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector
item::InliningTodo, boundscheck::Symbol, todo_bbs::Vector{Tuple{Int, Int}})
# Ok, do the inlining here
inlined_at = compact.result[idx][:line]
@assert inlined_at[2] == 0 "already inlined this instruction"
ssa_substitute = ir_prepare_inlining!(InsertHere(compact), compact, item.ir, item.di, item.mi, inlined_at[1], argexprs)
ssa_substitute = ir_prepare_inlining!(InsertHere(compact), compact, item.ir, item.di, item.mi, inlined_at, argexprs)
boundscheck = has_flag(compact.result[idx], IR_FLAG_INBOUNDS) ? :off : boundscheck

# If the iterator already moved on to the next basic block,
Expand Down Expand Up @@ -1771,7 +1769,7 @@ struct SSASubstitute
mi::MethodInstance
arg_replacements::Vector{Any}
spvals_ssa::Union{Nothing,SSAValue}
inlined_at::NTuple{2,Int32} # TODO: add a map also, so that ssaidx doesn't need to equal inlined_idx?
inlined_at::NTuple{3,Int32} # TODO: add a map also, so that ssaidx doesn't need to equal inlined_idx?
end

function insert_spval!(insert_node!::Inserter, spvals_ssa::SSAValue, spidx::Int, do_isdefined::Bool)
Expand Down
2 changes: 1 addition & 1 deletion base/compiler/ssair/passes.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1516,7 +1516,7 @@ function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int,
add_inlining_backedge!(et, mi)

# TODO: Should there be a special line number node for inlined finalizers?
inline_at = ir[SSAValue(idx)][:line][1]
inline_at = ir[SSAValue(idx)][:line]
ssa_substitute = ir_prepare_inlining!(InsertBefore(ir, SSAValue(idx)), ir, src, di, mi, inline_at, argexprs)

# TODO: Use the actual inliner here rather than open coding this special purpose inliner.
Expand Down
54 changes: 43 additions & 11 deletions test/compiler/inline.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1818,23 +1818,55 @@ let src = code_typed1((AtomicMemoryRef{Int},)) do a
@test count(isinvokemodify(:+), src.code) == 1
end



# apply `ssa_inlining_pass` multiple times
let interp = Core.Compiler.NativeInterpreter()
func_mul_int(a::Int, b::Int) = Core.Intrinsics.mul_int(a, b)
multi_inlining1(a::Int, b::Int) = @noinline func_mul_int(a, b)
let i::Int, continue_::Bool
interp = Core.Compiler.NativeInterpreter()
# check if callsite `@noinline` annotation works
ir, = Base.code_ircode((Int,Int); optimize_until="inlining", interp) do a, b
@noinline a*b
end |> only
i = findfirst(isinvoke(:*), ir.stmts.stmt)
ir, = only(Base.code_ircode(multi_inlining1, (Int,Int); optimize_until="inlining", interp))
i = findfirst(isinvoke(:func_mul_int), ir.stmts.stmt)
@test i !== nothing

# ok, now delete the callsite flag, and see the second inlining pass can inline the call
# now delete the callsite flag, and see the second inlining pass can inline the call
@eval Core.Compiler $ir.stmts[$i][:flag] &= ~IR_FLAG_NOINLINE
inlining = Core.Compiler.InliningState(interp)
ir = Core.Compiler.ssa_inlining_pass!(ir, inlining, false)
@test count(isinvoke(:*), ir.stmts.stmt) == 0
@test count(iscall((ir, Core.Intrinsics.mul_int)), ir.stmts.stmt) == 1
@test findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) === nothing
@test (i = findfirst(iscall((ir, Core.Intrinsics.mul_int)), ir.stmts.stmt)) !== nothing
lins = Base.IRShow.buildLineInfoNode(ir.debuginfo, nothing, i)
@test (continue_ = length(lins) == 2) # :multi_inlining1 -> :func_mul_int
if continue_
def1 = lins[1].method
@test def1 isa Core.MethodInstance && def1.def.name === :multi_inlining1
def2 = lins[2].method
@test def2 isa Core.MethodInstance && def2.def.name === :func_mul_int
end
end

call_func_mul_int(a::Int, b::Int) = @noinline func_mul_int(a, b)
multi_inlining2(a::Int, b::Int) = call_func_mul_int(a, b)
let i::Int, continue_::Bool
interp = Core.Compiler.NativeInterpreter()
# check if callsite `@noinline` annotation works
ir, = only(Base.code_ircode(multi_inlining2, (Int,Int); optimize_until="inlining", interp))
i = findfirst(isinvoke(:func_mul_int), ir.stmts.stmt)
@test i !== nothing
# now delete the callsite flag, and see the second inlining pass can inline the call
@eval Core.Compiler $ir.stmts[$i][:flag] &= ~IR_FLAG_NOINLINE
inlining = Core.Compiler.InliningState(interp)
ir = Core.Compiler.ssa_inlining_pass!(ir, inlining, false)
@test findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) === nothing
@test (i = findfirst(iscall((ir, Core.Intrinsics.mul_int)), ir.stmts.stmt)) !== nothing
lins = Base.IRShow.buildLineInfoNode(ir.debuginfo, nothing, i)
@test_broken (continue_ = length(lins) == 3) # see TODO in `ir_inline_linetable!`
if continue_
def1 = lins[1].method
@test def1 isa Core.MethodInstance && def1.def.name === :multi_inlining2
def2 = lins[2].method
@test def2 isa Core.MethodInstance && def2.def.name === :call_func_mul_int
def3 = lins[3].method
@test def3 isa Core.MethodInstance && def3.def.name === :call_func_mul_int
end
end

# Test special purpose inliner for Core.ifelse
Expand Down