diff --git a/base/atomics.jl b/base/atomics.jl index 97405d88fd408..e6d62c3fc807b 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -356,13 +356,13 @@ for typ in atomictypes rt = "$lt, $lt*" irt = "$ilt, $ilt*" @eval getindex(x::Atomic{$typ}) = - llvmcall($""" + GC.@preserve x llvmcall($""" %ptr = inttoptr i$WORD_SIZE %0 to $lt* %rv = load atomic $rt %ptr acquire, align $(gc_alignment(typ)) ret $lt %rv """, $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x)) @eval setindex!(x::Atomic{$typ}, v::$typ) = - llvmcall($""" + GC.@preserve x llvmcall($""" %ptr = inttoptr i$WORD_SIZE %0 to $lt* store atomic $lt %1, $lt* %ptr release, align $(gc_alignment(typ)) ret void @@ -371,7 +371,7 @@ for typ in atomictypes # Note: atomic_cas! succeeded (i.e. it stored "new") if and only if the result is "cmp" if typ <: Integer @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = - llvmcall($""" + GC.@preserve x llvmcall($""" %ptr = inttoptr i$WORD_SIZE %0 to $lt* %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 acq_rel acquire %rv = extractvalue { $lt, i1 } %rs, 0 @@ -380,7 +380,7 @@ for typ in atomictypes unsafe_convert(Ptr{$typ}, x), cmp, new) else @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = - llvmcall($""" + GC.@preserve x llvmcall($""" %iptr = inttoptr i$WORD_SIZE %0 to $ilt* %icmp = bitcast $lt %1 to $ilt %inew = bitcast $lt %2 to $ilt @@ -403,7 +403,7 @@ for typ in atomictypes if rmwop in arithmetic_ops && !(typ <: ArithmeticTypes) continue end if typ <: Integer @eval $fn(x::Atomic{$typ}, v::$typ) = - llvmcall($""" + GC.@preserve x llvmcall($""" %ptr = inttoptr i$WORD_SIZE %0 to $lt* %rv = atomicrmw $rmw $lt* %ptr, $lt %1 acq_rel ret $lt %rv @@ -411,7 +411,7 @@ for typ in atomictypes else rmwop === :xchg || continue @eval $fn(x::Atomic{$typ}, v::$typ) = - llvmcall($""" + GC.@preserve x llvmcall($""" %iptr = inttoptr i$WORD_SIZE %0 to $ilt* %ival = bitcast $lt %1 to $ilt %irv = atomicrmw $rmw $ilt* %iptr, $ilt %ival acq_rel diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index ede05572edc3a..d37ad96adfefb 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1249,6 +1249,8 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), return abstract_apply(interp, argtypes, sv, max_methods) elseif f === invoke return abstract_invoke(interp, argtypes, sv) + elseif f === modifyfield! + return abstract_modifyfield!(interp, argtypes, sv) end return CallMeta(abstract_call_builtin(interp, f, fargs, argtypes, sv, max_methods), false) elseif f === Core.kwfunc @@ -1515,7 +1517,8 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), return abstract_eval_special_value(interp, e, vtypes, sv) end e = e::Expr - if e.head === :call + ehead = e.head + if ehead === :call ea = e.args argtypes = collect_argtypes(interp, ea, vtypes, sv) if argtypes === nothing @@ -1525,7 +1528,7 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), sv.stmt_info[sv.currpc] = callinfo.info t = callinfo.rt end - elseif e.head === :new + elseif ehead === :new t = instanceof_tfunc(abstract_eval_value(interp, e.args[1], vtypes, sv))[1] if isconcretetype(t) && !ismutabletype(t) args = Vector{Any}(undef, length(e.args)-1) @@ -1562,7 +1565,7 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), end end end - elseif e.head === :splatnew + elseif ehead === :splatnew t = instanceof_tfunc(abstract_eval_value(interp, e.args[1], vtypes, sv))[1] if length(e.args) == 2 && isconcretetype(t) && !ismutabletype(t) at = abstract_eval_value(interp, e.args[2], vtypes, sv) @@ -1575,7 +1578,7 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), t = PartialStruct(t, at.fields::Vector{Any}) end end - elseif e.head === :new_opaque_closure + elseif ehead === :new_opaque_closure t = Union{} if length(e.args) >= 5 ea = e.args @@ -1594,7 +1597,7 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), end end end - elseif e.head === :foreigncall + elseif ehead === :foreigncall abstract_eval_value(interp, e.args[1], vtypes, sv) t = sp_type_rewrap(e.args[2], sv.linfo, true) for i = 3:length(e.args) @@ -1602,21 +1605,21 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), t = Bottom end end - elseif e.head === :cfunction + elseif ehead === :cfunction t = e.args[1] isa(t, Type) || (t = Any) abstract_eval_cfunction(interp, e, vtypes, sv) - elseif e.head === :method + elseif ehead === :method t = (length(e.args) == 1) ? Any : Nothing - elseif e.head === :copyast + elseif ehead === :copyast t = abstract_eval_value(interp, e.args[1], vtypes, sv) if t isa Const && t.val isa Expr # `copyast` makes copies of Exprs t = Expr end - elseif e.head === :invoke + elseif ehead === :invoke || ehead === :invoke_modify error("type inference data-flow error: tried to double infer a function") - elseif e.head === :isdefined + elseif ehead === :isdefined sym = e.args[1] t = Bool if isa(sym, SlotNumber) diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 09c778d57d695..1ffb9aa90f36a 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -491,7 +491,7 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp return 0 end return error_path ? params.inline_error_path_cost : params.inline_nonleaf_penalty - elseif head === :foreigncall || head === :invoke + elseif head === :foreigncall || head === :invoke || head == :invoke_modify # Calls whose "return type" is Union{} do not actually return: # they are errors. Since these are not part of the typical # run-time of the function, we omit them from diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index c76e74e50c6d0..35eb4a1631dc4 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -1141,6 +1141,22 @@ function process_simple!(ir::IRCode, todo::Vector{Pair{Int, Any}}, idx::Int, sta ir.stmts[idx][:inst] = res return nothing end + if (sig.f === modifyfield! || sig.ft ⊑ typeof(modifyfield!)) && 5 <= length(stmt.args) <= 6 + let info = ir.stmts[idx][:info] + info isa MethodResultPure && (info = info.info) + info isa ConstCallInfo && (info = info.call) + info isa MethodMatchInfo || return nothing + length(info.results) == 1 || return nothing + match = info.results[1]::MethodMatch + match.fully_covers || return nothing + case = compileable_specialization(state.et, match) + case === nothing && return nothing + stmt.head = :invoke_modify + pushfirst!(stmt.args, case) + ir.stmts[idx][:inst] = stmt + end + return nothing + end check_effect_free!(ir, stmt, calltype, idx) diff --git a/base/compiler/ssair/ir.jl b/base/compiler/ssair/ir.jl index bc268d33b1a30..a2eaf5c69cbdd 100644 --- a/base/compiler/ssair/ir.jl +++ b/base/compiler/ssair/ir.jl @@ -403,7 +403,8 @@ function getindex(x::UseRef) end function is_relevant_expr(e::Expr) - return e.head in (:call, :invoke, :new, :splatnew, :(=), :(&), + return e.head in (:call, :invoke, :invoke_modify, + :new, :splatnew, :(=), :(&), :gc_preserve_begin, :gc_preserve_end, :foreigncall, :isdefined, :copyast, :undefcheck, :throw_undef_if_not, diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index ba9bc34660485..ba83516ef2d8f 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -939,10 +939,40 @@ function modifyfield!_tfunc(o, f, op, v) @nospecialize T = _fieldtype_tfunc(o, isconcretetype(o), f) T === Bottom && return Bottom - # note: we could sometimes refine this to a PartialStruct if we analyzed `op(o.f, v)::T` PT = Const(Pair) return instanceof_tfunc(apply_type_tfunc(PT, T, T))[1] end +function abstract_modifyfield!(interp::AbstractInterpreter, argtypes::Vector{Any}, sv::InferenceState) + nargs = length(argtypes) + if !isempty(argtypes) && isvarargtype(argtypes[nargs]) + nargs - 1 <= 6 || return CallMeta(Bottom, false) + nargs > 3 || return CallMeta(Any, false) + else + 5 <= nargs <= 6 || return CallMeta(Bottom, false) + end + o = unwrapva(argtypes[2]) + f = unwrapva(argtypes[3]) + RT = modifyfield!_tfunc(o, f, Any, Any) + info = false + if nargs >= 5 && RT !== Bottom + # we may be able to refine this to a PartialStruct by analyzing `op(o.f, v)::T` + # as well as compute the info for the method matches + op = unwrapva(argtypes[4]) + v = unwrapva(argtypes[5]) + TF = getfield_tfunc(o, f) + push!(sv.ssavalue_uses[sv.currpc], sv.currpc) # temporarily disable `call_result_unused` check for this call + callinfo = abstract_call(interp, nothing, Any[op, TF, v], sv, #=max_methods=# 1) + pop!(sv.ssavalue_uses[sv.currpc], sv.currpc) + TF2 = tmeet(callinfo.rt, widenconst(TF)) + if TF2 === Bottom + RT = Bottom + elseif isconcretetype(RT) && has_nontrivial_const_info(TF2) # isconcrete condition required to form a PartialStruct + RT = PartialStruct(RT, Any[TF, TF2]) + end + info = callinfo.info + end + return CallMeta(RT, info) +end replacefield!_tfunc(o, f, x, v, success_order, failure_order) = (@nospecialize; replacefield!_tfunc(o, f, x, v)) replacefield!_tfunc(o, f, x, v, success_order) = (@nospecialize; replacefield!_tfunc(o, f, x, v)) function replacefield!_tfunc(o, f, x, v) diff --git a/base/compiler/validation.jl b/base/compiler/validation.jl index 6e05c96cd7936..b7e63900a3fdf 100644 --- a/base/compiler/validation.jl +++ b/base/compiler/validation.jl @@ -4,6 +4,7 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}( :call => 1:typemax(Int), :invoke => 2:typemax(Int), + :invoke_modify => 3:typemax(Int), :static_parameter => 1:1, :(&) => 1:1, :(=) => 2:2, @@ -78,7 +79,7 @@ end function _validate_val!(@nospecialize(x), errors, ssavals::BitSet) if isa(x, Expr) - if x.head === :call || x.head === :invoke + if x.head === :call || x.head === :invoke || x.head === :invoke_modify f = x.args[1] if f isa GlobalRef && (f.name === :cglobal) && x.head === :call # TODO: these are not yet linearized @@ -138,7 +139,8 @@ function validate_code!(errors::Vector{>:InvalidCodeError}, c::CodeInfo, is_top_ end validate_val!(lhs) validate_val!(rhs) - elseif head === :call || head === :invoke || head === :gc_preserve_end || head === :meta || + elseif head === :call || head === :invoke || x.head === :invoke_modify || + head === :gc_preserve_end || head === :meta || head === :inbounds || head === :foreigncall || head === :cfunction || head === :const || head === :enter || head === :leave || head === :pop_exception || head === :method || head === :global || head === :static_parameter || @@ -238,7 +240,7 @@ end function is_valid_rvalue(@nospecialize(x)) is_valid_argument(x) && return true - if isa(x, Expr) && x.head in (:new, :splatnew, :the_exception, :isdefined, :call, :invoke, :foreigncall, :cfunction, :gc_preserve_begin, :copyast) + if isa(x, Expr) && x.head in (:new, :splatnew, :the_exception, :isdefined, :call, :invoke, :invoke_modify, :foreigncall, :cfunction, :gc_preserve_begin, :copyast) return true end return false diff --git a/src/ast.c b/src/ast.c index 94bbf48dde17e..c33bf56d91379 100644 --- a/src/ast.c +++ b/src/ast.c @@ -28,6 +28,7 @@ extern "C" { // head symbols for each expression type jl_sym_t *call_sym; jl_sym_t *invoke_sym; +jl_sym_t *invoke_modify_sym; jl_sym_t *empty_sym; jl_sym_t *top_sym; jl_sym_t *module_sym; jl_sym_t *slot_sym; jl_sym_t *export_sym; jl_sym_t *import_sym; @@ -345,6 +346,7 @@ void jl_init_common_symbols(void) empty_sym = jl_symbol(""); call_sym = jl_symbol("call"); invoke_sym = jl_symbol("invoke"); + invoke_modify_sym = jl_symbol("invoke_modify"); foreigncall_sym = jl_symbol("foreigncall"); cfunction_sym = jl_symbol("cfunction"); quote_sym = jl_symbol("quote"); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index b2f8c88520b87..4e9917b6b0780 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1547,17 +1547,23 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Value *parent, // for the write barrier, NULL if no barrier needed bool isboxed, AtomicOrdering Order, AtomicOrdering FailOrder, unsigned alignment, bool needlock, bool issetfield, bool isreplacefield, bool isswapfield, bool ismodifyfield, - bool maybe_null_if_boxed, const std::string &fname) + bool maybe_null_if_boxed, const jl_cgval_t *modifyop, const std::string &fname) { auto newval = [&](const jl_cgval_t &lhs) { - jl_cgval_t argv[3] = { cmp, lhs, rhs }; - Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, JLCALL_F_CC); - argv[0] = mark_julia_type(ctx, callval, true, jl_any_type); - if (!jl_subtype(argv[0].typ, jltype)) { - emit_typecheck(ctx, argv[0], jltype, fname + "typed_store"); - argv[0] = update_julia_type(ctx, argv[0], jltype); - } - return argv[0]; + const jl_cgval_t argv[3] = { cmp, lhs, rhs }; + jl_cgval_t ret; + if (modifyop) { + ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + } + else { + Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, JLCALL_F_CC); + ret = mark_julia_type(ctx, callval, true, jl_any_type); + } + if (!jl_subtype(ret.typ, jltype)) { + emit_typecheck(ctx, ret, jltype, fname + "typed_store"); + ret = update_julia_type(ctx, ret, jltype); + } + return ret; }; assert(!needlock || parent != nullptr); Type *elty = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, jltype); @@ -1570,7 +1576,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, else if (isreplacefield) { Value *Success = emit_f_is(ctx, cmp, ghostValue(jltype)); Success = ctx.builder.CreateZExt(Success, T_int8); - jl_cgval_t argv[2] = {ghostValue(jltype), mark_julia_type(ctx, Success, false, jl_bool_type)}; + const jl_cgval_t argv[2] = {ghostValue(jltype), mark_julia_type(ctx, Success, false, jl_bool_type)}; jl_datatype_t *rettyp = jl_apply_cmpswap_type(jltype); return emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); } @@ -1579,7 +1585,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } else { // modifyfield jl_cgval_t oldval = ghostValue(jltype); - jl_cgval_t argv[2] = { oldval, newval(oldval) }; + const jl_cgval_t argv[2] = { oldval, newval(oldval) }; jl_datatype_t *rettyp = jl_apply_modify_type(jltype); return emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); } @@ -1862,7 +1868,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } } if (ismodifyfield) { - jl_cgval_t argv[2] = { oldval, rhs }; + const jl_cgval_t argv[2] = { oldval, rhs }; jl_datatype_t *rettyp = jl_apply_modify_type(jltype); oldval = emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); } @@ -1881,7 +1887,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, oldval = mark_julia_type(ctx, instr, isboxed, jltype); if (isreplacefield) { Success = ctx.builder.CreateZExt(Success, T_int8); - jl_cgval_t argv[2] = {oldval, mark_julia_type(ctx, Success, false, jl_bool_type)}; + const jl_cgval_t argv[2] = {oldval, mark_julia_type(ctx, Success, false, jl_bool_type)}; jl_datatype_t *rettyp = jl_apply_cmpswap_type(jltype); oldval = emit_new_struct(ctx, (jl_value_t*)rettyp, 2, argv); } @@ -3269,7 +3275,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, jl_cgval_t rhs, jl_cgval_t cmp, bool checked, bool wb, AtomicOrdering Order, AtomicOrdering FailOrder, bool needlock, bool issetfield, bool isreplacefield, bool isswapfield, bool ismodifyfield, - const std::string &fname) + const jl_cgval_t *modifyop, const std::string &fname) { if (!sty->name->mutabl && checked) { std::string msg = fname + "immutable struct of type " @@ -3309,9 +3315,14 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, if (ismodifyfield) { if (needlock) emit_lockstate_value(ctx, strct, false); - jl_cgval_t argv[3] = { cmp, oldval, rhs }; - Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, JLCALL_F_CC); - rhs = mark_julia_type(ctx, callval, true, jl_any_type); + const jl_cgval_t argv[3] = { cmp, oldval, rhs }; + if (modifyop) { + rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + } + else { + Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, JLCALL_F_CC); + rhs = mark_julia_type(ctx, callval, true, jl_any_type); + } if (!jl_subtype(rhs.typ, jfty)) { emit_typecheck(ctx, rhs, jfty, fname); rhs = update_julia_type(ctx, rhs, jfty); @@ -3364,7 +3375,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, return typed_store(ctx, addr, NULL, rhs, cmp, jfty, strct.tbaa, nullptr, wb ? maybe_bitcast(ctx, data_pointer(ctx, strct), T_pjlvalue) : nullptr, isboxed, Order, FailOrder, align, - needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, maybe_null, fname); + needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, maybe_null, modifyop, fname); } } @@ -3543,7 +3554,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg else need_wb = false; emit_typecheck(ctx, rhs, jl_svecref(sty->types, i), "new"); - emit_setfield(ctx, sty, strctinfo, i, rhs, jl_cgval_t(), false, need_wb, AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic, false, true, false, false, false, ""); + emit_setfield(ctx, sty, strctinfo, i, rhs, jl_cgval_t(), false, need_wb, AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic, false, true, false, false, false, nullptr, ""); } return strctinfo; } diff --git a/src/codegen.cpp b/src/codegen.cpp index e0b86c51bf7d1..5cc9f66ffaeff 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1165,12 +1165,13 @@ static Value *get_current_ptls(jl_codectx_t &ctx); static Value *get_current_signal_page(jl_codectx_t &ctx); static void CreateTrap(IRBuilder<> &irbuilder, bool create_new_block = true); static CallInst *emit_jlcall(jl_codectx_t &ctx, Function *theFptr, Value *theF, - jl_cgval_t *args, size_t nargs, CallingConv::ID cc); + const jl_cgval_t *args, size_t nargs, CallingConv::ID cc); static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction *theFptr, Value *theF, - jl_cgval_t *args, size_t nargs, CallingConv::ID cc); + const jl_cgval_t *args, size_t nargs, CallingConv::ID cc); static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2, Value *nullcheck1 = nullptr, Value *nullcheck2 = nullptr); static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, const jl_cgval_t *argv); +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const jl_cgval_t *argv, size_t nargs, jl_value_t *rt); static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G); @@ -2679,6 +2680,102 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva return emit_box_compare(ctx, arg1, arg2, nullcheck1, nullcheck2); } +static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, + const jl_cgval_t *argv, size_t nargs, const jl_cgval_t *modifyop) +{ + bool issetfield = f == jl_builtin_setfield; + bool isreplacefield = f == jl_builtin_replacefield; + bool isswapfield = f == jl_builtin_swapfield; + bool ismodifyfield = f == jl_builtin_modifyfield; + const jl_cgval_t undefval; + const jl_cgval_t &obj = argv[1]; + const jl_cgval_t &fld = argv[2]; + jl_cgval_t val = argv[isreplacefield || ismodifyfield ? 4 : 3]; + const jl_cgval_t &cmp = isreplacefield || ismodifyfield ? argv[3] : undefval; + enum jl_memory_order order = jl_memory_order_notatomic; + const std::string fname = issetfield ? "setfield!" : isreplacefield ? "replacefield!" : isswapfield ? "swapfield!" : "modifyfield!"; + if (nargs >= (isreplacefield || ismodifyfield ? 5 : 4)) { + const jl_cgval_t &ord = argv[isreplacefield || ismodifyfield ? 5 : 4]; + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); + if (!ord.constant) + return false; + order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetfield, true); + } + enum jl_memory_order fail_order = order; + if (isreplacefield && nargs == 6) { + const jl_cgval_t &ord = argv[6]; + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); + if (!ord.constant) + return false; + fail_order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); + } + if (order == jl_memory_order_invalid || fail_order == jl_memory_order_invalid || fail_order > order) { + emit_atomic_error(ctx, "invalid atomic ordering"); + *ret = jl_cgval_t(); // unreachable + return true; + } + + jl_datatype_t *uty = (jl_datatype_t*)jl_unwrap_unionall(obj.typ); + if (jl_is_datatype(uty) && jl_struct_try_layout(uty)) { + ssize_t idx = -1; + if (fld.constant && fld.typ == (jl_value_t*)jl_symbol_type) { + idx = jl_field_index(uty, (jl_sym_t*)fld.constant, 0); + } + else if (fld.constant && fld.typ == (jl_value_t*)jl_long_type) { + ssize_t i = jl_unbox_long(fld.constant); + if (i > 0 && i <= jl_datatype_nfields(uty)) + idx = i - 1; + } + if (idx != -1) { + jl_value_t *ft = jl_svecref(uty->types, idx); + if (!jl_has_free_typevars(ft)) { + if (!ismodifyfield && !jl_subtype(val.typ, ft)) { + emit_typecheck(ctx, val, ft, fname); + val = update_julia_type(ctx, val, ft); + } + // TODO: attempt better codegen for approximate types + bool isboxed = jl_field_isptr(uty, idx); + bool isatomic = jl_field_isatomic(uty, idx); + bool needlock = isatomic && !isboxed && jl_datatype_size(jl_field_type(uty, idx)) > MAX_ATOMIC_SIZE; + if (isatomic == (order == jl_memory_order_notatomic)) { + emit_atomic_error(ctx, + issetfield ? + (isatomic ? "setfield!: atomic field cannot be written non-atomically" + : "setfield!: non-atomic field cannot be written atomically") : + isreplacefield ? + (isatomic ? "replacefield!: atomic field cannot be written non-atomically" + : "replacefield!: non-atomic field cannot be written atomically") : + isswapfield ? + (isatomic ? "swapfield!: atomic field cannot be written non-atomically" + : "swapfield!: non-atomic field cannot be written atomically") : + (isatomic ? "modifyfield!: atomic field cannot be written non-atomically" + : "modifyfield!: non-atomic field cannot be written atomically")); + *ret = jl_cgval_t(); + return true; + } + if (isatomic == (fail_order == jl_memory_order_notatomic)) { + emit_atomic_error(ctx, + (isatomic ? "replacefield!: atomic field cannot be accessed non-atomically" + : "replacefield!: non-atomic field cannot be accessed atomically")); + *ret = jl_cgval_t(); + return true; + } + *ret = emit_setfield(ctx, uty, obj, idx, val, cmp, true, true, + (needlock || order <= jl_memory_order_notatomic) + ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 + : get_llvm_atomic_order(order), + (needlock || fail_order <= jl_memory_order_notatomic) + ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 + : get_llvm_atomic_order(fail_order), + needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, + modifyop, fname); + return true; + } + } + } + return false; +} + static std::pair, jl_llvm_functions_t> emit_function( jl_method_instance_t *lam, @@ -3008,6 +3105,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, false, false, false, + nullptr, ""); } } @@ -3151,97 +3249,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if ((f == jl_builtin_setfield && (nargs == 3 || nargs == 4)) || (f == jl_builtin_swapfield && (nargs == 3 || nargs == 4)) || (f == jl_builtin_replacefield && (nargs == 4 || nargs == 5 || nargs == 6)) || - (true && f == jl_builtin_modifyfield && (nargs == 4 || nargs == 5))) { - bool issetfield = f == jl_builtin_setfield; - bool isreplacefield = f == jl_builtin_replacefield; - bool isswapfield = f == jl_builtin_swapfield; - bool ismodifyfield = f == jl_builtin_modifyfield; - const jl_cgval_t undefval; - const jl_cgval_t &obj = argv[1]; - const jl_cgval_t &fld = argv[2]; - jl_cgval_t val = argv[isreplacefield || ismodifyfield ? 4 : 3]; - const jl_cgval_t &cmp = isreplacefield || ismodifyfield ? argv[3] : undefval; - enum jl_memory_order order = jl_memory_order_notatomic; - const std::string fname = issetfield ? "setfield!" : isreplacefield ? "replacefield!" : isswapfield ? "swapfield!" : "modifyfield!"; - if (nargs >= (isreplacefield || ismodifyfield ? 5 : 4)) { - const jl_cgval_t &ord = argv[isreplacefield || ismodifyfield ? 5 : 4]; - emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); - if (!ord.constant) - return false; - order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetfield, true); - } - enum jl_memory_order fail_order = order; - if (isreplacefield && nargs == 6) { - const jl_cgval_t &ord = argv[6]; - emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); - if (!ord.constant) - return false; - fail_order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); - } - if (order == jl_memory_order_invalid || fail_order == jl_memory_order_invalid || fail_order > order) { - emit_atomic_error(ctx, "invalid atomic ordering"); - *ret = jl_cgval_t(); // unreachable - return true; - } - - jl_datatype_t *uty = (jl_datatype_t*)jl_unwrap_unionall(obj.typ); - if (jl_is_datatype(uty) && jl_struct_try_layout(uty)) { - ssize_t idx = -1; - if (fld.constant && fld.typ == (jl_value_t*)jl_symbol_type) { - idx = jl_field_index(uty, (jl_sym_t*)fld.constant, 0); - } - else if (fld.constant && fld.typ == (jl_value_t*)jl_long_type) { - ssize_t i = jl_unbox_long(fld.constant); - if (i > 0 && i <= jl_datatype_nfields(uty)) - idx = i - 1; - } - if (idx != -1) { - jl_value_t *ft = jl_svecref(uty->types, idx); - if (!jl_has_free_typevars(ft)) { - if (!ismodifyfield && !jl_subtype(val.typ, ft)) { - emit_typecheck(ctx, val, ft, fname); - val = update_julia_type(ctx, val, ft); - } - // TODO: attempt better codegen for approximate types - bool isboxed = jl_field_isptr(uty, idx); - bool isatomic = jl_field_isatomic(uty, idx); - bool needlock = isatomic && !isboxed && jl_datatype_size(jl_field_type(uty, idx)) > MAX_ATOMIC_SIZE; - if (isatomic == (order == jl_memory_order_notatomic)) { - emit_atomic_error(ctx, - issetfield ? - (isatomic ? "setfield!: atomic field cannot be written non-atomically" - : "setfield!: non-atomic field cannot be written atomically") : - isreplacefield ? - (isatomic ? "replacefield!: atomic field cannot be written non-atomically" - : "replacefield!: non-atomic field cannot be written atomically") : - isswapfield ? - (isatomic ? "swapfield!: atomic field cannot be written non-atomically" - : "swapfield!: non-atomic field cannot be written atomically") : - (isatomic ? "modifyfield!: atomic field cannot be written non-atomically" - : "modifyfield!: non-atomic field cannot be written atomically")); - *ret = jl_cgval_t(); - return true; - } - if (isatomic == (fail_order == jl_memory_order_notatomic)) { - emit_atomic_error(ctx, - (isatomic ? "replacefield!: atomic field cannot be accessed non-atomically" - : "replacefield!: non-atomic field cannot be accessed atomically")); - *ret = jl_cgval_t(); - return true; - } - *ret = emit_setfield(ctx, uty, obj, idx, val, cmp, true, true, - (needlock || order <= jl_memory_order_notatomic) - ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 - : get_llvm_atomic_order(order), - (needlock || fail_order <= jl_memory_order_notatomic) - ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 - : get_llvm_atomic_order(fail_order), - needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, - fname); - return true; - } - } - } + (f == jl_builtin_modifyfield && (nargs == 4 || nargs == 5))) { + return emit_f_opfield(ctx, ret, f, argv, nargs, nullptr); } else if (f == jl_builtin_nfields && nargs == 1) { @@ -3468,7 +3477,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // Returns T_prjlvalue static CallInst *emit_jlcall(jl_codectx_t &ctx, Function *theFptr, Value *theF, - jl_cgval_t *argv, size_t nargs, CallingConv::ID cc) + const jl_cgval_t *argv, size_t nargs, CallingConv::ID cc) { // emit arguments SmallVector theArgs; @@ -3492,14 +3501,14 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, Function *theFptr, Value *theF, } // Returns T_prjlvalue static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction *theFptr, Value *theF, - jl_cgval_t *argv, size_t nargs, CallingConv::ID cc) + const jl_cgval_t *argv, size_t nargs, CallingConv::ID cc) { return emit_jlcall(ctx, prepare_call(theFptr), theF, argv, nargs, cc); } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_t *mi, jl_value_t *jlretty, StringRef specFunctionObject, - jl_cgval_t *argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) + const jl_cgval_t *argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) { // emit specialized call site bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; @@ -3579,7 +3588,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_ jl_cgval_t retval; switch (returninfo.cc) { case jl_returninfo_t::Boxed: - retval = mark_julia_type(ctx, call, true, inferred_retty); + retval = mark_julia_type(ctx, call, true, jlretty); break; case jl_returninfo_t::Register: retval = mark_julia_type(ctx, call, false, jlretty); @@ -3609,20 +3618,18 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_ break; } // see if inference has a different / better type for the call than the lambda - if (inferred_retty != retval.typ) - retval = update_julia_type(ctx, retval, inferred_retty); - return retval; + return update_julia_type(ctx, retval, inferred_retty); } -static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, StringRef specFunctionObject, - jl_cgval_t *argv, size_t nargs, jl_value_t *inferred_retty) +static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty, StringRef specFunctionObject, + const jl_cgval_t *argv, size_t nargs, jl_value_t *inferred_retty) { auto theFptr = cast( jl_Module->getOrInsertFunction(specFunctionObject, jl_func_sig).getCallee()); add_return_attr(theFptr, Attribute::NonNull); theFptr->addFnAttr(Thunk); Value *ret = emit_jlcall(ctx, theFptr, nullptr, argv, nargs, JLCALL_F_CC); - return mark_julia_type(ctx, ret, true, inferred_retty); + return update_julia_type(ctx, mark_julia_type(ctx, ret, true, jlretty), inferred_retty); } static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) @@ -3639,7 +3646,11 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) if (argv[i].typ == jl_bottom_type) return jl_cgval_t(); } + return emit_invoke(ctx, lival, argv, nargs, rt); +} +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const jl_cgval_t *argv, size_t nargs, jl_value_t *rt) +{ bool handled = false; jl_cgval_t result; if (lival.constant) { @@ -3651,7 +3662,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) FunctionType *ft = ctx.f->getFunctionType(); StringRef protoname = ctx.f->getName(); if (ft == jl_func_sig) { - result = emit_call_specfun_boxed(ctx, protoname, argv, nargs, rt); + result = emit_call_specfun_boxed(ctx, ctx.rettype, protoname, argv, nargs, rt); handled = true; } else if (ft != jl_func_sig_sparams) { @@ -3693,7 +3704,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) if (specsig) result = emit_call_specfun_other(ctx, mi, codeinst->rettype, protoname, argv, nargs, &cc, &return_roots, rt); else - result = emit_call_specfun_boxed(ctx, protoname, argv, nargs, rt); + result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, argv, nargs, rt); handled = true; if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); @@ -3712,6 +3723,40 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) return result; } +static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) +{ + jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); + size_t arglen = jl_array_dim0(ex->args); + size_t nargs = arglen - 1; + assert(arglen >= 2); + jl_cgval_t lival = emit_expr(ctx, args[0]); + jl_cgval_t *argv = (jl_cgval_t*)alloca(sizeof(jl_cgval_t) * nargs); + for (size_t i = 0; i < nargs; ++i) { + argv[i] = emit_expr(ctx, args[i + 1]); + if (argv[i].typ == jl_bottom_type) + return jl_cgval_t(); + } + const jl_cgval_t &f = argv[0]; + jl_cgval_t ret; + if (f.constant && f.constant == jl_builtin_modifyfield) { + if (emit_f_opfield(ctx, &ret, jl_builtin_modifyfield, argv, nargs - 1, &lival)) + return ret; + auto it = builtin_func_map.find(&jl_f_modifyfield); + assert(it != builtin_func_map.end()); + Value *oldnew = emit_jlcall(ctx, it->second, V_rnull, &argv[1], nargs - 1, JLCALL_F_CC); + return mark_julia_type(ctx, oldnew, true, rt); + } + if (f.constant && jl_typeis(f.constant, jl_intrinsic_type)) { + JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(f.constant); + if (fi == JL_I::atomic_pointermodify && jl_intrinsic_nargs((int)fi) == nargs - 1) + return emit_atomic_pointerop(ctx, fi, argv, nargs - 1, &lival); + } + + // emit function and arguments + Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs, JLCALL_F_CC); + return mark_julia_type(ctx, callval, true, rt); +} + static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) { jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); @@ -4558,6 +4603,12 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval) jl_array_ptr_ref(ctx.source->ssavaluetypes, ssaval); return emit_invoke(ctx, ex, expr_t); } + else if (head == invoke_modify_sym) { + assert(ssaval >= 0); + jl_value_t *expr_t = jl_is_long(ctx.source->ssavaluetypes) ? (jl_value_t*)jl_any_type : + jl_array_ptr_ref(ctx.source->ssavaluetypes, ssaval); + return emit_invoke_modify(ctx, ex, expr_t); + } else if (head == call_sym) { jl_value_t *expr_t; if (ssaval < 0) diff --git a/src/dump.c b/src/dump.c index 1cdf0787648c8..afadca3edad2a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -2750,7 +2750,7 @@ void jl_init_serializer(void) htable_new(&backref_table, 0); void *vals[] = { jl_emptysvec, jl_emptytuple, jl_false, jl_true, jl_nothing, jl_any_type, - call_sym, invoke_sym, goto_ifnot_sym, return_sym, jl_symbol("tuple"), + call_sym, invoke_sym, invoke_modify_sym, goto_ifnot_sym, return_sym, jl_symbol("tuple"), jl_an_empty_string, jl_an_empty_vec_any, // empirical list of very common symbols diff --git a/src/interpreter.c b/src/interpreter.c index f999542d68c4f..ea93527d88938 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -217,6 +217,9 @@ static jl_value_t *eval_value(jl_value_t *e, interpreter_state *s) else if (head == invoke_sym) { return do_invoke(args, nargs, s); } + else if (head == invoke_modify_sym) { + return do_call(args + 1, nargs - 1, s); + } else if (head == isdefined_sym) { jl_value_t *sym = args[0]; int defined = 0; diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 7883542c74a13..1847fc5c60e37 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -684,7 +684,7 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv) if (!type_is_ghost(ptrty)) { thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ); typed_store(ctx, thePtr, im1, x, jl_cgval_t(), ety, tbaa_data, nullptr, nullptr, isboxed, - AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic, align_nb, false, true, false, false, false, false, ""); + AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic, align_nb, false, true, false, false, false, false, nullptr, ""); } } return e; @@ -779,7 +779,7 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) // e[i] <= x (swap) // e[i] y => x (replace) // x(e[i], y) (modify) -static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, const jl_cgval_t *argv, int nargs) +static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, const jl_cgval_t *argv, int nargs, const jl_cgval_t *modifyop) { bool issetfield = f == atomic_pointerset; bool isreplacefield = f == atomic_pointerreplace; @@ -817,7 +817,7 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, const jl Value *thePtr = emit_unbox(ctx, T_pprjlvalue, e, e.typ); bool isboxed = true; jl_cgval_t ret = typed_store(ctx, thePtr, nullptr, x, y, ety, tbaa_data, nullptr, nullptr, isboxed, - llvm_order, llvm_failorder, sizeof(jl_value_t*), false, issetfield, isreplacefield, isswapfield, ismodifyfield, false, "atomic_pointermodify"); + llvm_order, llvm_failorder, sizeof(jl_value_t*), false, issetfield, isreplacefield, isswapfield, ismodifyfield, false, modifyop, "atomic_pointermodify"); if (issetfield) ret = e; return ret; @@ -851,7 +851,7 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, const jl assert(!isboxed); Value *thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ); jl_cgval_t ret = typed_store(ctx, thePtr, nullptr, x, y, ety, tbaa_data, nullptr, nullptr, isboxed, - llvm_order, llvm_failorder, nb, false, issetfield, isreplacefield, isswapfield, ismodifyfield, false, "atomic_pointermodify"); + llvm_order, llvm_failorder, nb, false, issetfield, isreplacefield, isswapfield, ismodifyfield, false, modifyop, "atomic_pointermodify"); if (issetfield) ret = e; return ret; @@ -1093,7 +1093,7 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar case atomic_pointerswap: case atomic_pointermodify: case atomic_pointerreplace: - return emit_atomic_pointerop(ctx, f, argv, nargs); + return emit_atomic_pointerop(ctx, f, argv, nargs, nullptr); case bitcast: return generic_bitcast(ctx, argv); case trunc_int: diff --git a/src/julia_internal.h b/src/julia_internal.h index a7943e11a8067..c0d492241cf0d 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1344,6 +1344,7 @@ void jl_log(int level, jl_value_t *module, jl_value_t *group, jl_value_t *id, int isabspath(const char *in) JL_NOTSAFEPOINT; extern jl_sym_t *call_sym; extern jl_sym_t *invoke_sym; +extern jl_sym_t *invoke_modify_sym; extern jl_sym_t *empty_sym; extern jl_sym_t *top_sym; extern jl_sym_t *module_sym; extern jl_sym_t *slot_sym; extern jl_sym_t *export_sym; extern jl_sym_t *import_sym; diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp index 18b54b117c323..f7130f6904479 100644 --- a/src/llvm-alloc-opt.cpp +++ b/src/llvm-alloc-opt.cpp @@ -628,6 +628,21 @@ void Optimizer::checkInst(Instruction *I) use_info.hasunknownmem = true; return true; } + if (isa(inst) || isa(inst)) { + // Only store value count + if (use->getOperandNo() != isa(inst) ? AtomicCmpXchgInst::getPointerOperandIndex() : AtomicRMWInst::getPointerOperandIndex()) { + use_info.escaped = true; + return false; + } + use_info.hasload = true; + auto storev = isa(inst) ? cast(inst)->getNewValOperand() : cast(inst)->getValOperand(); + if (cur.offset == UINT32_MAX || !use_info.addMemOp(inst, use->getOperandNo(), + cur.offset, storev->getType(), + true, *pass.DL)) + use_info.hasunknownmem = true; + use_info.refload = true; + return true; + } if (isa(inst) || isa(inst)) { push_inst(inst); return true; @@ -1331,6 +1346,22 @@ void Optimizer::splitOnStack(CallInst *orig_inst) store->eraseFromParent(); return; } + else if (isa(user) || isa(user)) { + auto slot_idx = find_slot(offset); + auto &slot = slots[slot_idx]; + assert(slot.offset <= offset && slot.offset + slot.size >= offset); + IRBuilder<> builder(user); + Value *newptr; + if (slot.isref) { + assert(slot.offset == offset); + newptr = slot.slot; + } + else { + Value *Val = isa(user) ? cast(user)->getNewValOperand() : cast(user)->getValOperand(); + newptr = slot_gep(slot, offset, Val->getType(), builder); + } + *use = newptr; + } else if (auto call = dyn_cast(user)) { auto callee = call->getCalledOperand(); assert(callee); // makes it clear for clang analyser that `callee` is not NULL