From e26b63cf41cff80783621ed193e794be5978be20 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 5 May 2023 14:35:48 -0400 Subject: [PATCH 1/3] convert some typeof tags to small integers The value here is two-fold. One, we speed up sysimg load times slightly, since we can entirely skip relocations for these values and avoid faulting in some writable pages. This should let us (later) move more content into the ConstData section, such as String objects. Secondly, some type-manipulation native code becomes simpler, since it is based on small consecutive constants instead of requiring extra pointer loads to check for pointer identity. This is similar to the existing small-union optimization, but for type tags, and only for specific ones. This makes use of the fact that the zero page (about 4096 byes) cannot be allocated in the usual C memory model, which lets us define up to 255 of these special types, after taking into account the 4 gc bits. As a current implementation limitation for performance, these cannot be parametric types. Note there are probably more places in Base that can be updated to use `jl_typetagof` instead of `jl_typeof`, where we know if it is a type or tag. For example, this optimize most of builtins.c file, except for the `jl_object_id` function. --- base/boot.jl | 19 +- cli/jl_exports.h | 4 + src/aotcompile.cpp | 18 +- src/array.c | 8 +- src/ast.c | 22 +-- src/builtins.c | 117 +++++++---- src/ccall.cpp | 14 +- src/cgutils.cpp | 212 ++++++++++++-------- src/codegen.cpp | 84 +++++--- src/datatype.c | 125 ++++++------ src/gc-debug.c | 8 +- src/gc.c | 360 ++++++++++++++++++---------------- src/gf.c | 2 +- src/init.c | 76 +------ src/interpreter.c | 6 +- src/intrinsics.cpp | 9 +- src/ircode.c | 35 ++-- src/jl_exported_funcs.inc | 2 +- src/jltypes.c | 144 ++++++++++++-- src/julia.expmap | 4 +- src/julia.h | 194 ++++++++++++------ src/julia_internal.h | 11 +- src/llvm-late-gc-lowering.cpp | 4 +- src/method.c | 12 +- src/module.c | 1 + src/partr.c | 2 +- src/processor.cpp | 2 + src/processor.h | 5 +- src/rtutils.c | 8 +- src/simplevector.c | 4 + src/stackwalk.c | 2 +- src/staticdata.c | 56 +++--- src/staticdata_utils.c | 4 +- src/subtype.c | 4 +- src/symbol.c | 4 +- src/task.c | 2 + src/toplevel.c | 4 +- test/ccall.jl | 2 +- 38 files changed, 937 insertions(+), 653 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index b9b54a8051fb0..9d44b98d46881 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -245,7 +245,6 @@ ccall(:jl_toplevel_eval_in, Any, (Any, Any), (f::typeof(Typeof))(x) = ($(_expr(:meta,:nospecialize,:x)); isa(x,Type) ? Type{x} : typeof(x)) end) - macro nospecialize(x) _expr(:meta, :nospecialize, x) end @@ -256,7 +255,15 @@ TypeVar(n::Symbol, @nospecialize(lb), @nospecialize(ub)) = _typevar(n, lb, ub) UnionAll(v::TypeVar, @nospecialize(t)) = ccall(:jl_type_unionall, Any, (Any, Any), v, t) -const Vararg = ccall(:jl_toplevel_eval_in, Any, (Any, Any), Core, _expr(:new, TypeofVararg)) +# simple convert for use by constructors of types in Core +# note that there is no actual conversion defined here, +# so the methods and ccall's in Core aren't permitted to use convert +convert(::Type{Any}, @nospecialize(x)) = x +convert(::Type{T}, x::T) where {T} = x +cconvert(::Type{T}, x) where {T} = convert(T, x) +unsafe_convert(::Type{T}, x::T) where {T} = x + +const Vararg = ccall(:jl_wrap_vararg, Any, (Int, Int), 0, 0) # dispatch token indicating a kwarg (keyword sorter) call function kwcall end @@ -448,14 +455,6 @@ function _Task(@nospecialize(f), reserved_stack::Int, completion_future) return ccall(:jl_new_task, Ref{Task}, (Any, Any, Int), f, completion_future, reserved_stack) end -# simple convert for use by constructors of types in Core -# note that there is no actual conversion defined here, -# so the methods and ccall's in Core aren't permitted to use convert -convert(::Type{Any}, @nospecialize(x)) = x -convert(::Type{T}, x::T) where {T} = x -cconvert(::Type{T}, x) where {T} = convert(T, x) -unsafe_convert(::Type{T}, x::T) where {T} = x - _is_internal(__module__) = __module__ === Core # can be used in place of `@assume_effects :foldable` (supposed to be used for bootstrapping) macro _foldable_meta() diff --git a/cli/jl_exports.h b/cli/jl_exports.h index e9be7c6f2f819..d28958c097edb 100644 --- a/cli/jl_exports.h +++ b/cli/jl_exports.h @@ -16,6 +16,10 @@ JL_EXPORTED_DATA_POINTERS(XX) JL_EXPORTED_DATA_SYMBOLS(XX) #undef XX +// define a copy of exported data +#define jl_max_tags 64 +JL_DLLEXPORT void *small_typeof[(jl_max_tags << 4) / sizeof(void*)]; // 16-bit aligned, like the GC + // Declare list of exported functions (sans type) #define XX(name) JL_DLLEXPORT void name(void); typedef void (anonfunc)(void); diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index b89cdf550171f..0c2aa4feb678d 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -1575,6 +1575,14 @@ void jl_dump_native_impl(void *native_code, GlobalVariable::ExternalLinkage, jlRTLD_DEFAULT_var, "jl_RTLD_DEFAULT_handle_pointer"), TheTriple); + + // let the compiler know we are going to internalize a copy of this, + // if it has a current usage with ExternalLinkage + auto small_typeof_copy = dataM->getGlobalVariable("small_typeof"); + if (small_typeof_copy) { + small_typeof_copy->setVisibility(GlobalValue::HiddenVisibility); + small_typeof_copy->setDSOLocal(true); + } } // Reserve space for the output files and names @@ -1651,13 +1659,21 @@ void jl_dump_native_impl(void *native_code, auto shards = emit_shard_table(*sysimageM, T_size, T_psize, threads); auto ptls = emit_ptls_table(*sysimageM, T_size, T_psize); auto header = emit_image_header(*sysimageM, threads, nfvars, ngvars); - auto AT = ArrayType::get(T_psize, 4); + auto AT = ArrayType::get(T_size, sizeof(small_typeof) / sizeof(void*)); + auto small_typeof_copy = new GlobalVariable(*sysimageM, AT, false, + GlobalVariable::ExternalLinkage, + Constant::getNullValue(AT), + "small_typeof"); + small_typeof_copy->setVisibility(GlobalValue::HiddenVisibility); + small_typeof_copy->setDSOLocal(true); + AT = ArrayType::get(T_psize, 5); auto pointers = new GlobalVariable(*sysimageM, AT, false, GlobalVariable::ExternalLinkage, ConstantArray::get(AT, { ConstantExpr::getBitCast(header, T_psize), ConstantExpr::getBitCast(shards, T_psize), ConstantExpr::getBitCast(ptls, T_psize), + ConstantExpr::getBitCast(small_typeof_copy, T_psize), ConstantExpr::getBitCast(target_ids, T_psize) }), "jl_image_pointers"); diff --git a/src/array.c b/src/array.c index 0b582296774b5..ae730bed7c4e8 100644 --- a/src/array.c +++ b/src/array.c @@ -509,7 +509,7 @@ JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len) jl_throw(jl_memory_exception); s = jl_gc_big_alloc_noinline(ptls, allocsz); } - jl_set_typeof(s, jl_string_type); + jl_set_typetagof(s, jl_string_tag, 0); maybe_record_alloc_to_profile(s, len, jl_string_type); *(size_t*)s = len; jl_string_data(s)[len] = 0; @@ -1255,7 +1255,7 @@ JL_DLLEXPORT void jl_array_ptr_copy(jl_array_t *dest, void **dest_p, JL_DLLEXPORT void jl_array_ptr_1d_push(jl_array_t *a, jl_value_t *item) { - assert(jl_typeis(a, jl_array_any_type)); + assert(jl_typetagis(a, jl_array_any_type)); jl_array_grow_end(a, 1); size_t n = jl_array_nrows(a); jl_array_ptr_set(a, n - 1, item); @@ -1263,8 +1263,8 @@ JL_DLLEXPORT void jl_array_ptr_1d_push(jl_array_t *a, jl_value_t *item) JL_DLLEXPORT void jl_array_ptr_1d_append(jl_array_t *a, jl_array_t *a2) { - assert(jl_typeis(a, jl_array_any_type)); - assert(jl_typeis(a2, jl_array_any_type)); + assert(jl_typetagis(a, jl_array_any_type)); + assert(jl_typetagis(a2, jl_array_any_type)); size_t i; size_t n = jl_array_nrows(a); size_t n2 = jl_array_nrows(a2); diff --git a/src/ast.c b/src/ast.c index 08c493c75b985..1574f920e06e9 100644 --- a/src/ast.c +++ b/src/ast.c @@ -700,11 +700,11 @@ static value_t julia_to_scm_noalloc(fl_context_t *fl_ctx, jl_value_t *v, int che if (julia_to_scm_noalloc1(fl_ctx, v, &retval)) return retval; assert(!jl_is_expr(v) && - !jl_typeis(v, jl_linenumbernode_type) && - !jl_typeis(v, jl_gotonode_type) && - !jl_typeis(v, jl_quotenode_type) && - !jl_typeis(v, jl_newvarnode_type) && - !jl_typeis(v, jl_globalref_type)); + !jl_typetagis(v, jl_linenumbernode_type) && + !jl_typetagis(v, jl_gotonode_type) && + !jl_typetagis(v, jl_quotenode_type) && + !jl_typetagis(v, jl_newvarnode_type) && + !jl_typetagis(v, jl_globalref_type)); return julia_to_scm_noalloc2(fl_ctx, v, check_valid); } @@ -745,7 +745,7 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali // GC Note: jl_fieldref(v, 0) allocates for GotoNode // but we don't need a GC root here because julia_to_list2_noalloc // shouldn't allocate in this case. - if (jl_typeis(v, jl_linenumbernode_type)) { + if (jl_typetagis(v, jl_linenumbernode_type)) { jl_value_t *file = jl_fieldref_noalloc(v,1); jl_value_t *line = jl_fieldref(v,0); value_t args = julia_to_list2_noalloc(fl_ctx, line, file, check_valid); @@ -755,13 +755,13 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali fl_free_gc_handles(fl_ctx, 1); return scmv; } - if (jl_typeis(v, jl_gotonode_type)) + if (jl_typetagis(v, jl_gotonode_type)) return julia_to_list2_noalloc(fl_ctx, (jl_value_t*)jl_goto_sym, jl_fieldref(v,0), check_valid); - if (jl_typeis(v, jl_quotenode_type)) + if (jl_typetagis(v, jl_quotenode_type)) return julia_to_list2(fl_ctx, (jl_value_t*)jl_inert_sym, jl_fieldref_noalloc(v,0), 0); - if (jl_typeis(v, jl_newvarnode_type)) + if (jl_typetagis(v, jl_newvarnode_type)) return julia_to_list2_noalloc(fl_ctx, (jl_value_t*)jl_newvar_sym, jl_fieldref(v,0), check_valid); - if (jl_typeis(v, jl_globalref_type)) { + if (jl_typetagis(v, jl_globalref_type)) { jl_module_t *m = jl_globalref_mod(v); jl_sym_t *sym = jl_globalref_name(v); if (m == jl_core_module) @@ -1011,7 +1011,7 @@ static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule // __source__ argument jl_value_t *lno = jl_array_ptr_ref(args, 1); margs[1] = lno; - if (!jl_typeis(lno, jl_linenumbernode_type)) { + if (!jl_typetagis(lno, jl_linenumbernode_type)) { margs[1] = jl_new_struct(jl_linenumbernode_type, jl_box_long(0), jl_nothing); } margs[2] = (jl_value_t*)inmodule; diff --git a/src/builtins.c b/src/builtins.c index d423e6c934780..799db9afcf685 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -35,8 +35,8 @@ extern "C" { static int bits_equal(const void *a, const void *b, int sz) JL_NOTSAFEPOINT { switch (sz) { - case 1: return *(int8_t*)a == *(int8_t*)b; - // Let compiler constant folds the following. + case 1: return *(uint8_t*)a == *(uint8_t*)b; + // Let compiler constant folds the following, though we may not know alignment of them case 2: return memcmp(a, b, 2) == 0; case 4: return memcmp(a, b, 4) == 0; case 8: return memcmp(a, b, 8) == 0; @@ -147,10 +147,10 @@ static int egal_types(const jl_value_t *a, const jl_value_t *b, jl_typeenv_t *en { if (a == b) return 1; - jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(a); - if (dt != (jl_datatype_t*)jl_typeof(b)) + uintptr_t dtag = jl_typetagof(a); + if (dtag != jl_typetagof(b)) return 0; - if (dt == jl_datatype_type) { + if (dtag == jl_datatype_tag << 4) { jl_datatype_t *dta = (jl_datatype_t*)a; jl_datatype_t *dtb = (jl_datatype_t*)b; if (dta->name != dtb->name) @@ -164,7 +164,7 @@ static int egal_types(const jl_value_t *a, const jl_value_t *b, jl_typeenv_t *en } return 1; } - if (dt == jl_tvar_type) { + if (dtag == jl_tvar_tag << 4) { jl_typeenv_t *pe = env; while (pe != NULL) { if (pe->var == (jl_tvar_t*)a) @@ -173,7 +173,7 @@ static int egal_types(const jl_value_t *a, const jl_value_t *b, jl_typeenv_t *en } return 0; } - if (dt == jl_unionall_type) { + if (dtag == jl_unionall_tag << 4) { jl_unionall_t *ua = (jl_unionall_t*)a; jl_unionall_t *ub = (jl_unionall_t*)b; if (tvar_names && ua->var->name != ub->var->name) @@ -183,11 +183,11 @@ static int egal_types(const jl_value_t *a, const jl_value_t *b, jl_typeenv_t *en jl_typeenv_t e = { ua->var, (jl_value_t*)ub->var, env }; return egal_types(ua->body, ub->body, &e, tvar_names); } - if (dt == jl_uniontype_type) { + if (dtag == jl_uniontype_tag << 4) { return egal_types(((jl_uniontype_t*)a)->a, ((jl_uniontype_t*)b)->a, env, tvar_names) && egal_types(((jl_uniontype_t*)a)->b, ((jl_uniontype_t*)b)->b, env, tvar_names); } - if (dt == jl_vararg_type) { + if (dtag == jl_vararg_tag << 4) { jl_vararg_t *vma = (jl_vararg_t*)a; jl_vararg_t *vmb = (jl_vararg_t*)b; jl_value_t *vmaT = vma->T ? vma->T : (jl_value_t*)jl_any_type; @@ -198,10 +198,8 @@ static int egal_types(const jl_value_t *a, const jl_value_t *b, jl_typeenv_t *en return egal_types(vma->N, vmb->N, env, tvar_names); return !vma->N && !vmb->N; } - if (dt == jl_symbol_type || dt == jl_module_type) - return 0; - assert(!dt->name->mutabl); - return jl_egal__bits(a, b, dt); + assert(dtag == jl_symbol_tag << 4 || dtag == jl_module_tag << 4 || !((jl_datatype_t*)jl_typeof(a))->name->mutabl); + return jl_egal__bitstag(a, b, dtag); } JL_DLLEXPORT int jl_types_egal(jl_value_t *a, jl_value_t *b) @@ -215,36 +213,72 @@ JL_DLLEXPORT int (jl_egal)(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value return jl_egal(a, b); } -JL_DLLEXPORT int jl_egal__unboxed(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, jl_datatype_t *dt) JL_NOTSAFEPOINT +JL_DLLEXPORT int jl_egal__unboxed(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, uintptr_t dtag) JL_NOTSAFEPOINT { // warning: a,b may NOT have been gc-rooted by the caller - return jl_egal__unboxed_(a, b, dt); -} - -int jl_egal__special(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, jl_datatype_t *dt) JL_NOTSAFEPOINT -{ - if (dt == jl_simplevector_type) - return compare_svec((jl_svec_t*)a, (jl_svec_t*)b); - if (dt == jl_datatype_type) { - jl_datatype_t *dta = (jl_datatype_t*)a; - jl_datatype_t *dtb = (jl_datatype_t*)b; - if (dta->name != dtb->name) - return 0; - if (dta->name != jl_tuple_typename && (dta->isconcretetype || dtb->isconcretetype)) - return 0; - return compare_svec(dta->parameters, dtb->parameters); - } - if (dt == jl_string_type) { - size_t l = jl_string_len(a); - if (jl_string_len(b) != l) + return jl_egal__unboxed_(a, b, dtag); +} + +JL_DLLEXPORT int jl_egal__bitstag(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, uintptr_t dtag) JL_NOTSAFEPOINT +{ + if (dtag < jl_max_tags << 4) { + switch ((enum jlsmall_typeof_tags)(dtag >> 4)) { + case jl_int8_tag: + case jl_uint8_tag: + return *(uint8_t*)a == *(uint8_t*)b; + case jl_int16_tag: + case jl_uint16_tag: + return *(uint16_t*)a == *(uint16_t*)b; + case jl_int32_tag: + case jl_uint32_tag: + case jl_char_tag: + return *(uint32_t*)a == *(uint32_t*)b; + case jl_int64_tag: + case jl_uint64_tag: + return *(uint64_t*)a == *(uint64_t*)b; + case jl_unionall_tag: + return egal_types(a, b, NULL, 1); + case jl_uniontype_tag: + return compare_fields(a, b, jl_uniontype_type); + case jl_vararg_tag: + return compare_fields(a, b, jl_vararg_type); + case jl_task_tag: + case jl_tvar_tag: + case jl_symbol_tag: + case jl_module_tag: + case jl_bool_tag: return 0; - return !memcmp(jl_string_data(a), jl_string_data(b), l); + case jl_simplevector_tag: + return compare_svec((jl_svec_t*)a, (jl_svec_t*)b); + case jl_string_tag: { + size_t l = jl_string_len(a); + if (jl_string_len(b) != l) + return 0; + return !memcmp(jl_string_data(a), jl_string_data(b), l); + } + case jl_datatype_tag: { + jl_datatype_t *dta = (jl_datatype_t*)a; + jl_datatype_t *dtb = (jl_datatype_t*)b; + if (dta->name != dtb->name) + return 0; + if (dta->name != jl_tuple_typename && (dta->isconcretetype || dtb->isconcretetype)) + return 0; + return compare_svec(dta->parameters, dtb->parameters); + } +#ifndef NDEBUG + default: +#endif + case jl_max_tags: + case jl_null_tag: + case jl_typeofbottom_tag: + case jl_tags_count: + abort(); + } } - assert(0 && "unreachable"); - return 0; + return jl_egal__bits(a, b, (jl_datatype_t*)dtag); } -int jl_egal__bits(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, jl_datatype_t *dt) JL_NOTSAFEPOINT +inline int jl_egal__bits(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, jl_datatype_t *dt) JL_NOTSAFEPOINT { size_t sz = jl_datatype_size(dt); if (sz == 0) @@ -252,8 +286,6 @@ int jl_egal__bits(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_ size_t nf = jl_datatype_nfields(dt); if (nf == 0 || !dt->layout->haspadding) return bits_equal(a, b, sz); - if (dt == jl_unionall_type) - return egal_types(a, b, NULL, 1); return compare_fields(a, b, dt); } @@ -1417,6 +1449,7 @@ JL_DLLEXPORT jl_tvar_t *jl_new_typevar(jl_sym_t *name, jl_value_t *lb, jl_value_ jl_type_error_rt("TypeVar", "upper bound", (jl_value_t *)jl_type_type, ub); jl_task_t *ct = jl_current_task; jl_tvar_t *tv = (jl_tvar_t *)jl_gc_alloc(ct->ptls, sizeof(jl_tvar_t), jl_tvar_type); + jl_set_typetagof(tv, jl_tvar_tag, 0); tv->name = name; tv->lb = lb; tv->ub = ub; @@ -1650,7 +1683,7 @@ static int equiv_field_types(jl_value_t *old, jl_value_t *ft) if (!jl_has_free_typevars(tb) || !jl_egal(ta, tb)) return 0; } - else if (jl_has_free_typevars(tb) || jl_typeof(ta) != jl_typeof(tb) || + else if (jl_has_free_typevars(tb) || jl_typetagof(ta) != jl_typetagof(tb) || !jl_types_equal(ta, tb)) { return 0; } @@ -1763,7 +1796,7 @@ static int equiv_type(jl_value_t *ta, jl_value_t *tb) if (!jl_is_datatype(dta)) return 0; jl_datatype_t *dtb = (jl_datatype_t*)jl_unwrap_unionall(tb); - if (!(jl_typeof(dta) == jl_typeof(dtb) && + if (!(jl_typetagof(dta) == jl_typetagof(dtb) && dta->name->name == dtb->name->name && dta->name->abstract == dtb->name->abstract && dta->name->mutabl == dtb->name->mutabl && @@ -1893,7 +1926,7 @@ static void add_intrinsic_properties(enum intrinsic f, unsigned nargs, void (*pf static void add_intrinsic(jl_module_t *inm, const char *name, enum intrinsic f) JL_GC_DISABLED { - jl_value_t *i = jl_permbox32(jl_intrinsic_type, (int32_t)f); + jl_value_t *i = jl_permbox32(jl_intrinsic_type, 0, (int32_t)f); jl_sym_t *sym = jl_symbol(name); jl_set_const(inm, sym, i); jl_module_export(inm, sym); diff --git a/src/ccall.cpp b/src/ccall.cpp index b67726eee8be1..6b2143579317f 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -803,7 +803,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar } ir = jl_fieldref(ir, 0); - if (!jl_is_string(ir) && !jl_typeis(ir, jl_array_uint8_type)) { + if (!jl_is_string(ir) && !jl_typetagis(ir, jl_array_uint8_type)) { emit_error(ctx, "Module IR passed to llvmcall must be a string or an array of bytes"); JL_GC_POP(); return jl_cgval_t(); @@ -1882,7 +1882,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) if (!val.isghost && !val.ispointer()) val = value_to_pointer(ctx, val); Value *args[] = { - emit_typeof_boxed(ctx, val), + emit_typeof(ctx, val), val.isghost ? ConstantPointerNull::get(T_pint8_derived) : ctx.builder.CreateBitCast( decay_derived(ctx, data_pointer(ctx, val)), @@ -1984,9 +1984,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( // XXX: result needs to be zero'd and given a GC root here // and has incorrect write barriers. // instead this code path should behave like `unsafe_load` - assert(jl_datatype_size(rt) > 0 && "sret shouldn't be a singleton instance"); - result = emit_allocobj(ctx, jl_datatype_size(rt), - literal_pointer_val(ctx, (jl_value_t*)rt)); + result = emit_allocobj(ctx, (jl_datatype_t*)rt); sretty = ctx.types().T_jlvalue; sretboxed = true; gc_uses.push_back(result); @@ -2148,15 +2146,13 @@ jl_cgval_t function_sig_t::emit_a_ccall( else if (jlretboxed && !retboxed) { assert(jl_is_datatype(rt)); if (static_rt) { - Value *runtime_bt = literal_pointer_val(ctx, rt); - size_t rtsz = jl_datatype_size(rt); - assert(rtsz > 0); - Value *strct = emit_allocobj(ctx, rtsz, runtime_bt); + Value *strct = emit_allocobj(ctx, (jl_datatype_t*)rt); MDNode *tbaa = jl_is_mutable(rt) ? ctx.tbaa().tbaa_mutab : ctx.tbaa().tbaa_immut; int boxalign = julia_alignment(rt); // copy the data from the return value to the new struct const DataLayout &DL = ctx.builder.GetInsertBlock()->getModule()->getDataLayout(); auto resultTy = result->getType(); + size_t rtsz = jl_datatype_size(rt); if (DL.getTypeStoreSize(resultTy) > rtsz) { // ARM and AArch64 can use a LLVM type larger than the julia type. // When this happens, cast through memory. diff --git a/src/cgutils.cpp b/src/cgutils.cpp index dc20167e14f95..29e3cf152625f 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -323,7 +323,7 @@ static Value *get_gc_root_for(const jl_cgval_t &x) static inline Constant *literal_static_pointer_val(const void *p, Type *T); -static Value *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr) +static Constant *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr) { // emit a GlobalVariable for a jl_value_t named "cname" // store the name given so we can reuse it (facilitating merging later) @@ -355,7 +355,7 @@ static Value *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr) return gv; } -static Value *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name, jl_module_t *mod, void *addr) +static Constant *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name, jl_module_t *mod, void *addr) { // emit a GlobalVariable for a jl_value_t, using the prefix, name, and module to // to create a readable name of the form prefixModA.ModB.name# @@ -384,7 +384,7 @@ static Value *julia_pgv(jl_codectx_t &ctx, const char *prefix, jl_sym_t *name, j } static JuliaVariable *julia_const_gv(jl_value_t *val); -static Value *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p) +static Constant *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p) { // emit a pointer to a jl_value_t* which will allow it to be valid across reloading code // also, try to give it a nice name for gdb, for easy identification @@ -404,6 +404,12 @@ static Value *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p) } if (jl_is_datatype(p)) { jl_datatype_t *addr = (jl_datatype_t*)p; + if (addr->smalltag) { + // some common builtin datatypes have a special pool for accessing them by smalltag id + Constant *tag = ConstantInt::get(getInt32Ty(ctx.builder.getContext()), addr->smalltag << 4); + Constant *smallp = ConstantExpr::getInBoundsGetElementPtr(getInt8Ty(ctx.builder.getContext()), prepare_global_in(jl_Module, jlsmall_typeof_var), tag); + return ConstantExpr::getBitCast(smallp, ctx.types().T_ppjlvalue); + } // DataTypes are prefixed with a + return julia_pgv(ctx, "+", addr->name->name, addr->name->module, p); } @@ -883,7 +889,8 @@ static bool is_uniontype_allunboxed(jl_value_t *typ) return for_each_uniontype_small([&](unsigned, jl_datatype_t*) {}, typ, counter); } -static Value *emit_typeof_boxed(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull=false); +static Value *emit_typeof(jl_codectx_t &ctx, Value *v, bool maybenull, bool justtag, bool notag=false); +static Value *emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull=false, bool justtag=false); static unsigned get_box_tindex(jl_datatype_t *jt, jl_value_t *ut) { @@ -1033,49 +1040,84 @@ static LoadInst *emit_nthptr_recast(jl_codectx_t &ctx, Value *v, Value *idx, MDN return load; } -static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &v, bool is_promotable=false); - -static Value *emit_typeof(jl_codectx_t &ctx, Value *v, bool maybenull); +static Value *emit_tagfrom(jl_codectx_t &ctx, jl_datatype_t *dt) +{ + if (dt->smalltag) + return ConstantInt::get(ctx.types().T_size, dt->smalltag << 4); + return ctx.builder.CreatePtrToInt(literal_pointer_val(ctx, (jl_value_t*)dt), ctx.types().T_size); +} -static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull) +// Returns justtag ? ctx.types.T_size : ctx.types().T_prjlvalue +static Value *emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull, bool justtag) { // given p, compute its type + jl_datatype_t *dt = NULL; if (p.constant) - return mark_julia_const(ctx, jl_typeof(p.constant)); - if (p.isboxed && !jl_is_concrete_type(p.typ)) { - if (jl_is_type_type(p.typ)) { - jl_value_t *tp = jl_tparam0(p.typ); - if (!jl_is_type(tp) || jl_is_concrete_type(tp)) { - // convert 1::Type{1} ==> typeof(1) ==> Int - return mark_julia_const(ctx, jl_typeof(tp)); - } + dt = (jl_datatype_t*)jl_typeof(p.constant); + else if (jl_is_concrete_type(p.typ)) + dt = (jl_datatype_t*)p.typ; + if (dt) { + if (justtag) + return emit_tagfrom(ctx, dt); + return track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)dt)); + } + auto notag = [justtag] (jl_value_t *typ) { + // compute if the tag is always a type (not a builtin tag) + // based on having no intersection with one of the special types + // this doesn't matter if the user just wants the tag value + if (justtag) + return false; + jl_value_t *uw = jl_unwrap_unionall(typ); + if (jl_is_datatype(uw)) { // quick path to catch common cases + jl_datatype_t *dt = (jl_datatype_t*)uw; + assert(!dt->smalltag); + if (!dt->name->abstract) + return true; + if (dt == jl_any_type) + return false; } - return mark_julia_type(ctx, emit_typeof(ctx, p.V, maybenull), true, jl_datatype_type); - } + if (jl_has_intersect_type_not_kind(typ)) + return false; + for (size_t i = 0; i < jl_tags_count; i++) { + jl_datatype_t *dt = small_typeof[(i << 4) / sizeof(*small_typeof)]; + if (dt && !jl_has_empty_intersection((jl_value_t*)dt, typ)) + return false; + } + return true; + }; + if (p.isboxed) + return emit_typeof(ctx, p.V, maybenull, justtag, notag(p.typ)); if (p.TIndex) { Value *tindex = ctx.builder.CreateAnd(p.TIndex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x7f)); bool allunboxed = is_uniontype_allunboxed(p.typ); - Value *datatype_or_p = ctx.emission_context.imaging ? Constant::getNullValue(ctx.types().T_ppjlvalue) : Constant::getNullValue(ctx.types().T_prjlvalue); + Type *expr_type = justtag ? ctx.types().T_size : ctx.emission_context.imaging ? ctx.types().T_pjlvalue : ctx.types().T_prjlvalue; + Value *datatype_or_p = Constant::getNullValue(ctx.emission_context.imaging ? expr_type->getPointerTo() : expr_type); unsigned counter = 0; for_each_uniontype_small( [&](unsigned idx, jl_datatype_t *jt) { Value *cmp = ctx.builder.CreateICmpEQ(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), idx)); - Value *ptr; - if (ctx.emission_context.imaging) { - ptr = literal_pointer_val_slot(ctx, (jl_value_t*)jt); - } - else { - ptr = track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jt)); + Constant *ptr; + if (justtag && jt->smalltag) { + ptr = ConstantInt::get(expr_type, jt->smalltag << 4); + if (ctx.emission_context.imaging) + ptr = get_pointer_to_constant(ctx.emission_context, ptr, "_j_tag", *jl_Module); } + else if (ctx.emission_context.imaging) + ptr = ConstantExpr::getBitCast(literal_pointer_val_slot(ctx, (jl_value_t*)jt), datatype_or_p->getType()); + else if (justtag) + ptr = ConstantInt::get(expr_type, (uintptr_t)jt); + else + ptr = ConstantExpr::getAddrSpaceCast(literal_static_pointer_val((jl_value_t*)jt, ctx.types().T_pjlvalue), expr_type); datatype_or_p = ctx.builder.CreateSelect(cmp, ptr, datatype_or_p); }, p.typ, counter); auto emit_unboxty = [&] () -> Value* { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); - if (ctx.emission_context.imaging) - return track_pjlvalue( - ctx, ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, datatype_or_p, Align(sizeof(void*))))); + if (ctx.emission_context.imaging) { + Value *datatype = ai.decorateInst(ctx.builder.CreateAlignedLoad(expr_type, datatype_or_p, Align(sizeof(void*)))); + return justtag ? datatype : track_pjlvalue(ctx, datatype); + } return datatype_or_p; }; Value *res; @@ -1086,7 +1128,7 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybe BasicBlock *mergeBB = BasicBlock::Create(ctx.builder.getContext(), "merge", ctx.f); ctx.builder.CreateCondBr(isnull, boxBB, unboxBB); ctx.builder.SetInsertPoint(boxBB); - auto boxTy = emit_typeof(ctx, p.Vboxed, maybenull); + auto boxTy = emit_typeof(ctx, p.Vboxed, maybenull, justtag, notag(p.typ)); ctx.builder.CreateBr(mergeBB); boxBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.SetInsertPoint(unboxBB); @@ -1094,7 +1136,7 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybe ctx.builder.CreateBr(mergeBB); unboxBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.SetInsertPoint(mergeBB); - auto phi = ctx.builder.CreatePHI(ctx.types().T_prjlvalue, 2); + auto phi = ctx.builder.CreatePHI(boxTy->getType(), 2); phi->addIncoming(boxTy, boxBB); phi->addIncoming(unboxTy, unboxBB); res = phi; @@ -1102,15 +1144,9 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybe else { res = emit_unboxty(); } - return mark_julia_type(ctx, res, true, jl_datatype_type); + return res; } - return mark_julia_const(ctx, p.typ); -} - -// Returns ctx.types().T_prjlvalue -static Value *emit_typeof_boxed(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull) -{ - return boxed(ctx, emit_typeof(ctx, p, maybenull)); + assert(0 && "what is this struct"); abort(); } static Value *emit_datatype_types(jl_codectx_t &ctx, Value *dt) @@ -1164,7 +1200,7 @@ static Value *emit_sizeof(jl_codectx_t &ctx, const jl_cgval_t &p) ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0)); ctx.builder.CreateCondBr(isboxed, dynloadBB, postBB); ctx.builder.SetInsertPoint(dynloadBB); - Value *datatype = emit_typeof(p.V); + Value *datatype = emit_typeof(ctx, p.V, false, false); Value *dyn_size = emit_datatype_size(ctx, datatype); ctx.builder.CreateBr(postBB); dynloadBB = ctx.builder.GetInsertBlock(); // could have changed @@ -1184,7 +1220,7 @@ static Value *emit_sizeof(jl_codectx_t &ctx, const jl_cgval_t &p) return ConstantInt::get(getInt32Ty(ctx.builder.getContext()), jl_datatype_size(p.typ)); } else { - Value *datatype = emit_typeof_boxed(ctx, p); + Value *datatype = emit_typeof(ctx, p, false, false); Value *dyn_size = emit_datatype_size(ctx, datatype); return dyn_size; } @@ -1369,21 +1405,38 @@ static Value *emit_nullcheck_guard2(jl_codectx_t &ctx, Value *nullcheck1, // Returns typeof(v), or null if v is a null pointer at run time and maybenull is true. // This is used when the value might have come from an undefined value (a PhiNode), -// yet we try to read its type to compute a union index when moving the value (a PiNode). +// yet jl_max_tags try to read its type to compute a union index when moving the value (a PiNode). // Returns a ctx.types().T_prjlvalue typed Value -static Value *emit_typeof(jl_codectx_t &ctx, Value *v, bool maybenull) +static Value *emit_typeof(jl_codectx_t &ctx, Value *v, bool maybenull, bool justtag, bool notag) { ++EmittedTypeof; assert(v != NULL && !isa(v) && "expected a conditionally boxed value"); + Value *nonnull = maybenull ? null_pointer_cmp(ctx, v) : ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 1); Function *typeof = prepare_call(jl_typeof_func); - if (maybenull) - return emit_guarded_test(ctx, null_pointer_cmp(ctx, v), Constant::getNullValue(typeof->getReturnType()), [&] { - // e.g. emit_typeof(ctx, v) - return ctx.builder.CreateCall(typeof, {v}); + return emit_guarded_test(ctx, nonnull, Constant::getNullValue(justtag ? ctx.types().T_size : typeof->getReturnType()), [&] { + // e.g. emit_typeof(ctx, v) + Value *typetag = ctx.builder.CreateCall(typeof, {v}); + if (notag) + return typetag; + Value *tag = ctx.builder.CreatePtrToInt(emit_pointer_from_objref(ctx, typetag), ctx.types().T_size); + if (justtag) + return tag; + auto issmall = ctx.builder.CreateICmpULT(tag, ConstantInt::get(tag->getType(), (uintptr_t)jl_max_tags << 4)); + return emit_guarded_test(ctx, issmall, typetag, [&] { + // we lied a bit: this wasn't really an object (though it was valid for GC rooting) + // and we need to use it as an index to get the real object now + Module *M = jl_Module; + Value *smallp = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), prepare_global_in(M, jlsmall_typeof_var), tag); + smallp = ctx.builder.CreateBitCast(smallp, typetag->getType()->getPointerTo(0)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + auto small = ctx.builder.CreateAlignedLoad(typetag->getType(), smallp, M->getDataLayout().getPointerABIAlignment(0)); + small->setMetadata(LLVMContext::MD_nonnull, MDNode::get(M->getContext(), None)); + return ai.decorateInst(small); }); - return ctx.builder.CreateCall(typeof, {v}); + }); } +static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &v, bool is_promotable=false); static void just_emit_type_error(jl_codectx_t &ctx, const jl_cgval_t &x, Value *type, const std::string &msg) { @@ -1431,11 +1484,11 @@ static bool can_optimize_isa_union(jl_uniontype_t *type) } // a simple case of emit_isa that is obvious not to include a safe-point -static Value *emit_exactly_isa(jl_codectx_t &ctx, const jl_cgval_t &arg, jl_value_t *dt) +static Value *emit_exactly_isa(jl_codectx_t &ctx, const jl_cgval_t &arg, jl_datatype_t *dt) { - assert(jl_is_concrete_type(dt)); + assert(jl_is_concrete_type((jl_value_t*)dt)); if (arg.TIndex) { - unsigned tindex = get_box_tindex((jl_datatype_t*)dt, arg.typ); + unsigned tindex = get_box_tindex(dt, arg.typ); if (tindex > 0) { // optimize more when we know that this is a split union-type where tindex = 0 is invalid Value *xtindex = ctx.builder.CreateAnd(arg.TIndex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x7f)); @@ -1449,8 +1502,7 @@ static Value *emit_exactly_isa(jl_codectx_t &ctx, const jl_cgval_t &arg, jl_valu BasicBlock *postBB = BasicBlock::Create(ctx.builder.getContext(), "post_isa", ctx.f); ctx.builder.CreateCondBr(isboxed, isaBB, postBB); ctx.builder.SetInsertPoint(isaBB); - Value *istype_boxed = ctx.builder.CreateICmpEQ(emit_typeof(ctx, arg.Vboxed, false), - track_pjlvalue(ctx, literal_pointer_val(ctx, dt))); + Value *istype_boxed = ctx.builder.CreateICmpEQ(emit_typeof(ctx, arg.Vboxed, false, true), emit_tagfrom(ctx, dt)); ctx.builder.CreateBr(postBB); isaBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.SetInsertPoint(postBB); @@ -1463,9 +1515,7 @@ static Value *emit_exactly_isa(jl_codectx_t &ctx, const jl_cgval_t &arg, jl_valu return ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 0); } } - return ctx.builder.CreateICmpEQ( - emit_typeof_boxed(ctx, arg), - track_pjlvalue(ctx, literal_pointer_val(ctx, dt))); + return ctx.builder.CreateICmpEQ(emit_typeof(ctx, arg, false, true), emit_tagfrom(ctx, dt)); } static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, @@ -1524,17 +1574,17 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, if (intersected_type == (jl_value_t*)jl_type_type) { // Inline jl_is_kind(jl_typeof(x)) // N.B. We do the comparison with untracked pointers, because that gives - // LLVM more optimization opportunities. That means it is poosible for + // LLVM more optimization opportunities. That means it is possible for // `typ` to get GC'ed, but we don't actually care, because we don't ever // dereference it. - Value *typ = emit_pointer_from_objref(ctx, emit_typeof_boxed(ctx, x)); + Value *typ = emit_typeof(ctx, x, false, true); auto val = ctx.builder.CreateOr( ctx.builder.CreateOr( - ctx.builder.CreateICmpEQ(typ, literal_pointer_val(ctx, (jl_value_t*)jl_uniontype_type)), - ctx.builder.CreateICmpEQ(typ, literal_pointer_val(ctx, (jl_value_t*)jl_datatype_type))), + ctx.builder.CreateICmpEQ(typ, emit_tagfrom(ctx, jl_uniontype_type)), + ctx.builder.CreateICmpEQ(typ, emit_tagfrom(ctx, jl_datatype_type))), ctx.builder.CreateOr( - ctx.builder.CreateICmpEQ(typ, literal_pointer_val(ctx, (jl_value_t*)jl_unionall_type)), - ctx.builder.CreateICmpEQ(typ, literal_pointer_val(ctx, (jl_value_t*)jl_typeofbottom_type)))); + ctx.builder.CreateICmpEQ(typ, emit_tagfrom(ctx, jl_unionall_type)), + ctx.builder.CreateICmpEQ(typ, emit_tagfrom(ctx, jl_typeofbottom_type)))); return std::make_pair(val, false); } // intersection with Type needs to be handled specially @@ -1550,15 +1600,16 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0)), false); } // tests for isa concretetype can be handled with pointer comparisons - if (jl_is_concrete_type(intersected_type)) - return std::make_pair(emit_exactly_isa(ctx, x, intersected_type), false); + if (jl_is_concrete_type(intersected_type)) { + return std::make_pair(emit_exactly_isa(ctx, x, (jl_datatype_t*)intersected_type), false); + } jl_datatype_t *dt = (jl_datatype_t*)jl_unwrap_unionall(intersected_type); if (jl_is_datatype(dt) && !dt->name->abstract && jl_subtype(dt->name->wrapper, type)) { // intersection is a supertype of all instances of its constructor, // so the isa test reduces to a comparison of the typename by pointer return std::make_pair( ctx.builder.CreateICmpEQ( - emit_datatype_name(ctx, emit_typeof_boxed(ctx, x)), + emit_datatype_name(ctx, emit_typeof(ctx, x, false, false)), literal_pointer_val(ctx, (jl_value_t*)dt->name)), false); } @@ -1587,7 +1638,7 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, // everything else can be handled via subtype tests return std::make_pair(ctx.builder.CreateICmpNE( ctx.builder.CreateCall(prepare_call(jlsubtype_func), - { emit_typeof_boxed(ctx, x), + { emit_typeof(ctx, x, false, false), track_pjlvalue(ctx, literal_pointer_val(ctx, type)) }), ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0)), false); } @@ -2938,7 +2989,7 @@ static Value *emit_array_nd_index( // --- boxing --- -static Value *emit_allocobj(jl_codectx_t &ctx, size_t static_size, Value *jt); +static Value *emit_allocobj(jl_codectx_t &ctx, jl_datatype_t *jt); static void init_bits_value(jl_codectx_t &ctx, Value *newv, Value *v, MDNode *tbaa, unsigned alignment = sizeof(void*)) // min alignment in julia's gc is pointer-aligned @@ -3129,14 +3180,14 @@ static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t return box; } -static Value *compute_box_tindex(jl_codectx_t &ctx, Value *datatype, jl_value_t *supertype, jl_value_t *ut) +static Value *compute_box_tindex(jl_codectx_t &ctx, Value *datatype_tag, jl_value_t *supertype, jl_value_t *ut) { Value *tindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0); unsigned counter = 0; for_each_uniontype_small( [&](unsigned idx, jl_datatype_t *jt) { if (jl_subtype((jl_value_t*)jt, supertype)) { - Value *cmp = ctx.builder.CreateICmpEQ(track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jt)), datatype); + Value *cmp = ctx.builder.CreateICmpEQ(emit_tagfrom(ctx, jt), datatype_tag); tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), idx), tindex); } }, @@ -3154,7 +3205,7 @@ static Value *compute_tindex_unboxed(jl_codectx_t &ctx, const jl_cgval_t &val, j return ConstantInt::get(getInt8Ty(ctx.builder.getContext()), get_box_tindex((jl_datatype_t*)jl_typeof(val.constant), typ)); if (val.TIndex) return ctx.builder.CreateAnd(val.TIndex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x7f)); - Value *typof = emit_typeof_boxed(ctx, val, maybenull); + Value *typof = emit_typeof(ctx, val, maybenull, true); return compute_box_tindex(ctx, typof, val.typ, typ); } @@ -3247,7 +3298,7 @@ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallB jl_cgval_t vinfo_r = jl_cgval_t(vinfo, (jl_value_t*)jt, NULL); box = _boxed_special(ctx, vinfo_r, t); if (!box) { - box = emit_allocobj(ctx, jl_datatype_size(jt), literal_pointer_val(ctx, (jl_value_t*)jt)); + box = emit_allocobj(ctx, jt); init_bits_cgval(ctx, box, vinfo_r, jl_is_mutable(jt) ? ctx.tbaa().tbaa_mutab : ctx.tbaa().tbaa_immut); } } @@ -3373,7 +3424,7 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab if (do_promote && is_promotable) { auto IP = ctx.builder.saveIP(); ctx.builder.SetInsertPoint(vinfo.promotion_point); - box = emit_allocobj(ctx, jl_datatype_size(jt), literal_pointer_val(ctx, (jl_value_t*)jt)); + box = emit_allocobj(ctx, (jl_datatype_t*)jt); Value *decayed = decay_derived(ctx, box); AllocaInst *originalAlloca = cast(vinfo.V); decayed = maybe_bitcast(ctx, decayed, PointerType::getWithSamePointeeType(originalAlloca->getType(), AddressSpace::Derived)); @@ -3385,7 +3436,7 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab originalAlloca->eraseFromParent(); ctx.builder.restoreIP(IP); } else { - box = emit_allocobj(ctx, jl_datatype_size(jt), literal_pointer_val(ctx, (jl_value_t*)jt)); + box = emit_allocobj(ctx, (jl_datatype_t*)jt); init_bits_cgval(ctx, box, vinfo, jl_is_mutable(jt) ? ctx.tbaa().tbaa_mutab : ctx.tbaa().tbaa_immut); } } @@ -3475,7 +3526,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con else { assert(src.isboxed && "expected boxed value for sizeof/alignment computation"); auto f = [&] { - Value *datatype = emit_typeof_boxed(ctx, src); + Value *datatype = emit_typeof(ctx, src, false, false); Value *copy_bytes = emit_datatype_size(ctx, datatype); emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src, copy_bytes, /*TODO: min-align*/1, isVolatile); return nullptr; @@ -3491,8 +3542,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con static void emit_cpointercheck(jl_codectx_t &ctx, const jl_cgval_t &x, const std::string &msg) { ++EmittedCPointerChecks; - Value *t = emit_typeof_boxed(ctx, x); - emit_typecheck(ctx, mark_julia_type(ctx, t, true, jl_any_type), (jl_value_t*)jl_datatype_type, msg); + Value *t = emit_typeof(ctx, x, false, false); Value *istype = ctx.builder.CreateICmpEQ(emit_datatype_name(ctx, t), @@ -3519,12 +3569,15 @@ static Value *emit_allocobj(jl_codectx_t &ctx, size_t static_size, Value *jt) auto call = ctx.builder.CreateCall(F, {current_task, ConstantInt::get(ctx.types().T_size, static_size), maybe_decay_untracked(ctx, jt)}); call->setAttributes(F->getAttributes()); if (static_size > 0) - { - call->addRetAttr(Attribute::getWithDereferenceableBytes(ctx.builder.getContext(),static_size)); - } + call->addRetAttr(Attribute::getWithDereferenceableBytes(ctx.builder.getContext(), static_size)); return call; } +static Value *emit_allocobj(jl_codectx_t &ctx, jl_datatype_t *jt) +{ + return emit_allocobj(ctx, jl_datatype_size(jt), ctx.builder.CreateIntToPtr(emit_tagfrom(ctx, jt), ctx.types().T_pjlvalue)); +} + // allocation for unknown object from an untracked pointer static Value *emit_new_bits(jl_codectx_t &ctx, Value *jt, Value *pval) { @@ -3902,8 +3955,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg return ret; } } - Value *strct = emit_allocobj(ctx, jl_datatype_size(sty), - literal_pointer_val(ctx, (jl_value_t*)ty)); + Value *strct = emit_allocobj(ctx, sty); jl_cgval_t strctinfo = mark_julia_type(ctx, strct, true, ty); strct = decay_derived(ctx, strct); undef_derived_strct(ctx, strct, sty, strctinfo.tbaa); diff --git a/src/codegen.cpp b/src/codegen.cpp index 16149325eb3e0..da69678dee6ff 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -643,6 +643,11 @@ static const auto jldlli_var = new JuliaVariable{ true, [](Type *T_size) -> Type * { return getInt8PtrTy(T_size->getContext()); }, }; +static const auto jlsmall_typeof_var = new JuliaVariable{ + XSTR(small_typeof), + true, + [](Type *T_size) -> Type * { return getInt8Ty(T_size->getContext()); }, +}; static const auto jlstack_chk_guard_var = new JuliaVariable{ XSTR(__stack_chk_guard), @@ -902,11 +907,11 @@ static const auto jl_excstack_state_func = new JuliaFunction{ +static const auto jlegalx_func = new JuliaFunction{ XSTR(jl_egal__unboxed), - [](LLVMContext &C) { + [](LLVMContext &C, Type *T_size) { Type *T = PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::Derived); - return FunctionType::get(getInt32Ty(C), {T, T, JuliaType::get_prjlvalue_ty(C)}, false); }, + return FunctionType::get(getInt32Ty(C), {T, T, T_size}, false); }, [](LLVMContext &C) { return AttributeList::get(C, Attributes(C, {Attribute::ReadOnly, Attribute::NoUnwind, Attribute::ArgMemOnly}), AttributeSet(), @@ -2146,7 +2151,7 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & if (!union_isaBB) { union_isaBB = BasicBlock::Create(ctx.builder.getContext(), "union_isa", ctx.f); ctx.builder.SetInsertPoint(union_isaBB); - union_box_dt = emit_typeof(ctx, v.Vboxed, skip != NULL); + union_box_dt = emit_typeof(ctx, v.Vboxed, skip != NULL, true); post_union_isaBB = ctx.builder.GetInsertBlock(); } }; @@ -2164,7 +2169,7 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & if (old_idx == 0) { // didn't handle this item before, select its new union index maybe_setup_union_isa(); - Value *cmp = ctx.builder.CreateICmpEQ(track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jt)), union_box_dt); + Value *cmp = ctx.builder.CreateICmpEQ(emit_tagfrom(ctx, jt), union_box_dt); union_box_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x80 | idx), union_box_tindex); } }, @@ -2881,8 +2886,8 @@ static Value *emit_box_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const } Value *neq = ctx.builder.CreateICmpNE(varg1, varg2); return emit_guarded_test(ctx, neq, true, [&] { - Value *dtarg = emit_typeof_boxed(ctx, arg1); - Value *dt_eq = ctx.builder.CreateICmpEQ(dtarg, emit_typeof_boxed(ctx, arg2)); + Value *dtarg = emit_typeof(ctx, arg1, false, true); + Value *dt_eq = ctx.builder.CreateICmpEQ(dtarg, emit_typeof(ctx, arg2, false, true)); return emit_guarded_test(ctx, dt_eq, false, [&] { return ctx.builder.CreateTrunc(ctx.builder.CreateCall(prepare_call(jlegalx_func), {varg1, varg2, dtarg}), getInt1Ty(ctx.builder.getContext())); @@ -3067,11 +3072,11 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva // since it is normalized to `::Type{Union{}}` instead... if (arg1.TIndex) return emit_nullcheck_guard(ctx, nullcheck1, [&] { - return emit_exactly_isa(ctx, arg1, rt2); // rt2 is a singleton type + return emit_exactly_isa(ctx, arg1, (jl_datatype_t*)rt2); // rt2 is a singleton type }); if (arg2.TIndex) return emit_nullcheck_guard(ctx, nullcheck2, [&] { - return emit_exactly_isa(ctx, arg2, rt1); // rt1 is a singleton type + return emit_exactly_isa(ctx, arg2, (jl_datatype_t*)rt1); // rt1 is a singleton type }); if (!(arg1.isboxed || arg1.constant) || !(arg2.isboxed || arg2.constant)) // not TIndex && not boxed implies it is an unboxed value of a different type from this singleton @@ -3094,8 +3099,8 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva bool justbits2 = jl_is_concrete_immutable(rt2); if (justbits1 || justbits2) { // whether this type is unique'd by value return emit_nullcheck_guard2(ctx, nullcheck1, nullcheck2, [&] () -> Value* { - jl_value_t *typ = justbits1 ? rt1 : rt2; - if (typ == (jl_value_t*)jl_bool_type) { // aka jl_pointer_egal + jl_datatype_t *typ = (jl_datatype_t*)(justbits1 ? rt1 : rt2); + if (typ == jl_bool_type) { // aka jl_pointer_egal // some optimizations for bool, since pointer comparison may be better if ((arg1.isboxed || arg1.constant) && (arg2.isboxed || arg2.constant)) { // aka have-fast-pointer Value *varg1 = arg1.constant ? literal_pointer_val(ctx, arg1.constant) : maybe_bitcast(ctx, arg1.Vboxed, ctx.types().T_pjlvalue); @@ -3105,14 +3110,14 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva } if (rt1 == rt2) return emit_bits_compare(ctx, arg1, arg2); - Value *same_type = emit_exactly_isa(ctx, (typ == rt2 ? arg1 : arg2), typ); + Value *same_type = emit_exactly_isa(ctx, (justbits1 ? arg2 : arg1), typ); BasicBlock *currBB = ctx.builder.GetInsertBlock(); BasicBlock *isaBB = BasicBlock::Create(ctx.builder.getContext(), "is", ctx.f); BasicBlock *postBB = BasicBlock::Create(ctx.builder.getContext(), "post_is", ctx.f); ctx.builder.CreateCondBr(same_type, isaBB, postBB); ctx.builder.SetInsertPoint(isaBB); - Value *bitcmp = emit_bits_compare(ctx, jl_cgval_t(arg1, typ, NULL), - jl_cgval_t(arg2, typ, NULL)); + Value *bitcmp = emit_bits_compare(ctx, jl_cgval_t(arg1, (jl_value_t*)typ, NULL), + jl_cgval_t(arg2, (jl_value_t*)typ, NULL)); isaBB = ctx.builder.GetInsertBlock(); // might have changed ctx.builder.CreateBr(postBB); ctx.builder.SetInsertPoint(postBB); @@ -3307,7 +3312,13 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } else if (f == jl_builtin_typeof && nargs == 1) { - *ret = emit_typeof(ctx, argv[1], false); + const jl_cgval_t &p = argv[1]; + if (p.constant) + *ret = mark_julia_const(ctx, jl_typeof(p.constant)); + else if (jl_is_concrete_type(p.typ)) + *ret = mark_julia_const(ctx, p.typ); + else + *ret = mark_julia_type(ctx, emit_typeof(ctx, p, false, false), true, jl_datatype_type); return true; } @@ -3743,9 +3754,10 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // the extra code for getting the length of the tuple if (!bounds_check_enabled(ctx, boundscheck)) { vidx = ctx.builder.CreateSub(vidx, ConstantInt::get(ctx.types().T_size, 1)); - } else { + } + else { vidx = emit_bounds_check(ctx, obj, (jl_value_t*)obj.typ, vidx, - emit_datatype_nfields(ctx, emit_typeof_boxed(ctx, obj)), + emit_datatype_nfields(ctx, emit_typeof(ctx, obj, false, false)), jl_true); } bool isboxed = !jl_datatype_isinlinealloc((jl_datatype_t*)jt, 0); @@ -3838,7 +3850,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (nf != -1) sz = ConstantInt::get(ctx.types().T_size, nf); else - sz = emit_datatype_nfields(ctx, emit_typeof_boxed(ctx, obj)); + sz = emit_datatype_nfields(ctx, emit_typeof(ctx, obj, false, false)); *ret = mark_julia_type(ctx, sz, false, jl_long_type); return true; } @@ -4144,6 +4156,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos for (size_t i = 0; i < nargs; i++) { jl_value_t *jt = jl_nth_slot_type(specTypes, i); + // n.b.: specTypes is required to be a datatype by construction for specsig jl_cgval_t arg = argv[i]; if (is_opaque_closure && i == 0) { Type *at = cft->getParamType(idx); @@ -4153,7 +4166,8 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos // rather than a boxed value. arg = value_to_pointer(ctx, arg); argvals[idx] = decay_derived(ctx, maybe_bitcast(ctx, data_pointer(ctx, arg), at)); - } else if (is_uniquerep_Type(jt)) { + } + else if (is_uniquerep_Type(jt)) { continue; } else { bool isboxed = deserves_argbox(jt); @@ -4421,7 +4435,7 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ Value *oldnew = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), &argv[1], nargs - 1, julia_call); return mark_julia_type(ctx, oldnew, true, rt); } - if (f.constant && jl_typeis(f.constant, jl_intrinsic_type)) { + if (f.constant && jl_typetagis(f.constant, jl_intrinsic_type)) { JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(f.constant); if (fi == JL_I::atomic_pointermodify && jl_intrinsic_nargs((int)fi) == nargs - 1) return emit_atomic_pointerop(ctx, fi, argv.data(), nargs - 1, &lival); @@ -4467,7 +4481,7 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo assert(nargs >= 1); jl_cgval_t f = emit_expr(ctx, args[0]); - if (f.constant && jl_typeis(f.constant, jl_intrinsic_type)) { + if (f.constant && jl_typetagis(f.constant, jl_intrinsic_type)) { JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(f.constant); return emit_intrinsic(ctx, fi, args, nargs - 1); } @@ -4631,8 +4645,7 @@ static jl_cgval_t emit_sparam(jl_codectx_t &ctx, size_t i) i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); Value *sp = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*)))); - Value *isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp, false), - track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); + Value *isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp, false, true), emit_tagfrom(ctx, jl_tvar_type)); jl_unionall_t *sparam = (jl_unionall_t*)ctx.linfo->def.method->sig; for (size_t j = 0; j < i; j++) { sparam = (jl_unionall_t*)sparam->body; @@ -4687,8 +4700,7 @@ static jl_cgval_t emit_isdefined(jl_codectx_t &ctx, jl_value_t *sym) i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); Value *sp = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*)))); - isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp, false), - track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); + isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp, false, true), emit_tagfrom(ctx, jl_tvar_type)); } else { jl_module_t *modu; @@ -5795,6 +5807,7 @@ static void emit_cfunc_invalidate( ++AI; for (size_t i = 0; i < nargs; i++) { jl_value_t *jt = jl_nth_slot_type(calltype, i); + // n.b. specTypes is required to be a datatype by construction for specsig bool isboxed = false; Type *et; if (i == 0 && is_for_opaque_closure) { @@ -5865,7 +5878,7 @@ static void emit_cfunc_invalidate( case jl_returninfo_t::Union: { Type *retty = gf_thunk->getReturnType(); Value *gf_retval = UndefValue::get(retty); - Value *tindex = compute_box_tindex(ctx, emit_typeof_boxed(ctx, gf_retbox), (jl_value_t*)jl_any_type, rettype); + Value *tindex = compute_box_tindex(ctx, emit_typeof(ctx, gf_retbox, false, true), (jl_value_t*)jl_any_type, rettype); tindex = ctx.builder.CreateOr(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x80)); gf_retval = ctx.builder.CreateInsertValue(gf_retval, gf_ret, 0); gf_retval = ctx.builder.CreateInsertValue(gf_retval, tindex, 1); @@ -6139,7 +6152,7 @@ static Function* gen_cfun_wrapper( BasicBlock *unboxedBB = BasicBlock::Create(ctx.builder.getContext(), "maybe-unboxed", cw); BasicBlock *isanyBB = BasicBlock::Create(ctx.builder.getContext(), "any", cw); BasicBlock *afterBB = BasicBlock::Create(ctx.builder.getContext(), "after", cw); - Value *isrtboxed = ctx.builder.CreateIsNull(val); + Value *isrtboxed = ctx.builder.CreateIsNull(val); // XXX: this is the wrong condition and should be inspecting runtime_dt intead ctx.builder.CreateCondBr(isrtboxed, boxedBB, loadBB); ctx.builder.SetInsertPoint(boxedBB); Value *p1 = ctx.builder.CreateBitCast(val, ctx.types().T_pjlvalue); @@ -6300,6 +6313,7 @@ static Function* gen_cfun_wrapper( Value *arg; jl_value_t *spect = (i == 0 && is_opaque_closure) ? (jl_value_t*)jl_any_type : jl_nth_slot_type(lam->specTypes, i); + // n.b. specTypes is required to be a datatype by construction for specsig bool isboxed = deserves_argbox(spect); Type *T = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, spect); if (is_uniquerep_Type(spect)) { @@ -6586,8 +6600,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con outboxed = (output_type != (jl_value_t*)jl_voidpointer_type); if (outboxed) { assert(jl_datatype_size(output_type) == sizeof(void*) * 4); - Value *strct = emit_allocobj(ctx, jl_datatype_size(output_type), - literal_pointer_val(ctx, (jl_value_t*)output_type)); + Value *strct = emit_allocobj(ctx, (jl_datatype_t*)output_type); Value *derived_strct = emit_bitcast(ctx, decay_derived(ctx, strct), ctx.types().T_size->getPointerTo()); MDNode *tbaa = best_tbaa(ctx.tbaa(), output_type); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); @@ -6721,6 +6734,7 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret for (size_t i = 0; i < jl_nparams(lam->specTypes) && idx < nfargs; ++i) { jl_value_t *ty = ((i == 0) && is_opaque_closure) ? (jl_value_t*)jl_any_type : jl_nth_slot_type(lam->specTypes, i); + // n.b. specTypes is required to be a datatype by construction for specsig bool isboxed = deserves_argbox(ty); Type *lty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, ty); if (type_is_ghost(lty) || is_uniquerep_Type(ty)) @@ -6992,6 +7006,7 @@ static jl_datatype_t *compute_va_type(jl_method_instance_t *lam, size_t nreq) JL_GC_PUSH1(&tupargs); for (size_t i = nreq; i < jl_nparams(lam->specTypes); ++i) { jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); + // n.b. specTypes is required to be a datatype by construction for specsig if (is_uniquerep_Type(argType)) argType = jl_typeof(jl_tparam0(argType)); else if (jl_has_intersect_type_not_kind(argType)) { @@ -7130,6 +7145,8 @@ static jl_llvm_functions_t if (argname == jl_unused_sym) continue; jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i); + // TODO: jl_nth_slot_type should call jl_rewrap_unionall + // specTypes is required to be a datatype by construction for specsig, but maybe not otherwise // OpaqueClosure implicitly loads the env if (i == 0 && ctx.is_opaque_closure) { if (jl_is_array(src->slottypes)) { @@ -7595,6 +7612,8 @@ static jl_llvm_functions_t for (i = 0; i < nreq; i++) { jl_sym_t *s = slot_symbol(ctx, i); jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); + // TODO: jl_nth_slot_type should call jl_rewrap_unionall? + // specTypes is required to be a datatype by construction for specsig, but maybe not otherwise bool isboxed = deserves_argbox(argType); Type *llvmArgType = NULL; if (i == 0 && ctx.is_opaque_closure) { @@ -7711,6 +7730,7 @@ static jl_llvm_functions_t SmallVector vargs(ctx.nvargs); for (size_t i = nreq; i < jl_nparams(lam->specTypes); ++i) { jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); + // n.b. specTypes is required to be a datatype by construction for specsig bool isboxed = deserves_argbox(argType); Type *llvmArgType = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, argType); vargs[i - nreq] = get_specsig_arg(argType, llvmArgType, isboxed); @@ -7785,7 +7805,7 @@ static jl_llvm_functions_t // LineInfoNode(mod::Module, method::Any, file::Symbol, line::Int32, inlined_at::Int32) jl_value_t *locinfo = jl_array_ptr_ref(src->linetable, i); DebugLineTable &info = linetable[i + 1]; - assert(jl_typeis(locinfo, jl_lineinfonode_type)); + assert(jl_typetagis(locinfo, jl_lineinfonode_type)); jl_module_t *module = (jl_module_t*)jl_fieldref_noalloc(locinfo, 0); jl_value_t *method = jl_fieldref_noalloc(locinfo, 1); jl_sym_t *filesym = (jl_sym_t*)jl_fieldref_noalloc(locinfo, 2); @@ -8669,7 +8689,7 @@ jl_llvm_functions_t jl_emit_codeinst( if (inferred != (jl_value_t*)src) { if (jl_is_method(def)) { src = (jl_code_info_t*)jl_compress_ir(def, src); - assert(jl_typeis(src, jl_array_uint8_type)); + assert(jl_typetagis(src, jl_array_uint8_type)); codeinst->relocatability = ((uint8_t*)jl_array_data(src))[jl_array_len(src)-1]; } jl_atomic_store_release(&codeinst->inferred, (jl_value_t*)src); @@ -8894,6 +8914,7 @@ static void init_f16_funcs(void) static void init_jit_functions(void) { + add_named_global(jlsmall_typeof_var, &small_typeof); add_named_global(jlstack_chk_guard_var, &__stack_chk_guard); add_named_global(jlRTLD_DEFAULT_var, &jl_RTLD_DEFAULT_handle); add_named_global(jlexe_var, &jl_exe_handle); @@ -8904,6 +8925,7 @@ static void init_jit_functions(void) }; global_jlvalue_to_llvm(new JuliaVariable{"jl_true", true, size2pjlvalue}, &jl_true); global_jlvalue_to_llvm(new JuliaVariable{"jl_false", true, size2pjlvalue}, &jl_false); + global_jlvalue_to_llvm(new JuliaVariable{"jl_nothing", true, size2pjlvalue}, &jl_nothing); global_jlvalue_to_llvm(new JuliaVariable{"jl_emptysvec", true, size2pjlvalue}, (jl_value_t**)&jl_emptysvec); global_jlvalue_to_llvm(new JuliaVariable{"jl_emptytuple", true, size2pjlvalue}, &jl_emptytuple); global_jlvalue_to_llvm(new JuliaVariable{"jl_diverror_exception", true, size2pjlvalue}, &jl_diverror_exception); diff --git a/src/datatype.c b/src/datatype.c index d500d762999a3..95c3b11c9abdc 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -96,6 +96,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(void) { jl_task_t *ct = jl_current_task; jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ct->ptls, sizeof(jl_datatype_t), jl_datatype_type); + jl_set_typetagof(t, jl_datatype_tag, 0); t->hash = 0; t->hasfreetypevars = 0; t->isdispatchtuple = 0; @@ -106,7 +107,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(void) t->maybe_subtype_of_cache = 1; t->ismutationfree = 0; t->isidentityfree = 0; - t->padding = 0; + t->smalltag = 0; t->name = NULL; t->super = NULL; t->parameters = NULL; @@ -964,6 +965,7 @@ JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *dt, const void *data) if (bt == jl_uint16_type) return jl_box_uint16(*(uint16_t*)data); if (bt == jl_char_type) return jl_box_char(*(uint32_t*)data); + assert(!bt->smalltag); jl_task_t *ct = jl_current_task; jl_value_t *v = jl_gc_alloc(ct->ptls, nb, bt); memcpy(jl_assume_aligned(v, sizeof(void*)), data, nb); @@ -989,6 +991,7 @@ JL_DLLEXPORT jl_value_t *jl_atomic_new_bits(jl_value_t *dt, const char *data) if (bt == jl_uint16_type) return jl_box_uint16(jl_atomic_load((_Atomic(uint16_t)*)data)); if (bt == jl_char_type) return jl_box_char(jl_atomic_load((_Atomic(uint32_t)*)data)); + assert(!bt->smalltag); jl_task_t *ct = jl_current_task; jl_value_t *v = jl_gc_alloc(ct->ptls, nb, bt); // data is aligned to the power of two, @@ -1056,6 +1059,7 @@ JL_DLLEXPORT jl_value_t *jl_atomic_swap_bits(jl_value_t *dt, char *dst, const jl if (bt == jl_uint16_type) return jl_box_uint16(jl_atomic_exchange((_Atomic(uint16_t)*)dst, *(uint16_t*)src)); if (bt == jl_char_type) return jl_box_char(jl_atomic_exchange((_Atomic(uint32_t)*)dst, *(uint32_t*)src)); + assert(!bt->smalltag); jl_task_t *ct = jl_current_task; jl_value_t *v = jl_gc_alloc(ct->ptls, jl_datatype_size(bt), bt); if (nb == 1) @@ -1224,30 +1228,29 @@ JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_datatype_t } // used by boot.jl -JL_DLLEXPORT jl_value_t *jl_typemax_uint(jl_value_t *bt) +JL_DLLEXPORT jl_value_t *jl_typemax_uint(jl_datatype_t *bt) { uint64_t data = 0xffffffffffffffffULL; jl_task_t *ct = jl_current_task; jl_value_t *v = jl_gc_alloc(ct->ptls, sizeof(size_t), bt); + if (bt->smalltag) + jl_set_typetagof(v, bt->smalltag, 0); memcpy(v, &data, sizeof(size_t)); return v; } -#define PERMBOXN_FUNC(nb,nw) \ - jl_value_t *jl_permbox##nb(jl_datatype_t *t, int##nb##_t x) \ +#define PERMBOXN_FUNC(nb) \ + jl_value_t *jl_permbox##nb(jl_datatype_t *t, uintptr_t tag, uint##nb##_t x) \ { /* n.b. t must be a concrete isbits datatype of the right size */ \ - jl_value_t *v = jl_gc_permobj(nw * sizeof(void*), t); \ - *(int##nb##_t*)jl_data_ptr(v) = x; \ + jl_value_t *v = jl_gc_permobj(LLT_ALIGN(nb, sizeof(void*)), t); \ + if (tag) jl_set_typetagof(v, tag, GC_OLD_MARKED); \ + *(uint##nb##_t*)jl_data_ptr(v) = x; \ return v; \ } -PERMBOXN_FUNC(8, 1) -PERMBOXN_FUNC(16, 1) -PERMBOXN_FUNC(32, 1) -#ifdef _P64 -PERMBOXN_FUNC(64, 1) -#else -PERMBOXN_FUNC(64, 2) -#endif +PERMBOXN_FUNC(8) +PERMBOXN_FUNC(16) +PERMBOXN_FUNC(32) +PERMBOXN_FUNC(64) #define UNBOX_FUNC(j_type,c_type) \ JL_DLLEXPORT c_type jl_unbox_##j_type(jl_value_t *v) \ @@ -1270,27 +1273,27 @@ UNBOX_FUNC(float64, double) UNBOX_FUNC(voidpointer, void*) UNBOX_FUNC(uint8pointer, uint8_t*) -#define BOX_FUNC(typ,c_type,pfx,nw) \ +#define BOX_FUNC(typ,c_type,pfx) \ JL_DLLEXPORT jl_value_t *pfx##_##typ(c_type x) \ { \ jl_task_t *ct = jl_current_task; \ - jl_value_t *v = jl_gc_alloc(ct->ptls, nw * sizeof(void*), \ + jl_value_t *v = jl_gc_alloc(ct->ptls, LLT_ALIGN(sizeof(x), sizeof(void*)), \ jl_##typ##_type); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } -BOX_FUNC(float32, float, jl_box, 1) -BOX_FUNC(voidpointer, void*, jl_box, 1) -BOX_FUNC(uint8pointer, uint8_t*, jl_box, 1) -#ifdef _P64 -BOX_FUNC(float64, double, jl_box, 1) -#else -BOX_FUNC(float64, double, jl_box, 2) -#endif +BOX_FUNC(float32, float, jl_box) +BOX_FUNC(float64, double, jl_box) +BOX_FUNC(voidpointer, void*, jl_box) +BOX_FUNC(uint8pointer, uint8_t*, jl_box) #define NBOX_C 1024 -#define SIBOX_FUNC(typ,c_type,nw)\ +// some shims to support UIBOX_FUNC definition +#define jl_ssavalue_tag (((uintptr_t)jl_ssavalue_type) >> 4) +#define jl_slotnumber_tag (((uintptr_t)jl_slotnumber_type) >> 4) + +#define SIBOX_FUNC(typ,c_type) \ static jl_value_t *boxed_##typ##_cache[NBOX_C]; \ JL_DLLEXPORT jl_value_t *jl_box_##typ(c_type x) \ { \ @@ -1298,36 +1301,33 @@ BOX_FUNC(float64, double, jl_box, 2) c_type idx = x+NBOX_C/2; \ if ((u##c_type)idx < (u##c_type)NBOX_C) \ return boxed_##typ##_cache[idx]; \ - jl_value_t *v = jl_gc_alloc(ct->ptls, nw * sizeof(void*), \ + jl_value_t *v = jl_gc_alloc(ct->ptls, LLT_ALIGN(sizeof(x), sizeof(void*)), \ jl_##typ##_type); \ + jl_set_typetagof(v, jl_##typ##_tag, 0); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } -#define UIBOX_FUNC(typ,c_type,nw) \ +#define UIBOX_FUNC(typ,c_type) \ static jl_value_t *boxed_##typ##_cache[NBOX_C]; \ JL_DLLEXPORT jl_value_t *jl_box_##typ(c_type x) \ { \ jl_task_t *ct = jl_current_task; \ if (x < NBOX_C) \ return boxed_##typ##_cache[x]; \ - jl_value_t *v = jl_gc_alloc(ct->ptls, nw * sizeof(void*), \ + jl_value_t *v = jl_gc_alloc(ct->ptls, LLT_ALIGN(sizeof(x), sizeof(void*)), \ jl_##typ##_type); \ + jl_set_typetagof(v, jl_##typ##_tag, 0); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } -SIBOX_FUNC(int16, int16_t, 1) -SIBOX_FUNC(int32, int32_t, 1) -UIBOX_FUNC(uint16, uint16_t, 1) -UIBOX_FUNC(uint32, uint32_t, 1) -UIBOX_FUNC(ssavalue, size_t, 1) -UIBOX_FUNC(slotnumber, size_t, 1) -#ifdef _P64 -SIBOX_FUNC(int64, int64_t, 1) -UIBOX_FUNC(uint64, uint64_t, 1) -#else -SIBOX_FUNC(int64, int64_t, 2) -UIBOX_FUNC(uint64, uint64_t, 2) -#endif +SIBOX_FUNC(int16, int16_t) +SIBOX_FUNC(int32, int32_t) +UIBOX_FUNC(uint16, uint16_t) +UIBOX_FUNC(uint32, uint32_t) +UIBOX_FUNC(ssavalue, size_t) +UIBOX_FUNC(slotnumber, size_t) +SIBOX_FUNC(int64, int64_t) +UIBOX_FUNC(uint64, uint64_t) static jl_value_t *boxed_char_cache[128]; JL_DLLEXPORT jl_value_t *jl_box_char(uint32_t x) @@ -1337,6 +1337,7 @@ JL_DLLEXPORT jl_value_t *jl_box_char(uint32_t x) if (u < 128) return boxed_char_cache[(uint8_t)u]; jl_value_t *v = jl_gc_alloc(ct->ptls, sizeof(void*), jl_char_type); + jl_set_typetagof(v, jl_char_tag, 0); *(uint32_t*)jl_data_ptr(v) = x; return v; } @@ -1356,35 +1357,35 @@ void jl_init_int32_int64_cache(void) { int64_t i; for(i=0; i < NBOX_C; i++) { - boxed_int32_cache[i] = jl_permbox32(jl_int32_type, i-NBOX_C/2); - boxed_int64_cache[i] = jl_permbox64(jl_int64_type, i-NBOX_C/2); + boxed_int32_cache[i] = jl_permbox32(jl_int32_type, jl_int32_tag, i-NBOX_C/2); + boxed_int64_cache[i] = jl_permbox64(jl_int64_type, jl_int64_tag, i-NBOX_C/2); #ifdef _P64 - boxed_ssavalue_cache[i] = jl_permbox64(jl_ssavalue_type, i); - boxed_slotnumber_cache[i] = jl_permbox64(jl_slotnumber_type, i); + boxed_ssavalue_cache[i] = jl_permbox64(jl_ssavalue_type, 0, i); + boxed_slotnumber_cache[i] = jl_permbox64(jl_slotnumber_type, 0, i); #else - boxed_ssavalue_cache[i] = jl_permbox32(jl_ssavalue_type, i); - boxed_slotnumber_cache[i] = jl_permbox32(jl_slotnumber_type, i); + boxed_ssavalue_cache[i] = jl_permbox32(jl_ssavalue_type, 0, i); + boxed_slotnumber_cache[i] = jl_permbox32(jl_slotnumber_type, 0, i); #endif } for(i=0; i < 256; i++) { - jl_boxed_uint8_cache[i] = jl_permbox8(jl_uint8_type, i); + jl_boxed_uint8_cache[i] = jl_permbox8(jl_uint8_type, jl_uint8_tag, i); } } void jl_init_box_caches(void) { - int64_t i; - for(i=0; i < 128; i++) { - boxed_char_cache[i] = jl_permbox32(jl_char_type, i << 24); + uint32_t i; + for (i = 0; i < 128; i++) { + boxed_char_cache[i] = jl_permbox32(jl_char_type, jl_char_tag, i << 24); } - for(i=0; i < 256; i++) { - jl_boxed_int8_cache[i] = jl_permbox8(jl_int8_type, i); + for (i = 0; i < 256; i++) { + jl_boxed_int8_cache[i] = jl_permbox8(jl_int8_type, jl_int8_tag, i); } - for(i=0; i < NBOX_C; i++) { - boxed_int16_cache[i] = jl_permbox16(jl_int16_type, i-NBOX_C/2); - boxed_uint16_cache[i] = jl_permbox16(jl_uint16_type, i); - boxed_uint32_cache[i] = jl_permbox32(jl_uint32_type, i); - boxed_uint64_cache[i] = jl_permbox64(jl_uint64_type, i); + for (i = 0; i < NBOX_C; i++) { + boxed_int16_cache[i] = jl_permbox16(jl_int16_type, jl_int16_tag, i-NBOX_C/2); + boxed_uint16_cache[i] = jl_permbox16(jl_uint16_type, jl_uint16_tag, i); + boxed_uint32_cache[i] = jl_permbox32(jl_uint32_type, jl_uint32_tag, i); + boxed_uint64_cache[i] = jl_permbox64(jl_uint64_type, jl_uint64_tag, i); } } @@ -1408,6 +1409,8 @@ JL_DLLEXPORT jl_value_t *jl_new_struct(jl_datatype_t *type, ...) size_t i, nf = jl_datatype_nfields(type); va_start(args, type); jl_value_t *jv = jl_gc_alloc(ct->ptls, jl_datatype_size(type), type); + if (type->smalltag) // TODO: move to callers? + jl_set_typetagof(jv, type->smalltag, 0); if (nf > 0 && jl_field_offset(type, 0) != 0) { memset(jv, 0, jl_field_offset(type, 0)); } @@ -1435,6 +1438,8 @@ JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, if (type->instance != NULL) return type->instance; jl_value_t *jv = jl_gc_alloc(ct->ptls, jl_datatype_size(type), type); + if (type->smalltag) // TODO: do we need this? + jl_set_typetagof(jv, type->smalltag, 0); if (jl_datatype_nfields(type) > 0) { if (jl_field_offset(type, 0) != 0) { memset(jl_data_ptr(jv), 0, jl_field_offset(type, 0)); @@ -1476,6 +1481,8 @@ JL_DLLEXPORT jl_value_t *jl_new_structt(jl_datatype_t *type, jl_value_t *tup) } size_t size = jl_datatype_size(type); jl_value_t *jv = jl_gc_alloc(ct->ptls, size, type); + if (type->smalltag) // TODO: do we need this? + jl_set_typetagof(jv, type->smalltag, 0); if (nf == 0) return jv; jl_value_t *fi = NULL; @@ -1509,6 +1516,8 @@ JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type) } size_t size = jl_datatype_size(type); jl_value_t *jv = jl_gc_alloc(ct->ptls, size, type); + if (type->smalltag) // TODO: do we need this? + jl_set_typetagof(jv, type->smalltag, 0); if (size > 0) memset(jl_data_ptr(jv), 0, size); return jv; diff --git a/src/gc-debug.c b/src/gc-debug.c index eeb170f0299a1..a5b779c8161b1 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -369,10 +369,10 @@ static void gc_verify_tags_page(jl_gc_pagemeta_t *pg) if (!in_freelist) { jl_value_t *dt = jl_typeof(jl_valueof(v)); if (dt != (jl_value_t*)jl_buff_tag && - // the following are used by the deserializer to invalidate objects - v->header != 0x10 && v->header != 0x20 && - v->header != 0x30 && v->header != 0x40 && - v->header != 0x50 && v->header != 0x60) { + // the following may be use (by the deserializer) to invalidate objects + v->header != 0xf10 && v->header != 0xf20 && + v->header != 0xf30 && v->header != 0xf40 && + v->header != 0xf50 && v->header != 0xf60) { assert(jl_typeof(dt) == (jl_value_t*)jl_datatype_type); } } diff --git a/src/gc.c b/src/gc.c index 60b110826ee80..f421ce4363c67 100644 --- a/src/gc.c +++ b/src/gc.c @@ -579,7 +579,7 @@ JL_DLLEXPORT void jl_gc_add_quiescent(jl_ptls_t ptls, void **v, void *f) JL_NOTS JL_DLLEXPORT void jl_gc_add_finalizer_th(jl_ptls_t ptls, jl_value_t *v, jl_function_t *f) JL_NOTSAFEPOINT { - if (__unlikely(jl_typeis(f, jl_voidpointer_type))) { + if (__unlikely(jl_typetagis(f, jl_voidpointer_type))) { jl_gc_add_ptr_finalizer(ptls, v, jl_unbox_voidpointer(f)); } else { @@ -2232,6 +2232,8 @@ STATIC_INLINE void gc_mark_stack(jl_ptls_t ptls, jl_gcframe_t *s, uint32_t nroot if (nroots & 1) { void **slot = (void **)gc_read_stack(&rts[i], offset, lb, ub); new_obj = (jl_value_t *)gc_read_stack(slot, offset, lb, ub); + if (new_obj == NULL) + continue; } else { new_obj = (jl_value_t *)gc_read_stack(&rts[i], offset, lb, ub); @@ -2243,11 +2245,13 @@ STATIC_INLINE void gc_mark_stack(jl_ptls_t ptls, jl_gcframe_t *s, uint32_t nroot } if (gc_ptr_tag(new_obj, 2)) continue; + // conservatively check for the presence of any smalltag type, instead of just NULL + // in the very unlikely event that codegen decides to root the result of julia.typeof + if (new_obj < (jl_value_t*)((uintptr_t)jl_max_tags << 4)) + continue; } - if (new_obj != NULL) { - gc_try_claim_and_push(mq, new_obj, NULL); - gc_heap_snapshot_record_frame_to_object_edge(s, new_obj); - } + gc_try_claim_and_push(mq, new_obj, NULL); + gc_heap_snapshot_record_frame_to_object_edge(s, new_obj); } jl_gcframe_t *sprev = (jl_gcframe_t *)gc_read_stack(&s->prev, offset, lb, ub); if (sprev == NULL) @@ -2389,7 +2393,7 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_ jl_raise_debugger(); #endif jl_taggedvalue_t *o = jl_astaggedvalue(new_obj); - jl_datatype_t *vt = (jl_datatype_t *)(o->header & ~(uintptr_t)0xf); + uintptr_t vtag = o->header & ~(uintptr_t)0xf; uint8_t bits = (gc_old(o->header) && !mark_reset_age) ? GC_OLD_MARKED : GC_MARKED; int update_meta = __likely(!meta_updated && !gc_verifying); int foreign_alloc = 0; @@ -2399,23 +2403,140 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_ update_meta = 0; } // Symbols are always marked - assert(vt != jl_symbol_type); - if (vt == jl_simplevector_type) { - size_t l = jl_svec_len(new_obj); - jl_value_t **data = jl_svec_data(new_obj); - size_t dtsz = l * sizeof(void *) + sizeof(jl_svec_t); - if (update_meta) - gc_setmark(ptls, o, bits, dtsz); - else if (foreign_alloc) - objprofile_count(vt, bits == GC_OLD_MARKED, dtsz); - jl_value_t *objary_parent = new_obj; - jl_value_t **objary_begin = data; - jl_value_t **objary_end = data + l; - uint32_t step = 1; - uintptr_t nptr = (l << 2) | (bits & GC_OLD); - gc_mark_objarray(ptls, objary_parent, objary_begin, objary_end, step, nptr); + assert(vtag != (uintptr_t)jl_symbol_type && vtag != jl_symbol_tag << 4); + if (vtag == (jl_datatype_tag << 4) || + vtag == (jl_unionall_tag << 4) || + vtag == (jl_uniontype_tag << 4) || + vtag == (jl_tvar_tag << 4) || + vtag == (jl_vararg_tag << 4)) { + // these objects have pointers in them, but no other special handling + // so we want these to fall through to the end + vtag = (uintptr_t)small_typeof[vtag / sizeof(*small_typeof)]; + } + else if (vtag < jl_max_tags << 4) { + // these objects either have specialing handling + if (vtag == jl_simplevector_tag << 4) { + size_t l = jl_svec_len(new_obj); + jl_value_t **data = jl_svec_data(new_obj); + size_t dtsz = l * sizeof(void *) + sizeof(jl_svec_t); + if (update_meta) + gc_setmark(ptls, o, bits, dtsz); + else if (foreign_alloc) + objprofile_count(jl_simplevector_type, bits == GC_OLD_MARKED, dtsz); + jl_value_t *objary_parent = new_obj; + jl_value_t **objary_begin = data; + jl_value_t **objary_end = data + l; + uint32_t step = 1; + uintptr_t nptr = (l << 2) | (bits & GC_OLD); + gc_mark_objarray(ptls, objary_parent, objary_begin, objary_end, step, nptr); + } + else if (vtag == jl_module_tag << 4) { + if (update_meta) + gc_setmark(ptls, o, bits, sizeof(jl_module_t)); + else if (foreign_alloc) + objprofile_count(jl_module_type, bits == GC_OLD_MARKED, sizeof(jl_module_t)); + jl_module_t *mb_parent = (jl_module_t *)new_obj; + jl_svec_t *bindings = jl_atomic_load_relaxed(&mb_parent->bindings); + jl_binding_t **table = (jl_binding_t**)jl_svec_data(bindings); + size_t bsize = jl_svec_len(bindings); + uintptr_t nptr = ((bsize + mb_parent->usings.len + 1) << 2) | (bits & GC_OLD); + jl_binding_t **mb_begin = table + 1; + jl_binding_t **mb_end = table + bsize; + gc_mark_module_binding(ptls, mb_parent, mb_begin, mb_end, nptr, bits); + } + else if (vtag == jl_task_tag << 4) { + if (update_meta) + gc_setmark(ptls, o, bits, sizeof(jl_task_t)); + else if (foreign_alloc) + objprofile_count(jl_task_type, bits == GC_OLD_MARKED, sizeof(jl_task_t)); + jl_task_t *ta = (jl_task_t *)new_obj; + gc_scrub_record_task(ta); + if (gc_cblist_task_scanner) { + int16_t tid = jl_atomic_load_relaxed(&ta->tid); + gc_invoke_callbacks(jl_gc_cb_task_scanner_t, gc_cblist_task_scanner, + (ta, tid != -1 && ta == gc_all_tls_states[tid]->root_task)); + } + #ifdef COPY_STACKS + void *stkbuf = ta->stkbuf; + if (stkbuf && ta->copy_stack) { + gc_setmark_buf_(ptls, stkbuf, bits, ta->bufsz); + // For gc_heap_snapshot_record: + // TODO: attribute size of stack + // TODO: edge to stack data + // TODO: synthetic node for stack data (how big is it?) + } + #endif + jl_gcframe_t *s = ta->gcstack; + size_t nroots; + uintptr_t offset = 0; + uintptr_t lb = 0; + uintptr_t ub = (uintptr_t)-1; + #ifdef COPY_STACKS + if (stkbuf && ta->copy_stack && !ta->ptls) { + int16_t tid = jl_atomic_load_relaxed(&ta->tid); + assert(tid >= 0); + jl_ptls_t ptls2 = gc_all_tls_states[tid]; + ub = (uintptr_t)ptls2->stackbase; + lb = ub - ta->copy_stack; + offset = (uintptr_t)stkbuf - lb; + } + #endif + if (s != NULL) { + nroots = gc_read_stack(&s->nroots, offset, lb, ub); + gc_heap_snapshot_record_task_to_frame_edge(ta, s); + assert(nroots <= UINT32_MAX); + gc_mark_stack(ptls, s, (uint32_t)nroots, offset, lb, ub); + } + if (ta->excstack) { + jl_excstack_t *excstack = ta->excstack; + gc_heap_snapshot_record_task_to_frame_edge(ta, excstack); + size_t itr = ta->excstack->top; + gc_setmark_buf_(ptls, excstack, bits, + sizeof(jl_excstack_t) + + sizeof(uintptr_t) * excstack->reserved_size); + gc_mark_excstack(ptls, excstack, itr); + } + const jl_datatype_layout_t *layout = jl_task_type->layout; + assert(layout->fielddesc_type == 0); + assert(layout->nfields > 0); + uint32_t npointers = layout->npointers; + char *obj8_parent = (char *)ta; + uint8_t *obj8_begin = (uint8_t *)jl_dt_layout_ptrs(layout); + uint8_t *obj8_end = obj8_begin + npointers; + // assume tasks always reference young objects: set lowest bit + uintptr_t nptr = (npointers << 2) | 1 | bits; + new_obj = gc_mark_obj8(ptls, obj8_parent, obj8_begin, obj8_end, nptr); + if (new_obj != NULL) { + if (!meta_updated) + goto mark_obj; + else + gc_ptr_queue_push(mq, new_obj); + } + } + else if (vtag == jl_string_tag << 4) { + size_t dtsz = jl_string_len(new_obj) + sizeof(size_t) + 1; + if (update_meta) + gc_setmark(ptls, o, bits, dtsz); + else if (foreign_alloc) + objprofile_count(jl_string_type, bits == GC_OLD_MARKED, dtsz); + } + else { + jl_datatype_t *vt = small_typeof[vtag / sizeof(*small_typeof)]; + size_t dtsz = jl_datatype_size(vt); + if (update_meta) + gc_setmark(ptls, o, bits, dtsz); + else if (foreign_alloc) + objprofile_count(vt, bits == GC_OLD_MARKED, dtsz); + } + return; + } + else { + jl_datatype_t *vt = (jl_datatype_t *)vtag; + if (__unlikely(!jl_is_datatype(vt) || vt->smalltag)) + gc_assert_datatype_fail(ptls, vt, mq); } - else if (vt->name == jl_array_typename) { + jl_datatype_t *vt = (jl_datatype_t *)vtag; + if (vt->name == jl_array_typename) { jl_array_t *a = (jl_array_t *)new_obj; jl_array_flags_t flags = a->flags; if (update_meta) { @@ -2504,82 +2625,27 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_ assert(0 && "unimplemented"); } } + return; } - else if (vt == jl_module_type) { - if (update_meta) - gc_setmark(ptls, o, bits, sizeof(jl_module_t)); - else if (foreign_alloc) - objprofile_count(vt, bits == GC_OLD_MARKED, sizeof(jl_module_t)); - jl_module_t *mb_parent = (jl_module_t *)new_obj; - jl_svec_t *bindings = jl_atomic_load_relaxed(&mb_parent->bindings); - jl_binding_t **table = (jl_binding_t**)jl_svec_data(bindings); - size_t bsize = jl_svec_len(bindings); - uintptr_t nptr = ((bsize + mb_parent->usings.len + 1) << 2) | (bits & GC_OLD); - jl_binding_t **mb_begin = table + 1; - jl_binding_t **mb_end = table + bsize; - gc_mark_module_binding(ptls, mb_parent, mb_begin, mb_end, nptr, bits); - } - else if (vt == jl_task_type) { - if (update_meta) - gc_setmark(ptls, o, bits, sizeof(jl_task_t)); - else if (foreign_alloc) - objprofile_count(vt, bits == GC_OLD_MARKED, sizeof(jl_task_t)); - jl_task_t *ta = (jl_task_t *)new_obj; - gc_scrub_record_task(ta); - if (gc_cblist_task_scanner) { - int16_t tid = jl_atomic_load_relaxed(&ta->tid); - gc_invoke_callbacks(jl_gc_cb_task_scanner_t, gc_cblist_task_scanner, - (ta, tid != -1 && ta == gc_all_tls_states[tid]->root_task)); - } - #ifdef COPY_STACKS - void *stkbuf = ta->stkbuf; - if (stkbuf && ta->copy_stack) { - gc_setmark_buf_(ptls, stkbuf, bits, ta->bufsz); - // For gc_heap_snapshot_record: - // TODO: attribute size of stack - // TODO: edge to stack data - // TODO: synthetic node for stack data (how big is it?) - } - #endif - jl_gcframe_t *s = ta->gcstack; - size_t nroots; - uintptr_t offset = 0; - uintptr_t lb = 0; - uintptr_t ub = (uintptr_t)-1; - #ifdef COPY_STACKS - if (stkbuf && ta->copy_stack && !ta->ptls) { - int16_t tid = jl_atomic_load_relaxed(&ta->tid); - assert(tid >= 0); - jl_ptls_t ptls2 = gc_all_tls_states[tid]; - ub = (uintptr_t)ptls2->stackbase; - lb = ub - ta->copy_stack; - offset = (uintptr_t)stkbuf - lb; - } - #endif - if (s != NULL) { - nroots = gc_read_stack(&s->nroots, offset, lb, ub); - gc_heap_snapshot_record_task_to_frame_edge(ta, s); - assert(nroots <= UINT32_MAX); - gc_mark_stack(ptls, s, (uint32_t)nroots, offset, lb, ub); - } - if (ta->excstack) { - jl_excstack_t *excstack = ta->excstack; - gc_heap_snapshot_record_task_to_frame_edge(ta, excstack); - size_t itr = ta->excstack->top; - gc_setmark_buf_(ptls, excstack, bits, - sizeof(jl_excstack_t) + - sizeof(uintptr_t) * excstack->reserved_size); - gc_mark_excstack(ptls, excstack, itr); - } - const jl_datatype_layout_t *layout = jl_task_type->layout; - assert(layout->fielddesc_type == 0); - assert(layout->nfields > 0); - uint32_t npointers = layout->npointers; - char *obj8_parent = (char *)ta; + size_t dtsz = jl_datatype_size(vt); + if (update_meta) + gc_setmark(ptls, o, bits, dtsz); + else if (foreign_alloc) + objprofile_count(vt, bits == GC_OLD_MARKED, dtsz); + if (vt == jl_weakref_type) + return; + const jl_datatype_layout_t *layout = vt->layout; + uint32_t npointers = layout->npointers; + if (npointers == 0) + return; + uintptr_t nptr = (npointers << 2 | (bits & GC_OLD)); + assert((layout->nfields > 0 || layout->fielddesc_type == 3) && + "opaque types should have been handled specially"); + if (layout->fielddesc_type == 0) { + char *obj8_parent = (char *)new_obj; uint8_t *obj8_begin = (uint8_t *)jl_dt_layout_ptrs(layout); uint8_t *obj8_end = obj8_begin + npointers; - // assume tasks always reference young objects: set lowest bit - uintptr_t nptr = (npointers << 2) | 1 | bits; + assert(obj8_begin < obj8_end); new_obj = gc_mark_obj8(ptls, obj8_parent, obj8_begin, obj8_end, nptr); if (new_obj != NULL) { if (!meta_updated) @@ -2588,80 +2654,42 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_ gc_ptr_queue_push(mq, new_obj); } } - else if (vt == jl_string_type) { - size_t dtsz = jl_string_len(new_obj) + sizeof(size_t) + 1; - if (update_meta) - gc_setmark(ptls, o, bits, dtsz); - else if (foreign_alloc) - objprofile_count(vt, bits == GC_OLD_MARKED, dtsz); - } - else { - if (__unlikely(!jl_is_datatype(vt))) - gc_assert_datatype_fail(ptls, vt, mq); - size_t dtsz = jl_datatype_size(vt); - if (update_meta) - gc_setmark(ptls, o, bits, dtsz); - else if (foreign_alloc) - objprofile_count(vt, bits == GC_OLD_MARKED, dtsz); - if (vt == jl_weakref_type) - return; - const jl_datatype_layout_t *layout = vt->layout; - uint32_t npointers = layout->npointers; - if (npointers == 0) - return; - uintptr_t nptr = (npointers << 2 | (bits & GC_OLD)); - assert((layout->nfields > 0 || layout->fielddesc_type == 3) && - "opaque types should have been handled specially"); - if (layout->fielddesc_type == 0) { - char *obj8_parent = (char *)new_obj; - uint8_t *obj8_begin = (uint8_t *)jl_dt_layout_ptrs(layout); - uint8_t *obj8_end = obj8_begin + npointers; - assert(obj8_begin < obj8_end); - new_obj = gc_mark_obj8(ptls, obj8_parent, obj8_begin, obj8_end, nptr); - if (new_obj != NULL) { - if (!meta_updated) - goto mark_obj; - else - gc_ptr_queue_push(mq, new_obj); - } - } - else if (layout->fielddesc_type == 1) { - char *obj16_parent = (char *)new_obj; - uint16_t *obj16_begin = (uint16_t *)jl_dt_layout_ptrs(layout); - uint16_t *obj16_end = obj16_begin + npointers; - assert(obj16_begin < obj16_end); - new_obj = gc_mark_obj16(ptls, obj16_parent, obj16_begin, obj16_end, nptr); - if (new_obj != NULL) { - if (!meta_updated) - goto mark_obj; - else - gc_ptr_queue_push(mq, new_obj); - } - } - else if (layout->fielddesc_type == 2) { - // This is very uncommon - // Do not do store to load forwarding to save some code size - char *obj32_parent = (char *)new_obj; - uint32_t *obj32_begin = (uint32_t *)jl_dt_layout_ptrs(layout); - uint32_t *obj32_end = obj32_begin + npointers; - assert(obj32_begin < obj32_end); - new_obj = gc_mark_obj32(ptls, obj32_parent, obj32_begin, obj32_end, nptr); - if (new_obj != NULL) { - if (!meta_updated) - goto mark_obj; - else - gc_ptr_queue_push(mq, new_obj); - } + else if (layout->fielddesc_type == 1) { + char *obj16_parent = (char *)new_obj; + uint16_t *obj16_begin = (uint16_t *)jl_dt_layout_ptrs(layout); + uint16_t *obj16_end = obj16_begin + npointers; + assert(obj16_begin < obj16_end); + new_obj = gc_mark_obj16(ptls, obj16_parent, obj16_begin, obj16_end, nptr); + if (new_obj != NULL) { + if (!meta_updated) + goto mark_obj; + else + gc_ptr_queue_push(mq, new_obj); } - else { - assert(layout->fielddesc_type == 3); - jl_fielddescdyn_t *desc = (jl_fielddescdyn_t *)jl_dt_layout_fields(layout); - int old = jl_astaggedvalue(new_obj)->bits.gc & 2; - uintptr_t young = desc->markfunc(ptls, new_obj); - if (old && young) - gc_mark_push_remset(ptls, new_obj, young * 4 + 3); + } + else if (layout->fielddesc_type == 2) { + // This is very uncommon + // Do not do store to load forwarding to save some code size + char *obj32_parent = (char *)new_obj; + uint32_t *obj32_begin = (uint32_t *)jl_dt_layout_ptrs(layout); + uint32_t *obj32_end = obj32_begin + npointers; + assert(obj32_begin < obj32_end); + new_obj = gc_mark_obj32(ptls, obj32_parent, obj32_begin, obj32_end, nptr); + if (new_obj != NULL) { + if (!meta_updated) + goto mark_obj; + else + gc_ptr_queue_push(mq, new_obj); } } + else { + assert(layout->fielddesc_type == 3); + jl_fielddescdyn_t *desc = (jl_fielddescdyn_t *)jl_dt_layout_fields(layout); + int old = jl_astaggedvalue(new_obj)->bits.gc & 2; + uintptr_t young = desc->markfunc(ptls, new_obj); + if (old && young) + gc_mark_push_remset(ptls, new_obj, young * 4 + 3); + } } } diff --git a/src/gf.c b/src/gf.c index 00a51f582a597..5df3e18ff8db7 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2998,7 +2998,7 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); entry = NULL; if (leafcache != (jl_array_t*)jl_an_empty_vec_any && - jl_typeis(jl_atomic_load_relaxed(&mt->cache), jl_typemap_level_type)) { + jl_typetagis(jl_atomic_load_relaxed(&mt->cache), jl_typemap_level_type)) { // hashing args is expensive, but looking at mt->cache is probably even more expensive tt = lookup_arg_type_tuple(F, args, nargs); if (tt != NULL) diff --git a/src/init.c b/src/init.c index 36e83fdd9c24d..02769e03c668e 100644 --- a/src/init.c +++ b/src/init.c @@ -381,7 +381,7 @@ JL_DLLEXPORT void jl_postoutput_hook(void) return; } -static void post_boot_hooks(void); +void post_boot_hooks(void); JL_DLLEXPORT void *jl_libjulia_internal_handle; JL_DLLEXPORT void *jl_libjulia_handle; @@ -894,80 +894,6 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_install_sigint_handler(); } -static jl_value_t *core(const char *name) -{ - return jl_get_global(jl_core_module, jl_symbol(name)); -} - -// fetch references to things defined in boot.jl -static void post_boot_hooks(void) -{ - jl_char_type = (jl_datatype_t*)core("Char"); - jl_int8_type = (jl_datatype_t*)core("Int8"); - jl_int16_type = (jl_datatype_t*)core("Int16"); - jl_float16_type = (jl_datatype_t*)core("Float16"); - jl_float32_type = (jl_datatype_t*)core("Float32"); - jl_float64_type = (jl_datatype_t*)core("Float64"); - jl_floatingpoint_type = (jl_datatype_t*)core("AbstractFloat"); - jl_number_type = (jl_datatype_t*)core("Number"); - jl_signed_type = (jl_datatype_t*)core("Signed"); - jl_datatype_t *jl_unsigned_type = (jl_datatype_t*)core("Unsigned"); - jl_datatype_t *jl_integer_type = (jl_datatype_t*)core("Integer"); - - jl_bool_type->super = jl_integer_type; - jl_uint8_type->super = jl_unsigned_type; - jl_uint16_type->super = jl_unsigned_type; - jl_uint32_type->super = jl_unsigned_type; - jl_uint64_type->super = jl_unsigned_type; - jl_int32_type->super = jl_signed_type; - jl_int64_type->super = jl_signed_type; - - jl_errorexception_type = (jl_datatype_t*)core("ErrorException"); - jl_stackovf_exception = jl_new_struct_uninit((jl_datatype_t*)core("StackOverflowError")); - jl_diverror_exception = jl_new_struct_uninit((jl_datatype_t*)core("DivideError")); - jl_undefref_exception = jl_new_struct_uninit((jl_datatype_t*)core("UndefRefError")); - jl_undefvarerror_type = (jl_datatype_t*)core("UndefVarError"); - jl_atomicerror_type = (jl_datatype_t*)core("ConcurrencyViolationError"); - jl_interrupt_exception = jl_new_struct_uninit((jl_datatype_t*)core("InterruptException")); - jl_boundserror_type = (jl_datatype_t*)core("BoundsError"); - jl_memory_exception = jl_new_struct_uninit((jl_datatype_t*)core("OutOfMemoryError")); - jl_readonlymemory_exception = jl_new_struct_uninit((jl_datatype_t*)core("ReadOnlyMemoryError")); - jl_typeerror_type = (jl_datatype_t*)core("TypeError"); - jl_argumenterror_type = (jl_datatype_t*)core("ArgumentError"); - jl_methoderror_type = (jl_datatype_t*)core("MethodError"); - jl_loaderror_type = (jl_datatype_t*)core("LoadError"); - jl_initerror_type = (jl_datatype_t*)core("InitError"); - jl_pair_type = core("Pair"); - jl_kwcall_func = core("kwcall"); - jl_kwcall_mt = ((jl_datatype_t*)jl_typeof(jl_kwcall_func))->name->mt; - jl_atomic_store_relaxed(&jl_kwcall_mt->max_args, 0); - - jl_weakref_type = (jl_datatype_t*)core("WeakRef"); - jl_vecelement_typename = ((jl_datatype_t*)jl_unwrap_unionall(core("VecElement")))->name; - - jl_init_box_caches(); - - // set module field of primitive types - jl_svec_t *bindings = jl_atomic_load_relaxed(&jl_core_module->bindings); - jl_value_t **table = jl_svec_data(bindings); - for (size_t i = 0; i < jl_svec_len(bindings); i++) { - if (table[i] != jl_nothing) { - jl_binding_t *b = (jl_binding_t*)table[i]; - jl_value_t *v = jl_atomic_load_relaxed(&b->value); - if (v) { - if (jl_is_unionall(v)) - v = jl_unwrap_unionall(v); - if (jl_is_datatype(v)) { - jl_datatype_t *tt = (jl_datatype_t*)v; - tt->name->module = jl_core_module; - if (tt->name->mt) - tt->name->mt->module = jl_core_module; - } - } - } - } -} - #ifdef __cplusplus } #endif diff --git a/src/interpreter.c b/src/interpreter.c index 7a699223d746e..c962abaa3a31a 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -102,7 +102,7 @@ static jl_value_t *eval_methoddef(jl_expr_t *ex, interpreter_state *s) fname = eval_value(args[0], s); jl_methtable_t *mt = NULL; - if (jl_typeis(fname, jl_methtable_type)) { + if (jl_typetagis(fname, jl_methtable_type)) { mt = (jl_methtable_t*)fname; } atypes = eval_value(args[1], s); @@ -663,7 +663,7 @@ jl_value_t *NOINLINE jl_fptr_interpret_call(jl_value_t *f, jl_value_t **args, ui size_t world = ct->world_age; jl_code_info_t *src = jl_code_for_interpreter(mi, world); jl_array_t *stmts = src->code; - assert(jl_typeis(stmts, jl_array_any_type)); + assert(jl_typetagis(stmts, jl_array_any_type)); unsigned nroots = jl_source_nslots(src) + jl_source_nssavalues(src) + 2; jl_value_t **locals = NULL; JL_GC_PUSHFRAME(s, locals, nroots); @@ -748,7 +748,7 @@ jl_value_t *NOINLINE jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t unsigned nroots = jl_source_nslots(src) + jl_source_nssavalues(src); JL_GC_PUSHFRAME(s, s->locals, nroots); jl_array_t *stmts = src->code; - assert(jl_typeis(stmts, jl_array_any_type)); + assert(jl_typetagis(stmts, jl_array_any_type)); s->src = src; s->module = m; s->sparam_vals = jl_emptysvec; diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 91a06f2f10524..9fd0561971d02 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -514,7 +514,7 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, const jl_cgval_t *argv) bool isboxed; Type *vxt = julia_type_to_llvm(ctx, v.typ, &isboxed); if (!jl_is_primitivetype(v.typ) || jl_datatype_size(v.typ) != nb) { - Value *typ = emit_typeof_boxed(ctx, v); + Value *typ = emit_typeof(ctx, v, false, false); if (!jl_is_primitivetype(v.typ)) { if (jl_is_datatype(v.typ) && !jl_is_abstracttype(v.typ)) { emit_error(ctx, "bitcast: value not a primitive type"); @@ -678,8 +678,7 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) else if (!jl_isbits(ety)) { assert(jl_is_datatype(ety)); uint64_t size = jl_datatype_size(ety); - Value *strct = emit_allocobj(ctx, size, - literal_pointer_val(ctx, ety)); + Value *strct = emit_allocobj(ctx, (jl_datatype_t*)ety); im1 = ctx.builder.CreateMul(im1, ConstantInt::get(ctx.types().T_size, LLT_ALIGN(size, jl_datatype_align(ety)))); Value *thePtr = emit_unbox(ctx, getInt8PtrTy(ctx.builder.getContext()), e, e.typ); @@ -823,9 +822,7 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) if (!jl_isbits(ety)) { assert(jl_is_datatype(ety)); - uint64_t size = jl_datatype_size(ety); - Value *strct = emit_allocobj(ctx, size, - literal_pointer_val(ctx, ety)); + Value *strct = emit_allocobj(ctx, (jl_datatype_t*)ety); Value *thePtr = emit_unbox(ctx, getInt8PtrTy(ctx.builder.getContext()), e, e.typ); Type *loadT = Type::getIntNTy(ctx.builder.getContext(), nb * 8); thePtr = emit_bitcast(ctx, thePtr, loadT->getPointerTo()); diff --git a/src/ircode.c b/src/ircode.c index 71ee292cbc397..e04ef3fa6d96c 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -147,7 +147,7 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) else if (v == (jl_value_t*)jl_base_module) { write_uint8(s->s, TAG_BASE); } - else if (jl_typeis(v, jl_string_type) && jl_string_len(v) == 0) { + else if (jl_typetagis(v, jl_string_tag << 4) && jl_string_len(v) == 0) { jl_encode_value(s, jl_an_empty_string); } else if (v == (jl_value_t*)s->method->module) { @@ -197,7 +197,7 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) write_uint8(s->s, TAG_LONG_SSAVALUE); write_uint16(s->s, ((jl_ssavalue_t*)v)->id); } - else if (jl_typeis(v, jl_slotnumber_type) && jl_slot_number(v) <= UINT16_MAX && jl_slot_number(v) >= 0) { + else if (jl_typetagis(v, jl_slotnumber_type) && jl_slot_number(v) <= UINT16_MAX && jl_slot_number(v) >= 0) { write_uint8(s->s, TAG_SLOTNUMBER); write_uint16(s->s, jl_slot_number(v)); } @@ -299,7 +299,7 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) else jl_encode_value(s, inner); } - else if (jl_typeis(v, jl_int64_type)) { + else if (jl_typetagis(v, jl_int64_tag << 4)) { void *data = jl_data_ptr(v); if (*(int64_t*)data >= INT16_MIN && *(int64_t*)data <= INT16_MAX) { write_uint8(s->s, TAG_SHORTER_INT64); @@ -314,14 +314,14 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) write_uint64(s->s, *(int64_t*)data); } } - else if (jl_typeis(v, jl_int32_type)) { + else if (jl_typetagis(v, jl_int32_tag << 4)) { jl_encode_int32(s, *(int32_t*)jl_data_ptr(v)); } - else if (jl_typeis(v, jl_uint8_type)) { + else if (jl_typetagis(v, jl_uint8_tag << 4)) { write_uint8(s->s, TAG_UINT8); write_int8(s->s, *(int8_t*)jl_data_ptr(v)); } - else if (jl_typeis(v, jl_lineinfonode_type)) { + else if (jl_typetagis(v, jl_lineinfonode_type)) { write_uint8(s->s, TAG_LINEINFO); for (i = 0; i < jl_datatype_nfields(jl_lineinfonode_type); i++) jl_encode_value(s, jl_get_nth_field(v, i)); @@ -330,7 +330,7 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) write_uint8(s->s, TAG_SINGLETON); jl_encode_value(s, jl_typeof(v)); } - else if (as_literal && jl_typeis(v, jl_string_type)) { + else if (as_literal && jl_typetagis(v, jl_string_tag << 4)) { write_uint8(s->s, TAG_STRING); write_int32(s->s, jl_string_len(v)); ios_write(s->s, jl_string_data(v), jl_string_len(v)); @@ -610,9 +610,12 @@ static jl_value_t *jl_decode_value_any(jl_ircode_state *s, uint8_t tag) JL_GC_DI { int32_t sz = (tag == TAG_SHORT_GENERAL ? read_uint8(s->s) : read_int32(s->s)); jl_value_t *v = jl_gc_alloc(s->ptls, sz, NULL); - jl_set_typeof(v, (void*)(intptr_t)0x50); + jl_set_typeof(v, (void*)(intptr_t)0xf50); jl_datatype_t *dt = (jl_datatype_t*)jl_decode_value(s); - jl_set_typeof(v, dt); + if (dt->smalltag) + jl_set_typetagof(v, dt->smalltag, 0); + else + jl_set_typeof(v, dt); char *data = (char*)jl_data_ptr(v); size_t i, np = dt->layout->npointers; char *start = data; @@ -858,7 +861,7 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t JL_TIMING(AST_UNCOMPRESS, AST_UNCOMPRESS); JL_LOCK(&m->writelock); // protect the roots array (Might GC) assert(jl_is_method(m)); - assert(jl_typeis(data, jl_array_uint8_type)); + assert(jl_typetagis(data, jl_array_uint8_type)); size_t i; ios_t src; ios_mem(&src, 0); @@ -940,7 +943,7 @@ JL_DLLEXPORT uint8_t jl_ir_flag_inferred(jl_array_t *data) { if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->inferred; - assert(jl_typeis(data, jl_array_uint8_type)); + assert(jl_typetagis(data, jl_array_uint8_type)); jl_code_info_flags_t flags; flags.packed = ((uint8_t*)data->data)[0]; return flags.bits.inferred; @@ -950,7 +953,7 @@ JL_DLLEXPORT uint8_t jl_ir_flag_inlining(jl_array_t *data) { if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->inlining; - assert(jl_typeis(data, jl_array_uint8_type)); + assert(jl_typetagis(data, jl_array_uint8_type)); jl_code_info_flags_t flags; flags.packed = ((uint8_t*)data->data)[0]; return flags.bits.inlining; @@ -960,7 +963,7 @@ JL_DLLEXPORT uint8_t jl_ir_flag_has_fcall(jl_array_t *data) { if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->has_fcall; - assert(jl_typeis(data, jl_array_uint8_type)); + assert(jl_typetagis(data, jl_array_uint8_type)); jl_code_info_flags_t flags; flags.packed = ((uint8_t*)data->data)[0]; return flags.bits.has_fcall; @@ -970,7 +973,7 @@ JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_array_t *data) { if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->inlining_cost; - assert(jl_typeis(data, jl_array_uint8_type)); + assert(jl_typetagis(data, jl_array_uint8_type)); uint16_t res = jl_load_unaligned_i16((char*)data->data + 2); return res; } @@ -1008,7 +1011,7 @@ JL_DLLEXPORT ssize_t jl_ir_nslots(jl_array_t *data) return jl_array_len(func->slotnames); } else { - assert(jl_typeis(data, jl_array_uint8_type)); + assert(jl_typetagis(data, jl_array_uint8_type)); int nslots = jl_load_unaligned_i32((char*)data->data + 2 + sizeof(uint16_t)); return nslots; } @@ -1019,7 +1022,7 @@ JL_DLLEXPORT uint8_t jl_ir_slotflag(jl_array_t *data, size_t i) assert(i < jl_ir_nslots(data)); if (jl_is_code_info(data)) return ((uint8_t*)((jl_code_info_t*)data)->slotflags->data)[i]; - assert(jl_typeis(data, jl_array_uint8_type)); + assert(jl_typetagis(data, jl_array_uint8_type)); return ((uint8_t*)data->data)[2 + sizeof(uint16_t) + sizeof(int32_t) + i]; } diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index c09f2aff4cb88..cce87f53368f4 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -120,7 +120,7 @@ XX(jl_check_pkgimage_clones) \ XX(jl_egal) \ XX(jl_egal__bits) \ - XX(jl_egal__special) \ + XX(jl_egal__bitstag) \ XX(jl_eh_restore_state) \ XX(jl_enter_handler) \ XX(jl_enter_threaded_region) \ diff --git a/src/jltypes.c b/src/jltypes.c index 85255f9247439..0549ee2c07e53 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -20,6 +20,7 @@ extern "C" { #endif _Atomic(jl_value_t*) cmpswap_names JL_GLOBALLY_ROOTED; +jl_datatype_t *small_typeof[(jl_max_tags << 4) / sizeof(*small_typeof)]; // 16-bit aligned, like the GC // compute empirical max-probe for a given size #define max_probe(size) ((size) <= 1024 ? 16 : (size) >> 6) @@ -51,7 +52,7 @@ static int typeenv_has_ne(jl_typeenv_t *env, jl_tvar_t *v) JL_NOTSAFEPOINT static int layout_uses_free_typevars(jl_value_t *v, jl_typeenv_t *env) { while (1) { - if (jl_typeis(v, jl_tvar_type)) + if (jl_is_typevar(v)) return !typeenv_has(env, (jl_tvar_t*)v); while (jl_is_unionall(v)) { jl_unionall_t *ua = (jl_unionall_t*)v; @@ -106,7 +107,7 @@ static int layout_uses_free_typevars(jl_value_t *v, jl_typeenv_t *env) static int has_free_typevars(jl_value_t *v, jl_typeenv_t *env) JL_NOTSAFEPOINT { while (1) { - if (jl_typeis(v, jl_tvar_type)) { + if (jl_is_typevar(v)) { return !typeenv_has(env, (jl_tvar_t*)v); } while (jl_is_unionall(v)) { @@ -160,7 +161,7 @@ JL_DLLEXPORT int jl_has_free_typevars(jl_value_t *v) JL_NOTSAFEPOINT static void find_free_typevars(jl_value_t *v, jl_typeenv_t *env, jl_array_t *out) { while (1) { - if (jl_typeis(v, jl_tvar_type)) { + if (jl_is_typevar(v)) { if (!typeenv_has(env, (jl_tvar_t*)v)) jl_array_ptr_1d_push(out, v); return; @@ -217,7 +218,7 @@ JL_DLLEXPORT jl_array_t *jl_find_free_typevars(jl_value_t *v) static int jl_has_bound_typevars(jl_value_t *v, jl_typeenv_t *env) JL_NOTSAFEPOINT { while (1) { - if (jl_typeis(v, jl_tvar_type)) { + if (jl_is_typevar(v)) { return typeenv_has_ne(env, (jl_tvar_t*)v); } while (jl_is_unionall(v)) { @@ -2353,7 +2354,7 @@ jl_datatype_t *jl_wrap_Type(jl_value_t *t) return (jl_datatype_t*)jl_instantiate_unionall(jl_type_type, t); } -jl_vararg_t *jl_wrap_vararg(jl_value_t *t, jl_value_t *n) +JL_DLLEXPORT jl_vararg_t *jl_wrap_vararg(jl_value_t *t, jl_value_t *n) { if (n) { if (jl_is_typevar(n) || jl_is_uniontype(jl_unwrap_unionall(n))) { @@ -2380,6 +2381,7 @@ jl_vararg_t *jl_wrap_vararg(jl_value_t *t, jl_value_t *n) } jl_task_t *ct = jl_current_task; jl_vararg_t *vm = (jl_vararg_t *)jl_gc_alloc(ct->ptls, sizeof(jl_vararg_t), jl_vararg_type); + jl_set_typetagof(vm, jl_vararg_tag, 0); vm->T = t; vm->N = n; return vm; @@ -2469,19 +2471,36 @@ static jl_tvar_t *tvar(const char *name) (jl_value_t*)jl_any_type); } +void export_small_typeof(void) +{ + void *copy; +#ifdef _OS_WINDOWS_ + jl_dlsym(jl_libjulia_handle, "small_typeof", ©, 1); +#else + jl_dlsym(jl_libjulia_internal_handle, "small_typeof", ©, 1); +#endif + memcpy(copy, &small_typeof, sizeof(small_typeof)); +} + +#define XX(name) \ + small_typeof[(jl_##name##_tag << 4) / sizeof(*small_typeof)] = jl_##name##_type; \ + jl_##name##_type->smalltag = jl_##name##_tag; void jl_init_types(void) JL_GC_DISABLED { jl_module_t *core = NULL; // will need to be assigned later // create base objects jl_datatype_type = jl_new_uninitialized_datatype(); - jl_set_typeof(jl_datatype_type, jl_datatype_type); + XX(datatype); jl_typename_type = jl_new_uninitialized_datatype(); jl_symbol_type = jl_new_uninitialized_datatype(); + XX(symbol); jl_simplevector_type = jl_new_uninitialized_datatype(); + XX(simplevector); jl_methtable_type = jl_new_uninitialized_datatype(); jl_emptysvec = (jl_svec_t*)jl_gc_permobj(sizeof(void*), jl_simplevector_type); + jl_set_typetagof(jl_emptysvec, jl_simplevector_tag, GC_OLD_MARKED); jl_svec_set_len_unsafe(jl_emptysvec, 0); jl_any_type = (jl_datatype_t*)jl_new_abstracttype((jl_value_t*)jl_symbol("Any"), core, NULL, jl_emptysvec); @@ -2604,18 +2623,22 @@ void jl_init_types(void) JL_GC_DISABLED jl_perm_symsvec(3, "name", "lb", "ub"), jl_svec(3, jl_symbol_type, jl_any_type, jl_any_type), jl_emptysvec, 0, 1, 3); + XX(tvar); const static uint32_t tvar_constfields[1] = { 0x00000007 }; // all fields are constant, even though TypeVar itself has identity jl_tvar_type->name->constfields = tvar_constfields; jl_typeofbottom_type = jl_new_datatype(jl_symbol("TypeofBottom"), core, type_type, jl_emptysvec, - jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 0, 0); - jl_bottom_type = jl_new_struct(jl_typeofbottom_type); + jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 0, 0); + XX(typeofbottom); + jl_bottom_type = jl_gc_permobj(0, jl_typeofbottom_type); + jl_set_typetagof(jl_bottom_type, jl_typeofbottom_tag, GC_OLD_MARKED); jl_typeofbottom_type->instance = jl_bottom_type; jl_unionall_type = jl_new_datatype(jl_symbol("UnionAll"), core, type_type, jl_emptysvec, jl_perm_symsvec(2, "var", "body"), jl_svec(2, jl_tvar_type, jl_any_type), jl_emptysvec, 0, 0, 2); + XX(unionall); // It seems like we probably usually end up needing the box for kinds (often used in an Any context), so force it to exist jl_unionall_type->name->mayinlinealloc = 0; @@ -2623,6 +2646,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_perm_symsvec(2, "a", "b"), jl_svec(2, jl_any_type, jl_any_type), jl_emptysvec, 0, 0, 2); + XX(uniontype); // It seems like we probably usually end up needing the box for kinds (often used in an Any context), so force it to exist jl_uniontype_type->name->mayinlinealloc = 0; @@ -2638,6 +2662,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_perm_symsvec(2, "T", "N"), jl_svec(2, jl_any_type, jl_any_type), jl_emptysvec, 0, 0, 0); + XX(vararg); // It seems like we probably usually end up needing the box for kinds (often used in an Any context), so force it to exist jl_vararg_type->name->mayinlinealloc = 0; @@ -2659,16 +2684,22 @@ void jl_init_types(void) JL_GC_DISABLED // non-primitive definitions follow jl_int32_type = jl_new_primitivetype((jl_value_t*)jl_symbol("Int32"), core, jl_any_type, jl_emptysvec, 32); + XX(int32); jl_int64_type = jl_new_primitivetype((jl_value_t*)jl_symbol("Int64"), core, jl_any_type, jl_emptysvec, 64); + XX(int64); jl_uint32_type = jl_new_primitivetype((jl_value_t*)jl_symbol("UInt32"), core, jl_any_type, jl_emptysvec, 32); + XX(uint32); jl_uint64_type = jl_new_primitivetype((jl_value_t*)jl_symbol("UInt64"), core, jl_any_type, jl_emptysvec, 64); + XX(uint64); jl_uint8_type = jl_new_primitivetype((jl_value_t*)jl_symbol("UInt8"), core, jl_any_type, jl_emptysvec, 8); + XX(uint8); jl_uint16_type = jl_new_primitivetype((jl_value_t*)jl_symbol("UInt16"), core, jl_any_type, jl_emptysvec, 16); + XX(uint16); jl_ssavalue_type = jl_new_datatype(jl_symbol("SSAValue"), core, jl_any_type, jl_emptysvec, jl_perm_symsvec(1, "id"), @@ -2690,12 +2721,14 @@ void jl_init_types(void) JL_GC_DISABLED jl_bool_type = NULL; jl_bool_type = jl_new_primitivetype((jl_value_t*)jl_symbol("Bool"), core, jl_any_type, jl_emptysvec, 8); - jl_false = jl_permbox8(jl_bool_type, 0); - jl_true = jl_permbox8(jl_bool_type, 1); + XX(bool); + jl_false = jl_permbox8(jl_bool_type, jl_bool_tag, 0); + jl_true = jl_permbox8(jl_bool_type, jl_bool_tag, 1); jl_abstractstring_type = jl_new_abstracttype((jl_value_t*)jl_symbol("AbstractString"), core, jl_any_type, jl_emptysvec); jl_string_type = jl_new_datatype(jl_symbol("String"), core, jl_abstractstring_type, jl_emptysvec, jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 1, 0); + XX(string); jl_string_type->instance = NULL; jl_compute_field_offsets(jl_string_type); jl_an_empty_string = jl_pchar_to_string("\0", 1); @@ -2796,6 +2829,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_module_type = jl_new_datatype(jl_symbol("Module"), core, jl_any_type, jl_emptysvec, jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 1, 0); + XX(module); jl_module_type->instance = NULL; jl_compute_field_offsets(jl_module_type); @@ -3166,11 +3200,9 @@ void jl_init_types(void) JL_GC_DISABLED jl_uint16_type), jl_emptysvec, 0, 1, 6); + XX(task); jl_value_t *listt = jl_new_struct(jl_uniontype_type, jl_task_type, jl_nothing_type); jl_svecset(jl_task_type->types, 0, listt); - jl_astaggedvalue(jl_current_task)->header = (uintptr_t)jl_task_type | jl_astaggedvalue(jl_current_task)->header; - - jl_value_t *pointer_void = jl_apply_type1((jl_value_t*)jl_pointer_type, (jl_value_t*)jl_nothing_type); jl_binding_type = jl_new_datatype(jl_symbol("Binding"), core, jl_any_type, jl_emptysvec, @@ -3188,6 +3220,8 @@ void jl_init_types(void) JL_GC_DISABLED jl_svec(3, jl_module_type, jl_symbol_type, jl_binding_type), jl_emptysvec, 0, 0, 3); + jl_value_t *pointer_void = jl_apply_type1((jl_value_t*)jl_pointer_type, (jl_value_t*)jl_nothing_type); + jl_voidpointer_type = (jl_datatype_t*)pointer_void; tv = jl_svec2(tvar("A"), tvar("R")); jl_opaque_closure_type = (jl_unionall_t*)jl_new_datatype(jl_symbol("OpaqueClosure"), core, jl_function_type, tv, // N.B.: OpaqueClosure call code relies on specptr being field 5. @@ -3204,7 +3238,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_emptysvec, 0, 0, 4); // complete builtin type metadata - jl_voidpointer_type = (jl_datatype_t*)pointer_void; jl_uint8pointer_type = (jl_datatype_t*)jl_apply_type1((jl_value_t*)jl_pointer_type, (jl_value_t*)jl_uint8_type); jl_svecset(jl_datatype_type->types, 5, jl_voidpointer_type); jl_svecset(jl_datatype_type->types, 6, jl_int32_type); @@ -3266,7 +3299,90 @@ void jl_init_types(void) JL_GC_DISABLED // override the preferred layout for a couple types jl_lineinfonode_type->name->mayinlinealloc = 0; // FIXME: assumed to be a pointer by codegen + export_small_typeof(); +} + +static jl_value_t *core(const char *name) +{ + return jl_get_global(jl_core_module, jl_symbol(name)); +} + +// fetch references to things defined in boot.jl +void post_boot_hooks(void) +{ + jl_char_type = (jl_datatype_t*)core("Char"); + XX(char); + jl_int8_type = (jl_datatype_t*)core("Int8"); + XX(int8); + jl_int16_type = (jl_datatype_t*)core("Int16"); + XX(int16); + jl_float16_type = (jl_datatype_t*)core("Float16"); + //XX(float16); + jl_float32_type = (jl_datatype_t*)core("Float32"); + //XX(float32); + jl_float64_type = (jl_datatype_t*)core("Float64"); + //XX(float64); + jl_floatingpoint_type = (jl_datatype_t*)core("AbstractFloat"); + jl_number_type = (jl_datatype_t*)core("Number"); + jl_signed_type = (jl_datatype_t*)core("Signed"); + jl_datatype_t *jl_unsigned_type = (jl_datatype_t*)core("Unsigned"); + jl_datatype_t *jl_integer_type = (jl_datatype_t*)core("Integer"); + + jl_bool_type->super = jl_integer_type; + jl_uint8_type->super = jl_unsigned_type; + jl_uint16_type->super = jl_unsigned_type; + jl_uint32_type->super = jl_unsigned_type; + jl_uint64_type->super = jl_unsigned_type; + jl_int32_type->super = jl_signed_type; + jl_int64_type->super = jl_signed_type; + + jl_errorexception_type = (jl_datatype_t*)core("ErrorException"); + jl_stackovf_exception = jl_new_struct_uninit((jl_datatype_t*)core("StackOverflowError")); + jl_diverror_exception = jl_new_struct_uninit((jl_datatype_t*)core("DivideError")); + jl_undefref_exception = jl_new_struct_uninit((jl_datatype_t*)core("UndefRefError")); + jl_undefvarerror_type = (jl_datatype_t*)core("UndefVarError"); + jl_atomicerror_type = (jl_datatype_t*)core("ConcurrencyViolationError"); + jl_interrupt_exception = jl_new_struct_uninit((jl_datatype_t*)core("InterruptException")); + jl_boundserror_type = (jl_datatype_t*)core("BoundsError"); + jl_memory_exception = jl_new_struct_uninit((jl_datatype_t*)core("OutOfMemoryError")); + jl_readonlymemory_exception = jl_new_struct_uninit((jl_datatype_t*)core("ReadOnlyMemoryError")); + jl_typeerror_type = (jl_datatype_t*)core("TypeError"); + jl_argumenterror_type = (jl_datatype_t*)core("ArgumentError"); + jl_methoderror_type = (jl_datatype_t*)core("MethodError"); + jl_loaderror_type = (jl_datatype_t*)core("LoadError"); + jl_initerror_type = (jl_datatype_t*)core("InitError"); + jl_pair_type = core("Pair"); + jl_kwcall_func = core("kwcall"); + jl_kwcall_mt = ((jl_datatype_t*)jl_typeof(jl_kwcall_func))->name->mt; + jl_atomic_store_relaxed(&jl_kwcall_mt->max_args, 0); + + jl_weakref_type = (jl_datatype_t*)core("WeakRef"); + jl_vecelement_typename = ((jl_datatype_t*)jl_unwrap_unionall(core("VecElement")))->name; + + jl_init_box_caches(); + + // set module field of primitive types + jl_svec_t *bindings = jl_atomic_load_relaxed(&jl_core_module->bindings); + jl_value_t **table = jl_svec_data(bindings); + for (size_t i = 0; i < jl_svec_len(bindings); i++) { + if (table[i] != jl_nothing) { + jl_binding_t *b = (jl_binding_t*)table[i]; + jl_value_t *v = jl_atomic_load_relaxed(&b->value); + if (v) { + if (jl_is_unionall(v)) + v = jl_unwrap_unionall(v); + if (jl_is_datatype(v)) { + jl_datatype_t *tt = (jl_datatype_t*)v; + tt->name->module = jl_core_module; + if (tt->name->mt) + tt->name->mt->module = jl_core_module; + } + } + } + } + export_small_typeof(); } +#undef XX #ifdef __cplusplus } diff --git a/src/julia.expmap b/src/julia.expmap index 7df813498182b..94b955e95981f 100644 --- a/src/julia.expmap +++ b/src/julia.expmap @@ -7,6 +7,7 @@ ios_*; arraylist_grow; small_arraylist_grow; + small_typeof; jl_*; ijl_*; _jl_mutex_*; @@ -18,10 +19,7 @@ memhash32; memhash32_seed; memhash_seed; - restore_arg_area_loc; restore_signals; - rl_clear_input; - save_arg_area_loc; u8_*; uv_*; add_library_mapping; diff --git a/src/julia.h b/src/julia.h index 89fea54fc428f..6ecb845f77afe 100644 --- a/src/julia.h +++ b/src/julia.h @@ -92,10 +92,11 @@ typedef struct _jl_value_t jl_value_t; struct _jl_taggedvalue_bits { uintptr_t gc:2; uintptr_t in_image:1; + uintptr_t unused:1; #ifdef _P64 - uintptr_t padding:61; + uintptr_t tag:60; #else - uintptr_t padding:29; + uintptr_t tag:28; #endif }; @@ -109,6 +110,7 @@ JL_EXTENSION struct _jl_taggedvalue_t { // jl_value_t value; }; +static inline jl_value_t *jl_to_typeof(uintptr_t t) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; #ifdef __clang_gcanalyzer__ JL_DLLEXPORT jl_taggedvalue_t *_jl_astaggedvalue(jl_value_t *v JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; #define jl_astaggedvalue(v) _jl_astaggedvalue((jl_value_t*)(v)) @@ -119,10 +121,10 @@ JL_DLLEXPORT jl_value_t *_jl_typeof(jl_value_t *v JL_PROPAGATES_ROOT) JL_NOTSAFE #else #define jl_astaggedvalue(v) \ ((jl_taggedvalue_t*)((char*)(v) - sizeof(jl_taggedvalue_t))) -#define jl_valueof(v) \ +#define jl_valueof(v) \ ((jl_value_t*)((char*)(v) + sizeof(jl_taggedvalue_t))) #define jl_typeof(v) \ - ((jl_value_t*)(jl_astaggedvalue(v)->header & ~(uintptr_t)15)) + jl_to_typeof(jl_typetagof(v)) #endif static inline void jl_set_typeof(void *v, void *t) JL_NOTSAFEPOINT { @@ -130,7 +132,11 @@ static inline void jl_set_typeof(void *v, void *t) JL_NOTSAFEPOINT jl_taggedvalue_t *tag = jl_astaggedvalue(v); jl_atomic_store_relaxed((_Atomic(jl_value_t*)*)&tag->type, (jl_value_t*)t); } +#define jl_typetagof(v) \ + ((jl_astaggedvalue(v)->header) & ~(uintptr_t)15) #define jl_typeis(v,t) (jl_typeof(v)==(jl_value_t*)(t)) +#define jl_typetagis(v,t) (jl_typetagof(v)==(uintptr_t)(t)) +#define jl_set_typetagof(v,t,gc) (jl_set_typeof((v), (void*)(((uintptr_t)(t) << 4) | (gc)))) // Symbols are interned strings (hash-consed) stored as an invasive binary tree. // The string data is nul-terminated and hangs off the end of the struct. @@ -562,7 +568,7 @@ typedef struct _jl_datatype_t { uint16_t isprimitivetype:1; // whether this is declared with 'primitive type' keyword (sized, no fields, and immutable) uint16_t ismutationfree:1; // whether any mutable memory is reachable through this type (in the type or via fields) uint16_t isidentityfree:1; // whether this type or any object reachable through its fields has non-content-based identity - uint16_t padding:6; + uint16_t smalltag:6; // whether this type has a small-tag optimization } jl_datatype_t; typedef struct _jl_vararg_t { @@ -694,6 +700,59 @@ typedef struct { // constants and type objects ------------------------------------------------- +#define JL_SMALL_TYPEOF(XX) \ + /* kinds */ \ + XX(typeofbottom) \ + XX(datatype) \ + XX(unionall) \ + XX(uniontype) \ + /* type parameter objects */ \ + XX(vararg) \ + XX(tvar) \ + XX(symbol) \ + XX(module) \ + /* special GC objects */ \ + XX(simplevector) \ + XX(string) \ + XX(task) \ + /* bits types with special allocators */ \ + XX(bool) \ + XX(char) \ + /*XX(float16)*/ \ + /*XX(float32)*/ \ + /*XX(float64)*/ \ + XX(int16) \ + XX(int32) \ + XX(int64) \ + XX(int8) \ + XX(uint16) \ + XX(uint32) \ + XX(uint64) \ + XX(uint8) \ + /* AST objects */ \ + /* XX(argument) */ \ + /* XX(newvarnode) */ \ + /* XX(slotnumber) */ \ + /* XX(ssavalue) */ \ + /* end of JL_SMALL_TYPEOF */ +enum jlsmall_typeof_tags { + jl_null_tag = 0, +#define XX(name) jl_##name##_tag, + JL_SMALL_TYPEOF(XX) +#undef XX + jl_tags_count, + jl_bitstags_first = jl_char_tag, // n.b. bool is not considered a bitstype, since it can be compared by pointer + jl_max_tags = 64 +}; +extern jl_datatype_t *small_typeof[(jl_max_tags << 4) / sizeof(jl_datatype_t*)]; +static inline jl_value_t *jl_to_typeof(uintptr_t t) +{ + if (t < (jl_max_tags << 4)) + return (jl_value_t*)small_typeof[t / sizeof(*small_typeof)]; + return (jl_value_t*)t; +} + + // kinds extern JL_DLLIMPORT jl_datatype_t *jl_typeofbottom_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_datatype_type JL_GLOBALLY_ROOTED; @@ -989,7 +1048,7 @@ STATIC_INLINE jl_value_t *jl_svecset( #else STATIC_INLINE jl_value_t *jl_svecref(void *t JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT { - assert(jl_typeis(t,jl_simplevector_type)); + assert(jl_typetagis(t,jl_simplevector_tag << 4)); assert(i < jl_svec_len(t)); // while svec is supposedly immutable, in practice we sometimes publish it first // and set the values lazily @@ -999,7 +1058,7 @@ STATIC_INLINE jl_value_t *jl_svecset( void *t JL_ROOTING_ARGUMENT JL_PROPAGATES_ROOT, size_t i, void *x JL_ROOTED_ARGUMENT) JL_NOTSAFEPOINT { - assert(jl_typeis(t,jl_simplevector_type)); + assert(jl_typetagis(t,jl_simplevector_tag << 4)); assert(i < jl_svec_len(t)); // while svec is supposedly immutable, in practice we sometimes publish it // first and set the values lazily. Those users occasionally might need to @@ -1055,13 +1114,13 @@ STATIC_INLINE jl_value_t *jl_array_ptr_set( STATIC_INLINE uint8_t jl_array_uint8_ref(void *a, size_t i) JL_NOTSAFEPOINT { assert(i < jl_array_len(a)); - assert(jl_typeis(a, jl_array_uint8_type)); + assert(jl_typetagis(a, jl_array_uint8_type)); return ((uint8_t*)(jl_array_data(a)))[i]; } STATIC_INLINE void jl_array_uint8_set(void *a, size_t i, uint8_t x) JL_NOTSAFEPOINT { assert(i < jl_array_len(a)); - assert(jl_typeis(a, jl_array_uint8_type)); + assert(jl_typetagis(a, jl_array_uint8_type)); ((uint8_t*)(jl_array_data(a)))[i] = x; } @@ -1230,56 +1289,57 @@ static inline int jl_is_layout_opaque(const jl_datatype_layout_t *l) JL_NOTSAFEP #define jl_is_nothing(v) (((jl_value_t*)(v)) == ((jl_value_t*)jl_nothing)) #define jl_is_tuple(v) (((jl_datatype_t*)jl_typeof(v))->name == jl_tuple_typename) #define jl_is_namedtuple(v) (((jl_datatype_t*)jl_typeof(v))->name == jl_namedtuple_typename) -#define jl_is_svec(v) jl_typeis(v,jl_simplevector_type) +#define jl_is_svec(v) jl_typetagis(v,jl_simplevector_tag<<4) #define jl_is_simplevector(v) jl_is_svec(v) -#define jl_is_datatype(v) jl_typeis(v,jl_datatype_type) +#define jl_is_datatype(v) jl_typetagis(v,jl_datatype_tag<<4) #define jl_is_mutable(t) (((jl_datatype_t*)t)->name->mutabl) #define jl_is_mutable_datatype(t) (jl_is_datatype(t) && (((jl_datatype_t*)t)->name->mutabl)) #define jl_is_immutable(t) (!((jl_datatype_t*)t)->name->mutabl) #define jl_is_immutable_datatype(t) (jl_is_datatype(t) && (!((jl_datatype_t*)t)->name->mutabl)) -#define jl_is_uniontype(v) jl_typeis(v,jl_uniontype_type) -#define jl_is_typevar(v) jl_typeis(v,jl_tvar_type) -#define jl_is_unionall(v) jl_typeis(v,jl_unionall_type) -#define jl_is_typename(v) jl_typeis(v,jl_typename_type) -#define jl_is_int8(v) jl_typeis(v,jl_int8_type) -#define jl_is_int16(v) jl_typeis(v,jl_int16_type) -#define jl_is_int32(v) jl_typeis(v,jl_int32_type) -#define jl_is_int64(v) jl_typeis(v,jl_int64_type) -#define jl_is_uint8(v) jl_typeis(v,jl_uint8_type) -#define jl_is_uint16(v) jl_typeis(v,jl_uint16_type) -#define jl_is_uint32(v) jl_typeis(v,jl_uint32_type) -#define jl_is_uint64(v) jl_typeis(v,jl_uint64_type) -#define jl_is_bool(v) jl_typeis(v,jl_bool_type) -#define jl_is_symbol(v) jl_typeis(v,jl_symbol_type) -#define jl_is_ssavalue(v) jl_typeis(v,jl_ssavalue_type) -#define jl_is_slotnumber(v) jl_typeis(v,jl_slotnumber_type) -#define jl_is_expr(v) jl_typeis(v,jl_expr_type) -#define jl_is_binding(v) jl_typeis(v,jl_binding_type) -#define jl_is_globalref(v) jl_typeis(v,jl_globalref_type) -#define jl_is_gotonode(v) jl_typeis(v,jl_gotonode_type) -#define jl_is_gotoifnot(v) jl_typeis(v,jl_gotoifnot_type) -#define jl_is_returnnode(v) jl_typeis(v,jl_returnnode_type) -#define jl_is_argument(v) jl_typeis(v,jl_argument_type) -#define jl_is_pinode(v) jl_typeis(v,jl_pinode_type) -#define jl_is_phinode(v) jl_typeis(v,jl_phinode_type) -#define jl_is_phicnode(v) jl_typeis(v,jl_phicnode_type) -#define jl_is_upsilonnode(v) jl_typeis(v,jl_upsilonnode_type) -#define jl_is_quotenode(v) jl_typeis(v,jl_quotenode_type) -#define jl_is_newvarnode(v) jl_typeis(v,jl_newvarnode_type) -#define jl_is_linenode(v) jl_typeis(v,jl_linenumbernode_type) -#define jl_is_method_instance(v) jl_typeis(v,jl_method_instance_type) -#define jl_is_code_instance(v) jl_typeis(v,jl_code_instance_type) -#define jl_is_code_info(v) jl_typeis(v,jl_code_info_type) -#define jl_is_method(v) jl_typeis(v,jl_method_type) -#define jl_is_module(v) jl_typeis(v,jl_module_type) -#define jl_is_mtable(v) jl_typeis(v,jl_methtable_type) -#define jl_is_task(v) jl_typeis(v,jl_task_type) -#define jl_is_string(v) jl_typeis(v,jl_string_type) +#define jl_is_uniontype(v) jl_typetagis(v,jl_uniontype_tag<<4) +#define jl_is_typevar(v) jl_typetagis(v,jl_tvar_tag<<4) +#define jl_is_unionall(v) jl_typetagis(v,jl_unionall_tag<<4) +#define jl_is_vararg(v) jl_typetagis(v,jl_vararg_tag<<4) +#define jl_is_typename(v) jl_typetagis(v,jl_typename_type) +#define jl_is_int8(v) jl_typetagis(v,jl_int8_tag<<4) +#define jl_is_int16(v) jl_typetagis(v,jl_int16_tag<<4) +#define jl_is_int32(v) jl_typetagis(v,jl_int32_tag<<4) +#define jl_is_int64(v) jl_typetagis(v,jl_int64_tag<<4) +#define jl_is_uint8(v) jl_typetagis(v,jl_uint8_tag<<4) +#define jl_is_uint16(v) jl_typetagis(v,jl_uint16_tag<<4) +#define jl_is_uint32(v) jl_typetagis(v,jl_uint32_tag<<4) +#define jl_is_uint64(v) jl_typetagis(v,jl_uint64_tag<<4) +#define jl_is_bool(v) jl_typetagis(v,jl_bool_tag<<4) +#define jl_is_symbol(v) jl_typetagis(v,jl_symbol_tag<<4) +#define jl_is_ssavalue(v) jl_typetagis(v,jl_ssavalue_type) +#define jl_is_slotnumber(v) jl_typetagis(v,jl_slotnumber_type) +#define jl_is_expr(v) jl_typetagis(v,jl_expr_type) +#define jl_is_binding(v) jl_typetagis(v,jl_binding_type) +#define jl_is_globalref(v) jl_typetagis(v,jl_globalref_type) +#define jl_is_gotonode(v) jl_typetagis(v,jl_gotonode_type) +#define jl_is_gotoifnot(v) jl_typetagis(v,jl_gotoifnot_type) +#define jl_is_returnnode(v) jl_typetagis(v,jl_returnnode_type) +#define jl_is_argument(v) jl_typetagis(v,jl_argument_type) +#define jl_is_pinode(v) jl_typetagis(v,jl_pinode_type) +#define jl_is_phinode(v) jl_typetagis(v,jl_phinode_type) +#define jl_is_phicnode(v) jl_typetagis(v,jl_phicnode_type) +#define jl_is_upsilonnode(v) jl_typetagis(v,jl_upsilonnode_type) +#define jl_is_quotenode(v) jl_typetagis(v,jl_quotenode_type) +#define jl_is_newvarnode(v) jl_typetagis(v,jl_newvarnode_type) +#define jl_is_linenode(v) jl_typetagis(v,jl_linenumbernode_type) +#define jl_is_method_instance(v) jl_typetagis(v,jl_method_instance_type) +#define jl_is_code_instance(v) jl_typetagis(v,jl_code_instance_type) +#define jl_is_code_info(v) jl_typetagis(v,jl_code_info_type) +#define jl_is_method(v) jl_typetagis(v,jl_method_type) +#define jl_is_module(v) jl_typetagis(v,jl_module_tag<<4) +#define jl_is_mtable(v) jl_typetagis(v,jl_methtable_type) +#define jl_is_task(v) jl_typetagis(v,jl_task_tag<<4) +#define jl_is_string(v) jl_typetagis(v,jl_string_tag<<4) #define jl_is_cpointer(v) jl_is_cpointer_type(jl_typeof(v)) #define jl_is_pointer(v) jl_is_cpointer_type(jl_typeof(v)) -#define jl_is_uint8pointer(v)jl_typeis(v,jl_uint8pointer_type) +#define jl_is_uint8pointer(v)jl_typetagis(v,jl_uint8pointer_type) #define jl_is_llvmpointer(v) (((jl_datatype_t*)jl_typeof(v))->name == jl_llvmpointer_typename) -#define jl_is_intrinsic(v) jl_typeis(v,jl_intrinsic_type) +#define jl_is_intrinsic(v) jl_typetagis(v,jl_intrinsic_type) #define jl_array_isbitsunion(a) (!(((jl_array_t*)(a))->flags.ptrarray) && jl_is_uniontype(jl_tparam0(jl_typeof(a)))) JL_DLLEXPORT int jl_subtype(jl_value_t *a, jl_value_t *b); @@ -1290,9 +1350,16 @@ STATIC_INLINE int jl_is_kind(jl_value_t *v) JL_NOTSAFEPOINT v==(jl_value_t*)jl_unionall_type || v==(jl_value_t*)jl_typeofbottom_type); } +STATIC_INLINE int jl_is_kindtag(uintptr_t t) JL_NOTSAFEPOINT +{ + t >>= 4; + return (t==(uintptr_t)jl_uniontype_tag || t==(uintptr_t)jl_datatype_tag || + t==(uintptr_t)jl_unionall_tag || t==(uintptr_t)jl_typeofbottom_tag); +} + STATIC_INLINE int jl_is_type(jl_value_t *v) JL_NOTSAFEPOINT { - return jl_is_kind(jl_typeof(v)); + return jl_is_kindtag(jl_typetagof(v)); } STATIC_INLINE int jl_is_primitivetype(void *v) JL_NOTSAFEPOINT @@ -1400,29 +1467,30 @@ STATIC_INLINE int jl_is_array_zeroinit(jl_array_t *a) JL_NOTSAFEPOINT // object identity JL_DLLEXPORT int jl_egal(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT; JL_DLLEXPORT int jl_egal__bits(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, jl_datatype_t *dt) JL_NOTSAFEPOINT; -JL_DLLEXPORT int jl_egal__special(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, jl_datatype_t *dt) JL_NOTSAFEPOINT; -JL_DLLEXPORT int jl_egal__unboxed(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, jl_datatype_t *dt) JL_NOTSAFEPOINT; +JL_DLLEXPORT int jl_egal__bitstag(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, uintptr_t dtag) JL_NOTSAFEPOINT; +JL_DLLEXPORT int jl_egal__unboxed(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, uintptr_t dtag) JL_NOTSAFEPOINT; JL_DLLEXPORT uintptr_t jl_object_id(jl_value_t *v) JL_NOTSAFEPOINT; JL_DLLEXPORT uintptr_t jl_type_hash(jl_value_t *v) JL_NOTSAFEPOINT; -STATIC_INLINE int jl_egal__unboxed_(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, jl_datatype_t *dt) JL_NOTSAFEPOINT +STATIC_INLINE int jl_egal__unboxed_(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, uintptr_t dtag) JL_NOTSAFEPOINT { - if (dt->name->mutabl) { - if (dt == jl_simplevector_type || dt == jl_string_type || dt == jl_datatype_type) - return jl_egal__special(a, b, dt); - return 0; + if (dtag < jl_max_tags << 4) { + if (dtag == jl_symbol_tag << 4 || dtag == jl_bool_tag << 4) + return 0; } - return jl_egal__bits(a, b, dt); + else if (((jl_datatype_t*)dtag)->name->mutabl) + return 0; + return jl_egal__bitstag(a, b, dtag); } STATIC_INLINE int jl_egal_(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT { if (a == b) return 1; - jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(a); - if (dt != (jl_datatype_t*)jl_typeof(b)) + uintptr_t dtag = jl_typetagof(a); + if (dtag != jl_typetagof(b)) return 0; - return jl_egal__unboxed_(a, b, dt); + return jl_egal__unboxed_(a, b, dtag); } #define jl_egal(a, b) jl_egal_((a), (b)) diff --git a/src/julia_internal.h b/src/julia_internal.h index c6c9f3f8e21df..1ce04cccf020f 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -512,10 +512,8 @@ STATIC_INLINE jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT o->header = tag | GC_OLD_MARKED; return jl_valueof(o); } -jl_value_t *jl_permbox8(jl_datatype_t *t, int8_t x); -jl_value_t *jl_permbox16(jl_datatype_t *t, int16_t x); -jl_value_t *jl_permbox32(jl_datatype_t *t, int32_t x); -jl_value_t *jl_permbox64(jl_datatype_t *t, int64_t x); +jl_value_t *jl_permbox8(jl_datatype_t *t, uintptr_t tag, uint8_t x); +jl_value_t *jl_permbox32(jl_datatype_t *t, uintptr_t tag, uint32_t x); jl_svec_t *jl_perm_symsvec(size_t n, ...); // this sizeof(__VA_ARGS__) trick can't be computed until C11, but that only matters to Clang in some situations @@ -798,11 +796,6 @@ typedef enum { JL_VARARG_UNBOUND = 3 } jl_vararg_kind_t; -STATIC_INLINE int jl_is_vararg(jl_value_t *v) JL_NOTSAFEPOINT -{ - return jl_typeof(v) == (jl_value_t*)jl_vararg_type; -} - STATIC_INLINE jl_value_t *jl_unwrap_vararg(jl_vararg_t *v JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { assert(jl_is_vararg((jl_value_t*)v)); diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index a8bab71ce91b5..a836ff1361768 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -2224,10 +2224,10 @@ Value *LateLowerGCFrame::EmitLoadTag(IRBuilder<> &builder, Type *T_size, Value * load->setMetadata(LLVMContext::MD_tbaa, tbaa_tag); MDBuilder MDB(load->getContext()); auto *NullInt = ConstantInt::get(T_size, 0); - // We can be sure that the tag is larger than page size. + // We can be sure that the tag is at least 16 (1<<4) // Hopefully this is enough to convince LLVM that the value is still not NULL // after masking off the tag bits - auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(T_size, 4096)); + auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(T_size, 16)); load->setMetadata(LLVMContext::MD_range, MDB.createRange(NonNullInt, NullInt)); return load; } diff --git a/src/method.c b/src/method.c index bed94319953c0..1b6795902d837 100644 --- a/src/method.c +++ b/src/method.c @@ -130,7 +130,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve rt = jl_interpret_toplevel_expr_in(module, rt, NULL, sparam_vals); } JL_CATCH { - if (jl_typeis(jl_current_exception(), jl_errorexception_type)) + if (jl_typetagis(jl_current_exception(), jl_errorexception_type)) jl_error("could not evaluate cfunction return type (it might depend on a local variable)"); else jl_rethrow(); @@ -142,7 +142,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve at = jl_interpret_toplevel_expr_in(module, at, NULL, sparam_vals); } JL_CATCH { - if (jl_typeis(jl_current_exception(), jl_errorexception_type)) + if (jl_typetagis(jl_current_exception(), jl_errorexception_type)) jl_error("could not evaluate cfunction argument type (it might depend on a local variable)"); else jl_rethrow(); @@ -163,7 +163,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve rt = jl_interpret_toplevel_expr_in(module, rt, NULL, sparam_vals); } JL_CATCH { - if (jl_typeis(jl_current_exception(), jl_errorexception_type)) + if (jl_typetagis(jl_current_exception(), jl_errorexception_type)) jl_error("could not evaluate ccall return type (it might depend on a local variable)"); else jl_rethrow(); @@ -175,7 +175,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve at = jl_interpret_toplevel_expr_in(module, at, NULL, sparam_vals); } JL_CATCH { - if (jl_typeis(jl_current_exception(), jl_errorexception_type)) + if (jl_typetagis(jl_current_exception(), jl_errorexception_type)) jl_error("could not evaluate ccall argument type (it might depend on a local variable)"); else jl_rethrow(); @@ -504,7 +504,7 @@ void jl_add_function_to_lineinfo(jl_code_info_t *ci, jl_value_t *func) JL_GC_PUSH3(&rt, &lno, &inl); for (i = 0; i < n; i++) { jl_value_t *ln = jl_array_ptr_ref(li, i); - assert(jl_typeis(ln, jl_lineinfonode_type)); + assert(jl_typetagis(ln, jl_lineinfonode_type)); jl_value_t *mod = jl_fieldref_noalloc(ln, 0); jl_value_t *file = jl_fieldref_noalloc(ln, 2); lno = jl_fieldref(ln, 3); @@ -689,7 +689,7 @@ static void jl_method_set_source(jl_method_t *m, jl_code_info_t *src) jl_array_t *copy = NULL; jl_svec_t *sparam_vars = jl_outer_unionall_vars(m->sig); JL_GC_PUSH3(©, &sparam_vars, &src); - assert(jl_typeis(src->code, jl_array_any_type)); + assert(jl_typetagis(src->code, jl_array_any_type)); jl_array_t *stmts = (jl_array_t*)src->code; size_t i, n = jl_array_len(stmts); copy = jl_alloc_vec_any(n); diff --git a/src/module.c b/src/module.c index d504cf7738767..04d3970f9b460 100644 --- a/src/module.c +++ b/src/module.c @@ -17,6 +17,7 @@ JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, ui const jl_uuid_t uuid_zero = {0, 0}; jl_module_t *m = (jl_module_t*)jl_gc_alloc(ct->ptls, sizeof(jl_module_t), jl_module_type); + jl_set_typetagof(m, jl_module_tag, 0); assert(jl_is_symbol(name)); m->name = name; m->parent = parent; diff --git a/src/partr.c b/src/partr.c index bed02cf80c219..403f911b1284f 100644 --- a/src/partr.c +++ b/src/partr.c @@ -293,7 +293,7 @@ static jl_task_t *get_next_task(jl_value_t *trypoptask, jl_value_t *q) { jl_gc_safepoint(); jl_task_t *task = (jl_task_t*)jl_apply_generic(trypoptask, &q, 1); - if (jl_typeis(task, jl_task_type)) { + if (jl_is_task(task)) { int self = jl_atomic_load_relaxed(&jl_current_task->tid); jl_set_task_tid(task, self); return task; diff --git a/src/processor.cpp b/src/processor.cpp index 88fcb813d8248..24a434af91ad3 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -812,6 +812,8 @@ static inline jl_image_t parse_sysimg(void *hdl, F &&callback) *tls_offset_idx = (uintptr_t)(jl_tls_offset == -1 ? 0 : jl_tls_offset); } + res.small_typeof = pointers->small_typeof; + return res; } diff --git a/src/processor.h b/src/processor.h index d2280068fb67d..3e83bbb2247d6 100644 --- a/src/processor.h +++ b/src/processor.h @@ -88,6 +88,7 @@ typedef struct { const int32_t *gvars_offsets; uint32_t ngvars; jl_image_fptrs_t fptrs; + void **small_typeof; } jl_image_t; // The header for each image @@ -194,8 +195,10 @@ typedef struct { const jl_image_header_t *header; // The shard table, contains per-shard data const jl_image_shard_t *shards; // points to header->nshards length array - // The TLS data + // The TLS data pointer const jl_image_ptls_t *ptls; + // A copy of small_typeof[] + void **small_typeof; // serialized target data // This contains the number of targets diff --git a/src/rtutils.c b/src/rtutils.c index a2ec34102f148..afea3ac2c2388 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -129,6 +129,8 @@ JL_DLLEXPORT void JL_NORETURN jl_type_error(const char *fname, JL_DLLEXPORT void JL_NORETURN jl_undefined_var_error(jl_sym_t *var) { + if (!jl_undefvarerror_type) + jl_errorf("UndefVarError(%s)", jl_symbol_name(var)); jl_throw(jl_new_struct(jl_undefvarerror_type, var)); } @@ -1215,10 +1217,10 @@ static size_t jl_static_show_next_(JL_STREAM *out, jl_value_t *v, jl_value_t *pr *newdepth = &this_item, *p = depth; while (p) { - if (jl_typeis(v, jl_typemap_entry_type) && newdepth == &this_item) { + if (jl_typetagis(v, jl_typemap_entry_type) && newdepth == &this_item) { jl_value_t *m = p->v; unsigned nid = 1; - while (m && jl_typeis(m, jl_typemap_entry_type)) { + while (m && jl_typetagis(m, jl_typemap_entry_type)) { if (m == v) { return jl_printf(out, "sig, depth, ctx) + @@ -1234,7 +1236,7 @@ static size_t jl_static_show_next_(JL_STREAM *out, jl_value_t *v, jl_value_t *pr jl_value_t *m2 = p->v; if (m2 == mnext) break; - while (m2 && jl_typeis(m2, jl_typemap_entry_type)) { + while (m2 && jl_typetagis(m2, jl_typemap_entry_type)) { jl_value_t *mnext2 = (jl_value_t*)jl_atomic_load_relaxed(&((jl_typemap_entry_t*)m2)->next); if (mnext2 == mnext) { if (m2 != m) diff --git a/src/simplevector.c b/src/simplevector.c index cb65646e00936..65217715ae55f 100644 --- a/src/simplevector.c +++ b/src/simplevector.c @@ -23,6 +23,7 @@ jl_svec_t *(jl_perm_symsvec)(size_t n, ...) { if (n == 0) return jl_emptysvec; jl_svec_t *jv = (jl_svec_t*)jl_gc_permobj((n + 1) * sizeof(void*), jl_simplevector_type); + jl_set_typetagof(jv, jl_simplevector_tag, jl_astaggedvalue(jv)->bits.gc); jl_svec_set_len_unsafe(jv, n); va_list args; va_start(args, n); @@ -37,6 +38,7 @@ JL_DLLEXPORT jl_svec_t *jl_svec1(void *a) jl_task_t *ct = jl_current_task; jl_svec_t *v = (jl_svec_t*)jl_gc_alloc(ct->ptls, sizeof(void*) * 2, jl_simplevector_type); + jl_set_typetagof(v, jl_simplevector_tag, 0); jl_svec_set_len_unsafe(v, 1); jl_svec_data(v)[0] = (jl_value_t*)a; return v; @@ -47,6 +49,7 @@ JL_DLLEXPORT jl_svec_t *jl_svec2(void *a, void *b) jl_task_t *ct = jl_current_task; jl_svec_t *v = (jl_svec_t*)jl_gc_alloc(ct->ptls, sizeof(void*) * 3, jl_simplevector_type); + jl_set_typetagof(v, jl_simplevector_tag, 0); jl_svec_set_len_unsafe(v, 2); jl_svec_data(v)[0] = (jl_value_t*)a; jl_svec_data(v)[1] = (jl_value_t*)b; @@ -59,6 +62,7 @@ JL_DLLEXPORT jl_svec_t *jl_alloc_svec_uninit(size_t n) if (n == 0) return jl_emptysvec; jl_svec_t *jv = (jl_svec_t*)jl_gc_alloc(ct->ptls, (n + 1) * sizeof(void*), jl_simplevector_type); + jl_set_typetagof(jv, jl_simplevector_tag, 0); jl_svec_set_len_unsafe(jv, n); return jv; } diff --git a/src/stackwalk.c b/src/stackwalk.c index 093467750d573..18bf4b2126938 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -673,7 +673,7 @@ void jl_print_bt_entry_codeloc(jl_bt_element_t *bt_entry) JL_NOTSAFEPOINT while (debuginfoloc != 0) { jl_line_info_node_t *locinfo = (jl_line_info_node_t*) jl_array_ptr_ref(src->linetable, debuginfoloc - 1); - assert(jl_typeis(locinfo, jl_lineinfonode_type)); + assert(jl_typetagis(locinfo, jl_lineinfonode_type)); const char *func_name = "Unknown"; jl_value_t *method = locinfo->method; if (jl_is_method_instance(method)) diff --git a/src/staticdata.c b/src/staticdata.c index 353382f11a067..1afb360aced1d 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -598,20 +598,20 @@ static int jl_needs_serialization(jl_serializer_state *s, jl_value_t *v) JL_NOTS if (v == NULL || jl_is_symbol(v) || v == jl_nothing) { return 0; } - else if (jl_typeis(v, jl_int64_type)) { + else if (jl_typetagis(v, jl_int64_tag << 4)) { int64_t i64 = *(int64_t*)v + NBOX_C / 2; if ((uint64_t)i64 < NBOX_C) return 0; } - else if (jl_typeis(v, jl_int32_type)) { + else if (jl_typetagis(v, jl_int32_tag << 4)) { int32_t i32 = *(int32_t*)v + NBOX_C / 2; if ((uint32_t)i32 < NBOX_C) return 0; } - else if (jl_typeis(v, jl_uint8_type)) { + else if (jl_typetagis(v, jl_uint8_tag << 4)) { return 0; } - else if (jl_typeis(v, jl_task_type)) { + else if (jl_typetagis(v, jl_task_tag << 4)) { return 0; } @@ -838,7 +838,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } } } - else if (jl_typeis(v, jl_module_type)) { + else if (jl_typetagis(v, jl_module_tag << 4)) { jl_queue_module_for_serialization(s, (jl_module_t*)v); } else if (layout->nfields > 0) { @@ -1024,17 +1024,17 @@ static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t * else if (v == jl_nothing) { return ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + 1; } - else if (jl_typeis(v, jl_int64_type)) { + else if (jl_typetagis(v, jl_int64_tag << 4)) { int64_t i64 = *(int64_t*)v + NBOX_C / 2; if ((uint64_t)i64 < NBOX_C) return ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + i64 + 2; } - else if (jl_typeis(v, jl_int32_type)) { + else if (jl_typetagis(v, jl_int32_tag << 4)) { int32_t i32 = *(int32_t*)v + NBOX_C / 2; if ((uint32_t)i32 < NBOX_C) return ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + i32 + 2 + NBOX_C; } - else if (jl_typeis(v, jl_uint8_type)) { + else if (jl_typetagis(v, jl_uint8_tag << 4)) { uint8_t u8 = *(uint8_t*)v; return ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + u8 + 2 + NBOX_C + NBOX_C; } @@ -1214,7 +1214,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED else if (s->incremental && needs_recaching(v)) { arraylist_push(jl_is_datatype(v) ? &s->fixup_types : &s->fixup_objs, (void*)reloc_offset); } - else if (s->incremental && jl_typeis(v, jl_binding_type)) { + else if (s->incremental && jl_typetagis(v, jl_binding_type)) { jl_binding_t *b = (jl_binding_t*)v; if (b->globalref == NULL || jl_object_in_image((jl_value_t*)b->globalref->mod)) jl_error("Binding cannot be serialized"); // no way (currently) to recover its identity @@ -1323,10 +1323,10 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } } - else if (jl_typeis(v, jl_module_type)) { + else if (jl_typetagis(v, jl_module_tag << 4)) { jl_write_module(s, item, (jl_module_t*)v); } - else if (jl_typeis(v, jl_task_type)) { + else if (jl_typetagis(v, jl_task_tag << 4)) { jl_error("Task cannot be serialized"); } else if (jl_is_svec(v)) { @@ -1350,7 +1350,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED assert(t->layout->npointers == 0); ios_write(s->s, (char*)v, jl_datatype_size(t)); } - else if (jl_bigint_type && jl_typeis(v, jl_bigint_type)) { + else if (jl_bigint_type && jl_typetagis(v, jl_bigint_type)) { // foreign types require special handling jl_value_t *sizefield = jl_get_nth_field(v, 1); int32_t sz = jl_unbox_int32(sizefield); @@ -1408,7 +1408,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED // A few objects need additional handling beyond the generic serialization above - if (s->incremental && jl_typeis(v, jl_typemap_entry_type)) { + if (s->incremental && jl_typetagis(v, jl_typemap_entry_type)) { jl_typemap_entry_t *newentry = (jl_typemap_entry_t*)&s->s->buf[reloc_offset]; if (newentry->max_world == ~(size_t)0) { if (newentry->min_world > 1) { @@ -1731,7 +1731,7 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas #else size_t depsidx = 0; #endif - assert(depsidx < jl_array_len(s->buildid_depmods_idxs)); + assert(s->buildid_depmods_idxs && depsidx < jl_array_len(s->buildid_depmods_idxs)); size_t i = ((uint32_t*)jl_array_data(s->buildid_depmods_idxs))[depsidx]; assert(2*i < jl_linkage_blobs.len); return (uintptr_t)jl_linkage_blobs.items[2*i] + offset*sizeof(void*); @@ -1832,6 +1832,8 @@ static void jl_read_reloclist(jl_serializer_state *s, jl_array_t *link_ids, uint uintptr_t *pv = (uintptr_t *)(base + pos); uintptr_t v = *pv; v = get_item_for_reloc(s, base, size, v, link_ids, &link_index); + if (bits && v && ((jl_datatype_t*)v)->smalltag) + v = (uintptr_t)((jl_datatype_t*)v)->smalltag << 4; // TODO: should we have a representation that supports sweep without a relocation step? *pv = v | bits; } assert(!link_ids || link_index == jl_array_len(link_ids)); @@ -1944,6 +1946,9 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) image->fptrs.base = NULL; if (fvars.base == NULL) return; + + memcpy(image->small_typeof, &small_typeof, sizeof(small_typeof)); + int img_fvars_max = s->fptr_record->size / sizeof(void*); size_t i; uintptr_t base = (uintptr_t)&s->s->buf[0]; @@ -2782,6 +2787,7 @@ JL_DLLEXPORT void jl_set_sysimg_so(void *handle) #endif extern void rebuild_image_blob_tree(void); +extern void export_small_typeof(void); static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_array_t *depmods, uint64_t checksum, /* outputs */ jl_array_t **restored, jl_array_t **init_order, @@ -2857,9 +2863,13 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl jl_value_t **tag = tags[i]; *tag = jl_read_value(&s); } +#define XX(name) \ + small_typeof[(jl_##name##_tag << 4) / sizeof(*small_typeof)] = jl_##name##_type; + JL_SMALL_TYPEOF(XX) +#undef XX + export_small_typeof(); jl_global_roots_table = (jl_array_t*)jl_read_value(&s); // set typeof extra-special values now that we have the type set by tags above - jl_astaggedvalue(jl_current_task)->header = (uintptr_t)jl_task_type | jl_astaggedvalue(jl_current_task)->header; jl_astaggedvalue(jl_nothing)->header = (uintptr_t)jl_nothing_type | jl_astaggedvalue(jl_nothing)->header; s.ptls->root_task->tls = jl_read_value(&s); jl_gc_wb(s.ptls->root_task, s.ptls->root_task->tls); @@ -2995,9 +3005,9 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl continue; } } - jl_value_t *otyp = jl_typeof(obj); // the original type of the object that was written here + uintptr_t otyp = jl_typetagof(obj); // the original type of the object that was written here assert(image_base < (char*)obj && (char*)obj <= image_base + sizeof_sysimg + sizeof(uintptr_t)); - if (otyp == (jl_value_t*)jl_datatype_type) { + if (otyp == jl_datatype_tag << 4) { jl_datatype_t *dt = (jl_datatype_t*)obj[0], *newdt; if (jl_is_datatype(dt)) { newdt = dt; // already done @@ -3044,7 +3054,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl else *pfld = (uintptr_t)newobj; assert(!(image_base < (char*)newobj && (char*)newobj <= image_base + sizeof_sysimg + sizeof(uintptr_t))); - assert(jl_typeis(obj, otyp)); + assert(jl_typetagis(obj, otyp)); } // A few fields (reached via super) might be self-recursive. This is rare, but handle them now. // They cannot be instances though, since the type must fully exist before the singleton field can be allocated @@ -3121,8 +3131,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl pfld = (uintptr_t*)(image_base + item); obj = *(jl_value_t***)pfld; } - jl_value_t *otyp = jl_typeof(obj); // the original type of the object that was written here - if (otyp == (jl_value_t*)jl_method_instance_type) { + uintptr_t otyp = jl_typetagof(obj); // the original type of the object that was written here + if (otyp == (uintptr_t)jl_method_instance_type) { assert(image_base < (char*)obj && (char*)obj <= image_base + sizeof_sysimg + sizeof(uintptr_t)); jl_value_t *m = obj[0]; if (jl_is_method_instance(m)) { @@ -3141,7 +3151,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl } *pfld = (uintptr_t)newobj; assert(!(image_base < (char*)newobj && (char*)newobj <= image_base + sizeof_sysimg + sizeof(uintptr_t))); - assert(jl_typeis(obj, otyp)); + assert(jl_typetagis(obj, otyp)); } arraylist_free(&s.uniquing_types); arraylist_free(&s.uniquing_objs); @@ -3157,7 +3167,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl for (size_t i = 0; i < s.fixup_objs.len; i++) { uintptr_t item = (uintptr_t)s.fixup_objs.items[i]; jl_value_t *obj = (jl_value_t*)(image_base + item); - if (jl_typeis(obj, jl_typemap_entry_type)) { + if (jl_typetagis(obj, jl_typemap_entry_type)) { jl_typemap_entry_t *entry = (jl_typemap_entry_t*)obj; entry->min_world = world; } @@ -3198,7 +3208,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl // rehash IdDict //assert(((jl_datatype_t*)(jl_typeof(obj)))->name == jl_idtable_typename); jl_array_t **a = (jl_array_t**)obj; - assert(jl_typeis(*a, jl_array_any_type)); + assert(jl_typetagis(*a, jl_array_any_type)); *a = jl_idtable_rehash(*a, jl_array_len(*a)); jl_gc_wb(obj, *a); } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 03dc3a82acd93..bf1a830b608de 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -949,7 +949,7 @@ static jl_array_t *jl_verify_methods(jl_array_t *edges, jl_array_t *maxvalids) jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method)); jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, 2 * i + 1); - assert(jl_typeis((jl_value_t*)callee_ids, jl_array_int32_type)); + assert(jl_typetagis((jl_value_t*)callee_ids, jl_array_int32_type)); if (callee_ids == NULL) { // serializing the edges had failed maxvalids2_data[i] = 0; @@ -999,7 +999,7 @@ static int jl_verify_graph_edge(size_t *maxvalids2_data, jl_array_t *edges, size size_t depth = stack->len; visited->items[idx] = (void*)(1 + depth); jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, idx * 2 + 1); - assert(jl_typeis((jl_value_t*)callee_ids, jl_array_int32_type)); + assert(jl_typetagis((jl_value_t*)callee_ids, jl_array_int32_type)); int32_t *idxs = (int32_t*)jl_array_data(callee_ids); size_t i, n = jl_array_len(callee_ids); cycle = depth; diff --git a/src/subtype.c b/src/subtype.c index 2ff5f0f75d09d..ce44402dfbc59 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -2286,7 +2286,9 @@ int jl_has_intersect_kind_not_type(jl_value_t *t) JL_DLLEXPORT int jl_isa(jl_value_t *x, jl_value_t *t) { - if (jl_typeis(x,t) || t == (jl_value_t*)jl_any_type) + if (t == (jl_value_t*)jl_any_type || jl_typetagis(x,t)) + return 1; + if (jl_typetagof(x) < (jl_max_tags << 4) && jl_is_datatype(t) && jl_typetagis(x,((jl_datatype_t*)t)->smalltag << 4)) return 1; if (jl_is_type(x)) { if (t == (jl_value_t*)jl_type_type) diff --git a/src/symbol.c b/src/symbol.c index 14606c82b9778..c9c0c0e533924 100644 --- a/src/symbol.c +++ b/src/symbol.c @@ -35,12 +35,10 @@ static jl_sym_t *mk_symbol(const char *str, size_t len) JL_NOTSAFEPOINT { jl_sym_t *sym; size_t nb = symbol_nbytes(len); - assert(jl_symbol_type && "not initialized"); - jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc_nolock(nb, 0, sizeof(void*), 0); sym = (jl_sym_t*)jl_valueof(tag); // set to old marked so that we won't look at it in the GC or write barrier. - tag->header = ((uintptr_t)jl_symbol_type) | GC_OLD_MARKED; + jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED); jl_atomic_store_relaxed(&sym->left, NULL); jl_atomic_store_relaxed(&sym->right, NULL); sym->hash = hash_symbol(str, len); diff --git a/src/task.c b/src/task.c index 9678cf2f3fe4e..3b2cf69237ba6 100644 --- a/src/task.c +++ b/src/task.c @@ -1039,6 +1039,7 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion { jl_task_t *ct = jl_current_task; jl_task_t *t = (jl_task_t*)jl_gc_alloc(ct->ptls, sizeof(jl_task_t), jl_task_type); + jl_set_typetagof(t, jl_task_tag, 0); JL_PROBE_RT_NEW_TASK(ct, t); t->copy_stack = 0; if (ssize == 0) { @@ -1636,6 +1637,7 @@ jl_task_t *jl_init_root_task(jl_ptls_t ptls, void *stack_lo, void *stack_hi) if (jl_nothing == NULL) // make a placeholder jl_nothing = jl_gc_permobj(0, jl_nothing_type); jl_task_t *ct = (jl_task_t*)jl_gc_alloc(ptls, sizeof(jl_task_t), jl_task_type); + jl_set_typetagof(ct, jl_task_tag, 0); memset(ct, 0, sizeof(jl_task_t)); void *stack = stack_lo; size_t ssize = (char*)stack_hi - (char*)stack_lo; diff --git a/src/toplevel.c b/src/toplevel.c index ad282a50d91ac..200d0ad220231 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -413,7 +413,7 @@ static void expr_attributes(jl_value_t *v, int *has_ccall, int *has_defs, int *h int jl_code_requires_compiler(jl_code_info_t *src, int include_force_compile) { jl_array_t *body = src->code; - assert(jl_typeis(body, jl_array_any_type)); + assert(jl_typetagis(body, jl_array_any_type)); size_t i; int has_ccall = 0, has_defs = 0, has_opaque = 0; if (include_force_compile && jl_has_meta(body, jl_force_compile_sym)) @@ -875,7 +875,7 @@ jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_value_t *e, int int has_ccall = 0, has_defs = 0, has_loops = 0, has_opaque = 0, forced_compile = 0; assert(head == jl_thunk_sym); thk = (jl_code_info_t*)jl_exprarg(ex, 0); - if (!jl_is_code_info(thk) || !jl_typeis(thk->code, jl_array_any_type)) { + if (!jl_is_code_info(thk) || !jl_typetagis(thk->code, jl_array_any_type)) { jl_eval_errorf(m, "malformed \"thunk\" statement"); } body_attributes((jl_array_t*)thk->code, &has_ccall, &has_defs, &has_loops, &has_opaque, &forced_compile); diff --git a/test/ccall.jl b/test/ccall.jl index eee48c5576465..0266dabd6332b 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -802,7 +802,7 @@ if cfunction_closure verbose && println("Testing cfunction closures: ") # helper Type for testing that constructors work -# with cfucntion and that object identity is preserved +# with cfunction and that object identity is preserved mutable struct IdentityTestKV{K, V} (T::Type{<:IdentityTestKV})(S) = (@test T === S; T) end From 160d2615db7f899b95d520f1232c3dc9d66e7d84 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 21 Apr 2023 09:20:32 -0400 Subject: [PATCH 2/3] move simple (isbits) objects into the constdata section --- src/jltypes.c | 2 +- src/staticdata.c | 216 ++++++++++++++++++++++++++++------------------- 2 files changed, 129 insertions(+), 89 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index 0549ee2c07e53..24809bc534819 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2730,7 +2730,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 1, 0); XX(string); jl_string_type->instance = NULL; - jl_compute_field_offsets(jl_string_type); + jl_compute_field_offsets(jl_string_type); // re-compute now that we assigned jl_string_type jl_an_empty_string = jl_pchar_to_string("\0", 1); *(size_t*)jl_an_empty_string = 0; diff --git a/src/staticdata.c b/src/staticdata.c index 1afb360aced1d..3494aa93a8725 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -453,7 +453,7 @@ static const jl_fptr_args_t id_to_fptrs[] = { typedef struct { ios_t *s; // the main stream - ios_t *const_data; // codegen-invisible internal data (e.g., datatype layouts, list-like typename fields, foreign types, internal arrays) + ios_t *const_data; // GC-invisible internal data (e.g., datatype layouts, list-like typename fields, foreign types, internal arrays) ios_t *symbols; // names (char*) of symbols (some may be referenced by pointer in generated code) ios_t *relocs; // for (de)serializing relocs_list and gctags_list ios_t *gvar_record; // serialized array mapping gvid => spos @@ -1090,6 +1090,7 @@ static void write_gctaggedfield(jl_serializer_state *s, jl_datatype_t *ref) JL_N write_pointer(s->s); } + // Special handling from `jl_write_values` for modules static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t *m) JL_GC_DISABLED { @@ -1187,41 +1188,57 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED assert(!(s->incremental && jl_object_in_image(v))); jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v); assert((t->instance == NULL || t->instance == v) && "detected singleton construction corruption"); + ios_t *f = s->s; + if (t->smalltag) { + if (t->layout->npointers == 0 || t == jl_string_type) { + if (jl_datatype_nfields(t) == 0 || t->name->mutabl == 0 || t == jl_string_type) { + f = s->const_data; + } + } + } + // realign stream to expected gc alignment (16 bytes) - uintptr_t skip_header_pos = ios_pos(s->s) + sizeof(jl_taggedvalue_t); - write_padding(s->s, LLT_ALIGN(skip_header_pos, 16) - skip_header_pos); + uintptr_t skip_header_pos = ios_pos(f) + sizeof(jl_taggedvalue_t); + write_padding(f, LLT_ALIGN(skip_header_pos, 16) - skip_header_pos); // write header if (s->incremental && jl_needs_serialization(s, (jl_value_t*)t) && needs_uniquing((jl_value_t*)t)) - arraylist_push(&s->uniquing_types, (void*)(uintptr_t)(ios_pos(s->s)|1)); - write_gctaggedfield(s, t); - size_t reloc_offset = ios_pos(s->s); + arraylist_push(&s->uniquing_types, (void*)(uintptr_t)(ios_pos(f)|1)); + if (f == s->const_data) + write_uint(s->const_data, ((uintptr_t)t->smalltag << 4) | GC_OLD_MARKED); + else + write_gctaggedfield(s, t); + size_t reloc_offset = ios_pos(f); assert(item < layout_table.len && layout_table.items[item] == NULL); - layout_table.items[item] = (void*)reloc_offset; // store the inverse mapping of `serialization_order` (`id` => object-as-streampos) - - if (s->incremental && needs_uniquing(v)) { - if (jl_is_method_instance(v)) { - jl_method_instance_t *mi = (jl_method_instance_t*)v; - write_pointerfield(s, mi->def.value); - write_pointerfield(s, mi->specTypes); - write_pointerfield(s, (jl_value_t*)mi->sparam_vals); - continue; + layout_table.items[item] = (void*)(reloc_offset | (f == s->const_data)); // store the inverse mapping of `serialization_order` (`id` => object-as-streampos) + + if (s->incremental) { + if (needs_uniquing(v)) { + if (jl_is_method_instance(v)) { + assert(f == s->s); + jl_method_instance_t *mi = (jl_method_instance_t*)v; + write_pointerfield(s, mi->def.value); + write_pointerfield(s, mi->specTypes); + write_pointerfield(s, (jl_value_t*)mi->sparam_vals); + continue; + } + else if (!jl_is_datatype(v)) { + assert(jl_is_datatype_singleton(t) && "unreachable"); + } } - else if (!jl_is_datatype(v)) { - assert(jl_is_datatype_singleton(t) && "unreachable"); + else if (needs_recaching(v)) { + arraylist_push(jl_is_datatype(v) ? &s->fixup_types : &s->fixup_objs, (void*)reloc_offset); + } + else if (jl_typetagis(v, jl_binding_type)) { + jl_binding_t *b = (jl_binding_t*)v; + if (b->globalref == NULL || jl_object_in_image((jl_value_t*)b->globalref->mod)) + jl_error("Binding cannot be serialized"); // no way (currently) to recover its identity } - } - else if (s->incremental && needs_recaching(v)) { - arraylist_push(jl_is_datatype(v) ? &s->fixup_types : &s->fixup_objs, (void*)reloc_offset); - } - else if (s->incremental && jl_typetagis(v, jl_binding_type)) { - jl_binding_t *b = (jl_binding_t*)v; - if (b->globalref == NULL || jl_object_in_image((jl_value_t*)b->globalref->mod)) - jl_error("Binding cannot be serialized"); // no way (currently) to recover its identity } // write data if (jl_is_array(v)) { + assert(f == s->s); // Internal data for types in julia.h with `jl_array_t` field(s) #define JL_ARRAY_ALIGN(jl_value, nbytes) LLT_ALIGN(jl_value, nbytes) jl_array_t *ar = (jl_array_t*)v; @@ -1237,12 +1254,12 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED int ndimwords = jl_array_ndimwords(ar->flags.ndims); size_t headersize = sizeof(jl_array_t) + ndimwords*sizeof(size_t); // copy header - ios_write(s->s, (char*)v, headersize); + ios_write(f, (char*)v, headersize); size_t alignment_amt = JL_SMALL_BYTE_ALIGNMENT; if (tot >= ARRAY_CACHE_ALIGN_THRESHOLD) alignment_amt = JL_CACHE_BYTE_ALIGNMENT; // make some header modifications in-place - jl_array_t *newa = (jl_array_t*)&s->s->buf[reloc_offset]; + jl_array_t *newa = (jl_array_t*)&f->buf[reloc_offset]; if (newa->flags.ndims == 1) newa->maxsize = alen; newa->offset = 0; @@ -1284,17 +1301,17 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } else { // Pointer eltypes are encoded in the mutable data section - size_t data = LLT_ALIGN(ios_pos(s->s), alignment_amt); - size_t padding_amt = data - ios_pos(s->s); + size_t data = LLT_ALIGN(ios_pos(f), alignment_amt); + size_t padding_amt = data - ios_pos(f); headersize += padding_amt; newa->data = (void*)headersize; // relocation offset arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item)); // relocation target - write_padding(s->s, padding_amt); + write_padding(f, padding_amt); if (ar->flags.hasptr) { // copy all of the data first const char *data = (const char*)jl_array_data(ar); - ios_write(s->s, data, datasize); + ios_write(f, data, datasize); // the rewrite all of the embedded pointers to null+relocation uint16_t elsz = ar->elsize; size_t j, np = ((jl_datatype_t*)et)->layout->npointers; @@ -1309,7 +1326,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED arraylist_push(&s->relocs_list, (void*)backref_id(s, fld, s->link_ids_relocs)); // relocation target record_uniquing(s, fld, fld_pos); } - memset(&s->s->buf[fld_pos], 0, sizeof(fld)); // relocation offset (none) + memset(&f->buf[fld_pos], 0, sizeof(fld)); // relocation offset (none) } } } @@ -1323,14 +1340,16 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } } - else if (jl_typetagis(v, jl_module_tag << 4)) { + else if (jl_typeis(v, jl_module_type)) { + assert(f == s->s); jl_write_module(s, item, (jl_module_t*)v); } else if (jl_typetagis(v, jl_task_tag << 4)) { jl_error("Task cannot be serialized"); } else if (jl_is_svec(v)) { - ios_write(s->s, (char*)v, sizeof(void*)); + assert(f == s->s); + ios_write(f, (char*)v, sizeof(void*)); size_t ii, l = jl_svec_len(v); assert(l > 0 || (jl_svec_t*)v == jl_emptysvec); for (ii = 0; ii < l; ii++) { @@ -1338,8 +1357,8 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } else if (jl_is_string(v)) { - ios_write(s->s, (char*)v, sizeof(void*) + jl_string_len(v)); - write_uint8(s->s, '\0'); // null-terminated strings for easier C-compatibility + ios_write(f, (char*)v, sizeof(void*) + jl_string_len(v)); + write_uint8(f, '\0'); // null-terminated strings for easier C-compatibility } else if (jl_is_foreign_type(t) == 1) { jl_error("Cannot serialize instances of foreign datatypes"); @@ -1348,16 +1367,17 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED // The object has no fields, so we just snapshot its byte representation assert(!t->layout->npointers); assert(t->layout->npointers == 0); - ios_write(s->s, (char*)v, jl_datatype_size(t)); + ios_write(f, (char*)v, jl_datatype_size(t)); } else if (jl_bigint_type && jl_typetagis(v, jl_bigint_type)) { // foreign types require special handling + assert(f == s->s); jl_value_t *sizefield = jl_get_nth_field(v, 1); int32_t sz = jl_unbox_int32(sizefield); int32_t nw = (sz == 0 ? 1 : (sz < 0 ? -sz : sz)); size_t nb = nw * gmp_limb_size; - ios_write(s->s, (char*)&nw, sizeof(int32_t)); - ios_write(s->s, (char*)&sz, sizeof(int32_t)); + ios_write(f, (char*)&nw, sizeof(int32_t)); + ios_write(f, (char*)&sz, sizeof(int32_t)); uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), 8); write_padding(s->const_data, data - ios_pos(s->const_data)); data /= sizeof(void*); @@ -1366,7 +1386,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target void *pdata = jl_unbox_voidpointer(jl_get_nth_field(v, 2)); ios_write(s->const_data, (char*)pdata, nb); - write_pointer(s->s); + write_pointer(f); } else { // Generic object::DataType serialization by field @@ -1376,16 +1396,16 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED for (i = 0; i < nf; i++) { size_t offset = jl_field_offset(t, i); const char *slot = data + offset; - write_padding(s->s, offset - tot); + write_padding(f, offset - tot); tot = offset; size_t fsz = jl_field_size(t, i); if (t->name->mutabl && jl_is_cpointer_type(jl_field_type(t, i)) && *(intptr_t*)slot != -1) { // reset Ptr fields to C_NULL (but keep MAP_FAILED / INVALID_HANDLE) assert(!jl_field_isptr(t, i)); - write_pointer(s->s); + write_pointer(f); } else if (fsz > 0) { - ios_write(s->s, slot, fsz); + ios_write(f, slot, fsz); } tot += fsz; } @@ -1403,12 +1423,13 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED arraylist_push(&s->relocs_list, (void*)backref_id(s, fld, s->link_ids_relocs)); // relocation target record_uniquing(s, fld, fld_pos); } - memset(&s->s->buf[fld_pos], 0, sizeof(fld)); // relocation offset (none) + memset(&f->buf[fld_pos], 0, sizeof(fld)); // relocation offset (none) } // A few objects need additional handling beyond the generic serialization above if (s->incremental && jl_typetagis(v, jl_typemap_entry_type)) { + assert(f == s->s); jl_typemap_entry_t *newentry = (jl_typemap_entry_t*)&s->s->buf[reloc_offset]; if (newentry->max_world == ~(size_t)0) { if (newentry->min_world > 1) { @@ -1423,9 +1444,10 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } else if (jl_is_method(v)) { - write_padding(s->s, sizeof(jl_method_t) - tot); // hidden fields + assert(f == s->s); + write_padding(f, sizeof(jl_method_t) - tot); // hidden fields jl_method_t *m = (jl_method_t*)v; - jl_method_t *newm = (jl_method_t*)&s->s->buf[reloc_offset]; + jl_method_t *newm = (jl_method_t*)&f->buf[reloc_offset]; if (s->incremental) { if (newm->deleted_world != ~(size_t)0) newm->deleted_world = 1; @@ -1439,13 +1461,16 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED arraylist_push(&s->ccallable_list, (void*)reloc_offset); } else if (jl_is_method_instance(v)) { - jl_method_instance_t *newmi = (jl_method_instance_t*)&s->s->buf[reloc_offset]; + assert(f == s->s); + jl_method_instance_t *newmi = (jl_method_instance_t*)&f->buf[reloc_offset]; jl_atomic_store_relaxed(&newmi->precompiled, 0); } else if (jl_is_code_instance(v)) { + assert(f == s->s); // Handle the native-code pointers + assert(f == s->s); jl_code_instance_t *m = (jl_code_instance_t*)v; - jl_code_instance_t *newm = (jl_code_instance_t*)&s->s->buf[reloc_offset]; + jl_code_instance_t *newm = (jl_code_instance_t*)&f->buf[reloc_offset]; if (s->incremental) { arraylist_push(&s->fixup_objs, (void*)reloc_offset); @@ -1525,8 +1550,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } else if (jl_is_datatype(v)) { + assert(f == s->s); jl_datatype_t *dt = (jl_datatype_t*)v; - jl_datatype_t *newdt = (jl_datatype_t*)&s->s->buf[reloc_offset]; + jl_datatype_t *newdt = (jl_datatype_t*)&f->buf[reloc_offset]; if (dt->layout != NULL) { size_t nf = dt->layout->nfields; @@ -1557,8 +1583,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } else if (jl_is_typename(v)) { + assert(f == s->s); jl_typename_t *tn = (jl_typename_t*)v; - jl_typename_t *newtn = (jl_typename_t*)&s->s->buf[reloc_offset]; + jl_typename_t *newtn = (jl_typename_t*)&f->buf[reloc_offset]; if (tn->atomicfields != NULL) { size_t nb = (jl_svec_len(tn->names) + 31) / 32 * sizeof(uint32_t); uintptr_t layout = LLT_ALIGN(ios_pos(s->const_data), sizeof(void*)); @@ -1581,6 +1608,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } else if (jl_is_globalref(v)) { + assert(f == s->s); jl_globalref_t *gr = (jl_globalref_t*)v; if (s->incremental && jl_object_in_image((jl_value_t*)gr->mod)) { // will need to populate the binding field later @@ -1588,11 +1616,12 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } else if (((jl_datatype_t*)(jl_typeof(v)))->name == jl_idtable_typename) { + assert(f == s->s); // will need to rehash this, later (after types are fully constructed) arraylist_push(&s->fixup_objs, (void*)reloc_offset); } else { - write_padding(s->s, jl_datatype_size(t) - tot); + write_padding(f, jl_datatype_size(t) - tot); } } } @@ -1628,6 +1657,14 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) assert(reloc_item < layout_table.len); uintptr_t reloc_base = (uintptr_t)layout_table.items[reloc_item]; assert(reloc_base != 0 && "layout offset missing for relocation item"); + if (reloc_base & 1) { + // convert to a ConstDataRef + tag = ConstDataRef; + reloc_base &= ~(uintptr_t)1; + assert(LLT_ALIGN(reloc_base, sizeof(void*)) == reloc_base); + reloc_base /= sizeof(void*); + assert(reloc_offset == 0); + } // write reloc_offset into s->s at pos return ((uintptr_t)tag << RELOC_TAG_OFFSET) + reloc_base + reloc_offset; } @@ -1668,16 +1705,18 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) } // Compute target location at deserialization -static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, size_t size, uintptr_t reloc_id, jl_array_t *link_ids, int *link_index) JL_NOTSAFEPOINT +static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, uintptr_t reloc_id, jl_array_t *link_ids, int *link_index) JL_NOTSAFEPOINT { enum RefTags tag = (enum RefTags)(reloc_id >> RELOC_TAG_OFFSET); size_t offset = (reloc_id & (((uintptr_t)1 << RELOC_TAG_OFFSET) - 1)); switch (tag) { case DataRef: - assert(offset <= size); - return base + offset; + assert(offset <= s->s->size); + return (uintptr_t)base + offset; case ConstDataRef: - return (uintptr_t)s->const_data->buf + (offset * sizeof(void*)); + offset *= sizeof(void*); + assert(offset <= s->const_data->size); + return (uintptr_t)s->const_data->buf + offset; case SymbolRef: assert(offset < deser_sym.len && deser_sym.items[offset] && "corrupt relocation item id"); return (uintptr_t)deser_sym.items[offset]; @@ -1752,16 +1791,23 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas } -static void jl_write_offsetlist(ios_t *s, char *base, size_t size, arraylist_t *list) +static void jl_finish_relocs(char *base, size_t size, arraylist_t *list) { for (size_t i = 0; i < list->len; i += 2) { - size_t last_pos = i ? (size_t)list->items[i - 2] : 0; size_t pos = (size_t)list->items[i]; size_t item = (size_t)list->items[i + 1]; // item is tagref-encoded uintptr_t *pv = (uintptr_t*)(base + pos); assert(pos < size && pos != 0); *pv = get_reloc_for_item(item, *pv); + } +} +static void jl_write_offsetlist(ios_t *s, size_t size, arraylist_t *list) +{ + for (size_t i = 0; i < list->len; i += 2) { + size_t last_pos = i ? (size_t)list->items[i - 2] : 0; + size_t pos = (size_t)list->items[i]; + assert(pos < size && pos != 0); // write pos as compressed difference. size_t pos_diff = pos - last_pos; while (pos_diff) { @@ -1790,23 +1836,9 @@ static void jl_write_arraylist(ios_t *s, arraylist_t *list) ios_write(s, (const char*)list->items, list->len * sizeof(void*)); } -static void jl_write_relocations(jl_serializer_state *s) -{ - char *base = &s->s->buf[0]; - jl_write_offsetlist(s->relocs, base, s->s->size, &s->gctags_list); - jl_write_offsetlist(s->relocs, base, s->s->size, &s->relocs_list); - if (s->incremental) { - jl_write_arraylist(s->relocs, &s->uniquing_types); - jl_write_arraylist(s->relocs, &s->uniquing_objs); - jl_write_arraylist(s->relocs, &s->fixup_types); - } - jl_write_arraylist(s->relocs, &s->fixup_objs); -} - static void jl_read_reloclist(jl_serializer_state *s, jl_array_t *link_ids, uint8_t bits) { uintptr_t base = (uintptr_t)s->s->buf; - size_t size = s->s->size; uintptr_t last_pos = 0; uint8_t *current = (uint8_t *)(s->relocs->buf + s->relocs->bpos); int link_index = 0; @@ -1831,7 +1863,7 @@ static void jl_read_reloclist(jl_serializer_state *s, jl_array_t *link_ids, uint last_pos = pos; uintptr_t *pv = (uintptr_t *)(base + pos); uintptr_t v = *pv; - v = get_item_for_reloc(s, base, size, v, link_ids, &link_index); + v = get_item_for_reloc(s, base, v, link_ids, &link_index); if (bits && v && ((jl_datatype_t*)v)->smalltag) v = (uintptr_t)((jl_datatype_t*)v)->smalltag << 4; // TODO: should we have a representation that supports sweep without a relocation step? *pv = v | bits; @@ -1900,13 +1932,12 @@ static void _jl_write_value(jl_serializer_state *s, jl_value_t *v) static jl_value_t *jl_read_value(jl_serializer_state *s) { - uintptr_t base = (uintptr_t)&s->s->buf[0]; - size_t size = s->s->size; + uintptr_t base = (uintptr_t)s->s->buf; uintptr_t offset = *(reloc_t*)(base + (uintptr_t)s->s->bpos); s->s->bpos += sizeof(reloc_t); if (offset == 0) return NULL; - return (jl_value_t*)get_item_for_reloc(s, base, size, offset, NULL, NULL); + return (jl_value_t*)get_item_for_reloc(s, base, offset, NULL, NULL); } // The next two, `jl_read_offset` and `jl_delayed_reloc`, are essentially a split version @@ -1929,10 +1960,9 @@ static jl_value_t *jl_delayed_reloc(jl_serializer_state *s, uintptr_t offset) JL { if (!offset) return NULL; - uintptr_t base = (uintptr_t)&s->s->buf[0]; - size_t size = s->s->size; + uintptr_t base = (uintptr_t)s->s->buf; int link_index = 0; - jl_value_t *ret = (jl_value_t*)get_item_for_reloc(s, base, size, offset, s->link_ids_relocs, &link_index); + jl_value_t *ret = (jl_value_t*)get_item_for_reloc(s, base, offset, s->link_ids_relocs, &link_index); assert(!s->link_ids_relocs || link_index < jl_array_len(s->link_ids_relocs)); return ret; } @@ -2020,10 +2050,9 @@ static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image, uint3 { if (image->gvars_base == NULL) return; + uintptr_t base = (uintptr_t)s->s->buf; size_t i = 0; size_t l = s->gvar_record->size / sizeof(reloc_t); - uintptr_t base = (uintptr_t)&s->s->buf[0]; - size_t size = s->s->size; reloc_t *gvars = (reloc_t*)&s->gvar_record->buf[0]; int gvar_link_index = 0; int external_fns_link_index = 0; @@ -2032,10 +2061,10 @@ static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image, uint3 uintptr_t offset = gvars[i]; uintptr_t v = 0; if (i < external_fns_begin) { - v = get_item_for_reloc(s, base, size, offset, s->link_ids_gvars, &gvar_link_index); + v = get_item_for_reloc(s, base, offset, s->link_ids_gvars, &gvar_link_index); } else { - v = get_item_for_reloc(s, base, size, offset, s->link_ids_external_fnvars, &external_fns_link_index); + v = get_item_for_reloc(s, base, offset, s->link_ids_external_fnvars, &external_fns_link_index); } uintptr_t *gv = sysimg_gvars(image->gvars_base, image->gvars_offsets, i); *gv = v; @@ -2371,7 +2400,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, ios_mem(&relocs, 0); ios_mem(&gvar_record, 0); ios_mem(&fptr_record, 0); - jl_serializer_state s; + jl_serializer_state s = {0}; s.incremental = !(worklist == NULL); s.s = &sysimg; s.const_data = &const_data; @@ -2487,7 +2516,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, write_padding(&sysimg, sizeof(uintptr_t)); jl_write_values(&s); external_fns_begin = write_gvars(&s, &gvars, &external_fns); - jl_write_relocations(&s); } // This ensures that we can use the low bit of addresses for @@ -2516,9 +2544,12 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, // step 3: combine all of the sections into one file assert(ios_pos(f) % JL_CACHE_BYTE_ALIGNMENT == 0); + ssize_t sysimg_offset = ios_pos(f); write_uint(f, sysimg.size - sizeof(uintptr_t)); ios_seek(&sysimg, sizeof(uintptr_t)); ios_copyall(f, &sysimg); + size_t sysimg_size = s.s->size; + assert(ios_pos(f) - sysimg_offset == sysimg_size); ios_close(&sysimg); write_uint(f, const_data.size); @@ -2534,6 +2565,18 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, ios_copyall(f, &symbols); ios_close(&symbols); + // Prepare and write the relocations sections, now that the rest of the image is laid out + char *base = &f->buf[0]; + jl_finish_relocs(base + sysimg_offset, sysimg_size, &s.gctags_list); + jl_finish_relocs(base + sysimg_offset, sysimg_size, &s.relocs_list); + jl_write_offsetlist(s.relocs, sysimg_size, &s.gctags_list); + jl_write_offsetlist(s.relocs, sysimg_size, &s.relocs_list); + if (s.incremental) { + jl_write_arraylist(s.relocs, &s.uniquing_types); + jl_write_arraylist(s.relocs, &s.uniquing_objs); + jl_write_arraylist(s.relocs, &s.fixup_types); + } + jl_write_arraylist(s.relocs, &s.fixup_objs); write_uint(f, relocs.size); write_padding(f, LLT_ALIGN(ios_pos(f), 8) - ios_pos(f)); ios_seek(&relocs, 0); @@ -2798,7 +2841,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl { int en = jl_gc_enable(0); ios_t sysimg, const_data, symbols, relocs, gvar_record, fptr_record; - jl_serializer_state s; + jl_serializer_state s = {0}; s.incremental = restored != NULL; // jl_linkage_blobs.len > 0; s.image = image; s.s = NULL; @@ -2808,9 +2851,6 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl s.gvar_record = &gvar_record; s.fptr_record = &fptr_record; s.ptls = jl_current_task->ptls; - arraylist_new(&s.relocs_list, 0); - arraylist_new(&s.gctags_list, 0); - s.link_ids_relocs = s.link_ids_gctags = s.link_ids_gvars = s.link_ids_external_fnvars = NULL; jl_value_t **const*const tags = get_tags(); htable_t new_dt_objs; htable_new(&new_dt_objs, 0); From 4be81cdb15077ea06b5af8d876526e25c81ead82 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 27 Jan 2023 14:17:47 -0500 Subject: [PATCH 3/3] change CodeInfo compression from Array to String String is simpler and smaller (since we do not yet have a Buffer{UInt8} type, which might be almost as simple) making it easier to manipulate. Combined with the previous changes, this lets us eliminate many more relocations from the image (previously every Array->data), which is quite a few megabytes of array header objects simply gone. MASTER: sysimg size breakdown: sys data: 79207516 isbits data: 58603452 symbols: 463610 tags list: 1078555 reloc list: 4491653 gvar list: 118848 fptr list: 214000 .text 11109176 .data 144184704 BEFORE (moving existing Strings): sysimg size breakdown: sys data: 75146604 isbits data: 62679532 symbols: 463411 tags list: 1015525 reloc list: 4481370 gvar list: 118688 fptr list: 213952 .text 11129192 .data 144126152 AFTER (making CodeInfo into Strings): sys data: 71348124 isbits data: 63789244 symbols: 463411 tags list: 933984 reloc list: 4398377 gvar list: 118688 fptr list: 213904 .text 11132968 .data 141272824 --- base/compiler/inferencestate.jl | 2 +- base/compiler/optimize.jl | 2 +- base/compiler/ssair/inlining.jl | 6 ++-- base/compiler/typeinfer.jl | 16 +++++++---- base/compiler/utilities.jl | 2 +- base/reflection.jl | 4 +-- src/aotcompile.cpp | 15 +++++----- src/codegen.cpp | 12 ++++---- src/gf.c | 2 +- src/interpreter.c | 4 +-- src/ircode.c | 50 +++++++++++++++++---------------- src/jitlayers.cpp | 6 ++-- src/julia.h | 16 +++++------ src/precompile_utils.c | 4 +-- src/staticdata.c | 29 ++++++++++--------- test/compiler/contextual.jl | 4 +-- test/compiler/inline.jl | 8 +++--- test/compiler/ssair.jl | 11 ++++---- 18 files changed, 103 insertions(+), 90 deletions(-) diff --git a/base/compiler/inferencestate.jl b/base/compiler/inferencestate.jl index 0e0409f755a0b..35469b9f29524 100644 --- a/base/compiler/inferencestate.jl +++ b/base/compiler/inferencestate.jl @@ -688,7 +688,7 @@ function IRInterpretationState(interp::AbstractInterpreter, code::CodeInstance, mi::MethodInstance, argtypes::Vector{Any}, world::UInt) @assert code.def === mi src = @atomic :monotonic code.inferred - if isa(src, Vector{UInt8}) + if isa(src, String) src = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), mi.def, C_NULL, src)::CodeInfo else isa(src, CodeInfo) || return nothing diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 0ab1b14d4a185..71eeb15d53eb0 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -43,7 +43,7 @@ const TOP_TUPLE = GlobalRef(Core, :tuple) const InlineCostType = UInt16 const MAX_INLINE_COST = typemax(InlineCostType) const MIN_INLINE_COST = InlineCostType(10) -const MaybeCompressed = Union{CodeInfo, Vector{UInt8}} +const MaybeCompressed = Union{CodeInfo, String} is_inlineable(@nospecialize src::MaybeCompressed) = ccall(:jl_ir_inlining_cost, InlineCostType, (Any,), src) != MAX_INLINE_COST diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 746a64b3a0996..3c444894dd4b6 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -944,7 +944,7 @@ end function may_have_fcalls(m::Method) isdefined(m, :source) || return true src = m.source - isa(src, CodeInfo) || isa(src, Vector{UInt8}) || return true + isa(src, MaybeCompressed) || return true return ccall(:jl_ir_flag_has_fcall, Bool, (Any,), src) end @@ -982,8 +982,8 @@ function analyze_method!(match::MethodMatch, argtypes::Vector{Any}, return resolve_todo(mi, match, argtypes, info, flag, state; invokesig) end -function retrieve_ir_for_inlining(mi::MethodInstance, src::Array{UInt8, 1}) - src = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), mi.def, C_NULL, src::Vector{UInt8})::CodeInfo +function retrieve_ir_for_inlining(mi::MethodInstance, src::String) + src = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), mi.def, C_NULL, src)::CodeInfo return inflate_ir!(src, mi) end retrieve_ir_for_inlining(mi::MethodInstance, src::CodeInfo) = inflate_ir(src, mi) diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 180c10d8340ad..7d983ec5420db 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -325,9 +325,15 @@ function CodeInstance(interp::AbstractInterpreter, result::InferenceResult, const_flags = 0x00 end end - relocatability = isa(inferred_result, Vector{UInt8}) ? inferred_result[end] : - inferred_result === nothing ? UInt8(1) : UInt8(0) - # relocatability = isa(inferred_result, Vector{UInt8}) ? inferred_result[end] : UInt8(0) + relocatability = 0x0 + if isa(inferred_result, String) + t = @_gc_preserve_begin inferred_result + relocatability = unsafe_load(unsafe_convert(Ptr{UInt8}, inferred_result), Core.sizeof(inferred_result)) + @_gc_preserve_end t + elseif inferred_result === nothing + relocatability = 0x1 + end + # relocatability = isa(inferred_result, String) ? inferred_result[end] : UInt8(0) return CodeInstance(result.linfo, widenconst(result_type), rettype_const, inferred_result, const_flags, first(valid_worlds), last(valid_worlds), @@ -352,7 +358,7 @@ function maybe_compress_codeinfo(interp::AbstractInterpreter, linfo::MethodInsta nslots = length(ci.slotflags) resize!(ci.slottypes::Vector{Any}, nslots) resize!(ci.slotnames, nslots) - return ccall(:jl_compress_ir, Vector{UInt8}, (Any, Any), def, ci) + return ccall(:jl_compress_ir, String, (Any, Any), def, ci) else return ci end @@ -1031,7 +1037,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance) inf.rettype = code.rettype end return inf - elseif isa(inf, Vector{UInt8}) + elseif isa(inf, String) ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) inf = _uncompressed_ir(code, inf) return inf diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index 80d3feaf363be..836c370b98bd4 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -137,7 +137,7 @@ function retrieve_code_info(linfo::MethodInstance, world::UInt) if src === nothing # can happen in images built with --strip-ir return nothing - elseif isa(src, Array{UInt8,1}) + elseif isa(src, String) c = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), m, C_NULL, src) else c = copy(src::CodeInfo) diff --git a/base/reflection.jl b/base/reflection.jl index 0156db00c9f52..a3e98e11a5f04 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -1163,8 +1163,8 @@ uncompressed_ir(m::Method) = isdefined(m, :source) ? _uncompressed_ir(m, m.sourc isdefined(m, :generator) ? error("Method is @generated; try `code_lowered` instead.") : error("Code for this Method is not available.") _uncompressed_ir(m::Method, s::CodeInfo) = copy(s) -_uncompressed_ir(m::Method, s::Array{UInt8,1}) = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), m, C_NULL, s)::CodeInfo -_uncompressed_ir(ci::Core.CodeInstance, s::Array{UInt8,1}) = ccall(:jl_uncompress_ir, Any, (Any, Any, Any), ci.def.def::Method, ci, s)::CodeInfo +_uncompressed_ir(m::Method, s::String) = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), m, C_NULL, s)::CodeInfo +_uncompressed_ir(ci::Core.CodeInstance, s::String) = ccall(:jl_uncompress_ir, Any, (Any, Any, Any), ci.def.def::Method, ci, s)::CodeInfo # for backwards compat const uncompressed_ast = uncompressed_ir const _uncompressed_ast = _uncompressed_ir diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 0c2aa4feb678d..01f023d5bef5e 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -238,7 +238,7 @@ static void jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_instance if ((jl_value_t*)*src_out == jl_nothing) *src_out = NULL; if (*src_out && jl_is_method(def)) - *src_out = jl_uncompress_ir(def, codeinst, (jl_array_t*)*src_out); + *src_out = jl_uncompress_ir(def, codeinst, (jl_value_t*)*src_out); } if (*src_out == NULL || !jl_is_code_info(*src_out)) { if (cgparams.lookup != jl_rettype_inferred) { @@ -2028,17 +2028,18 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, siz jl_value_t *jlrettype = (jl_value_t*)jl_any_type; jl_code_info_t *src = NULL; JL_GC_PUSH2(&src, &jlrettype); - if (jl_is_method(mi->def.method) && mi->def.method->source != NULL && jl_ir_flag_inferred((jl_array_t*)mi->def.method->source)) { + if (jl_is_method(mi->def.method) && mi->def.method->source != NULL && mi->def.method->source != jl_nothing && jl_ir_flag_inferred(mi->def.method->source)) { src = (jl_code_info_t*)mi->def.method->source; if (src && !jl_is_code_info(src)) - src = jl_uncompress_ir(mi->def.method, NULL, (jl_array_t*)src); - } else { + src = jl_uncompress_ir(mi->def.method, NULL, (jl_value_t*)src); + } + else { jl_value_t *ci = jl_rettype_inferred(mi, world, world); if (ci != jl_nothing) { jl_code_instance_t *codeinst = (jl_code_instance_t*)ci; src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); if ((jl_value_t*)src != jl_nothing && !jl_is_code_info(src) && jl_is_method(mi->def.method)) - src = jl_uncompress_ir(mi->def.method, codeinst, (jl_array_t*)src); + src = jl_uncompress_ir(mi->def.method, codeinst, (jl_value_t*)src); jlrettype = codeinst->rettype; } if (!src || (jl_value_t*)src == jl_nothing) { @@ -2047,8 +2048,8 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, siz jlrettype = src->rettype; else if (jl_is_method(mi->def.method)) { src = mi->def.method->generator ? jl_code_for_staged(mi, world) : (jl_code_info_t*)mi->def.method->source; - if (src && !jl_is_code_info(src) && jl_is_method(mi->def.method)) - src = jl_uncompress_ir(mi->def.method, NULL, (jl_array_t*)src); + if (src && (jl_value_t*)src != jl_nothing && !jl_is_code_info(src) && jl_is_method(mi->def.method)) + src = jl_uncompress_ir(mi->def.method, NULL, (jl_value_t*)src); } // TODO: use mi->uninferred } diff --git a/src/codegen.cpp b/src/codegen.cpp index da69678dee6ff..a9d2cb0c60333 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5252,7 +5252,7 @@ static std::pair get_oc_function(jl_codectx_t &ctx, jl_met } ++EmittedOpaqueClosureFunctions; - ir = jl_uncompress_ir(closure_method, ci, (jl_array_t*)inferred); + ir = jl_uncompress_ir(closure_method, ci, (jl_value_t*)inferred); // TODO: Emit this inline and outline it late using LLVM's coroutine support. orc::ThreadSafeModule closure_m = jl_create_ts_module( @@ -8648,7 +8648,7 @@ jl_llvm_functions_t jl_emit_codeinst( return jl_emit_oc_wrapper(m, params, codeinst->def, codeinst->rettype); } if (src && (jl_value_t*)src != jl_nothing && jl_is_method(def)) - src = jl_uncompress_ir(def, codeinst, (jl_array_t*)src); + src = jl_uncompress_ir(def, codeinst, (jl_value_t*)src); if (!src || !jl_is_code_info(src)) { JL_GC_POP(); m = orc::ThreadSafeModule(); @@ -8677,7 +8677,7 @@ jl_llvm_functions_t jl_emit_codeinst( } if (params.world) {// don't alter `inferred` when the code is not directly being used - auto inferred = jl_atomic_load_relaxed(&codeinst->inferred); + jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); // don't change inferred state if (inferred) { jl_method_t *def = codeinst->def->def.method; @@ -8689,8 +8689,8 @@ jl_llvm_functions_t jl_emit_codeinst( if (inferred != (jl_value_t*)src) { if (jl_is_method(def)) { src = (jl_code_info_t*)jl_compress_ir(def, src); - assert(jl_typetagis(src, jl_array_uint8_type)); - codeinst->relocatability = ((uint8_t*)jl_array_data(src))[jl_array_len(src)-1]; + assert(jl_is_string(src)); + codeinst->relocatability = jl_string_data(src)[jl_string_len(src)-1]; } jl_atomic_store_release(&codeinst->inferred, (jl_value_t*)src); jl_gc_wb(codeinst, src); @@ -8701,7 +8701,7 @@ jl_llvm_functions_t jl_emit_codeinst( inferred != jl_nothing && // don't delete inlineable code, unless it is constant (jl_atomic_load_relaxed(&codeinst->invoke) == jl_fptr_const_return_addr || - (jl_ir_inlining_cost((jl_array_t*)inferred) == UINT16_MAX)) && + (jl_ir_inlining_cost(inferred) == UINT16_MAX)) && // don't delete code when generating a precompile file !(params.imaging || jl_options.incremental)) { // if not inlineable, code won't be needed again diff --git a/src/gf.c b/src/gf.c index 5df3e18ff8db7..b8bb21c36b2a7 100644 --- a/src/gf.c +++ b/src/gf.c @@ -443,7 +443,7 @@ JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_method_instance_t *mi, size_t mi while (codeinst) { if (codeinst->min_world <= min_world && max_world <= codeinst->max_world) { jl_value_t *code = jl_atomic_load_relaxed(&codeinst->inferred); - if (code && (code == jl_nothing || jl_ir_flag_inferred((jl_array_t*)code))) + if (code && (code == jl_nothing || jl_ir_flag_inferred(code))) return (jl_value_t*)codeinst; } codeinst = jl_atomic_load_relaxed(&codeinst->next); diff --git a/src/interpreter.c b/src/interpreter.c index c962abaa3a31a..f5ec0ce6858c8 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -641,7 +641,7 @@ jl_code_info_t *jl_code_for_interpreter(jl_method_instance_t *mi, size_t world) } if (src && (jl_value_t*)src != jl_nothing) { JL_GC_PUSH1(&src); - src = jl_uncompress_ir(mi->def.method, NULL, (jl_array_t*)src); + src = jl_uncompress_ir(mi->def.method, NULL, (jl_value_t*)src); jl_atomic_store_release(&mi->uninferred, (jl_value_t*)src); jl_gc_wb(mi, src); JL_GC_POP(); @@ -703,7 +703,7 @@ JL_DLLEXPORT jl_callptr_t jl_fptr_interpret_call_addr = &jl_fptr_interpret_call; jl_value_t *jl_interpret_opaque_closure(jl_opaque_closure_t *oc, jl_value_t **args, size_t nargs) { jl_method_t *source = oc->source; - jl_code_info_t *code = jl_uncompress_ir(source, NULL, (jl_array_t*)source->source); + jl_code_info_t *code = jl_uncompress_ir(source, NULL, (jl_value_t*)source->source); interpreter_state *s; unsigned nroots = jl_source_nslots(code) + jl_source_nssavalues(code) + 2; jl_task_t *ct = jl_current_task; diff --git a/src/ircode.c b/src/ircode.c index e04ef3fa6d96c..4121d6691aa5b 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -761,7 +761,9 @@ static jl_value_t *jl_decode_value(jl_ircode_state *s) JL_GC_DISABLED // --- entry points --- -JL_DLLEXPORT jl_array_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) +typedef jl_value_t jl_string_t; // for local expressibility + +JL_DLLEXPORT jl_string_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) { JL_TIMING(AST_COMPRESS, AST_COMPRESS); JL_LOCK(&m->writelock); // protect the roots array (Might GC) @@ -841,7 +843,7 @@ JL_DLLEXPORT jl_array_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) write_uint8(s.s, s.relocatability); ios_flush(s.s); - jl_array_t *v = jl_take_buffer(&dest); + jl_string_t *v = jl_pchar_to_string(s.s->buf, s.s->size); ios_close(s.s); if (jl_array_len(m->roots) == 0) { m->roots = NULL; @@ -854,19 +856,19 @@ JL_DLLEXPORT jl_array_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) return v; } -JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t *metadata, jl_array_t *data) +JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t *metadata, jl_string_t *data) { if (jl_is_code_info(data)) return (jl_code_info_t*)data; JL_TIMING(AST_UNCOMPRESS, AST_UNCOMPRESS); JL_LOCK(&m->writelock); // protect the roots array (Might GC) assert(jl_is_method(m)); - assert(jl_typetagis(data, jl_array_uint8_type)); + assert(jl_is_string(data)); size_t i; ios_t src; ios_mem(&src, 0); - ios_setbuf(&src, (char*)data->data, jl_array_len(data), 0); - src.size = jl_array_len(data); + ios_setbuf(&src, (char*)jl_string_data(data), jl_string_len(data), 0); + src.size = jl_string_len(data); int en = jl_gc_enable(0); // Might GC jl_ircode_state s = { &src, @@ -939,42 +941,42 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t return code; } -JL_DLLEXPORT uint8_t jl_ir_flag_inferred(jl_array_t *data) +JL_DLLEXPORT uint8_t jl_ir_flag_inferred(jl_string_t *data) { if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->inferred; - assert(jl_typetagis(data, jl_array_uint8_type)); + assert(jl_is_string(data)); jl_code_info_flags_t flags; - flags.packed = ((uint8_t*)data->data)[0]; + flags.packed = jl_string_data(data)[0]; return flags.bits.inferred; } -JL_DLLEXPORT uint8_t jl_ir_flag_inlining(jl_array_t *data) +JL_DLLEXPORT uint8_t jl_ir_flag_inlining(jl_string_t *data) { if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->inlining; - assert(jl_typetagis(data, jl_array_uint8_type)); + assert(jl_is_string(data)); jl_code_info_flags_t flags; - flags.packed = ((uint8_t*)data->data)[0]; + flags.packed = jl_string_data(data)[0]; return flags.bits.inlining; } -JL_DLLEXPORT uint8_t jl_ir_flag_has_fcall(jl_array_t *data) +JL_DLLEXPORT uint8_t jl_ir_flag_has_fcall(jl_string_t *data) { if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->has_fcall; - assert(jl_typetagis(data, jl_array_uint8_type)); + assert(jl_is_string(data)); jl_code_info_flags_t flags; - flags.packed = ((uint8_t*)data->data)[0]; + flags.packed = jl_string_data(data)[0]; return flags.bits.has_fcall; } -JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_array_t *data) +JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_string_t *data) { if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->inlining_cost; - assert(jl_typetagis(data, jl_array_uint8_type)); - uint16_t res = jl_load_unaligned_i16((char*)data->data + 2); + assert(jl_is_string(data)); + uint16_t res = jl_load_unaligned_i16(jl_string_data(data) + 2); return res; } @@ -1004,26 +1006,26 @@ JL_DLLEXPORT jl_value_t *jl_compress_argnames(jl_array_t *syms) return str; } -JL_DLLEXPORT ssize_t jl_ir_nslots(jl_array_t *data) +JL_DLLEXPORT ssize_t jl_ir_nslots(jl_value_t *data) { if (jl_is_code_info(data)) { jl_code_info_t *func = (jl_code_info_t*)data; return jl_array_len(func->slotnames); } else { - assert(jl_typetagis(data, jl_array_uint8_type)); - int nslots = jl_load_unaligned_i32((char*)data->data + 2 + sizeof(uint16_t)); + assert(jl_is_string(data)); + int nslots = jl_load_unaligned_i32(jl_string_data(data) + 2 + sizeof(uint16_t)); return nslots; } } -JL_DLLEXPORT uint8_t jl_ir_slotflag(jl_array_t *data, size_t i) +JL_DLLEXPORT uint8_t jl_ir_slotflag(jl_string_t *data, size_t i) { assert(i < jl_ir_nslots(data)); if (jl_is_code_info(data)) return ((uint8_t*)((jl_code_info_t*)data)->slotflags->data)[i]; - assert(jl_typetagis(data, jl_array_uint8_type)); - return ((uint8_t*)data->data)[2 + sizeof(uint16_t) + sizeof(int32_t) + i]; + assert(jl_is_string(data)); + return jl_string_data(data)[2 + sizeof(uint16_t) + sizeof(int32_t) + i]; } JL_DLLEXPORT jl_array_t *jl_uncompress_argnames(jl_value_t *syms) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index c6aef2d35839c..643f0468457ae 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -455,7 +455,7 @@ jl_code_instance_t *jl_generate_fptr_impl(jl_method_instance_t *mi JL_PROPAGATES if ((jl_value_t*)src == jl_nothing) src = NULL; else if (jl_is_method(mi->def.method)) - src = jl_uncompress_ir(mi->def.method, codeinst, (jl_array_t*)src); + src = jl_uncompress_ir(mi->def.method, codeinst, (jl_value_t*)src); } else { // identify whether this is an invalidated method that is being recompiled @@ -546,7 +546,7 @@ void jl_generate_fptr_for_unspecialized_impl(jl_code_instance_t *unspec) src = jl_code_for_staged(unspec->def, ~(size_t)0); } if (src && (jl_value_t*)src != jl_nothing) - src = jl_uncompress_ir(def, NULL, (jl_array_t*)src); + src = jl_uncompress_ir(def, NULL, (jl_value_t*)src); } else { src = (jl_code_info_t*)jl_atomic_load_relaxed(&unspec->def->uninferred); @@ -606,7 +606,7 @@ jl_value_t *jl_dump_method_asm_impl(jl_method_instance_t *mi, size_t world, src = def->generator ? jl_code_for_staged(mi, world) : (jl_code_info_t*)def->source; } if (src && (jl_value_t*)src != jl_nothing) - src = jl_uncompress_ir(mi->def.method, codeinst, (jl_array_t*)src); + src = jl_uncompress_ir(mi->def.method, codeinst, (jl_value_t*)src); } fptr = (uintptr_t)jl_atomic_load_acquire(&codeinst->invoke); specfptr = (uintptr_t)jl_atomic_load_relaxed(&codeinst->specptr.fptr); diff --git a/src/julia.h b/src/julia.h index 6ecb845f77afe..8774889e71e07 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1918,14 +1918,14 @@ JL_DLLEXPORT void jl_register_newmeth_tracer(void (*callback)(jl_method_t *trace JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr JL_MAYBE_UNROOTED); // IR representation -JL_DLLEXPORT jl_array_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code); -JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t *metadata, jl_array_t *data); -JL_DLLEXPORT uint8_t jl_ir_flag_inferred(jl_array_t *data) JL_NOTSAFEPOINT; -JL_DLLEXPORT uint8_t jl_ir_flag_inlining(jl_array_t *data) JL_NOTSAFEPOINT; -JL_DLLEXPORT uint8_t jl_ir_flag_has_fcall(jl_array_t *data) JL_NOTSAFEPOINT; -JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_array_t *data) JL_NOTSAFEPOINT; -JL_DLLEXPORT ssize_t jl_ir_nslots(jl_array_t *data) JL_NOTSAFEPOINT; -JL_DLLEXPORT uint8_t jl_ir_slotflag(jl_array_t *data, size_t i) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code); +JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t *metadata, jl_value_t *data); +JL_DLLEXPORT uint8_t jl_ir_flag_inferred(jl_value_t *data) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint8_t jl_ir_flag_inlining(jl_value_t *data) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint8_t jl_ir_flag_has_fcall(jl_value_t *data) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_value_t *data) JL_NOTSAFEPOINT; +JL_DLLEXPORT ssize_t jl_ir_nslots(jl_value_t *data) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint8_t jl_ir_slotflag(jl_value_t *data, size_t i) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_compress_argnames(jl_array_t *syms); JL_DLLEXPORT jl_array_t *jl_uncompress_argnames(jl_value_t *syms); JL_DLLEXPORT jl_value_t *jl_uncompress_argname_n(jl_value_t *syms, size_t i); diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 9e513a1cfed3a..055ec4b3330f1 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -186,8 +186,8 @@ static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closur jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); if (inferred && inferred != jl_nothing && - jl_ir_flag_inferred((jl_array_t*)inferred) && - (jl_ir_inlining_cost((jl_array_t*)inferred) == UINT16_MAX)) { + jl_ir_flag_inferred(inferred) && + (jl_ir_inlining_cost(inferred) == UINT16_MAX)) { do_compile = 1; } else if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL || jl_atomic_load_relaxed(&codeinst->precompile)) { diff --git a/src/staticdata.c b/src/staticdata.c index 3494aa93a8725..1728e0642551b 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -2162,7 +2162,7 @@ static jl_value_t *strip_codeinfo_meta(jl_method_t *m, jl_value_t *ci_, int orig int compressed = 0; if (!jl_is_code_info(ci_)) { compressed = 1; - ci = jl_uncompress_ir(m, NULL, (jl_array_t*)ci_); + ci = jl_uncompress_ir(m, NULL, (jl_value_t*)ci_); } else { ci = (jl_code_info_t*)ci_; @@ -2521,6 +2521,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, // This ensures that we can use the low bit of addresses for // identifying end pointers in gc's eytzinger search. write_padding(&sysimg, 4 - (sysimg.size % 4)); + write_padding(&const_data, 4 - (const_data.size % 4)); if (sysimg.size > ((uintptr_t)1 << RELOC_TAG_OFFSET)) { jl_printf( @@ -2858,9 +2859,9 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl // step 1: read section map assert(ios_pos(f) == 0 && f->bm == bm_mem); - size_t sizeof_sysimg = read_uint(f); - ios_static_buffer(&sysimg, f->buf, sizeof_sysimg + sizeof(uintptr_t)); - ios_skip(f, sizeof_sysimg); + size_t sizeof_sysdata = read_uint(f); + ios_static_buffer(&sysimg, f->buf, sizeof_sysdata + sizeof(uintptr_t)); + ios_skip(f, sizeof_sysdata); size_t sizeof_constdata = read_uint(f); // realign stream to max-alignment for data @@ -2868,6 +2869,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl ios_static_buffer(&const_data, f->buf + f->bpos, sizeof_constdata); ios_skip(f, sizeof_constdata); + size_t sizeof_sysimg = f->bpos; + size_t sizeof_symbols = read_uint(f); ios_seek(f, LLT_ALIGN(ios_pos(f), 8)); ios_static_buffer(&symbols, f->buf + f->bpos, sizeof_symbols); @@ -3046,7 +3049,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl } } uintptr_t otyp = jl_typetagof(obj); // the original type of the object that was written here - assert(image_base < (char*)obj && (char*)obj <= image_base + sizeof_sysimg + sizeof(uintptr_t)); + assert(image_base < (char*)obj && (char*)obj <= image_base + sizeof_sysimg); if (otyp == jl_datatype_tag << 4) { jl_datatype_t *dt = (jl_datatype_t*)obj[0], *newdt; if (jl_is_datatype(dt)) { @@ -3083,7 +3086,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl newobj = (jl_value_t*)newdt; } else { - assert(!(image_base < (char*)otyp && (char*)otyp <= image_base + sizeof_sysimg + sizeof(uintptr_t))); + assert(!(image_base < (char*)otyp && (char*)otyp <= image_base + sizeof_sysimg)); assert(jl_is_datatype_singleton((jl_datatype_t*)otyp) && "unreachable"); newobj = ((jl_datatype_t*)otyp)->instance; assert(newobj != jl_nothing); @@ -3093,7 +3096,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl *pfld = (uintptr_t)newobj | GC_OLD | GC_IN_IMAGE; else *pfld = (uintptr_t)newobj; - assert(!(image_base < (char*)newobj && (char*)newobj <= image_base + sizeof_sysimg + sizeof(uintptr_t))); + assert(!(image_base < (char*)newobj && (char*)newobj <= image_base + sizeof_sysimg)); assert(jl_typetagis(obj, otyp)); } // A few fields (reached via super) might be self-recursive. This is rare, but handle them now. @@ -3106,7 +3109,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl assert(jl_is_datatype(dt)); jl_value_t *newobj = (jl_value_t*)dt; *pfld = (uintptr_t)newobj; - assert(!(image_base < (char*)newobj && (char*)newobj <= image_base + sizeof_sysimg + sizeof(uintptr_t))); + assert(!(image_base < (char*)newobj && (char*)newobj <= image_base + sizeof_sysimg)); } arraylist_free(&delay_list); // now that all the fields of dt are assigned and unique, copy them into @@ -3173,7 +3176,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl } uintptr_t otyp = jl_typetagof(obj); // the original type of the object that was written here if (otyp == (uintptr_t)jl_method_instance_type) { - assert(image_base < (char*)obj && (char*)obj <= image_base + sizeof_sysimg + sizeof(uintptr_t)); + assert(image_base < (char*)obj && (char*)obj <= image_base + sizeof_sysimg); jl_value_t *m = obj[0]; if (jl_is_method_instance(m)) { newobj = m; // already done @@ -3190,7 +3193,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl abort(); // should be unreachable } *pfld = (uintptr_t)newobj; - assert(!(image_base < (char*)newobj && (char*)newobj <= image_base + sizeof_sysimg + sizeof(uintptr_t))); + assert(!(image_base < (char*)newobj && (char*)newobj <= image_base + sizeof_sysimg)); assert(jl_typetagis(obj, otyp)); } arraylist_free(&s.uniquing_types); @@ -3288,7 +3291,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl " reloc list: %8u\n" " gvar list: %8u\n" " fptr list: %8u\n", - (unsigned)sizeof_sysimg, + (unsigned)sizeof_sysdata, (unsigned)sizeof_constdata, (unsigned)sizeof_symbols, (unsigned)sizeof_tags, @@ -3297,7 +3300,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl (unsigned)sizeof_fptr_record); } if (cachesizes) { - cachesizes->sysdata = sizeof_sysimg; + cachesizes->sysdata = sizeof_sysdata; cachesizes->isbitsdata = sizeof_constdata; cachesizes->symboldata = sizeof_symbols; cachesizes->tagslist = sizeof_tags; @@ -3325,7 +3328,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl // Prepare for later external linkage against the sysimg // Also sets up images for protection against garbage collection arraylist_push(&jl_linkage_blobs, (void*)image_base); - arraylist_push(&jl_linkage_blobs, (void*)(image_base + sizeof_sysimg + sizeof(uintptr_t))); + arraylist_push(&jl_linkage_blobs, (void*)(image_base + sizeof_sysimg)); arraylist_push(&jl_image_relocs, (void*)relocs_base); rebuild_image_blob_tree(); diff --git a/test/compiler/contextual.jl b/test/compiler/contextual.jl index 16332555a0c3a..0e8fe27591a5e 100644 --- a/test/compiler/contextual.jl +++ b/test/compiler/contextual.jl @@ -192,12 +192,12 @@ try Baz = Base.require(Main, :Baz) @test length(Bar.mt) == 1 finally + filter!((≠)(load_path), LOAD_PATH) + filter!((≠)(depot_path), DEPOT_PATH) rm(load_path, recursive=true, force=true) try rm(depot_path, force=true, recursive=true) catch err @show err end - filter!((≠)(load_path), LOAD_PATH) - filter!((≠)(depot_path), DEPOT_PATH) end diff --git a/test/compiler/inline.jl b/test/compiler/inline.jl index e41b0cb2dca6f..7920212537608 100644 --- a/test/compiler/inline.jl +++ b/test/compiler/inline.jl @@ -684,9 +684,9 @@ begin end # https://github.com/JuliaLang/julia/issues/42246 -@test mktempdir() do dir +mktempdir() do dir cd(dir) do - code = quote + code = """ issue42246() = @noinline IOBuffer("a") let ci, rt = only(code_typed(issue42246)) @@ -699,9 +699,9 @@ end exit(1) end end - end |> string + """ cmd = `$(Base.julia_cmd()) --code-coverage=tmp.info -e $code` - success(pipeline(Cmd(cmd); stdout=stdout, stderr=stderr)) + @test success(pipeline(cmd; stdout, stderr)) end end diff --git a/test/compiler/ssair.jl b/test/compiler/ssair.jl index 38862c123f160..43f17d4ad69f2 100644 --- a/test/compiler/ssair.jl +++ b/test/compiler/ssair.jl @@ -321,8 +321,8 @@ end f_if_typecheck() = (if nothing; end; unsafe_load(Ptr{Int}(0))) @test_throws TypeError f_if_typecheck() -@test let # https://github.com/JuliaLang/julia/issues/42258 - code = quote +let # https://github.com/JuliaLang/julia/issues/42258 + code = """ function foo() a = @noinline rand(rand(0:10)) if isempty(a) @@ -335,10 +335,11 @@ f_if_typecheck() = (if nothing; end; unsafe_load(Ptr{Int}(0))) code_typed(foo; optimize=true) code_typed(Core.Compiler.setindex!, (Core.Compiler.UseRef,Core.Compiler.NewSSAValue); optimize=true) - end |> string + """ cmd = `$(Base.julia_cmd()) -g 2 -e $code` - stderr = IOBuffer() - success(pipeline(Cmd(cmd); stdout=stdout, stderr=stderr)) && isempty(String(take!(stderr))) + stderr = Base.BufferStream() + @test success(pipeline(Cmd(cmd); stdout, stderr)) + @test readchomp(stderr) == "" end @testset "code_ircode" begin