Merge pull request #49556 from JuliaLang/jn/small-typeof

convert some typeof tags to small integers
JuliaLang · May 8, 2023 · 8bcea42 · 8bcea42
2 parents 2cad2a7 + 4be81cd
commit 8bcea42
Show file tree

Hide file tree

Showing 49 changed files with 1,158 additions and 821 deletions.
diff --git a/base/boot.jl b/base/boot.jl
@@ -245,7 +245,6 @@ ccall(:jl_toplevel_eval_in, Any, (Any, Any),
       (f::typeof(Typeof))(x) = ($(_expr(:meta,:nospecialize,:x)); isa(x,Type) ? Type{x} : typeof(x))
       end)
 
-
 macro nospecialize(x)
     _expr(:meta, :nospecialize, x)
 end
@@ -256,7 +255,15 @@ TypeVar(n::Symbol, @nospecialize(lb), @nospecialize(ub)) = _typevar(n, lb, ub)
 
 UnionAll(v::TypeVar, @nospecialize(t)) = ccall(:jl_type_unionall, Any, (Any, Any), v, t)
 
-const Vararg = ccall(:jl_toplevel_eval_in, Any, (Any, Any), Core, _expr(:new, TypeofVararg))
+# simple convert for use by constructors of types in Core
+# note that there is no actual conversion defined here,
+# so the methods and ccall's in Core aren't permitted to use convert
+convert(::Type{Any}, @nospecialize(x)) = x
+convert(::Type{T}, x::T) where {T} = x
+cconvert(::Type{T}, x) where {T} = convert(T, x)
+unsafe_convert(::Type{T}, x::T) where {T} = x
+
+const Vararg = ccall(:jl_wrap_vararg, Any, (Int, Int), 0, 0)
 
 # dispatch token indicating a kwarg (keyword sorter) call
 function kwcall end
@@ -448,14 +455,6 @@ function _Task(@nospecialize(f), reserved_stack::Int, completion_future)
     return ccall(:jl_new_task, Ref{Task}, (Any, Any, Int), f, completion_future, reserved_stack)
 end
 
-# simple convert for use by constructors of types in Core
-# note that there is no actual conversion defined here,
-# so the methods and ccall's in Core aren't permitted to use convert
-convert(::Type{Any}, @nospecialize(x)) = x
-convert(::Type{T}, x::T) where {T} = x
-cconvert(::Type{T}, x) where {T} = convert(T, x)
-unsafe_convert(::Type{T}, x::T) where {T} = x
-
 _is_internal(__module__) = __module__ === Core
 # can be used in place of `@assume_effects :foldable` (supposed to be used for bootstrapping)
 macro _foldable_meta()

diff --git a/base/compiler/inferencestate.jl b/base/compiler/inferencestate.jl
@@ -688,7 +688,7 @@ function IRInterpretationState(interp::AbstractInterpreter,
     code::CodeInstance, mi::MethodInstance, argtypes::Vector{Any}, world::UInt)
     @assert code.def === mi
     src = @atomic :monotonic code.inferred
-    if isa(src, Vector{UInt8})
+    if isa(src, String)
         src = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), mi.def, C_NULL, src)::CodeInfo
     else
         isa(src, CodeInfo) || return nothing

diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl
@@ -43,7 +43,7 @@ const TOP_TUPLE = GlobalRef(Core, :tuple)
 const InlineCostType = UInt16
 const MAX_INLINE_COST = typemax(InlineCostType)
 const MIN_INLINE_COST = InlineCostType(10)
-const MaybeCompressed = Union{CodeInfo, Vector{UInt8}}
+const MaybeCompressed = Union{CodeInfo, String}
 
 is_inlineable(@nospecialize src::MaybeCompressed) =
     ccall(:jl_ir_inlining_cost, InlineCostType, (Any,), src) != MAX_INLINE_COST

diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl
@@ -944,7 +944,7 @@ end
 function may_have_fcalls(m::Method)
     isdefined(m, :source) || return true
     src = m.source
-    isa(src, CodeInfo) || isa(src, Vector{UInt8}) || return true
+    isa(src, MaybeCompressed) || return true
     return ccall(:jl_ir_flag_has_fcall, Bool, (Any,), src)
 end
 
@@ -982,8 +982,8 @@ function analyze_method!(match::MethodMatch, argtypes::Vector{Any},
     return resolve_todo(mi, match, argtypes, info, flag, state; invokesig)
 end
 
-function retrieve_ir_for_inlining(mi::MethodInstance, src::Array{UInt8, 1})
-    src = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), mi.def, C_NULL, src::Vector{UInt8})::CodeInfo
+function retrieve_ir_for_inlining(mi::MethodInstance, src::String)
+    src = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), mi.def, C_NULL, src)::CodeInfo
     return inflate_ir!(src, mi)
 end
 retrieve_ir_for_inlining(mi::MethodInstance, src::CodeInfo) = inflate_ir(src, mi)

diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl
@@ -325,9 +325,15 @@ function CodeInstance(interp::AbstractInterpreter, result::InferenceResult,
             const_flags = 0x00
         end
     end
-    relocatability = isa(inferred_result, Vector{UInt8}) ? inferred_result[end] :
-                     inferred_result === nothing ? UInt8(1) : UInt8(0)
-    # relocatability = isa(inferred_result, Vector{UInt8}) ? inferred_result[end] : UInt8(0)
+    relocatability = 0x0
+    if isa(inferred_result, String)
+        t = @_gc_preserve_begin inferred_result
+        relocatability = unsafe_load(unsafe_convert(Ptr{UInt8}, inferred_result), Core.sizeof(inferred_result))
+        @_gc_preserve_end t
+    elseif inferred_result === nothing
+        relocatability = 0x1
+    end
+    # relocatability = isa(inferred_result, String) ? inferred_result[end] : UInt8(0)
     return CodeInstance(result.linfo,
         widenconst(result_type), rettype_const, inferred_result,
         const_flags, first(valid_worlds), last(valid_worlds),
@@ -352,7 +358,7 @@ function maybe_compress_codeinfo(interp::AbstractInterpreter, linfo::MethodInsta
             nslots = length(ci.slotflags)
             resize!(ci.slottypes::Vector{Any}, nslots)
             resize!(ci.slotnames, nslots)
-            return ccall(:jl_compress_ir, Vector{UInt8}, (Any, Any), def, ci)
+            return ccall(:jl_compress_ir, String, (Any, Any), def, ci)
         else
             return ci
         end
@@ -1031,7 +1037,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance)
                 inf.rettype = code.rettype
             end
             return inf
-        elseif isa(inf, Vector{UInt8})
+        elseif isa(inf, String)
             ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time)
             inf = _uncompressed_ir(code, inf)
             return inf

diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl
@@ -137,7 +137,7 @@ function retrieve_code_info(linfo::MethodInstance, world::UInt)
         if src === nothing
             # can happen in images built with --strip-ir
             return nothing
-        elseif isa(src, Array{UInt8,1})
+        elseif isa(src, String)
             c = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), m, C_NULL, src)
         else
             c = copy(src::CodeInfo)

diff --git a/base/reflection.jl b/base/reflection.jl
@@ -1163,8 +1163,8 @@ uncompressed_ir(m::Method) = isdefined(m, :source) ? _uncompressed_ir(m, m.sourc
                              isdefined(m, :generator) ? error("Method is @generated; try `code_lowered` instead.") :
                              error("Code for this Method is not available.")
 _uncompressed_ir(m::Method, s::CodeInfo) = copy(s)
-_uncompressed_ir(m::Method, s::Array{UInt8,1}) = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), m, C_NULL, s)::CodeInfo
-_uncompressed_ir(ci::Core.CodeInstance, s::Array{UInt8,1}) = ccall(:jl_uncompress_ir, Any, (Any, Any, Any), ci.def.def::Method, ci, s)::CodeInfo
+_uncompressed_ir(m::Method, s::String) = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), m, C_NULL, s)::CodeInfo
+_uncompressed_ir(ci::Core.CodeInstance, s::String) = ccall(:jl_uncompress_ir, Any, (Any, Any, Any), ci.def.def::Method, ci, s)::CodeInfo
 # for backwards compat
 const uncompressed_ast = uncompressed_ir
 const _uncompressed_ast = _uncompressed_ir

diff --git a/cli/jl_exports.h b/cli/jl_exports.h
@@ -16,6 +16,10 @@ JL_EXPORTED_DATA_POINTERS(XX)
 JL_EXPORTED_DATA_SYMBOLS(XX)
 #undef XX
 
+// define a copy of exported data
+#define jl_max_tags 64
+JL_DLLEXPORT void *small_typeof[(jl_max_tags << 4) / sizeof(void*)]; // 16-bit aligned, like the GC
+
 // Declare list of exported functions (sans type)
 #define XX(name)    JL_DLLEXPORT void name(void);
 typedef void (anonfunc)(void);

diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp
@@ -238,7 +238,7 @@ static void jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_instance
         if ((jl_value_t*)*src_out == jl_nothing)
             *src_out = NULL;
         if (*src_out && jl_is_method(def))
-            *src_out = jl_uncompress_ir(def, codeinst, (jl_array_t*)*src_out);
+            *src_out = jl_uncompress_ir(def, codeinst, (jl_value_t*)*src_out);
     }
     if (*src_out == NULL || !jl_is_code_info(*src_out)) {
         if (cgparams.lookup != jl_rettype_inferred) {
@@ -1575,6 +1575,14 @@ void jl_dump_native_impl(void *native_code,
                                      GlobalVariable::ExternalLinkage,
                                      jlRTLD_DEFAULT_var,
                                      "jl_RTLD_DEFAULT_handle_pointer"), TheTriple);
+
+        // let the compiler know we are going to internalize a copy of this,
+        // if it has a current usage with ExternalLinkage
+        auto small_typeof_copy = dataM->getGlobalVariable("small_typeof");
+        if (small_typeof_copy) {
+            small_typeof_copy->setVisibility(GlobalValue::HiddenVisibility);
+            small_typeof_copy->setDSOLocal(true);
+        }
     }
 
     // Reserve space for the output files and names
@@ -1651,13 +1659,21 @@ void jl_dump_native_impl(void *native_code,
         auto shards = emit_shard_table(*sysimageM, T_size, T_psize, threads);
         auto ptls = emit_ptls_table(*sysimageM, T_size, T_psize);
         auto header = emit_image_header(*sysimageM, threads, nfvars, ngvars);
-        auto AT = ArrayType::get(T_psize, 4);
+        auto AT = ArrayType::get(T_size, sizeof(small_typeof) / sizeof(void*));
+        auto small_typeof_copy = new GlobalVariable(*sysimageM, AT, false,
+                                                    GlobalVariable::ExternalLinkage,
+                                                    Constant::getNullValue(AT),
+                                                    "small_typeof");
+        small_typeof_copy->setVisibility(GlobalValue::HiddenVisibility);
+        small_typeof_copy->setDSOLocal(true);
+        AT = ArrayType::get(T_psize, 5);
         auto pointers = new GlobalVariable(*sysimageM, AT, false,
                                            GlobalVariable::ExternalLinkage,
                                            ConstantArray::get(AT, {
                                                 ConstantExpr::getBitCast(header, T_psize),
                                                 ConstantExpr::getBitCast(shards, T_psize),
                                                 ConstantExpr::getBitCast(ptls, T_psize),
+                                                ConstantExpr::getBitCast(small_typeof_copy, T_psize),
                                                 ConstantExpr::getBitCast(target_ids, T_psize)
                                            }),
                                            "jl_image_pointers");
@@ -2012,17 +2028,18 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, siz
     jl_value_t *jlrettype = (jl_value_t*)jl_any_type;
     jl_code_info_t *src = NULL;
     JL_GC_PUSH2(&src, &jlrettype);
-    if (jl_is_method(mi->def.method) && mi->def.method->source != NULL && jl_ir_flag_inferred((jl_array_t*)mi->def.method->source)) {
+    if (jl_is_method(mi->def.method) && mi->def.method->source != NULL && mi->def.method->source != jl_nothing && jl_ir_flag_inferred(mi->def.method->source)) {
         src = (jl_code_info_t*)mi->def.method->source;
         if (src && !jl_is_code_info(src))
-            src = jl_uncompress_ir(mi->def.method, NULL, (jl_array_t*)src);
-    } else {
+            src = jl_uncompress_ir(mi->def.method, NULL, (jl_value_t*)src);
+    }
+    else {
         jl_value_t *ci = jl_rettype_inferred(mi, world, world);
         if (ci != jl_nothing) {
             jl_code_instance_t *codeinst = (jl_code_instance_t*)ci;
             src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred);
             if ((jl_value_t*)src != jl_nothing && !jl_is_code_info(src) && jl_is_method(mi->def.method))
-                src = jl_uncompress_ir(mi->def.method, codeinst, (jl_array_t*)src);
+                src = jl_uncompress_ir(mi->def.method, codeinst, (jl_value_t*)src);
             jlrettype = codeinst->rettype;
         }
         if (!src || (jl_value_t*)src == jl_nothing) {
@@ -2031,8 +2048,8 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, siz
                 jlrettype = src->rettype;
             else if (jl_is_method(mi->def.method)) {
                 src = mi->def.method->generator ? jl_code_for_staged(mi, world) : (jl_code_info_t*)mi->def.method->source;
-                if (src && !jl_is_code_info(src) && jl_is_method(mi->def.method))
-                    src = jl_uncompress_ir(mi->def.method, NULL, (jl_array_t*)src);
+                if (src && (jl_value_t*)src != jl_nothing && !jl_is_code_info(src) && jl_is_method(mi->def.method))
+                    src = jl_uncompress_ir(mi->def.method, NULL, (jl_value_t*)src);
             }
             // TODO: use mi->uninferred
         }

diff --git a/src/array.c b/src/array.c
@@ -509,7 +509,7 @@ JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len)
             jl_throw(jl_memory_exception);
         s = jl_gc_big_alloc_noinline(ptls, allocsz);
     }
-    jl_set_typeof(s, jl_string_type);
+    jl_set_typetagof(s, jl_string_tag, 0);
     maybe_record_alloc_to_profile(s, len, jl_string_type);
     *(size_t*)s = len;
     jl_string_data(s)[len] = 0;
@@ -1255,16 +1255,16 @@ JL_DLLEXPORT void jl_array_ptr_copy(jl_array_t *dest, void **dest_p,
 
 JL_DLLEXPORT void jl_array_ptr_1d_push(jl_array_t *a, jl_value_t *item)
 {
-    assert(jl_typeis(a, jl_array_any_type));
+    assert(jl_typetagis(a, jl_array_any_type));
     jl_array_grow_end(a, 1);
     size_t n = jl_array_nrows(a);
     jl_array_ptr_set(a, n - 1, item);
 }
 
 JL_DLLEXPORT void jl_array_ptr_1d_append(jl_array_t *a, jl_array_t *a2)
 {
-    assert(jl_typeis(a, jl_array_any_type));
-    assert(jl_typeis(a2, jl_array_any_type));
+    assert(jl_typetagis(a, jl_array_any_type));
+    assert(jl_typetagis(a2, jl_array_any_type));
     size_t i;
     size_t n = jl_array_nrows(a);
     size_t n2 = jl_array_nrows(a2);

diff --git a/src/ast.c b/src/ast.c
@@ -700,11 +700,11 @@ static value_t julia_to_scm_noalloc(fl_context_t *fl_ctx, jl_value_t *v, int che
     if (julia_to_scm_noalloc1(fl_ctx, v, &retval))
         return retval;
     assert(!jl_is_expr(v) &&
-           !jl_typeis(v, jl_linenumbernode_type) &&
-           !jl_typeis(v, jl_gotonode_type) &&
-           !jl_typeis(v, jl_quotenode_type) &&
-           !jl_typeis(v, jl_newvarnode_type) &&
-           !jl_typeis(v, jl_globalref_type));
+           !jl_typetagis(v, jl_linenumbernode_type) &&
+           !jl_typetagis(v, jl_gotonode_type) &&
+           !jl_typetagis(v, jl_quotenode_type) &&
+           !jl_typetagis(v, jl_newvarnode_type) &&
+           !jl_typetagis(v, jl_globalref_type));
     return julia_to_scm_noalloc2(fl_ctx, v, check_valid);
 }
 
@@ -745,7 +745,7 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali
     // GC Note: jl_fieldref(v, 0) allocates for GotoNode
     //          but we don't need a GC root here because julia_to_list2_noalloc
     //          shouldn't allocate in this case.
-    if (jl_typeis(v, jl_linenumbernode_type)) {
+    if (jl_typetagis(v, jl_linenumbernode_type)) {
         jl_value_t *file = jl_fieldref_noalloc(v,1);
         jl_value_t *line = jl_fieldref(v,0);
         value_t args = julia_to_list2_noalloc(fl_ctx, line, file, check_valid);
@@ -755,13 +755,13 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali
         fl_free_gc_handles(fl_ctx, 1);
         return scmv;
     }
-    if (jl_typeis(v, jl_gotonode_type))
+    if (jl_typetagis(v, jl_gotonode_type))
         return julia_to_list2_noalloc(fl_ctx, (jl_value_t*)jl_goto_sym, jl_fieldref(v,0), check_valid);
-    if (jl_typeis(v, jl_quotenode_type))
+    if (jl_typetagis(v, jl_quotenode_type))
         return julia_to_list2(fl_ctx, (jl_value_t*)jl_inert_sym, jl_fieldref_noalloc(v,0), 0);
-    if (jl_typeis(v, jl_newvarnode_type))
+    if (jl_typetagis(v, jl_newvarnode_type))
         return julia_to_list2_noalloc(fl_ctx, (jl_value_t*)jl_newvar_sym, jl_fieldref(v,0), check_valid);
-    if (jl_typeis(v, jl_globalref_type)) {
+    if (jl_typetagis(v, jl_globalref_type)) {
         jl_module_t *m = jl_globalref_mod(v);
         jl_sym_t *sym = jl_globalref_name(v);
         if (m == jl_core_module)
@@ -1011,7 +1011,7 @@ static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule
     // __source__ argument
     jl_value_t *lno = jl_array_ptr_ref(args, 1);
     margs[1] = lno;
-    if (!jl_typeis(lno, jl_linenumbernode_type)) {
+    if (!jl_typetagis(lno, jl_linenumbernode_type)) {
         margs[1] = jl_new_struct(jl_linenumbernode_type, jl_box_long(0), jl_nothing);
     }
     margs[2] = (jl_value_t*)inmodule;