Skip to content

Commit

Permalink
Merge pull request #48881 from JuliaLang/vc/backports-1.9
Browse files Browse the repository at this point in the history 
[release-1.9] Backport #48751 and #48832
  • Loading branch information
@KristofferC
KristofferC authored Mar 6, 2023
2 parents fcde569 + c1cb058 commit be9df3f
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 127 deletions.
2 changes: 2 additions & 0 deletions src/method.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,8 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo)
for (int i = 0; i < jl_array_len(func->code); ++i) {
jl_value_t *stmt = jl_array_ptr_ref(func->code, i);
if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == jl_new_opaque_closure_sym) {
if (jl_options.incremental && jl_generating_output())
jl_error("Impossible to correctly handle OpaqueClosure inside @generated returned during precompile process.");
linfo->uninferred = jl_copy_ast((jl_value_t*)func);
jl_gc_wb(linfo, linfo->uninferred);
break;
Expand Down
95 changes: 52 additions & 43 deletions src/staticdata.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -608,10 +608,10 @@ static uintptr_t jl_fptr_id(void *fptr)

// `jl_queue_for_serialization` adds items to `serialization_order`
#define jl_queue_for_serialization(s, v) jl_queue_for_serialization_((s), (jl_value_t*)(v), 1, 0)
static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate);
static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED;


static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_t *m)
static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_t *m) JL_GC_DISABLED
{
jl_queue_for_serialization(s, m->name);
jl_queue_for_serialization(s, m->parent);
Expand Down Expand Up @@ -648,7 +648,7 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_
// you want to handle uniquing of `Dict{String,Float64}` before you tackle `Vector{Dict{String,Float64}}`.
// Uniquing is done in `serialization_order`, so the very first mention of such an object must
// be the "source" rather than merely a cross-reference.
static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate)
static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED
{
jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v);
jl_queue_for_serialization_(s, (jl_value_t*)t, 1, immediate);
Expand All @@ -672,23 +672,34 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_
immediate = 0; // do not handle remaining fields immediately (just field types remains)
}
if (s->incremental && jl_is_method_instance(v)) {
jl_method_instance_t *mi = (jl_method_instance_t*)v;
jl_value_t *def = mi->def.value;
if (needs_uniquing(v)) {
// we only need 3 specific fields of this (the rest are not used)
jl_method_instance_t *mi = (jl_method_instance_t*)v;
jl_queue_for_serialization(s, mi->def.value);
jl_queue_for_serialization(s, mi->specTypes);
jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals);
recursive = 0;
goto done_fields;
}
else if (needs_recaching(v)) {
else if (jl_is_method(def) && jl_object_in_image(def)) {
// we only need 3 specific fields of this (the rest are restored afterward, if valid)
jl_method_instance_t *mi = (jl_method_instance_t*)v;
// in particular, cache is repopulated by jl_mi_cache_insert for all foreign function,
// so must not be present here
record_field_change((jl_value_t**)&mi->uninferred, NULL);
record_field_change((jl_value_t**)&mi->backedges, NULL);
record_field_change((jl_value_t**)&mi->callbacks, NULL);
record_field_change((jl_value_t**)&mi->cache, NULL);
}
else {
assert(!needs_recaching(v));
}
// n.b. opaque closures cannot be inspected and relied upon like a
// normal method since they can get improperly introduced by generated
// functions, so if they appeared at all, we will probably serialize
// them wrong and segfault. The jl_code_for_staged function should
// prevent this from happening, so we do not need to detect that user
// error now.
}
if (jl_is_typename(v)) {
jl_typename_t *tn = (jl_typename_t*)v;
Expand All @@ -700,6 +711,15 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_
assert(!jl_object_in_image((jl_value_t*)tn->wrapper));
}
}
if (s->incremental && jl_is_code_instance(v)) {
jl_code_instance_t *ci = (jl_code_instance_t*)v;
// make sure we don't serialize other reachable cache entries of foreign methods
if (jl_object_in_image((jl_value_t*)ci->def->def.value)) {
// TODO: if (ci in ci->defs->cache)
record_field_change((jl_value_t**)&ci->next, NULL);
}
}


if (immediate) // must be things that can be recursively handled, and valid as type parameters
assert(jl_is_immutable(t) || jl_is_typevar(v) || jl_is_symbol(v) || jl_is_svec(v));
Expand Down Expand Up @@ -775,7 +795,7 @@ done_fields: ;
*bp = (void*)((char*)HT_NOTFOUND + 1 + idx);
}

static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate)
static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED
{
if (!jl_needs_serialization(s, v))
return;
Expand Down Expand Up @@ -818,7 +838,7 @@ static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, i
// Do a pre-order traversal of the to-serialize worklist, in the identical order
// to the calls to jl_queue_for_serialization would occur in a purely recursive
// implementation, but without potentially running out of stack.
static void jl_serialize_reachable(jl_serializer_state *s)
static void jl_serialize_reachable(jl_serializer_state *s) JL_GC_DISABLED
{
size_t i, prevlen = 0;
while (object_worklist.len) {
Expand Down Expand Up @@ -2877,10 +2897,11 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl
*method_roots_list = (jl_array_t*)jl_delayed_reloc(&s, offset_method_roots_list);
*ext_targets = (jl_array_t*)jl_delayed_reloc(&s, offset_ext_targets);
*edges = (jl_array_t*)jl_delayed_reloc(&s, offset_edges);
if (!*new_specializations)
*new_specializations = jl_alloc_vec_any(0);
}
s.s = NULL;


// step 3: apply relocations
assert(!ios_eof(f));
jl_read_symbols(&s);
Expand Down Expand Up @@ -3142,19 +3163,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl
jl_code_instance_t *ci = (jl_code_instance_t*)obj;
assert(s.incremental);
ci->min_world = world;
if (ci->max_world == 1) { // sentinel value: has edges to external callables
ptrhash_put(&new_code_instance_validate, ci, (void*)(~(uintptr_t)HT_NOTFOUND)); // "HT_FOUND"
}
else if (ci->max_world) {
// It's valid, but it may not be connected
if (!ci->def->cache)
ci->def->cache = ci;
}
else {
// Ensure this code instance is not connected
if (ci->def->cache == ci)
ci->def->cache = NULL;
}
if (ci->max_world != 0)
jl_array_ptr_1d_push(*new_specializations, (jl_value_t*)ci);
}
else if (jl_is_globalref(obj)) {
continue; // wait until all the module binding tables have been initialized
Expand Down Expand Up @@ -3298,7 +3308,7 @@ static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_
}

// TODO?: refactor to make it easier to create the "package inspector"
static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *image, jl_array_t *depmods, int complete)
static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *image, jl_array_t *depmods, int completeinfo)
{
uint64_t checksum = 0;
int64_t dataendpos = 0;
Expand Down Expand Up @@ -3330,7 +3340,6 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im
else {
ios_close(f);
ios_static_buffer(f, sysimg, len);
htable_new(&new_code_instance_validate, 0);
pkgcachesizes cachesizes;
jl_restore_system_image_from_stream_(f, image, depmods, checksum, (jl_array_t**)&restored, &init_order, &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges, &base, &ccallable_list, &cachesizes);
JL_SIGATOMIC_END();
Expand All @@ -3342,25 +3351,25 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im
jl_copy_roots(method_roots_list, jl_worklist_key((jl_array_t*)restored));
// Handle edges
jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)ext_targets, (jl_array_t*)new_specializations); // restore external backedges (needs to be last)
// check new CodeInstances and validate any that lack external backedges
validate_new_code_instances();
// reinit ccallables
jl_reinit_ccallable(&ccallable_list, base, NULL);
arraylist_free(&ccallable_list);
htable_free(&new_code_instance_validate);
if (complete) {
cachesizes_sv = jl_alloc_svec_uninit(7);
jl_svec_data(cachesizes_sv)[0] = jl_box_long(cachesizes.sysdata);
jl_svec_data(cachesizes_sv)[1] = jl_box_long(cachesizes.isbitsdata);
jl_svec_data(cachesizes_sv)[2] = jl_box_long(cachesizes.symboldata);
jl_svec_data(cachesizes_sv)[3] = jl_box_long(cachesizes.tagslist);
jl_svec_data(cachesizes_sv)[4] = jl_box_long(cachesizes.reloclist);
jl_svec_data(cachesizes_sv)[5] = jl_box_long(cachesizes.gvarlist);
jl_svec_data(cachesizes_sv)[6] = jl_box_long(cachesizes.fptrlist);

if (completeinfo) {
cachesizes_sv = jl_alloc_svec(7);
jl_svecset(cachesizes_sv, 0, jl_box_long(cachesizes.sysdata));
jl_svecset(cachesizes_sv, 1, jl_box_long(cachesizes.isbitsdata));
jl_svecset(cachesizes_sv, 2, jl_box_long(cachesizes.symboldata));
jl_svecset(cachesizes_sv, 3, jl_box_long(cachesizes.tagslist));
jl_svecset(cachesizes_sv, 4, jl_box_long(cachesizes.reloclist));
jl_svecset(cachesizes_sv, 5, jl_box_long(cachesizes.gvarlist));
jl_svecset(cachesizes_sv, 6, jl_box_long(cachesizes.fptrlist));
restored = (jl_value_t*)jl_svec(8, restored, init_order, extext_methods, new_specializations, method_roots_list,
ext_targets, edges, cachesizes_sv);
} else
}
else {
restored = (jl_value_t*)jl_svec(2, restored, init_order);
}
}
}

Expand All @@ -3373,24 +3382,24 @@ static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, uin
jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
}

JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int complete)
JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo)
{
ios_t f;
ios_static_buffer(&f, (char*)buf, sz);
jl_value_t *ret = jl_restore_package_image_from_stream(&f, image, depmods, complete);
jl_value_t *ret = jl_restore_package_image_from_stream(&f, image, depmods, completeinfo);
ios_close(&f);
return ret;
}

JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int complete)
JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int completeinfo)
{
ios_t f;
if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) {
return jl_get_exceptionf(jl_errorexception_type,
"Cache file \"%s\" not found.\n", fname);
}
jl_image_t pkgimage = {};
jl_value_t *ret = jl_restore_package_image_from_stream(&f, &pkgimage, depmods, complete);
jl_value_t *ret = jl_restore_package_image_from_stream(&f, &pkgimage, depmods, completeinfo);
ios_close(&f);
return ret;
}
Expand Down Expand Up @@ -3440,7 +3449,7 @@ JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len)
JL_SIGATOMIC_END();
}

JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods, int complete)
JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods, int completeinfo)
{
void *pkgimg_handle = jl_dlopen(fname, JL_RTLD_LAZY);
if (!pkgimg_handle) {
Expand Down Expand Up @@ -3492,7 +3501,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j
}
#endif

jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, complete);
jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, completeinfo);

return mod;
}
Expand Down
Loading

0 comments on commit be9df3f

Please sign in to comment.