staticdata: fix unreferenced data assertion (#39246)

Need to sequence operations correctly here. We need to first finish
marking referenced data before we start deleting unreferenced data. This
otherwise can lead to an assertion failure in some situations.

Refs #37650

(cherry picked from commit f91bb74)

src/staticdata.c

@@ -1427,30 +1427,33 @@ static void jl_finalize_deserializer(jl_serializer_state *s) JL_GC_DISABLED
 
 
 
-// --- helper functions ---
-
-// remove cached types not referenced in the stream
-static int keep_type_cache_entry(jl_value_t *ti)
+static void jl_scan_type_cache_gv(jl_serializer_state *s, jl_svec_t *cache)
 {
-    if (ptrhash_get(&backref_table, ti) != HT_NOTFOUND || jl_get_llvm_gv(native_functions, ti) != 0)
-        return 1;
-    if (jl_is_datatype(ti)) {
-        jl_value_t *singleton = ((jl_datatype_t*)ti)->instance;
-        if (singleton && (ptrhash_get(&backref_table, singleton) != HT_NOTFOUND ||
-                    jl_get_llvm_gv(native_functions, singleton) != 0))
-            return 1;
-    }
-    return 0;
+    size_t l = jl_svec_len(cache), i;
+    for (i = 0; i < l; i++) {
+        jl_value_t *ti = jl_svecref(cache, i);
+        if (ti == NULL || ti == jl_nothing)
+            continue;
+        if (jl_get_llvm_gv(native_functions, ti)) {
+            jl_serialize_value(s, ti);
+        }
+        else if (jl_is_datatype(ti)) {
+            jl_value_t *singleton = ((jl_datatype_t*)ti)->instance;
+            if (singleton && jl_get_llvm_gv(native_functions, singleton))
+                jl_serialize_value(s, ti);
+        }
+    }
 }
 
+// remove cached types not referenced in the stream
 static void jl_prune_type_cache_hash(jl_svec_t *cache)
 {
     size_t l = jl_svec_len(cache), i;
     for (i = 0; i < l; i++) {
         jl_value_t *ti = jl_svecref(cache, i);
         if (ti == NULL || ti == jl_nothing)
             continue;
-        if (!keep_type_cache_entry(ti))
+        if (ptrhash_get(&backref_table, ti) == HT_NOTFOUND)
             jl_svecset(cache, i, jl_nothing);
     }
 }
@@ -1462,7 +1465,7 @@ static void jl_prune_type_cache_linear(jl_svec_t *cache)
         jl_value_t *ti = jl_svecref(cache, i);
         if (ti == NULL)
             break;
-        if (keep_type_cache_entry(ti))
+        if (ptrhash_get(&backref_table, ti) != HT_NOTFOUND)
             jl_svecset(cache, ins++, ti);
     }
     if (i > ins) {
@@ -1528,14 +1531,28 @@ static void jl_save_system_image_to_stream(ios_t *f) JL_GC_DISABLED
             jl_value_t *tag = *tags[i];
             jl_serialize_value(&s, tag);
         }
-        // prune unused entries from built-in type caches
+        // step 1.1: check for values only found in the generated code
+        arraylist_t typenames;
+        arraylist_new(&typenames, 0);
         for (i = 0; i < backref_table.size; i += 2) {
             jl_typename_t *tn = (jl_typename_t*)backref_table.table[i];
             if (tn == HT_NOTFOUND || !jl_is_typename(tn))
                 continue;
+            arraylist_push(&typenames, tn);
+        }
+        for (i = 0; i < typenames.len; i++) {
+            jl_typename_t *tn = (jl_typename_t*)typenames.items[i];
+            jl_scan_type_cache_gv(&s, tn->cache);
+            jl_scan_type_cache_gv(&s, tn->linearcache);
+        }
+        // step 1.2: prune (garbage collect) some special weak references from
+        // built-in type caches
+        for (i = 0; i < typenames.len; i++) {
+            jl_typename_t *tn = (jl_typename_t*)typenames.items[i];
             jl_prune_type_cache_hash(tn->cache);
             jl_prune_type_cache_linear(tn->linearcache);
         }
+        arraylist_free(&typenames);
     }
 
     { // step 2: build all the sysimg sections