change incremental serialize to use deferred errors

this gives the caller more control over when an how errors get printed

base/loading.jl

@@ -141,15 +141,18 @@ function find_all_in_cache_path(mod::Symbol)
     return paths
 end
 
+# these return either the array of modules loaded from the path / content given
+# or an Exception that describes why it couldn't be loaded
 function _include_from_serialized(content::Vector{UInt8})
     return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content))
 end
-
 function _include_from_serialized(path::String)
     return ccall(:jl_restore_incremental, Any, (Cstring,), path)
 end
 
-# returns an array of modules loaded, or nothing if failed
+# returns an array of modules loaded, or an Exception that describes why it failed
+# and also attempts to load the same file across all nodes (if toplevel_node and myid() == master)
+# and it reconnects the Base.Docs.META
 function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, toplevel_load::Bool)
     local restored = nothing
     local content::Vector{UInt8}
@@ -161,13 +164,23 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t
             content = remotecall_fetch(open, node, read, path_to_try)
         end
         restored = _include_from_serialized(content)
-        if restored !== nothing
-            others = filter(x -> x != myid(), procs())
-            refs = Any[ @spawnat p (nothing !== _include_from_serialized(content)) for p in others]
-            for (id, ref) in zip(others, refs)
-                if !fetch(ref)
-                    warn("node state is inconsistent: node $id failed to load cache from $path_to_try")
-                end
+        isa(restored, Exception) && return restored
+        others = filter(x -> x != myid(), procs())
+        refs = Any[
+            (p, @spawnat(p,
+                let m = try
+                            _include_from_serialized(content)
+                        catch ex
+                            isa(ex, Exception) ? ex : ErrorException(string(ex))
+                        end
+                    isa(m, Exception) ? m : nothing
+                end))
+            for p in others ]
+        for (id, ref) in refs
+            m = fetch(ref)
+            if m !== nothing
+                warn("Node state is inconsistent: node $id failed to load cache from $path_to_try. Got:")
+                warn(m)
             end
         end
     elseif node == myid()
@@ -177,8 +190,8 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t
         restored = _include_from_serialized(content)
     end
 
-    if restored !== nothing
-        for M in restored
+    if !isa(restored, Exception)
+        for M in restored::Vector{Any}
             if isdefined(M, Base.Docs.META)
                 push!(Base.Docs.modules, M)
             end
@@ -197,17 +210,29 @@ function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::Str
         paths = @fetchfrom node find_all_in_cache_path(mod)
     end
 
+    local restored = nothing
     for path_to_try in paths
         if stale_cachefile(sourcepath, path_to_try)
             continue
         end
         restored = _require_from_serialized(node, mod, path_to_try, toplevel_load)
-        if restored === nothing
-            warn("deserialization checks failed while attempting to load cache from $path_to_try")
+        if isa(restored, Exception)
+            if isa(restored, ErrorException) && endswith(restored.msg, " uuid did not match cache file.")
+                # can't use this cache due to a module uuid mismatch,
+                # defer reporting error until after trying all of the possible matches
+                continue
+            end
+            warn("Deserialization checks failed while attempting to load cache from $path_to_try.")
+            error(restored)
         else
             return restored
         end
     end
+    if isa(restored, Exception)
+        warn("""Deserialization checks failed while attempting to load cache from $path_to_try.
+             This is likely because module %s does not support precompilation but is imported by a module that does.""")
+        warn(restored)
+    end
     return !isempty(paths)
 end
 
@@ -346,8 +371,10 @@ function require(mod::Symbol)
                 # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls
                 # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
                 cachefile = compilecache(mod)
-                if nothing === _require_from_serialized(1, mod, cachefile, last)
-                    warn("compilecache failed to create a usable precompiled cache file for module $name.")
+                m = _require_from_serialized(1, mod, cachefile, last)
+                if !isa(m, Exception)
+                    warn("Compilecache failed to create a usable precompiled cache file for module $name. Got:")
+                    warn(m)
                 else
                     return # success
                 end
@@ -371,7 +398,9 @@ function require(mod::Symbol)
                 rethrow() # rethrow non-precompilable=true errors
             end
             cachefile = compilecache(mod)
-            if nothing === _require_from_serialized(1, mod, cachefile, last)
+            m = _require_from_serialized(1, mod, cachefile, last)
+            if isa(m, Exception)
+                warn(m)
                 error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.")
             end
         end
@@ -572,15 +601,6 @@ function stale_cachefile(modpath, cachefile)
                 return true
             end
         end
-        # files are not stale, so module list is valid and needs checking
-        for (M,uuid) in modules
-            if !isdefined(Main, M)
-                require(M) # should recursively recompile module M if stale
-            end
-            if module_uuid(getfield(Main, M)) != uuid
-                return true
-            end
-        end
         return false # fresh cachefile
     finally
         close(io)

src/builtins.c

@@ -44,8 +44,8 @@ JL_DLLEXPORT void JL_NORETURN jl_error(const char *str)
 
 extern int vasprintf(char **str, const char *fmt, va_list ap);
 
-static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type,
-                                       const char *fmt, va_list args)
+static jl_value_t *jl_vexceptionf(jl_datatype_t *exception_type,
+                                  const char *fmt, va_list args)
 {
     if (exception_type == NULL) {
         jl_printf(JL_STDERR, "ERROR: ");
@@ -64,24 +64,38 @@ static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type,
         free(str);
     }
     JL_GC_PUSH1(&msg);
-    jl_throw(jl_new_struct(exception_type, msg));
+    jl_value_t *e = jl_new_struct(exception_type, msg);
+    JL_GC_POP();
+    return e;
 }
 
 JL_DLLEXPORT void JL_NORETURN jl_errorf(const char *fmt, ...)
 {
     va_list args;
     va_start(args, fmt);
-    jl_vexceptionf(jl_errorexception_type, fmt, args);
+    jl_value_t *e = jl_vexceptionf(jl_errorexception_type, fmt, args);
     va_end(args);
+    jl_throw(e);
 }
 
 JL_DLLEXPORT void JL_NORETURN jl_exceptionf(jl_datatype_t *exception_type,
                                             const char *fmt, ...)
 {
     va_list args;
     va_start(args, fmt);
-    jl_vexceptionf(exception_type, fmt, args);
+    jl_value_t *e = jl_vexceptionf(exception_type, fmt, args);
+    va_end(args);
+    jl_throw(e);
+}
+
+jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type,
+                              const char *fmt, ...)
+{
+    va_list args;
+    va_start(args, fmt);
+    jl_value_t *e = jl_vexceptionf(exception_type, fmt, args);
     va_end(args);
+    return e;
 }
 
 JL_DLLEXPORT void JL_NORETURN jl_too_few_args(const char *fname, int min)

src/dump.c

@@ -1711,16 +1711,16 @@ static void jl_deserialize_lambdas_from_mod(jl_serializer_state *s)
     }
 }
 
-static int read_verify_mod_list(ios_t *s)
+static jl_value_t *read_verify_mod_list(ios_t *s)
 {
     if (!jl_main_module->uuid) {
-        jl_printf(JL_STDERR, "ERROR: Main module uuid state is invalid for module deserialization.\n");
-        return 0;
+        return jl_get_exceptionf(jl_errorexception_type,
+                "Main module uuid state is invalid for module deserialization.");
     }
     while (1) {
         size_t len = read_int32(s);
         if (len == 0)
-            return 1;
+            return NULL;
         char *name = (char*)alloca(len+1);
         ios_read(s, name, len);
         name[len] = '\0';
@@ -1742,20 +1742,17 @@ static int read_verify_mod_list(ios_t *s)
             m = (jl_module_t*)jl_get_global(jl_main_module, sym);
         }
         if (!m) {
-            jl_printf(JL_STDERR, "ERROR: requiring \"%s\" did not define a corresponding module\n", name);
-            return 0;
+            return jl_get_exceptionf(jl_errorexception_type,
+                    "Requiring \"%s\" did not define a corresponding module.", name);
         }
         if (!jl_is_module(m)) {
             ios_close(s);
-            jl_errorf("invalid module path (%s does not name a module)", name);
+            return jl_get_exceptionf(jl_errorexception_type,
+                "Invalid module path (%s does not name a module).", name);
         }
         if (m->uuid != uuid) {
-            jl_printf(JL_STDERR,
-                      "WARNING: Module %s uuid did not match cache file\n"
-                      "  This is likely because module %s does not support\n"
-                      "  precompilation but is imported by a module that does.\n",
-                      name, name);
-            return 0;
+            return jl_get_exceptionf(jl_errorexception_type,
+                "Module %s uuid did not match cache file.", name);
         }
     }
 }
@@ -1825,13 +1822,13 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list)
                 jl_declare_constant(b); // this can throw
                 if (b->value != NULL) {
                     if (!jl_is_module(b->value)) {
-                        jl_errorf("invalid redefinition of constant %s",
+                        jl_errorf("Invalid redefinition of constant %s.",
                                   jl_symbol_name(mod->name)); // this also throws
                     }
                     if (jl_generating_output() && jl_options.incremental) {
-                        jl_errorf("cannot replace module %s during incremental precompile", jl_symbol_name(mod->name));
+                        jl_errorf("Cannot replace module %s during incremental precompile.", jl_symbol_name(mod->name));
                     }
-                    jl_printf(JL_STDERR, "WARNING: replacing module %s\n",
+                    jl_printf(JL_STDERR, "WARNING: replacing module %s.\n",
                               jl_symbol_name(mod->name));
                 }
                 b->value = v;
@@ -2098,14 +2095,14 @@ JL_DLLEXPORT void jl_restore_system_image(const char *fname)
         int err = jl_load_sysimg_so();
         if (err != 0) {
             if (jl_sysimg_handle == 0)
-                jl_errorf("system image file \"%s\" not found", fname);
-            jl_errorf("library \"%s\" does not contain a valid system image", fname);
+                jl_errorf("System image file \"%s\" not found.", fname);
+            jl_errorf("Library \"%s\" does not contain a valid system image.", fname);
         }
     }
     else {
         ios_t f;
         if (ios_file(&f, fname, 1, 0, 0, 0) == NULL)
-            jl_errorf("system image file \"%s\" not found", fname);
+            jl_errorf("System image file \"%s\" not found.", fname);
         JL_SIGATOMIC_BEGIN();
         jl_restore_system_image_from_stream(&f);
         ios_close(&f);
@@ -2363,16 +2360,17 @@ static int trace_method(jl_typemap_entry_t *entry, void *closure)
     return 1;
 }
 
-static jl_array_t *_jl_restore_incremental(ios_t *f)
+static jl_value_t *_jl_restore_incremental(ios_t *f)
 {
-    if (ios_eof(f)) {
+    if (ios_eof(f) || !jl_read_verify_header(f)) {
         ios_close(f);
-        return NULL;
+        return jl_get_exceptionf(jl_errorexception_type,
+                "Precompile file header verification checks failed.");
     }
-    if (!jl_read_verify_header(f) ||
-        !read_verify_mod_list(f)) {
+    jl_value_t *verify_error = read_verify_mod_list(f);
+    if (verify_error) {
         ios_close(f);
-        return NULL;
+        return verify_error;
     }
     size_t deplen = read_uint64(f);
     ios_skip(f, deplen); // skip past the dependency list
@@ -2418,28 +2416,24 @@ static jl_array_t *_jl_restore_incremental(ios_t *f)
     jl_init_restored_modules(init_order);
     JL_GC_POP();
 
-    return restored;
+    return (jl_value_t*)restored;
 }
 
 JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz)
 {
     ios_t f;
-    jl_array_t *modules;
     ios_static_buffer(&f, (char*)buf, sz);
-    modules = _jl_restore_incremental(&f);
-    return modules ? (jl_value_t*) modules : jl_nothing;
+    return _jl_restore_incremental(&f);
 }
 
 JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname)
 {
     ios_t f;
-    jl_array_t *modules;
     if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) {
-        jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname);
-        return jl_nothing;
+        return jl_get_exceptionf(jl_errorexception_type,
+            "Cache file \"%s\" not found.\n", fname);
     }
-    modules = _jl_restore_incremental(&f);
-    return modules ? (jl_value_t*) modules : jl_nothing;
+    return _jl_restore_incremental(&f);
 }
 
 // --- init ---

src/julia_internal.h

@@ -240,6 +240,7 @@ void jl_set_gs_ctr(uint32_t ctr);
 
 void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args);
 void JL_NORETURN jl_method_error(jl_function_t *f, jl_value_t **args, size_t na);
+jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, const char *fmt, ...);
 
 JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t);