From 5abccb645185804f60244971230200c0f922772e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 19 Aug 2016 15:47:01 -0400 Subject: [PATCH] change incremental serialize to use deferred errors this gives the caller more control over when an how errors get printed --- base/loading.jl | 70 ++++++++++++++++++++++++++++---------------- src/builtins.c | 24 +++++++++++---- src/dump.c | 62 ++++++++++++++++++--------------------- src/julia_internal.h | 1 + 4 files changed, 93 insertions(+), 64 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 317ffd1800af5..bbc2b926fdaa8 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -141,15 +141,18 @@ function find_all_in_cache_path(mod::Symbol) return paths end +# these return either the array of modules loaded from the path / content given +# or an Exception that describes why it couldn't be loaded function _include_from_serialized(content::Vector{UInt8}) return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content)) end - function _include_from_serialized(path::String) return ccall(:jl_restore_incremental, Any, (Cstring,), path) end -# returns an array of modules loaded, or nothing if failed +# returns an array of modules loaded, or an Exception that describes why it failed +# and also attempts to load the same file across all nodes (if toplevel_node and myid() == master) +# and it reconnects the Base.Docs.META function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, toplevel_load::Bool) local restored = nothing local content::Vector{UInt8} @@ -161,13 +164,23 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t content = remotecall_fetch(open, node, read, path_to_try) end restored = _include_from_serialized(content) - if restored !== nothing - others = filter(x -> x != myid(), procs()) - refs = Any[ @spawnat p (nothing !== _include_from_serialized(content)) for p in others] - for (id, ref) in zip(others, refs) - if !fetch(ref) - warn("node state is inconsistent: node $id failed to load cache from $path_to_try") - end + isa(restored, Exception) && return restored + others = filter(x -> x != myid(), procs()) + refs = Any[ + (p, @spawnat(p, + let m = try + _include_from_serialized(content) + catch ex + isa(ex, Exception) ? ex : ErrorException(string(ex)) + end + isa(m, Exception) ? m : nothing + end)) + for p in others ] + for (id, ref) in refs + m = fetch(ref) + if m !== nothing + warn("Node state is inconsistent: node $id failed to load cache from $path_to_try. Got:") + warn(m) end end elseif node == myid() @@ -177,8 +190,8 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t restored = _include_from_serialized(content) end - if restored !== nothing - for M in restored + if !isa(restored, Exception) + for M in restored::Vector{Any} if isdefined(M, Base.Docs.META) push!(Base.Docs.modules, M) end @@ -197,17 +210,29 @@ function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::Str paths = @fetchfrom node find_all_in_cache_path(mod) end + local restored = nothing for path_to_try in paths if stale_cachefile(sourcepath, path_to_try) continue end restored = _require_from_serialized(node, mod, path_to_try, toplevel_load) - if restored === nothing - warn("deserialization checks failed while attempting to load cache from $path_to_try") + if isa(restored, Exception) + if isa(restored, ErrorException) && endswith(restored.msg, " uuid did not match cache file.") + # can't use this cache due to a module uuid mismatch, + # defer reporting error until after trying all of the possible matches + continue + end + warn("Deserialization checks failed while attempting to load cache from $path_to_try.") + error(restored) else return restored end end + if isa(restored, Exception) + warn("""Deserialization checks failed while attempting to load cache from $path_to_try. + This is likely because module %s does not support precompilation but is imported by a module that does.""") + warn(restored) + end return !isempty(paths) end @@ -346,8 +371,10 @@ function require(mod::Symbol) # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) cachefile = compilecache(mod) - if nothing === _require_from_serialized(1, mod, cachefile, last) - warn("compilecache failed to create a usable precompiled cache file for module $name.") + m = _require_from_serialized(1, mod, cachefile, last) + if !isa(m, Exception) + warn("Compilecache failed to create a usable precompiled cache file for module $name. Got:") + warn(m) else return # success end @@ -371,7 +398,9 @@ function require(mod::Symbol) rethrow() # rethrow non-precompilable=true errors end cachefile = compilecache(mod) - if nothing === _require_from_serialized(1, mod, cachefile, last) + m = _require_from_serialized(1, mod, cachefile, last) + if isa(m, Exception) + warn(m) error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.") end end @@ -572,15 +601,6 @@ function stale_cachefile(modpath, cachefile) return true end end - # files are not stale, so module list is valid and needs checking - for (M,uuid) in modules - if !isdefined(Main, M) - require(M) # should recursively recompile module M if stale - end - if module_uuid(getfield(Main, M)) != uuid - return true - end - end return false # fresh cachefile finally close(io) diff --git a/src/builtins.c b/src/builtins.c index 4698cab3afcbb..a89403454fb86 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -44,8 +44,8 @@ JL_DLLEXPORT void JL_NORETURN jl_error(const char *str) extern int vasprintf(char **str, const char *fmt, va_list ap); -static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type, - const char *fmt, va_list args) +static jl_value_t *jl_vexceptionf(jl_datatype_t *exception_type, + const char *fmt, va_list args) { if (exception_type == NULL) { jl_printf(JL_STDERR, "ERROR: "); @@ -64,15 +64,18 @@ static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type, free(str); } JL_GC_PUSH1(&msg); - jl_throw(jl_new_struct(exception_type, msg)); + jl_value_t *e = jl_new_struct(exception_type, msg); + JL_GC_POP(); + return e; } JL_DLLEXPORT void JL_NORETURN jl_errorf(const char *fmt, ...) { va_list args; va_start(args, fmt); - jl_vexceptionf(jl_errorexception_type, fmt, args); + jl_value_t *e = jl_vexceptionf(jl_errorexception_type, fmt, args); va_end(args); + jl_throw(e); } JL_DLLEXPORT void JL_NORETURN jl_exceptionf(jl_datatype_t *exception_type, @@ -80,8 +83,19 @@ JL_DLLEXPORT void JL_NORETURN jl_exceptionf(jl_datatype_t *exception_type, { va_list args; va_start(args, fmt); - jl_vexceptionf(exception_type, fmt, args); + jl_value_t *e = jl_vexceptionf(exception_type, fmt, args); + va_end(args); + jl_throw(e); +} + +jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, + const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + jl_value_t *e = jl_vexceptionf(exception_type, fmt, args); va_end(args); + return e; } JL_DLLEXPORT void JL_NORETURN jl_too_few_args(const char *fname, int min) diff --git a/src/dump.c b/src/dump.c index 58905f0f3136f..712874975f3c0 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1711,16 +1711,16 @@ static void jl_deserialize_lambdas_from_mod(jl_serializer_state *s) } } -static int read_verify_mod_list(ios_t *s) +static jl_value_t *read_verify_mod_list(ios_t *s) { if (!jl_main_module->uuid) { - jl_printf(JL_STDERR, "ERROR: Main module uuid state is invalid for module deserialization.\n"); - return 0; + return jl_get_exceptionf(jl_errorexception_type, + "Main module uuid state is invalid for module deserialization."); } while (1) { size_t len = read_int32(s); if (len == 0) - return 1; + return NULL; char *name = (char*)alloca(len+1); ios_read(s, name, len); name[len] = '\0'; @@ -1742,20 +1742,17 @@ static int read_verify_mod_list(ios_t *s) m = (jl_module_t*)jl_get_global(jl_main_module, sym); } if (!m) { - jl_printf(JL_STDERR, "ERROR: requiring \"%s\" did not define a corresponding module\n", name); - return 0; + return jl_get_exceptionf(jl_errorexception_type, + "Requiring \"%s\" did not define a corresponding module.", name); } if (!jl_is_module(m)) { ios_close(s); - jl_errorf("invalid module path (%s does not name a module)", name); + return jl_get_exceptionf(jl_errorexception_type, + "Invalid module path (%s does not name a module).", name); } if (m->uuid != uuid) { - jl_printf(JL_STDERR, - "WARNING: Module %s uuid did not match cache file\n" - " This is likely because module %s does not support\n" - " precompilation but is imported by a module that does.\n", - name, name); - return 0; + return jl_get_exceptionf(jl_errorexception_type, + "Module %s uuid did not match cache file.", name); } } } @@ -1825,13 +1822,13 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list) jl_declare_constant(b); // this can throw if (b->value != NULL) { if (!jl_is_module(b->value)) { - jl_errorf("invalid redefinition of constant %s", + jl_errorf("Invalid redefinition of constant %s.", jl_symbol_name(mod->name)); // this also throws } if (jl_generating_output() && jl_options.incremental) { - jl_errorf("cannot replace module %s during incremental precompile", jl_symbol_name(mod->name)); + jl_errorf("Cannot replace module %s during incremental precompile.", jl_symbol_name(mod->name)); } - jl_printf(JL_STDERR, "WARNING: replacing module %s\n", + jl_printf(JL_STDERR, "WARNING: replacing module %s.\n", jl_symbol_name(mod->name)); } b->value = v; @@ -2098,14 +2095,14 @@ JL_DLLEXPORT void jl_restore_system_image(const char *fname) int err = jl_load_sysimg_so(); if (err != 0) { if (jl_sysimg_handle == 0) - jl_errorf("system image file \"%s\" not found", fname); - jl_errorf("library \"%s\" does not contain a valid system image", fname); + jl_errorf("System image file \"%s\" not found.", fname); + jl_errorf("Library \"%s\" does not contain a valid system image.", fname); } } else { ios_t f; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) - jl_errorf("system image file \"%s\" not found", fname); + jl_errorf("System image file \"%s\" not found.", fname); JL_SIGATOMIC_BEGIN(); jl_restore_system_image_from_stream(&f); ios_close(&f); @@ -2363,16 +2360,17 @@ static int trace_method(jl_typemap_entry_t *entry, void *closure) return 1; } -static jl_array_t *_jl_restore_incremental(ios_t *f) +static jl_value_t *_jl_restore_incremental(ios_t *f) { - if (ios_eof(f)) { + if (ios_eof(f) || !jl_read_verify_header(f)) { ios_close(f); - return NULL; + return jl_get_exceptionf(jl_errorexception_type, + "Precompile file header verification checks failed."); } - if (!jl_read_verify_header(f) || - !read_verify_mod_list(f)) { + jl_value_t *verify_error = read_verify_mod_list(f); + if (verify_error) { ios_close(f); - return NULL; + return verify_error; } size_t deplen = read_uint64(f); ios_skip(f, deplen); // skip past the dependency list @@ -2418,28 +2416,24 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) jl_init_restored_modules(init_order); JL_GC_POP(); - return restored; + return (jl_value_t*)restored; } JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz) { ios_t f; - jl_array_t *modules; ios_static_buffer(&f, (char*)buf, sz); - modules = _jl_restore_incremental(&f); - return modules ? (jl_value_t*) modules : jl_nothing; + return _jl_restore_incremental(&f); } JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname) { ios_t f; - jl_array_t *modules; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { - jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname); - return jl_nothing; + return jl_get_exceptionf(jl_errorexception_type, + "Cache file \"%s\" not found.\n", fname); } - modules = _jl_restore_incremental(&f); - return modules ? (jl_value_t*) modules : jl_nothing; + return _jl_restore_incremental(&f); } // --- init --- diff --git a/src/julia_internal.h b/src/julia_internal.h index d4e0f32e51058..9851f3a14f6ab 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -240,6 +240,7 @@ void jl_set_gs_ctr(uint32_t ctr); void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args); void JL_NORETURN jl_method_error(jl_function_t *f, jl_value_t **args, size_t na); +jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, const char *fmt, ...); JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t);