diff --git a/base/loading.jl b/base/loading.jl index 6bda601ef549f..ddca6283d848f 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -115,9 +115,9 @@ find_in_path(name::AbstractString, wd = pwd()) = find_in_path(String(name), wd) function find_in_node_path(name::String, srcpath, node::Int=1) if myid() == node - find_in_path(name, srcpath) + return find_in_path(name, srcpath) else - remotecall_fetch(find_in_path, node, name, srcpath) + return remotecall_fetch(find_in_path, node, name, srcpath) end end @@ -126,33 +126,37 @@ function find_source_file(file::String) file2 = find_in_path(file) file2 !== nothing && return file2 file2 = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file) - isfile(file2) ? file2 : nothing + return isfile(file2) ? file2 : nothing end function find_all_in_cache_path(mod::Symbol) name = string(mod) - paths = AbstractString[] + paths = String[] for prefix in LOAD_CACHE_PATH path = joinpath(prefix, name*".ji") if isfile_casesensitive(path) push!(paths, path) end end - paths + return paths end +# these return either the array of modules loaded from the path / content given +# or an Exception that describes why it couldn't be loaded function _include_from_serialized(content::Vector{UInt8}) - return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8},Int), content, sizeof(content)) + return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content)) +end +function _include_from_serialized(path::String) + return ccall(:jl_restore_incremental, Any, (Cstring,), path) end -# returns an array of modules loaded, or nothing if failed +# returns an array of modules loaded, or an Exception that describes why it failed +# and also attempts to load the same file across all nodes (if toplevel_node and myid() == master) +# and it reconnects the Base.Docs.META function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, toplevel_load::Bool) - if JLOptions().use_compilecache == 0 - return nothing - end - restored = nothing + local restored = nothing + local content::Vector{UInt8} if toplevel_load && myid() == 1 && nprocs() > 1 - recompile_stale(mod, path_to_try) # broadcast top-level import/using from node 1 (only) if node == myid() content = open(read, path_to_try) @@ -160,26 +164,34 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t content = remotecall_fetch(open, node, read, path_to_try) end restored = _include_from_serialized(content) - if restored !== nothing - others = filter(x -> x != myid(), procs()) - refs = Any[ @spawnat p (nothing !== _include_from_serialized(content)) for p in others] - for (id, ref) in zip(others, refs) - if !fetch(ref) - warn("node state is inconsistent: node $id failed to load cache from $path_to_try") - end + isa(restored, Exception) && return restored + others = filter(x -> x != myid(), procs()) + refs = Any[ + (p, @spawnat(p, + let m = try + _include_from_serialized(content) + catch ex + isa(ex, Exception) ? ex : ErrorException(string(ex)) + end + isa(m, Exception) ? m : nothing + end)) + for p in others ] + for (id, ref) in refs + m = fetch(ref) + if m !== nothing + warn("Node state is inconsistent: node $id failed to load cache from $path_to_try. Got:") + warn(m, prefix="WARNING: ") end end elseif node == myid() - myid() == 1 && recompile_stale(mod, path_to_try) - restored = ccall(:jl_restore_incremental, Any, (Cstring,), path_to_try) + restored = _include_from_serialized(path_to_try) else content = remotecall_fetch(open, node, read, path_to_try) restored = _include_from_serialized(content) end - # otherwise, continue search - if restored !== nothing - for M in restored + if !isa(restored, Exception) + for M in restored::Vector{Any} if isdefined(M, Base.Docs.META) push!(Base.Docs.modules, M) end @@ -188,37 +200,53 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t return restored end -function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool) - if JLOptions().use_compilecache == 0 - return nothing - end +# returns `true` if require found a precompile cache for this mod, but couldn't load it +# returns `false` if the module isn't known to be precompilable +# returns the set of modules restored if the cache load succeeded +function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::String, toplevel_load::Bool) if node == myid() paths = find_all_in_cache_path(mod) else paths = @fetchfrom node find_all_in_cache_path(mod) end - sort!(paths, by=mtime, rev=true) # try newest cachefiles first - for path_to_try in paths + + local restored = nothing, failedpath = "" + for path_to_try in paths::Vector{String} + if stale_cachefile(sourcepath, path_to_try) + continue + end restored = _require_from_serialized(node, mod, path_to_try, toplevel_load) - if restored === nothing - warn("deserialization checks failed while attempting to load cache from $path_to_try") + if isa(restored, Exception) + if isa(restored, ErrorException) && endswith(restored.msg, " uuid did not match cache file.") + # can't use this cache due to a module uuid mismatch, + # defer reporting error until after trying all of the possible matches + failedpath = path_to_try + continue + end + warn("Deserialization checks failed while attempting to load cache from $path_to_try.") + throw(restored) else return restored end end - return nothing + if isa(restored, Exception) + warn("Deserialization checks failed while attempting to load cache from $failedpath.") + warn(restored, prefix="WARNING: ") + end + return !isempty(paths) end # to synchronize multiple tasks trying to import/using something const package_locks = Dict{Symbol,Condition}() # used to optionally track dependencies when requiring a module: -const _require_dependencies = Tuple{String,Float64}[] -const _track_dependencies = [false] +const _concrete_dependencies = Any[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them +const _require_dependencies = Any[] # a list of (path, mtime) tuples that are the file dependencies of the module currently being precompiled +const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies function _include_dependency(_path::AbstractString) prev = source_path(nothing) - path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev),_path) - if myid() == 1 && _track_dependencies[1] + path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev), _path) + if myid() == 1 && _track_dependencies[] apath = abspath(path) push!(_require_dependencies, (apath, mtime(apath))) end @@ -314,8 +342,8 @@ toplevel_load = true function require(mod::Symbol) # dependency-tracking is only used for one top-level include(path), # and is not applied recursively to imported modules: - old_track_dependencies = _track_dependencies[1] - _track_dependencies[1] = false + old_track_dependencies = _track_dependencies[] + _track_dependencies[] = false global toplevel_load loading = get(package_locks, mod, false) @@ -329,22 +357,46 @@ function require(mod::Symbol) last = toplevel_load::Bool try toplevel_load = false - if nothing !== _require_from_serialized(1, mod, last) - return - end - if JLOptions().incremental != 0 - # spawn off a new incremental precompile task from node 1 for recursive `require` calls - cachefile = compilecache(mod) - if nothing === _require_from_serialized(1, mod, cachefile, last) - warn("require failed to create a precompiled cache file") - end - return - end + # perform the search operation to select the module file require intends to load name = string(mod) path = find_in_node_path(name, nothing, 1) if path === nothing - throw(ArgumentError("$name not found in path.\nRun Pkg.add(\"$name\") to install the $name package")) + throw(ArgumentError("module $name not found in current path.\nRun `Pkg.add(\"$name\")` to install the $name package.")) + end + + # attempt to load the module file via the precompile cache locations + doneprecompile = false + if JLOptions().use_compilecache != 0 + doneprecompile = _require_search_from_serialized(1, mod, path, last) + if !isa(doneprecompile, Bool) + return # success + end end + + # if the module being required was supposed to have a particular version + # but it was not handled by the precompile loader, complain + for (concrete_mod, concrete_uuid) in _concrete_dependencies + if mod === concrete_mod + warn("""Module $mod with uuid $concrete_uuid is missing from the cache. + This may mean module $mod does not support precompilation but is imported by a module that does.""") + end + end + + if doneprecompile === true || JLOptions().incremental != 0 + # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls + # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) + cachefile = compilecache(mod) + m = _require_from_serialized(1, mod, cachefile, last) + if isa(m, Exception) + warn("Compilecache failed to create a usable precompiled cache file for module $name. Got:") + warn(m, prefix="WARNING: ") + else + return # success + end + end + + # just load the file normally via include + # for unknown dependencies try if last && myid() == 1 && nprocs() > 1 # include on node 1 first to check for PrecompilableErrors @@ -357,20 +409,22 @@ function require(mod::Symbol) eval(Main, :(Base.include_from_node1($path))) end catch ex - if !precompilableerror(ex, true) + if doneprecompile === true || JLOptions().use_compilecache == 0 || !precompilableerror(ex, true) rethrow() # rethrow non-precompilable=true errors end - isinteractive() && info("Precompiling module $mod...") + # the file requested `__precompile__`, so try to build a cache file and use that cachefile = compilecache(mod) - if nothing === _require_from_serialized(1, mod, cachefile, last) - error("__precompile__(true) but require failed to create a precompiled cache file") + m = _require_from_serialized(1, mod, cachefile, last) + if isa(m, Exception) + warn(m, prefix="WARNING: ") + error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.") end end finally toplevel_load = last loading = pop!(package_locks, mod) notify(loading, all=true) - _track_dependencies[1] = old_track_dependencies + _track_dependencies[] = old_track_dependencies end nothing end @@ -414,7 +468,8 @@ task-local include path is set to the directory containing the file. Nested call in parallel, and files will be fetched from node 1. This function is typically used to load source interactively, or to combine files in packages that are broken into multiple source files. """ -function include_from_node1(_path::AbstractString) +include_from_node1(path::AbstractString) = include_from_node1(String(path)) +function include_from_node1(_path::String) path, prev = _include_dependency(_path) tls = task_local_storage() tls[:SOURCE_PATH] = path @@ -448,7 +503,7 @@ function evalfile(path::AbstractString, args::Vector{String}=String[]) end evalfile(path::AbstractString, args::Vector) = evalfile(path, String[args...]) -function create_expr_cache(input::AbstractString, output::AbstractString) +function create_expr_cache(input::String, output::String, concrete_deps::Vector{Any}) rm(output, force=true) # Remove file if it exists code_object = """ while !eof(STDIN) @@ -469,6 +524,9 @@ function create_expr_cache(input::AbstractString, output::AbstractString) append!(Base.LOAD_CACHE_PATH, $LOAD_CACHE_PATH) empty!(Base.DL_LOAD_PATH) append!(Base.DL_LOAD_PATH, $DL_LOAD_PATH) + empty!(Base._concrete_dependencies) + append!(Base._concrete_dependencies, $concrete_deps) + Base._track_dependencies[] = true end) source = source_path(nothing) if source !== nothing @@ -476,7 +534,6 @@ function create_expr_cache(input::AbstractString, output::AbstractString) task_local_storage()[:SOURCE_PATH] = $(source) end) end - serialize(io, :(Base._track_dependencies[1] = true)) serialize(io, :(Base.include($(abspath(input))))) if source !== nothing serialize(io, :(delete!(task_local_storage(), :SOURCE_PATH))) @@ -494,14 +551,38 @@ end compilecache(mod::Symbol) = compilecache(string(mod)) function compilecache(name::String) myid() == 1 || error("can only precompile from node 1") + # decide where to get the source file from path = find_in_path(name, nothing) path === nothing && throw(ArgumentError("$name not found in path")) + path = String(path) + # decide where to put the resulting cache file cachepath = LOAD_CACHE_PATH[1] if !isdir(cachepath) mkpath(cachepath) end - cachefile = abspath(cachepath, name*".ji") - if !success(create_expr_cache(path, cachefile)) + cachefile::String = abspath(cachepath, name*".ji") + # build up the list of modules that we want the precompile process to preserve + concrete_deps = copy(_concrete_dependencies) + for existing in names(Main) + if isdefined(Main, existing) + mod = getfield(Main, existing) + if isa(mod, Module) && !(mod === Main || mod === Core || mod === Base) + mod = mod::Module + if module_parent(mod) === Main && module_name(mod) === existing + push!(concrete_deps, (existing, module_uuid(mod))) + end + end + end + end + # run the expression and cache the result + if isinteractive() + if isfile(cachefile) + info("Recompiling stale cache file $cachefile for module $name.") + else + info("Precompiling module $name.") + end + end + if !success(create_expr_cache(path, cachefile, concrete_deps)) error("Failed to precompile $name to $cachefile") end return cachefile @@ -511,26 +592,55 @@ module_uuid(m::Module) = ccall(:jl_module_uuid, UInt64, (Any,), m) isvalid_cache_header(f::IOStream) = 0 != ccall(:jl_read_verify_header, Cint, (Ptr{Void},), f.ios) -function cache_dependencies(f::IO) - modules = Tuple{Symbol,UInt64}[] - files = Tuple{String,Float64}[] +function parse_cache_header(f::IO) + modules = Dict{Symbol,UInt64}() while true n = ntoh(read(f, Int32)) n == 0 && break - push!(modules, - (Symbol(read(f, n)), # module symbol - ntoh(read(f, UInt64)))) # module UUID (timestamp) + sym = Symbol(read(f, n)) # module symbol + uuid = ntoh(read(f, UInt64)) # module UUID (mostly just a timestamp) + modules[sym] = uuid end - read(f, Int64) # total bytes for file dependencies + totbytes = ntoh(read(f, Int64)) # total bytes for file dependencies + # read the list of files + files = Tuple{String,Float64}[] while true n = ntoh(read(f, Int32)) n == 0 && break + totbytes -= 4 + n + 8 + if n < 0 # probably means this wasn't a valid file to be read by Base.parse_cache_header + error("EOF while reading cache header") + end push!(files, (String(read(f, n)), ntoh(read(f, Float64)))) end + @assert totbytes == 4 "header of cache file appears to be corrupt" return modules, files end -function cache_dependencies(cachefile::AbstractString) +function parse_cache_header(cachefile::String) + io = open(cachefile, "r") + try + !isvalid_cache_header(io) && throw(ArgumentError("invalid cache file $cachefile")) + return parse_cache_header(io) + finally + close(io) + end +end + +function cache_dependencies(f::IO) + defs, files = parse_cache_header(f) + modules = [] + while true + n = ntoh(read(f, Int32)) + n == 0 && break + sym = Symbol(read(f, n)) # module symbol + uuid = ntoh(read(f, UInt64)) # module UUID (mostly just a timestamp) + push!(modules, (sym, uuid)) + end + return modules, files +end + +function cache_dependencies(cachefile::String) io = open(cachefile, "r") try !isvalid_cache_header(io) && throw(ArgumentError("invalid cache file $cachefile")) @@ -540,44 +650,37 @@ function cache_dependencies(cachefile::AbstractString) end end -function stale_cachefile(modpath, cachefile) +function stale_cachefile(modpath::String, cachefile::String) io = open(cachefile, "r") try if !isvalid_cache_header(io) return true # invalid cache file end - modules, files = cache_dependencies(io) + modules, files = parse_cache_header(io) + + # check if this file is going to provide one of our concrete dependencies + provides_concrete = false + for (mod, uuid) in _concrete_dependencies + if get(modules, mod, UInt64(0)) === uuid + provides_concrete = true + else + return false # cachefile doesn't provide the required version of the dependency + end + end + provides_concrete && return false # this is the file we want + + # now check if this file is fresh relative to its source files if files[1][1] != modpath return true # cache file was compiled from a different path end - for (f,ftime) in files + for (f, ftime) in files # Issue #13606: compensate for Docker images rounding mtimes if mtime(f) ∉ (ftime, floor(ftime)) return true end end - # files are not stale, so module list is valid and needs checking - for (M,uuid) in modules - if !isdefined(Main, M) - require(M) # should recursively recompile module M if stale - end - if module_uuid(getfield(Main, M)) != uuid - return true - end - end return false # fresh cachefile finally close(io) end end - -function recompile_stale(mod, cachefile) - path = find_in_path(string(mod), nothing) - if path === nothing - error("module $mod not found in current path; you should rm(\"$(escape_string(cachefile))\") to remove the orphaned cache file") - end - if stale_cachefile(path, cachefile) - info("Recompiling stale cache file $cachefile for module $mod.") - compilecache(mod) - end -end diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index 72241c2ea00a7..9985d6d2be557 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -257,24 +257,33 @@ the statements in a module often involves compiling a large amount of code. Julia provides the ability to create precompiled versions of modules to reduce this time. -There are two mechanisms that can achieve this: -incremental compile and custom system image. - -To create a custom system image that can be used when starting Julia with the ``-J`` option, -recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules. - To create an incremental precompiled module file, add -``__precompile__()`` at the top of your module file (before the -``module`` starts). This will cause it to be automatically compiled -the first time it is imported. Alternatively, you can manually call -``Base.compilecache(modulename)``. The resulting cache files will be -stored in ``Base.LOAD_CACHE_PATH[1]``. Subsequently, the module is -automatically recompiled upon ``import`` whenever any of its -dependencies change; dependencies are modules it imports, the Julia -build, files it includes, or explicit dependencies declared by -``include_dependency(path)`` in the module file(s). Precompiling a -module also recursively precompiles any modules that are imported -therein. If you know that it is *not* safe to precompile your module +``__precompile__()`` at the top of your module file +(before the ``module`` starts). +This will cause it to be automatically compiled the first time it is imported. +Alternatively, you can manually call ``Base.compilecache(modulename)``. +The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``. +Subsequently, the module is automatically recompiled upon ``import`` +whenever any of its dependencies change; +dependencies are modules it imports, the Julia build, files it includes, +or explicit dependencies declared by ``include_dependency(path)`` in the module file(s). + +For file dependencies, a change is determined by examining whether the modification time (mtime) +of each file loaded by ``include`` or added explicity by ``include_dependency`` is unchanged, +or equal to the modification time truncated to the nearest second +(to accommodate systems that can't copy mtime with sub-second accuracy). +It also takes into account whether the path to the file chosen by the search logic in ``require`` +matches the path that had created the precompile file. + +It also takes into account the set of dependencies already loaded into the current process +and won't recompile those modules, even if their files change or disappear, +in order to avoid creating incompatibilities between the running system and the precompile cache. +If you want to have changes to the source reflected in the running system, +you should call ``reload("Module")`` on the module you changed, +and any module that depended on it in which you want to see the change reflected. + +Precompiling a module also recursively precompiles any modules that are imported therein. +If you know that it is *not* safe to precompile your module (for the reasons described below), you should put ``__precompile__(false)`` in the module file to cause ``Base.compilecache`` to throw an error (and thereby prevent the module from being imported by @@ -290,12 +299,22 @@ initialization steps that must occur at *runtime* from steps that can occur at *compile time*. For this purpose, Julia allows you to define an ``__init__()`` function in your module that executes any initialization steps that must occur at runtime. +This function will not be called during compilation +(``--output-*`` or ``__precompile__()``). +You may, of course, call it manually if necessary, +but the default is to assume this function deals with computing state for +the local machine, which does not need to be -- or even should not be -- +captured in the compiled image. +It will be called after the module is loaded into a process, +including if it is being loaded into an incremental compile +(``--output-incremental=yes``), but not if it is being loaded +into a full-compilation process. In particular, if you define a ``function __init__()`` in a module, then Julia will call ``__init__()`` immediately *after* the module is loaded (e.g., by ``import``, ``using``, or ``require``) at runtime for the *first* time (i.e., ``__init__`` is only called once, and only -after all statements in the module have been executed). Because it is +after all statements in the module have been executed). Because it is called after the module is fully imported, any submodules or other imported modules have their ``__init__`` functions called *before* the ``__init__`` of the enclosing module. @@ -304,7 +323,7 @@ Two typical uses of ``__init__`` are calling runtime initialization functions of external C libraries and initializing global constants that involve pointers returned by external libraries. For example, suppose that we are calling a C library ``libfoo`` that requires us -to call a ``foo_init()`` initialization function at runtime. Suppose +to call a ``foo_init()`` initialization function at runtime. Suppose that we also want to define a global constant ``foo_data_ptr`` that holds the return value of a ``void *foo_data()`` function defined by ``libfoo`` — this constant must be initialized at runtime (not at compile @@ -312,14 +331,17 @@ time) because the pointer address will change from run to run. You could accomplish this by defining the following ``__init__`` function in your module:: + const foo_data_ptr = Ref{Ptr{Void}}(0) function __init__() - ccall((:foo_init,:libfoo), Void, ()) - global const foo_data_ptr = ccall((:foo_data,:libfoo), Ptr{Void}, ()) + ccall((:foo_init, :libfoo), Void, ()) + foo_data_ptr[] = ccall((:foo_data, :libfoo), Ptr{Void}, ()) end Notice that it is perfectly possible to define a global inside a function like ``__init__``; this is one of the advantages of using a -dynamic language. +dynamic language. But by making it a constant at global scope, +we can ensure that the type is known to the compiler and allow it to generate +better optimized code. Obviously, any other globals in your module that depends on ``foo_data_ptr`` would also have to be initialized in ``__init__``. diff --git a/src/builtins.c b/src/builtins.c index 4698cab3afcbb..a89403454fb86 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -44,8 +44,8 @@ JL_DLLEXPORT void JL_NORETURN jl_error(const char *str) extern int vasprintf(char **str, const char *fmt, va_list ap); -static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type, - const char *fmt, va_list args) +static jl_value_t *jl_vexceptionf(jl_datatype_t *exception_type, + const char *fmt, va_list args) { if (exception_type == NULL) { jl_printf(JL_STDERR, "ERROR: "); @@ -64,15 +64,18 @@ static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type, free(str); } JL_GC_PUSH1(&msg); - jl_throw(jl_new_struct(exception_type, msg)); + jl_value_t *e = jl_new_struct(exception_type, msg); + JL_GC_POP(); + return e; } JL_DLLEXPORT void JL_NORETURN jl_errorf(const char *fmt, ...) { va_list args; va_start(args, fmt); - jl_vexceptionf(jl_errorexception_type, fmt, args); + jl_value_t *e = jl_vexceptionf(jl_errorexception_type, fmt, args); va_end(args); + jl_throw(e); } JL_DLLEXPORT void JL_NORETURN jl_exceptionf(jl_datatype_t *exception_type, @@ -80,8 +83,19 @@ JL_DLLEXPORT void JL_NORETURN jl_exceptionf(jl_datatype_t *exception_type, { va_list args; va_start(args, fmt); - jl_vexceptionf(exception_type, fmt, args); + jl_value_t *e = jl_vexceptionf(exception_type, fmt, args); + va_end(args); + jl_throw(e); +} + +jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, + const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + jl_value_t *e = jl_vexceptionf(exception_type, fmt, args); va_end(args); + return e; } JL_DLLEXPORT void JL_NORETURN jl_too_few_args(const char *fname, int min) diff --git a/src/dump.c b/src/dump.c index 58905f0f3136f..b28c88ad6f402 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1084,7 +1084,7 @@ static void write_mod_list(ios_t *s) } // "magic" string and version header of .ji file -static const int JI_FORMAT_VERSION = 2; +static const int JI_FORMAT_VERSION = 3; static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature static const uint16_t BOM = 0xFEFF; // byte-order marker static void write_header(ios_t *s) @@ -1101,6 +1101,22 @@ static void write_header(ios_t *s) ios_write(s, commit, strlen(commit)+1); } +// serialize information about the result of deserializing this file +static void write_work_list(ios_t *s) +{ + int i, l = jl_array_len(serializer_worklist); + for (i = 0; i < l; i++) { + jl_module_t *workmod = (jl_module_t*)jl_array_ptr_ref(serializer_worklist, i); + if (workmod->parent == jl_main_module) { + size_t l = strlen(jl_symbol_name(workmod->name)); + write_int32(s, l); + ios_write(s, jl_symbol_name(workmod->name), l); + write_uint64(s, workmod->uuid); + } + } + write_int32(s, 0); +} + // serialize the global _require_dependencies array of pathnames that // are include depenencies static void write_dependency_list(ios_t *s) @@ -1711,16 +1727,16 @@ static void jl_deserialize_lambdas_from_mod(jl_serializer_state *s) } } -static int read_verify_mod_list(ios_t *s) +static jl_value_t *read_verify_mod_list(ios_t *s) { if (!jl_main_module->uuid) { - jl_printf(JL_STDERR, "ERROR: Main module uuid state is invalid for module deserialization.\n"); - return 0; + return jl_get_exceptionf(jl_errorexception_type, + "Main module uuid state is invalid for module deserialization."); } while (1) { size_t len = read_int32(s); if (len == 0) - return 1; + return NULL; char *name = (char*)alloca(len+1); ios_read(s, name, len); name[len] = '\0'; @@ -1742,20 +1758,17 @@ static int read_verify_mod_list(ios_t *s) m = (jl_module_t*)jl_get_global(jl_main_module, sym); } if (!m) { - jl_printf(JL_STDERR, "ERROR: requiring \"%s\" did not define a corresponding module\n", name); - return 0; + return jl_get_exceptionf(jl_errorexception_type, + "Requiring \"%s\" did not define a corresponding module.", name); } if (!jl_is_module(m)) { ios_close(s); - jl_errorf("invalid module path (%s does not name a module)", name); + return jl_get_exceptionf(jl_errorexception_type, + "Invalid module path (%s does not name a module).", name); } if (m->uuid != uuid) { - jl_printf(JL_STDERR, - "WARNING: Module %s uuid did not match cache file\n" - " This is likely because module %s does not support\n" - " precompilation but is imported by a module that does.\n", - name, name); - return 0; + return jl_get_exceptionf(jl_errorexception_type, + "Module %s uuid did not match cache file.", name); } } } @@ -1825,13 +1838,13 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list) jl_declare_constant(b); // this can throw if (b->value != NULL) { if (!jl_is_module(b->value)) { - jl_errorf("invalid redefinition of constant %s", + jl_errorf("Invalid redefinition of constant %s.", jl_symbol_name(mod->name)); // this also throws } if (jl_generating_output() && jl_options.incremental) { - jl_errorf("cannot replace module %s during incremental precompile", jl_symbol_name(mod->name)); + jl_errorf("Cannot replace module %s during incremental precompile.", jl_symbol_name(mod->name)); } - jl_printf(JL_STDERR, "WARNING: replacing module %s\n", + jl_printf(JL_STDERR, "WARNING: replacing module %s.\n", jl_symbol_name(mod->name)); } b->value = v; @@ -2098,14 +2111,14 @@ JL_DLLEXPORT void jl_restore_system_image(const char *fname) int err = jl_load_sysimg_so(); if (err != 0) { if (jl_sysimg_handle == 0) - jl_errorf("system image file \"%s\" not found", fname); - jl_errorf("library \"%s\" does not contain a valid system image", fname); + jl_errorf("System image file \"%s\" not found.", fname); + jl_errorf("Library \"%s\" does not contain a valid system image.", fname); } } else { ios_t f; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) - jl_errorf("system image file \"%s\" not found", fname); + jl_errorf("System image file \"%s\" not found.", fname); JL_SIGATOMIC_BEGIN(); jl_restore_system_image_from_stream(&f); ios_close(&f); @@ -2200,8 +2213,10 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) } serializer_worklist = worklist; write_header(&f); - write_mod_list(&f); // this can throw, keep it early (before any actual initialization) + write_work_list(&f); write_dependency_list(&f); + write_mod_list(&f); // this can return errors during deserialize, + // best to keep it early (before any actual initialization) arraylist_new(&reinit_list, 0); htable_new(&backref_table, 5000); @@ -2363,19 +2378,31 @@ static int trace_method(jl_typemap_entry_t *entry, void *closure) return 1; } -static jl_array_t *_jl_restore_incremental(ios_t *f) +static jl_value_t *_jl_restore_incremental(ios_t *f) { - if (ios_eof(f)) { + if (ios_eof(f) || !jl_read_verify_header(f)) { ios_close(f); - return NULL; + return jl_get_exceptionf(jl_errorexception_type, + "Precompile file header verification checks failed."); + } + { // skip past the mod list + size_t len; + while ((len = read_int32(f))) + ios_skip(f, len + sizeof(uint64_t)); + } + { // skip past the dependency list + size_t deplen = read_uint64(f); + ios_skip(f, deplen); } - if (!jl_read_verify_header(f) || - !read_verify_mod_list(f)) { + + // verify that the system state is valid + jl_value_t *verify_error = read_verify_mod_list(f); + if (verify_error) { ios_close(f); - return NULL; + return verify_error; } - size_t deplen = read_uint64(f); - ios_skip(f, deplen); // skip past the dependency list + + // prepare to deserialize arraylist_new(&backref_list, 4000); arraylist_push(&backref_list, jl_main_module); arraylist_new(&flagref_list, 0); @@ -2418,28 +2445,24 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) jl_init_restored_modules(init_order); JL_GC_POP(); - return restored; + return (jl_value_t*)restored; } JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz) { ios_t f; - jl_array_t *modules; ios_static_buffer(&f, (char*)buf, sz); - modules = _jl_restore_incremental(&f); - return modules ? (jl_value_t*) modules : jl_nothing; + return _jl_restore_incremental(&f); } JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname) { ios_t f; - jl_array_t *modules; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { - jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname); - return jl_nothing; + return jl_get_exceptionf(jl_errorexception_type, + "Cache file \"%s\" not found.\n", fname); } - modules = _jl_restore_incremental(&f); - return modules ? (jl_value_t*) modules : jl_nothing; + return _jl_restore_incremental(&f); } // --- init --- diff --git a/src/julia_internal.h b/src/julia_internal.h index d4e0f32e51058..9851f3a14f6ab 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -240,6 +240,7 @@ void jl_set_gs_ctr(uint32_t ctr); void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args); void JL_NORETURN jl_method_error(jl_function_t *f, jl_value_t **args, size_t na); +jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, const char *fmt, ...); JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t); diff --git a/src/module.c b/src/module.c index 6fbee0f969345..ac134d6e1ad5b 100644 --- a/src/module.c +++ b/src/module.c @@ -26,7 +26,9 @@ JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name) m->name = name; m->parent = NULL; m->istopmod = 0; - m->uuid = jl_hrtime(); + static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing + m->uuid = jl_hrtime() + (++mcounter); + if (!m->uuid) m->uuid++; // uuid 0 is invalid m->counter = 0; htable_new(&m->bindings, 0); arraylist_new(&m->usings, 0); diff --git a/test/compile.jl b/test/compile.jl index 09dc51390edc6..9c4d1c7bdc8ed 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -2,10 +2,18 @@ using Base.Test -function redirected_stderr() +function redirected_stderr(expected) rd, wr = redirect_stderr() - @async readstring(rd) # make sure the kernel isn't being forced to buffer the output - nothing + t = @async begin + read = readstring(rd) # also makes sure the kernel isn't being forced to buffer the output + if !contains(read, expected) + @show expected + @show read + @test false + end + nothing + end + return t end olderr = STDERR @@ -86,12 +94,14 @@ try # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: + t = redirected_stderr("WARNING: replacing module Foo4b3a94a1a081a8cb.\nWARNING: Method definition ") try - redirected_stderr() - @test nothing !== Base._require_from_serialized(myid(), Foo_module, #=broadcast-load=#false) + @test isa(Base._require_from_serialized(myid(), Foo_module, cachefile, #=broadcast-load=#false), Array{Any,1}) finally + close(STDERR) redirect_stderr(olderr) end + wait(t) let Foo = eval(Main, Foo_module) @test Foo.foo(17) == 18 @@ -101,10 +111,14 @@ try @test stringmime("text/plain", Base.Docs.doc(Foo.foo)) == "foo function\n" @test stringmime("text/plain", Base.Docs.doc(Foo.Bar.bar)) == "bar function\n" - deps = Base.cache_dependencies(cachefile) - @test sort(deps[1]) == map(s -> (s, Base.module_uuid(eval(s))), - [:Base,:Core,:Main]) - @test map(x -> x[1], sort(deps[2])) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")] + modules, deps = Base.parse_cache_header(cachefile) + @test modules == Dict(Foo_module => Base.module_uuid(Foo)) + @test map(x -> x[1], sort(deps)) == [Foo_file, joinpath(dir, "bar.jl"), joinpath(dir, "foo.jl")] + + modules, deps1 = Base.cache_dependencies(cachefile) + @test sort(modules) == map(s -> (s, Base.module_uuid(eval(s))), + [:Base, :Core, :Main]) + @test deps == deps1 @test current_task()(0x01, 0x4000, 0x30031234) == 2 @test nothing(0x01, 0x4000, 0x30031234) == 52 @@ -139,17 +153,28 @@ try end """) + t = redirected_stderr("ERROR: LoadError: __precompile__(false) is not allowed in files that are being precompiled\n in __precompile__") try - redirected_stderr() Base.compilecache("Baz") # from __precompile__(false) error("__precompile__ disabled test failed") catch exc + close(STDERR) redirect_stderr(olderr) isa(exc, ErrorException) || rethrow(exc) !isempty(search(exc.msg, "__precompile__(false)")) && rethrow(exc) end + wait(t) # Issue #12720 + FooBar1_file = joinpath(dir, "FooBar1.jl") + write(FooBar1_file, + """ + __precompile__(true) + module FooBar1 + using FooBar + end + """) + sleep(2) # give FooBar and FooBar1 different timestamps, in reverse order too FooBar_file = joinpath(dir, "FooBar.jl") write(FooBar_file, """ @@ -159,17 +184,64 @@ try """) Base.compilecache("FooBar") - sleep(2) @test isfile(joinpath(dir, "FooBar.ji")) + @test !Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) + @test !isdefined(Main, :FooBar) + @test !isdefined(Main, :FooBar1) - touch(FooBar_file) + @eval using FooBar + fb_uuid = Base.module_uuid(Main.FooBar) + sleep(2); touch(FooBar_file) insert!(Base.LOAD_CACHE_PATH, 1, dir2) - Base.recompile_stale(:FooBar, joinpath(dir, "FooBar.ji")) - sleep(2) + @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) + @eval using FooBar1 + @test !isfile(joinpath(dir2, "FooBar.ji")) + @test !isfile(joinpath(dir, "FooBar1.ji")) + @test isfile(joinpath(dir2, "FooBar1.ji")) + @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) + @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) + @test fb_uuid == Base.module_uuid(Main.FooBar) + fb_uuid1 = Base.module_uuid(Main.FooBar1) + @test fb_uuid != fb_uuid1 + + t = redirected_stderr("WARNING: replacing module FooBar.") + try + reload("FooBar") + finally + close(STDERR) + redirect_stderr(olderr) + end + wait(t) + @test fb_uuid != Base.module_uuid(Main.FooBar) + @test fb_uuid1 == Base.module_uuid(Main.FooBar1) + fb_uuid = Base.module_uuid(Main.FooBar) + @test isfile(joinpath(dir2, "FooBar.ji")) + @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) + @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) + @test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji")) + + t = redirected_stderr(""" + WARNING: Deserialization checks failed while attempting to load cache from $(joinpath(dir2, "FooBar1.ji")). + WARNING: Module FooBar uuid did not match cache file. + WARNING: replacing module FooBar1. + """) + try + reload("FooBar1") + finally + close(STDERR) + redirect_stderr(olderr) + end + wait(t) + @test fb_uuid == Base.module_uuid(Main.FooBar) + @test fb_uuid1 != Base.module_uuid(Main.FooBar1) + @test isfile(joinpath(dir2, "FooBar.ji")) + @test isfile(joinpath(dir2, "FooBar1.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) @test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji")) + @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) + # test behavior of precompile modules that throw errors write(FooBar_file, """ __precompile__(true) @@ -177,18 +249,20 @@ try error("break me") end """) - + t = redirected_stderr("ERROR: LoadError: break me\n in error") try - redirected_stderr() Base.require(:FooBar) error("\"LoadError: break me\" test failed") catch exc + close(STDERR) redirect_stderr(olderr) isa(exc, ErrorException) || rethrow(exc) !isempty(search(exc.msg, "ERROR: LoadError: break me")) && rethrow(exc) end + wait(t) finally if STDERR != olderr + close(STDERR) redirect_stderr(olderr) end splice!(Base.LOAD_CACHE_PATH, 1:2) @@ -244,7 +318,7 @@ end let module_name = string("a",randstring()) insert!(LOAD_PATH, 1, pwd()) file_name = string(module_name, ".jl") - touch(file_name) + sleep(2); touch(file_name) code = """module $(module_name)\nend\n""" write(file_name, code) reload(module_name)