From 10c8ec127778766e49f914e22f7e928420efe2c4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 19 Aug 2016 14:16:25 -0400 Subject: [PATCH] fix sort / fallback order for incremental precompile fix #18069 --- base/loading.jl | 97 ++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 6bda601ef549f..317ffd1800af5 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -115,9 +115,9 @@ find_in_path(name::AbstractString, wd = pwd()) = find_in_path(String(name), wd) function find_in_node_path(name::String, srcpath, node::Int=1) if myid() == node - find_in_path(name, srcpath) + return find_in_path(name, srcpath) else - remotecall_fetch(find_in_path, node, name, srcpath) + return remotecall_fetch(find_in_path, node, name, srcpath) end end @@ -126,7 +126,7 @@ function find_source_file(file::String) file2 = find_in_path(file) file2 !== nothing && return file2 file2 = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file) - isfile(file2) ? file2 : nothing + return isfile(file2) ? file2 : nothing end function find_all_in_cache_path(mod::Symbol) @@ -138,21 +138,22 @@ function find_all_in_cache_path(mod::Symbol) push!(paths, path) end end - paths + return paths end function _include_from_serialized(content::Vector{UInt8}) - return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8},Int), content, sizeof(content)) + return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content)) +end + +function _include_from_serialized(path::String) + return ccall(:jl_restore_incremental, Any, (Cstring,), path) end # returns an array of modules loaded, or nothing if failed function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, toplevel_load::Bool) - if JLOptions().use_compilecache == 0 - return nothing - end - restored = nothing + local restored = nothing + local content::Vector{UInt8} if toplevel_load && myid() == 1 && nprocs() > 1 - recompile_stale(mod, path_to_try) # broadcast top-level import/using from node 1 (only) if node == myid() content = open(read, path_to_try) @@ -170,13 +171,11 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t end end elseif node == myid() - myid() == 1 && recompile_stale(mod, path_to_try) - restored = ccall(:jl_restore_incremental, Any, (Cstring,), path_to_try) + restored = _include_from_serialized(path_to_try) else content = remotecall_fetch(open, node, read, path_to_try) restored = _include_from_serialized(content) end - # otherwise, continue search if restored !== nothing for M in restored @@ -188,17 +187,20 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t return restored end -function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool) - if JLOptions().use_compilecache == 0 - return nothing - end +# returns `true` if require found a precompile cache for this mod, but couldn't load it +# returns `false` if the module isn't known to be precompilable +# returns the set of modules restored if the cache load succeeded +function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::String, toplevel_load::Bool) if node == myid() paths = find_all_in_cache_path(mod) else paths = @fetchfrom node find_all_in_cache_path(mod) end - sort!(paths, by=mtime, rev=true) # try newest cachefiles first + for path_to_try in paths + if stale_cachefile(sourcepath, path_to_try) + continue + end restored = _require_from_serialized(node, mod, path_to_try, toplevel_load) if restored === nothing warn("deserialization checks failed while attempting to load cache from $path_to_try") @@ -206,7 +208,7 @@ function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool) return restored end end - return nothing + return !isempty(paths) end # to synchronize multiple tasks trying to import/using something @@ -329,22 +331,30 @@ function require(mod::Symbol) last = toplevel_load::Bool try toplevel_load = false - if nothing !== _require_from_serialized(1, mod, last) - return - end - if JLOptions().incremental != 0 - # spawn off a new incremental precompile task from node 1 for recursive `require` calls - cachefile = compilecache(mod) - if nothing === _require_from_serialized(1, mod, cachefile, last) - warn("require failed to create a precompiled cache file") - end - return - end name = string(mod) path = find_in_node_path(name, nothing, 1) if path === nothing - throw(ArgumentError("$name not found in path.\nRun Pkg.add(\"$name\") to install the $name package")) + throw(ArgumentError("module $name not found in current path.\nRun `Pkg.add(\"$name\")` to install the $name package.")) end + + doneprecompile = false + if JLOptions().use_compilecache != 0 + doneprecompile = _require_search_from_serialized(1, mod, path, last) + if !isa(doneprecompile, Bool) + return # success + elseif doneprecompile === true || JLOptions().incremental != 0 + # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls + # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) + cachefile = compilecache(mod) + if nothing === _require_from_serialized(1, mod, cachefile, last) + warn("compilecache failed to create a usable precompiled cache file for module $name.") + else + return # success + end + end + # fall-through to attempting to load the source file + end + try if last && myid() == 1 && nprocs() > 1 # include on node 1 first to check for PrecompilableErrors @@ -357,13 +367,12 @@ function require(mod::Symbol) eval(Main, :(Base.include_from_node1($path))) end catch ex - if !precompilableerror(ex, true) + if doneprecompile === true || JLOptions().use_compilecache == 0 || !precompilableerror(ex, true) rethrow() # rethrow non-precompilable=true errors end - isinteractive() && info("Precompiling module $mod...") cachefile = compilecache(mod) if nothing === _require_from_serialized(1, mod, cachefile, last) - error("__precompile__(true) but require failed to create a precompiled cache file") + error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.") end end finally @@ -501,6 +510,13 @@ function compilecache(name::String) mkpath(cachepath) end cachefile = abspath(cachepath, name*".ji") + if isinteractive() + if isfile(cachepath) + info("Recompiling stale cache file $cachefile for module $name.") + else + info("Precompiling module $name.") + end + end if !success(create_expr_cache(path, cachefile)) error("Failed to precompile $name to $cachefile") end @@ -550,7 +566,7 @@ function stale_cachefile(modpath, cachefile) if files[1][1] != modpath return true # cache file was compiled from a different path end - for (f,ftime) in files + for (f, ftime) in files # Issue #13606: compensate for Docker images rounding mtimes if mtime(f) ∉ (ftime, floor(ftime)) return true @@ -570,14 +586,3 @@ function stale_cachefile(modpath, cachefile) close(io) end end - -function recompile_stale(mod, cachefile) - path = find_in_path(string(mod), nothing) - if path === nothing - error("module $mod not found in current path; you should rm(\"$(escape_string(cachefile))\") to remove the orphaned cache file") - end - if stale_cachefile(path, cachefile) - info("Recompiling stale cache file $cachefile for module $mod.") - compilecache(mod) - end -end