Skip to content

Commit

Permalink
fix sort / fallback order for incremental precompile
Browse files Browse the repository at this point in the history 
fix #18069
  • Loading branch information
@vtjnash
vtjnash committed Aug 24, 2016
1 parent 60f20bd commit 94ce57d
Showing 1 changed file with 51 additions and 46 deletions.
97 changes: 51 additions & 46 deletions base/loading.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,9 @@ find_in_path(name::AbstractString, wd = pwd()) = find_in_path(String(name), wd)

function find_in_node_path(name::String, srcpath, node::Int=1)
if myid() == node
find_in_path(name, srcpath)
return find_in_path(name, srcpath)
else
remotecall_fetch(find_in_path, node, name, srcpath)
return remotecall_fetch(find_in_path, node, name, srcpath)
end
end

Expand All @@ -126,7 +126,7 @@ function find_source_file(file::String)
file2 = find_in_path(file)
file2 !== nothing && return file2
file2 = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file)
isfile(file2) ? file2 : nothing
return isfile(file2) ? file2 : nothing
end

function find_all_in_cache_path(mod::Symbol)
Expand All @@ -138,21 +138,22 @@ function find_all_in_cache_path(mod::Symbol)
push!(paths, path)
end
end
paths
return paths
end

function _include_from_serialized(content::Vector{UInt8})
return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8},Int), content, sizeof(content))
return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content))
end

function _include_from_serialized(path::String)
return ccall(:jl_restore_incremental, Any, (Cstring,), path)
end

# returns an array of modules loaded, or nothing if failed
function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, toplevel_load::Bool)
if JLOptions().use_compilecache == 0
return nothing
end
restored = nothing
local restored = nothing
local content::Vector{UInt8}
if toplevel_load && myid() == 1 && nprocs() > 1
recompile_stale(mod, path_to_try)
# broadcast top-level import/using from node 1 (only)
if node == myid()
content = open(read, path_to_try)
Expand All @@ -170,13 +171,11 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t
end
end
elseif node == myid()
myid() == 1 && recompile_stale(mod, path_to_try)
restored = ccall(:jl_restore_incremental, Any, (Cstring,), path_to_try)
restored = _include_from_serialized(path_to_try)
else
content = remotecall_fetch(open, node, read, path_to_try)
restored = _include_from_serialized(content)
end
# otherwise, continue search

if restored !== nothing
for M in restored
Expand All @@ -188,25 +187,28 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t
return restored
end

function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool)
if JLOptions().use_compilecache == 0
return nothing
end
# returns `true` if require found a precompile cache for this mod, but couldn't load it
# returns `false` if the module isn't known to be precompilable
# returns the set of modules restored if the cache load succeeded
function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::String, toplevel_load::Bool)
if node == myid()
paths = find_all_in_cache_path(mod)
else
paths = @fetchfrom node find_all_in_cache_path(mod)
end
sort!(paths, by=mtime, rev=true) # try newest cachefiles first

for path_to_try in paths
if stale_cachefile(sourcepath, path_to_try)
continue
end
restored = _require_from_serialized(node, mod, path_to_try, toplevel_load)
if restored === nothing
warn("deserialization checks failed while attempting to load cache from $path_to_try")
else
return restored
end
end
return nothing
return !isempty(paths)
end

# to synchronize multiple tasks trying to import/using something
Expand Down Expand Up @@ -329,22 +331,30 @@ function require(mod::Symbol)
last = toplevel_load::Bool
try
toplevel_load = false
if nothing !== _require_from_serialized(1, mod, last)
return
end
if JLOptions().incremental != 0
# spawn off a new incremental precompile task from node 1 for recursive `require` calls
cachefile = compilecache(mod)
if nothing === _require_from_serialized(1, mod, cachefile, last)
warn("require failed to create a precompiled cache file")
end
return
end
name = string(mod)
path = find_in_node_path(name, nothing, 1)
if path === nothing
throw(ArgumentError("$name not found in path.\nRun Pkg.add(\"$name\") to install the $name package"))
throw(ArgumentError("module $name not found in current path.\nRun `Pkg.add(\"$name\")` to install the $name package."))
end

doneprecompile = false
if JLOptions().use_compilecache != 0
doneprecompile = _require_search_from_serialized(1, mod, path, last)
if !isa(doneprecompile, Bool)
return # success
elseif doneprecompile === true || JLOptions().incremental != 0
# spawn off a new incremental pre-compile task from node 1 for recursive `require` calls
# or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
cachefile = compilecache(mod)
if nothing === _require_from_serialized(1, mod, cachefile, last)
warn("compilecache failed to create a usable precompiled cache file for module $name.")
else
return # success
end
end
# fall-through to attempting to load the source file
end

try
if last && myid() == 1 && nprocs() > 1
# include on node 1 first to check for PrecompilableErrors
Expand All @@ -357,13 +367,12 @@ function require(mod::Symbol)
eval(Main, :(Base.include_from_node1($path)))
end
catch ex
if !precompilableerror(ex, true)
if doneprecompile === true || JLOptions().use_compilecache == 0 || !precompilableerror(ex, true)
rethrow() # rethrow non-precompilable=true errors
end
isinteractive() && info("Precompiling module $mod...")
cachefile = compilecache(mod)
if nothing === _require_from_serialized(1, mod, cachefile, last)
error("__precompile__(true) but require failed to create a precompiled cache file")
error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.")
end
end
finally
Expand Down Expand Up @@ -501,6 +510,13 @@ function compilecache(name::String)
mkpath(cachepath)
end
cachefile = abspath(cachepath, name*".ji")
if isinteractive()
if isfile(cachepath)
info("Recompiling stale cache file $cachefile for module $name.")
else
info("Precompiling module $name.")
end
end
if !success(create_expr_cache(path, cachefile))
error("Failed to precompile $name to $cachefile")
end
Expand Down Expand Up @@ -550,7 +566,7 @@ function stale_cachefile(modpath, cachefile)
if files[1][1] != modpath
return true # cache file was compiled from a different path
end
for (f,ftime) in files
for (f, ftime) in files
# Issue #13606: compensate for Docker images rounding mtimes
if mtime(f) ∉ (ftime, floor(ftime))
return true
Expand All @@ -570,14 +586,3 @@ function stale_cachefile(modpath, cachefile)
close(io)
end
end

function recompile_stale(mod, cachefile)
path = find_in_path(string(mod), nothing)
if path === nothing
error("module $mod not found in current path; you should rm(\"$(escape_string(cachefile))\") to remove the orphaned cache file")
end
if stale_cachefile(path, cachefile)
info("Recompiling stale cache file $cachefile for module $mod.")
compilecache(mod)
end
end

0 comments on commit 94ce57d

Please sign in to comment.