Skip to content

Commit

Permalink
remove distributed loading, disentangle loading from source search.
Browse files Browse the repository at this point in the history
  • Loading branch information
@StefanKarpinski
StefanKarpinski committed Jan 6, 2017
1 parent da4534b commit 014cee8
Show file tree
Hide file tree
Showing 8 changed files with 63 additions and 295 deletions.
163 changes: 32 additions & 131 deletions base/loading.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,13 @@ elseif is_apple()
path, attr_list, buf, sizeof(buf), FSOPT_NOFOLLOW)
systemerror(:getattrlist, ret ≠ 0)
filename_length = unsafe_load(
convert(Ptr{UInt32}, pointer(buf) + 8))
convert(Ptr{UInt32}, pointer(buf) + 8))
if (filename_length + header_size) > length(buf)
resize!(buf, filename_length + header_size)
continue
end
casepreserved_basename =
view(buf, (header_size+1):(header_size+filename_length-1))
view(buf, (header_size+1):(header_size+filename_length-1))
break
end
# Hack to compensate for inability to create a string from a subarray with no allocations.
Expand Down Expand Up @@ -90,44 +90,15 @@ function try_path(prefix::String, base::String, name::String)
return nothing
end

# `wd` is a working directory to search. defaults to current working directory.
# if `wd === nothing`, no extra path is searched.
function find_in_path(name::String, wd)
isabspath(name) && return name
base = name
if endswith(name,".jl")
base = name[1:end-3]
else
name = string(base,".jl")
end
if wd !== nothing
isfile_casesensitive(joinpath(wd,name)) && return joinpath(wd,name)
end
p = try_path(Pkg.dir(), base, name)
p !== nothing && return p
for prefix in LOAD_PATH
function find_in_path(base::String)
name = "$base.jl"
for prefix in [Pkg.dir(); LOAD_PATH]
p = try_path(prefix, base, name)
p !== nothing && return p
end
return nothing
end
find_in_path(name::AbstractString, wd = pwd()) = find_in_path(String(name), wd)

function find_in_node_path(name::String, srcpath, node::Int=1)
if myid() == node
return find_in_path(name, srcpath)
else
return remotecall_fetch(find_in_path, node, name, srcpath)
end
end

function find_source_file(file::String)
(isabspath(file) || isfile(file)) && return file
file2 = find_in_path(file)
file2 !== nothing && return file2
file2 = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file)
return isfile(file2) ? file2 : nothing
end
find_in_path(name::AbstractString) = find_in_path(String(name))

function find_all_in_cache_path(mod::Symbol)
name = string(mod)
Expand All @@ -151,45 +122,9 @@ function _include_from_serialized(path::String)
end

# returns an array of modules loaded, or an Exception that describes why it failed
# and also attempts to load the same file across all nodes (if toplevel_node and myid() == master)
# and it reconnects the Base.Docs.META
function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, toplevel_load::Bool)
local restored = nothing
local content::Vector{UInt8}
if toplevel_load && myid() == 1 && nprocs() > 1
# broadcast top-level import/using from node 1 (only)
if node == myid()
content = open(read, path_to_try)
else
content = remotecall_fetch(open, node, read, path_to_try)
end
restored = _include_from_serialized(content)
isa(restored, Exception) && return restored
others = filter(x -> x != myid(), procs())
refs = Any[
(p, @spawnat(p,
let m = try
_include_from_serialized(content)
catch ex
isa(ex, Exception) ? ex : ErrorException(string(ex))
end
isa(m, Exception) ? m : nothing
end))
for p in others ]
for (id, ref) in refs
m = fetch(ref)
if m !== nothing
warn("Node state is inconsistent: node $id failed to load cache from $path_to_try. Got:")
warn(m, prefix="WARNING: ")
end
end
elseif node == myid()
restored = _include_from_serialized(path_to_try)
else
content = remotecall_fetch(open, node, read, path_to_try)
restored = _include_from_serialized(content)
end

function _require_from_serialized(path_to_try::String)
restored = _include_from_serialized(path_to_try)
if !isa(restored, Exception)
for M in restored::Vector{Any}
if isdefined(M, Base.Docs.META)
Expand All @@ -203,18 +138,13 @@ end
# returns `true` if require found a precompile cache for this mod, but couldn't load it
# returns `false` if the module isn't known to be precompilable
# returns the set of modules restored if the cache load succeeded
function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::String, toplevel_load::Bool)
if node == myid()
paths = find_all_in_cache_path(mod)
else
paths = @fetchfrom node find_all_in_cache_path(mod)
end

function _require_search_from_serialized(mod::Symbol, sourcepath::String)
paths = find_all_in_cache_path(mod)
for path_to_try in paths::Vector{String}
if stale_cachefile(sourcepath, path_to_try)
continue
end
restored = _require_from_serialized(node, mod, path_to_try, toplevel_load)
restored = _require_from_serialized(path_to_try)
if isa(restored, Exception)
if isa(restored, ErrorException) && endswith(restored.msg, " uuid did not match cache file.")
# can't use this cache due to a module uuid mismatch,
Expand Down Expand Up @@ -246,7 +176,7 @@ const _track_dependencies = Ref(false) # set this to true to track the list of f
function _include_dependency(_path::AbstractString)
prev = source_path(nothing)
path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev), _path)
if myid() == 1 && _track_dependencies[]
if _track_dependencies[]
apath = abspath(path)
push!(_require_dependencies, (apath, mtime(apath)))
end
Expand Down Expand Up @@ -304,12 +234,13 @@ order to throw an error if Julia attempts to precompile it.
using `__precompile__()`. Failure to do so can result in a runtime error when loading the module.
"""
function __precompile__(isprecompilable::Bool=true)
if (myid() == 1 &&
JLOptions().use_compilecache != 0 &&
isprecompilable != (0 != ccall(:jl_generating_output, Cint, ())) &&
!(isprecompilable && toplevel_load::Bool))
throw(PrecompilableError(isprecompilable))
JLOptions().use_compilecache == 0 && return nothing
if 0 != ccall(:jl_generating_output, Cint, ())
isprecompilable || throw(PrecompilableError(false))
else
isprecompilable && current_module() == Main && throw(PrecompilableError(true))
end
return nothing
end

function require_modname(name::AbstractString)
Expand Down Expand Up @@ -348,9 +279,6 @@ function reload(name::AbstractString)
end
end

# require always works in Main scope and loads files from node 1
toplevel_load = true

"""
require(module::Symbol)
Expand All @@ -359,11 +287,10 @@ already defined in `Main`. It can also be called directly to force reloading a m
regardless of whether it has been loaded before (for example, when interactively developing
libraries).
Loads a source file, in the context of the `Main` module, on every active node, searching
standard locations for files. `require` is considered a top-level operation, so it sets the
current `include` path but does not use it to search for files (see help for `include`).
This function is typically used to load library code, and is implicitly called by `using` to
load packages.
Loads a source file, in the context of the `Main` module, searching standard locations for
files. `require` is considered a top-level operation, so it sets the current `include` path
but does not use it to search for files (see help for `include`). This function is
typically used to load library code, and is implicitly called by `using` to load packages.
When searching for files, `require` first looks for package code under `Pkg.dir()`,
then tries paths in the global array `LOAD_PATH`. `require` is case-sensitive on
Expand All @@ -377,7 +304,6 @@ function require(mod::Symbol)
_track_dependencies[] = false
DEBUG_LOADING[] = haskey(ENV, "JULIA_DEBUG_LOADING")

global toplevel_load
loading = get(package_locks, mod, false)
if loading !== false
# load already in progress for this module
Expand All @@ -386,20 +312,18 @@ function require(mod::Symbol)
end
package_locks[mod] = Condition()

last = toplevel_load::Bool
try
toplevel_load = false
# perform the search operation to select the module file require intends to load
name = string(mod)
path = find_in_node_path(name, nothing, 1)
path = find_in_path(name)
if path === nothing
throw(ArgumentError("Module $name not found in current path.\nRun `Pkg.add(\"$name\")` to install the $name package."))
end

# attempt to load the module file via the precompile cache locations
doneprecompile = false
if JLOptions().use_compilecache != 0
doneprecompile = _require_search_from_serialized(1, mod, path, last)
doneprecompile = _require_search_from_serialized(mod, path)
if !isa(doneprecompile, Bool)
return # success
end
Expand All @@ -422,7 +346,7 @@ function require(mod::Symbol)
# spawn off a new incremental pre-compile task from node 1 for recursive `require` calls
# or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
cachefile = compilecache(mod)
m = _require_from_serialized(1, mod, cachefile, last)
m = _require_from_serialized(cachefile)
if isa(m, Exception)
warn("The call to compilecache failed to create a usable precompiled cache file for module $name. Got:")
warn(m, prefix="WARNING: ")
Expand All @@ -434,41 +358,28 @@ function require(mod::Symbol)

# just load the file normally via include
# for unknown dependencies
try
if last && myid() == 1 && nprocs() > 1
# include on node 1 first to check for PrecompilableErrors
eval(Main, :(Base.include_from_node1($path)))

# broadcast top-level import/using from node 1 (only)
refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in filter(x -> x != 1, procs()) ]
for r in refs; wait(r); end
else
eval(Main, :(Base.include_from_node1($path)))
end
try eval(Main, :(Base._include($path)))
catch ex
if doneprecompile === true || JLOptions().use_compilecache == 0 || !precompilableerror(ex, true)
rethrow() # rethrow non-precompilable=true errors
end
# the file requested `__precompile__`, so try to build a cache file and use that
cachefile = compilecache(mod)
m = _require_from_serialized(1, mod, cachefile, last)
m = _require_from_serialized(cachefile)
if isa(m, Exception)
warn(m, prefix="WARNING: ")
# TODO: disable __precompile__(true) error and do normal include instead of error
error("Module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.")
end
end
finally
toplevel_load = last
loading = pop!(package_locks, mod)
notify(loading, all=true)
_track_dependencies[] = old_track_dependencies
end
nothing
end

# remote/parallel load

"""
include_string(code::AbstractString, filename::AbstractString="string")
Expand Down Expand Up @@ -519,29 +430,20 @@ evaluated by `julia -e <expr>`.
"""
macro __DIR__() source_dir() end

include_from_node1(path::AbstractString) = include_from_node1(String(path))
function include_from_node1(_path::String)
_include(path::AbstractString) = _include(String(path))

function _include(_path::String)
path, prev = _include_dependency(_path)
tls = task_local_storage()
tls[:SOURCE_PATH] = path
local result
try
if myid()==1
# sleep a bit to process file requests from other nodes
nprocs()>1 && sleep(0.005)
result = Core.include(path)
nprocs()>1 && sleep(0.005)
else
result = include_string(remotecall_fetch(readstring, 1, path), path)
end
try Core.include(path)
finally
if prev === nothing
delete!(tls, :SOURCE_PATH)
else
tls[:SOURCE_PATH] = prev
end
end
result
end

"""
Expand Down Expand Up @@ -629,9 +531,8 @@ This can be used to reduce package load times. Cache files are stored in
for important notes.
"""
function compilecache(name::String)
myid() == 1 || error("can only precompile from node 1")
# decide where to get the source file from
path = find_in_path(name, nothing)
path = find_in_path(name)
path === nothing && throw(ArgumentError("$name not found in path"))
path = String(path)
# decide where to put the resulting cache file
Expand Down
2 changes: 1 addition & 1 deletion base/precompile.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ precompile(Base.ht_keyindex2, (Dict{UInt8, Any}, UInt8))
precompile(Base.in, (Char, String))
precompile(Base.in, (Int, Base.UnitRange{Int}))
precompile(Base.in, (UInt8, Base.KeyIterator{Dict{UInt8, Any}}))
precompile(Base.include_from_node1, (String,))
precompile(Base.include, (String,))
precompile(Base.init_stdio, (Ptr{Void},))
precompile(Base.input_color, ())
precompile(Base.insert!, (Array{Any,1}, Int, Base.GlobalRef))
Expand Down
6 changes: 6 additions & 0 deletions base/reflection.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -720,6 +720,12 @@ Get the name of a generic `Function` as a symbol, or `:anonymous`.
"""
function_name(f::Function) = typeof(f).name.mt.name

function find_source_file(file::String)
(isabspath(file) || isfile(file)) && return file
file = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file)
return isfile(file) ? file : nothing
end

functionloc(m::Core.MethodInstance) = functionloc(m.def)

"""
Expand Down
Loading

0 comments on commit 014cee8

Please sign in to comment.