From 014cee8eda0f8744040ca687861ea0efd0c679ca Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 6 Jan 2017 15:36:47 -0500 Subject: [PATCH] remove distributed loading, disentangle loading from source search. --- base/loading.jl | 163 +++++++++----------------------------------- base/precompile.jl | 2 +- base/reflection.jl | 6 ++ base/require.jl | 135 ------------------------------------ base/sysimg.jl | 36 +++++----- test/cmdlineargs.jl | 6 +- test/compile.jl | 2 +- test/loading.jl | 8 +-- 8 files changed, 63 insertions(+), 295 deletions(-) delete mode 100644 base/require.jl diff --git a/base/loading.jl b/base/loading.jl index 50b0600e72991..cdcf9ac22c395 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -54,13 +54,13 @@ elseif is_apple() path, attr_list, buf, sizeof(buf), FSOPT_NOFOLLOW) systemerror(:getattrlist, ret ≠ 0) filename_length = unsafe_load( - convert(Ptr{UInt32}, pointer(buf) + 8)) + convert(Ptr{UInt32}, pointer(buf) + 8)) if (filename_length + header_size) > length(buf) resize!(buf, filename_length + header_size) continue end casepreserved_basename = - view(buf, (header_size+1):(header_size+filename_length-1)) + view(buf, (header_size+1):(header_size+filename_length-1)) break end # Hack to compensate for inability to create a string from a subarray with no allocations. @@ -90,44 +90,15 @@ function try_path(prefix::String, base::String, name::String) return nothing end -# `wd` is a working directory to search. defaults to current working directory. -# if `wd === nothing`, no extra path is searched. -function find_in_path(name::String, wd) - isabspath(name) && return name - base = name - if endswith(name,".jl") - base = name[1:end-3] - else - name = string(base,".jl") - end - if wd !== nothing - isfile_casesensitive(joinpath(wd,name)) && return joinpath(wd,name) - end - p = try_path(Pkg.dir(), base, name) - p !== nothing && return p - for prefix in LOAD_PATH +function find_in_path(base::String) + name = "$base.jl" + for prefix in [Pkg.dir(); LOAD_PATH] p = try_path(prefix, base, name) p !== nothing && return p end return nothing end -find_in_path(name::AbstractString, wd = pwd()) = find_in_path(String(name), wd) - -function find_in_node_path(name::String, srcpath, node::Int=1) - if myid() == node - return find_in_path(name, srcpath) - else - return remotecall_fetch(find_in_path, node, name, srcpath) - end -end - -function find_source_file(file::String) - (isabspath(file) || isfile(file)) && return file - file2 = find_in_path(file) - file2 !== nothing && return file2 - file2 = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file) - return isfile(file2) ? file2 : nothing -end +find_in_path(name::AbstractString) = find_in_path(String(name)) function find_all_in_cache_path(mod::Symbol) name = string(mod) @@ -151,45 +122,9 @@ function _include_from_serialized(path::String) end # returns an array of modules loaded, or an Exception that describes why it failed -# and also attempts to load the same file across all nodes (if toplevel_node and myid() == master) # and it reconnects the Base.Docs.META -function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, toplevel_load::Bool) - local restored = nothing - local content::Vector{UInt8} - if toplevel_load && myid() == 1 && nprocs() > 1 - # broadcast top-level import/using from node 1 (only) - if node == myid() - content = open(read, path_to_try) - else - content = remotecall_fetch(open, node, read, path_to_try) - end - restored = _include_from_serialized(content) - isa(restored, Exception) && return restored - others = filter(x -> x != myid(), procs()) - refs = Any[ - (p, @spawnat(p, - let m = try - _include_from_serialized(content) - catch ex - isa(ex, Exception) ? ex : ErrorException(string(ex)) - end - isa(m, Exception) ? m : nothing - end)) - for p in others ] - for (id, ref) in refs - m = fetch(ref) - if m !== nothing - warn("Node state is inconsistent: node $id failed to load cache from $path_to_try. Got:") - warn(m, prefix="WARNING: ") - end - end - elseif node == myid() - restored = _include_from_serialized(path_to_try) - else - content = remotecall_fetch(open, node, read, path_to_try) - restored = _include_from_serialized(content) - end - +function _require_from_serialized(path_to_try::String) + restored = _include_from_serialized(path_to_try) if !isa(restored, Exception) for M in restored::Vector{Any} if isdefined(M, Base.Docs.META) @@ -203,18 +138,13 @@ end # returns `true` if require found a precompile cache for this mod, but couldn't load it # returns `false` if the module isn't known to be precompilable # returns the set of modules restored if the cache load succeeded -function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::String, toplevel_load::Bool) - if node == myid() - paths = find_all_in_cache_path(mod) - else - paths = @fetchfrom node find_all_in_cache_path(mod) - end - +function _require_search_from_serialized(mod::Symbol, sourcepath::String) + paths = find_all_in_cache_path(mod) for path_to_try in paths::Vector{String} if stale_cachefile(sourcepath, path_to_try) continue end - restored = _require_from_serialized(node, mod, path_to_try, toplevel_load) + restored = _require_from_serialized(path_to_try) if isa(restored, Exception) if isa(restored, ErrorException) && endswith(restored.msg, " uuid did not match cache file.") # can't use this cache due to a module uuid mismatch, @@ -246,7 +176,7 @@ const _track_dependencies = Ref(false) # set this to true to track the list of f function _include_dependency(_path::AbstractString) prev = source_path(nothing) path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev), _path) - if myid() == 1 && _track_dependencies[] + if _track_dependencies[] apath = abspath(path) push!(_require_dependencies, (apath, mtime(apath))) end @@ -304,12 +234,13 @@ order to throw an error if Julia attempts to precompile it. using `__precompile__()`. Failure to do so can result in a runtime error when loading the module. """ function __precompile__(isprecompilable::Bool=true) - if (myid() == 1 && - JLOptions().use_compilecache != 0 && - isprecompilable != (0 != ccall(:jl_generating_output, Cint, ())) && - !(isprecompilable && toplevel_load::Bool)) - throw(PrecompilableError(isprecompilable)) + JLOptions().use_compilecache == 0 && return nothing + if 0 != ccall(:jl_generating_output, Cint, ()) + isprecompilable || throw(PrecompilableError(false)) + else + isprecompilable && current_module() == Main && throw(PrecompilableError(true)) end + return nothing end function require_modname(name::AbstractString) @@ -348,9 +279,6 @@ function reload(name::AbstractString) end end -# require always works in Main scope and loads files from node 1 -toplevel_load = true - """ require(module::Symbol) @@ -359,11 +287,10 @@ already defined in `Main`. It can also be called directly to force reloading a m regardless of whether it has been loaded before (for example, when interactively developing libraries). -Loads a source file, in the context of the `Main` module, on every active node, searching -standard locations for files. `require` is considered a top-level operation, so it sets the -current `include` path but does not use it to search for files (see help for `include`). -This function is typically used to load library code, and is implicitly called by `using` to -load packages. +Loads a source file, in the context of the `Main` module, searching standard locations for +files. `require` is considered a top-level operation, so it sets the current `include` path +but does not use it to search for files (see help for `include`). This function is +typically used to load library code, and is implicitly called by `using` to load packages. When searching for files, `require` first looks for package code under `Pkg.dir()`, then tries paths in the global array `LOAD_PATH`. `require` is case-sensitive on @@ -377,7 +304,6 @@ function require(mod::Symbol) _track_dependencies[] = false DEBUG_LOADING[] = haskey(ENV, "JULIA_DEBUG_LOADING") - global toplevel_load loading = get(package_locks, mod, false) if loading !== false # load already in progress for this module @@ -386,12 +312,10 @@ function require(mod::Symbol) end package_locks[mod] = Condition() - last = toplevel_load::Bool try - toplevel_load = false # perform the search operation to select the module file require intends to load name = string(mod) - path = find_in_node_path(name, nothing, 1) + path = find_in_path(name) if path === nothing throw(ArgumentError("Module $name not found in current path.\nRun `Pkg.add(\"$name\")` to install the $name package.")) end @@ -399,7 +323,7 @@ function require(mod::Symbol) # attempt to load the module file via the precompile cache locations doneprecompile = false if JLOptions().use_compilecache != 0 - doneprecompile = _require_search_from_serialized(1, mod, path, last) + doneprecompile = _require_search_from_serialized(mod, path) if !isa(doneprecompile, Bool) return # success end @@ -422,7 +346,7 @@ function require(mod::Symbol) # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) cachefile = compilecache(mod) - m = _require_from_serialized(1, mod, cachefile, last) + m = _require_from_serialized(cachefile) if isa(m, Exception) warn("The call to compilecache failed to create a usable precompiled cache file for module $name. Got:") warn(m, prefix="WARNING: ") @@ -434,24 +358,14 @@ function require(mod::Symbol) # just load the file normally via include # for unknown dependencies - try - if last && myid() == 1 && nprocs() > 1 - # include on node 1 first to check for PrecompilableErrors - eval(Main, :(Base.include_from_node1($path))) - - # broadcast top-level import/using from node 1 (only) - refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in filter(x -> x != 1, procs()) ] - for r in refs; wait(r); end - else - eval(Main, :(Base.include_from_node1($path))) - end + try eval(Main, :(Base._include($path))) catch ex if doneprecompile === true || JLOptions().use_compilecache == 0 || !precompilableerror(ex, true) rethrow() # rethrow non-precompilable=true errors end # the file requested `__precompile__`, so try to build a cache file and use that cachefile = compilecache(mod) - m = _require_from_serialized(1, mod, cachefile, last) + m = _require_from_serialized(cachefile) if isa(m, Exception) warn(m, prefix="WARNING: ") # TODO: disable __precompile__(true) error and do normal include instead of error @@ -459,7 +373,6 @@ function require(mod::Symbol) end end finally - toplevel_load = last loading = pop!(package_locks, mod) notify(loading, all=true) _track_dependencies[] = old_track_dependencies @@ -467,8 +380,6 @@ function require(mod::Symbol) nothing end -# remote/parallel load - """ include_string(code::AbstractString, filename::AbstractString="string") @@ -519,21 +430,13 @@ evaluated by `julia -e `. """ macro __DIR__() source_dir() end -include_from_node1(path::AbstractString) = include_from_node1(String(path)) -function include_from_node1(_path::String) +_include(path::AbstractString) = _include(String(path)) + +function _include(_path::String) path, prev = _include_dependency(_path) tls = task_local_storage() tls[:SOURCE_PATH] = path - local result - try - if myid()==1 - # sleep a bit to process file requests from other nodes - nprocs()>1 && sleep(0.005) - result = Core.include(path) - nprocs()>1 && sleep(0.005) - else - result = include_string(remotecall_fetch(readstring, 1, path), path) - end + try Core.include(path) finally if prev === nothing delete!(tls, :SOURCE_PATH) @@ -541,7 +444,6 @@ function include_from_node1(_path::String) tls[:SOURCE_PATH] = prev end end - result end """ @@ -629,9 +531,8 @@ This can be used to reduce package load times. Cache files are stored in for important notes. """ function compilecache(name::String) - myid() == 1 || error("can only precompile from node 1") # decide where to get the source file from - path = find_in_path(name, nothing) + path = find_in_path(name) path === nothing && throw(ArgumentError("$name not found in path")) path = String(path) # decide where to put the resulting cache file diff --git a/base/precompile.jl b/base/precompile.jl index c8ce400142c17..2a4a2ed05a75e 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -237,7 +237,7 @@ precompile(Base.ht_keyindex2, (Dict{UInt8, Any}, UInt8)) precompile(Base.in, (Char, String)) precompile(Base.in, (Int, Base.UnitRange{Int})) precompile(Base.in, (UInt8, Base.KeyIterator{Dict{UInt8, Any}})) -precompile(Base.include_from_node1, (String,)) +precompile(Base.include, (String,)) precompile(Base.init_stdio, (Ptr{Void},)) precompile(Base.input_color, ()) precompile(Base.insert!, (Array{Any,1}, Int, Base.GlobalRef)) diff --git a/base/reflection.jl b/base/reflection.jl index 2f3f50f4a7e6a..8726350598593 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -720,6 +720,12 @@ Get the name of a generic `Function` as a symbol, or `:anonymous`. """ function_name(f::Function) = typeof(f).name.mt.name +function find_source_file(file::String) + (isabspath(file) || isfile(file)) && return file + file = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file) + return isfile(file) ? file : nothing +end + functionloc(m::Core.MethodInstance) = functionloc(m.def) """ diff --git a/base/require.jl b/base/require.jl deleted file mode 100644 index f9142e11617f3..0000000000000 --- a/base/require.jl +++ /dev/null @@ -1,135 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -# deprecated require -module OldRequire - -function find_in_path(name::AbstractString) - isabspath(name) && return name - isfile(name) && return abspath(name) - base = name - if endswith(name,".jl") - base = name[1:end-3] - else - name = string(base,".jl") - isfile(name) && return abspath(name) - end - for prefix in [Pkg.dir(); LOAD_PATH] - path = joinpath(prefix, name) - isfile(path) && return abspath(path) - path = joinpath(prefix, base, "src", name) - isfile(path) && return abspath(path) - path = joinpath(prefix, name, "src", name) - isfile(path) && return abspath(path) - end - return nothing -end - -find_in_node1_path(name) = myid()==1 ? - find_in_path(name) : remotecall_fetch(find_in_path, 1, name) - -# Store list of files and their load time -package_list = Dict{String,Float64}() -# to synchronize multiple tasks trying to require something -package_locks = Dict{String,Any}() - -# only broadcast top-level (not nested) requires and reloads -toplevel_load = true - -require(fname::AbstractString) = require(String(fname)) -require(f::AbstractString, fs::AbstractString...) = (require(f); for x in fs require(x); end) - -function require(name::String) - path = find_in_node1_path(name) - path === nothing && error("$name not found") - - if myid() == 1 && toplevel_load - refs = Any[ @spawnat p _require(path) for p in filter(x->x!=1, procs()) ] - _require(path) - for r in refs; wait(r); end - else - _require(path) - end - nothing -end - -function _require(path) - global toplevel_load - if haskey(package_list,path) - wait(package_locks[path]) - else - last = toplevel_load - toplevel_load = false - try - reload_path(path) - finally - toplevel_load = last - end - end -end - -# remote/parallel load - -function source_path(default::Union{AbstractString,Void}="") - t = current_task() - while true - s = t.storage - if s !== nothing && haskey(s, :SOURCE_PATH) - return s[:SOURCE_PATH] - end - if t === t.parent - return default - end - t = t.parent - end -end - -function include_from_node1(path::AbstractString) - prev = source_path(nothing) - path = (prev === nothing) ? abspath(path) : joinpath(dirname(prev),path) - tls = task_local_storage() - tls[:SOURCE_PATH] = path - local result - try - if myid()==1 - # sleep a bit to process file requests from other nodes - nprocs()>1 && sleep(0.005) - result = Core.include(path) - nprocs()>1 && sleep(0.005) - else - result = include_string(remotecall_fetch(readstring, 1, path), path) - end - finally - if prev === nothing - delete!(tls, :SOURCE_PATH) - else - tls[:SOURCE_PATH] = prev - end - end - result -end - -function reload_path(path::AbstractString) - had = haskey(package_list, path) - if !had - package_locks[path] = RemoteChannel() - end - package_list[path] = time() - tls = task_local_storage() - prev = pop!(tls, :SOURCE_PATH, nothing) - try - eval(Main, :(Base.include_from_node1($path))) - catch e - had || delete!(package_list, path) - rethrow(e) - finally - if prev !== nothing - tls[:SOURCE_PATH] = prev - end - end - if !isready(package_locks[path]) - put!(package_locks[path],nothing) - end - nothing -end - -end # module diff --git a/base/sysimg.jl b/base/sysimg.jl index 12a051bec0e37..4bb6d28210644 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -4,16 +4,21 @@ baremodule Base using Core.Intrinsics ccall(:jl_set_istopmod, Void, (Bool,), true) -function include(path::AbstractString) - local result - if INCLUDE_STATE === 1 - result = Core.include(path) - elseif INCLUDE_STATE === 2 - result = _include(path) - elseif INCLUDE_STATE === 3 - result = include_from_node1(path) + +let SOURCE_PATH = "" + global function include(path::AbstractString) + if INCLUDE_STATE === 1 + return Core.include(path) + elseif INCLUDE_STATE === 2 + prev = SOURCE_PATH + path = joinpath(dirname(prev),path) + SOURCE_PATH = path + value = Core.include(path) + SOURCE_PATH = prev + return value + end + _include(path) # INCLUDE_STATE === 3 end - result end INCLUDE_STATE = 1 # include = Core.include @@ -227,16 +232,7 @@ importall .Math const (√)=sqrt const (∛)=cbrt -let SOURCE_PATH = "" - global function _include(path) - prev = SOURCE_PATH - path = joinpath(dirname(prev),path) - SOURCE_PATH = path - Core.include(path) - SOURCE_PATH = prev - end -end -INCLUDE_STATE = 2 # include = _include (from lines above) +INCLUDE_STATE = 2 # Core.include with try-less source-path wrapper # reduction along dims include("reducedim.jl") # macros in this file relies on string.jl @@ -393,7 +389,7 @@ function __init__() init_threadcall() end -INCLUDE_STATE = 3 # include = include_from_node1 +INCLUDE_STATE = 3 # include = Base._include (standard include) include("precompile.jl") end # baremodule Base diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 3d0844cfa0f9a..805aad4bb6338 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -366,12 +366,12 @@ end for precomp in ("yes", "no") bt = readstring(pipeline(ignorestatus(`$(Base.julia_cmd()) --startup-file=no --precompiled=$precomp -E 'include("____nonexistent_file")'`), stderr=catcmd)) - @test contains(bt, "include_from_node1") + @test contains(bt, "_include") if is_windows() && Sys.WORD_SIZE == 32 && precomp == "yes" # fixme, issue #17251 - @test_broken contains(bt, "include_from_node1(::String) at $(joinpath(".","loading.jl"))") + @test_broken contains(bt, "_include(::String) at $(joinpath(".","loading.jl"))") else - @test contains(bt, "include_from_node1(::String) at $(joinpath(".","loading.jl"))") + @test contains(bt, "_include(::String) at $(joinpath(".","loading.jl"))") end lno = match(r"at \.[\/\\]loading\.jl:(\d+)", bt) @test length(lno.captures) == 1 diff --git a/test/compile.jl b/test/compile.jl index fd7be47f004b0..ab6965693d21d 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -138,7 +138,7 @@ try # the module doesn't reload from the image: t = redirected_stderr("WARNING: replacing module Foo4b3a94a1a081a8cb.\nWARNING: Method definition ") try - @test isa(Base._require_from_serialized(myid(), Foo_module, cachefile, #=broadcast-load=#false), Array{Any,1}) + @test isa(Base._require_from_serialized(cachefile), Array{Any,1}) finally close(STDERR) redirect_stderr(olderr) diff --git a/test/loading.jl b/test/loading.jl index 7e7f707d1b1f9..ffb9d1d810106 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -40,7 +40,7 @@ mktempdir() do dir end end -let paddedname = "Ztest_sourcepath.jl" - filename = SubString(paddedname, 2, length(paddedname)) - @test Base.find_in_path(filename) == abspath(paddedname[2:end]) -end +# let paddedname = "Ztest_sourcepath.jl" +# filename = SubString(paddedname, 2, length(paddedname)) +# @test Base.find_in_path(filename) == abspath(paddedname[2:end]) +# end