From bd6b7eb0ee88aa12558205165a89801ea6f9874e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 5 Mar 2024 02:07:34 +0100 Subject: [PATCH] Move parallel precompilation to Base (#53403) Parallel precompilation is more or less now required in order to use somewhat large packages unless you want to wait an obscene long time for it to complete. Right now, we even start a parallel precompilation on a package load if we notice that the package you are loading is not precompiled. This functionally has typically been implemented in Pkg but with Pkg not being in the sysimage it becomes a bit awkward because we then need to load Pkg from Base. The only real reason this functionality has been implemented in Pkg is that Pkg has some useful features for parsing environments. Moving precompilation to Base has typically been stalled on such an environment parser not existing in Base. However, in https://github.com/JuliaLang/julia/pull/46690 I started implemented code loading on top of a more up front environment parser (instead of the "incremental" one that currently exists in `loading.jl`) and we can retro fit this to be used as the basis of parallel precompilation. At some later point code loading could be implemented on top of it but that is for now considered future work. This PR thus adds the environment parser from the codeloading PR and implementes the parallel precompilation feature from Pkg on top of it (instead of on top of the `EnvCache` in Pkg). Some points to bring up here: - This copy pastes the progress bar implementation in Pkg into here. It is probably a bit excessive to use so we can simplify that significantly. - Parallel precompilation uses the `FileWatching` module to avoid different processes trying to precompile the same package concurrently. Right now, I used grab this from `Base.loaded_modules` relying on it being in the sysimage. - This removes the "suspended" functionality from the Pkg precompilation which does not try to precompile packages if they have "recently" failed which is unclear how useful it is in practice. This also requires the Serialization stdlib and uses data structures defined in Pkg so it is hard to keep when moving this to Base. --------- Co-authored-by: Ian Butterworth (cherry picked from commit 67451604cf463c912c2325eb5990ad46262468c2) --- base/Base.jl | 2 + base/loading.jl | 15 +- base/precompilation.jl | 948 +++++++++++++++++++++++++++++++++++++++++ doc/make.jl | 3 +- 4 files changed, 960 insertions(+), 8 deletions(-) create mode 100644 base/precompilation.jl diff --git a/base/Base.jl b/base/Base.jl index e02b132aae6e4..c6a1113996fc3 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -542,6 +542,8 @@ if isdefined(Core, :Compiler) && is_primary_base_module Docs.loaddocs(Core.Compiler.CoreDocs.DOCS) end +include("precompilation.jl") + # finally, now make `include` point to the full version for m in methods(include) delete_method(m) diff --git a/base/loading.jl b/base/loading.jl index 2be97237238dd..5f3c846131810 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2147,8 +2147,6 @@ end require(uuidkey::PkgId) = @lock require_lock _require_prelocked(uuidkey) -const REPL_PKGID = PkgId(UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL") - function _require_prelocked(uuidkey::PkgId, env=nothing) if _require_world_age[] != typemax(UInt) Base.invoke_in_world(_require_world_age[], __require_prelocked, uuidkey, env) @@ -2262,8 +2260,9 @@ function set_pkgorigin_version_path(pkg::PkgId, path::String) nothing end -# A hook to allow code load to use Pkg.precompile +# Unused const PKG_PRECOMPILE_HOOK = Ref{Function}() +disable_parallel_precompile::Bool = false # Returns `nothing` or the new(ish) module function _require(pkg::PkgId, env=nothing) @@ -2284,7 +2283,7 @@ function _require(pkg::PkgId, env=nothing) end set_pkgorigin_version_path(pkg, path) - pkg_precompile_attempted = false # being safe to avoid getting stuck in a Pkg.precompile loop + parallel_precompile_attempted = false # being safe to avoid getting stuck in a precompilepkgs loop reasons = Dict{String,Int}() # attempt to load the module file via the precompile cache locations if JLOptions().use_compiled_modules != 0 @@ -2314,11 +2313,13 @@ function _require(pkg::PkgId, env=nothing) if JLOptions().use_compiled_modules == 1 if !generating_output(#=incremental=#false) - if !pkg_precompile_attempted && isinteractive() && isassigned(PKG_PRECOMPILE_HOOK) - pkg_precompile_attempted = true + project = active_project() + if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) && project !== nothing && + isfile(project) && project_file_manifest_path(project) !== nothing + parallel_precompile_attempted = true unlock(require_lock) try - @invokelatest PKG_PRECOMPILE_HOOK[](pkg.name, _from_loading = true) + Precompilation.precompilepkgs([pkg.name]; _from_loading=true) finally lock(require_lock) end diff --git a/base/precompilation.jl b/base/precompilation.jl new file mode 100644 index 0000000000000..4efd35836a29d --- /dev/null +++ b/base/precompilation.jl @@ -0,0 +1,948 @@ +module Precompilation + +using Base: PkgId, UUID, SHA1, parsed_toml, project_file_name_uuid, project_names, + project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive + +# This is currently only used for pkgprecompile but the plan is to use this in code loading in the future +# see the `kc/codeloading2.0` branch +struct ExplicitEnv + path::String + project_deps::Dict{String, UUID} # [deps] in Project.toml + project_weakdeps::Dict{String, UUID} # [weakdeps] in Project.toml + project_extras::Dict{String, UUID} # [extras] in Project.toml + project_extensions::Dict{String, Vector{UUID}} # [exts] in Project.toml + deps::Dict{UUID, Vector{UUID}} # all dependencies in Manifest.toml + weakdeps::Dict{UUID, Vector{UUID}} # all weak dependencies in Manifest.toml + extensions::Dict{UUID, Dict{String, Vector{UUID}}} + # Lookup name for a UUID + names::Dict{UUID, String} + lookup_strategy::Dict{UUID, Union{ + SHA1, # `git-tree-sha1` entry + String, # `path` entry + Nothing, # stdlib (no `path` nor `git-tree-sha1`) + Missing}} # not present in the manifest + #prefs::Union{Nothing, Dict{String, Any}} + #local_prefs::Union{Nothing, Dict{String, Any}} +end + +function ExplicitEnv(envpath::String=Base.active_project()) + envpath = abspath(envpath) + project_d = parsed_toml(envpath) + + # TODO: Perhaps verify that two packages with the same UUID do not have different names? + names = Dict{UUID, String}() + project_uuid_to_name = Dict{String, UUID}() + + project_deps = Dict{String, UUID}() + project_weakdeps = Dict{String, UUID}() + project_extras = Dict{String, UUID}() + + # Collect all direct dependencies of the project + for key in ["deps", "weakdeps", "extras"] + for (name, _uuid) in get(Dict{String, Any}, project_d, key)::Dict{String, Any} + v = key == "deps" ? project_deps : + key == "weakdeps" ? project_weakdeps : + key == "extras" ? project_extras : + error() + uuid = UUID(_uuid) + v[name] = uuid + names[UUID(uuid)] = name + project_uuid_to_name[name] = UUID(uuid) + end + end + + project_extensions = Dict{String, Vector{UUID}}() + # Collect all extensions of the project + for (name, triggers::Union{String, Vector{String}}) in get(Dict{String, Any}, project_d, "extensions")::Dict{String, Any} + if triggers isa String + triggers = [triggers] + end + uuids = UUID[] + for trigger in triggers + uuid = get(project_uuid_to_name, trigger, nothing) + if uuid === nothing + error("Trigger $trigger for extension $name not found in project") + end + push!(uuids, uuid) + end + project_extensions[name] = uuids + end + + # This project might be a package, in that case, that is also a "dependency" + # of the project. + proj_name = get(project_d, "name", nothing)::Union{String, Nothing} + _proj_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing} + proj_uuid = _proj_uuid === nothing ? nothing : UUID(_proj_uuid) + + if proj_name !== nothing && proj_uuid !== nothing + # TODO: Error on missing uuid? + project_deps[proj_name] = UUID(proj_uuid) + names[UUID(proj_uuid)] = proj_name + end + + manifest = project_file_manifest_path(envpath) + manifest_d = manifest === nothing ? Dict{String, Any}() : parsed_toml(manifest) + + # Dependencies in a manifest can either be stored compressed (when name is unique among all packages) + # in which case it is a `Vector{String}` or expanded where it is a `name => uuid` mapping. + deps = Dict{UUID, Union{Vector{String}, Vector{UUID}}}() + weakdeps = Dict{UUID, Union{Vector{String}, Vector{UUID}}}() + extensions = Dict{UUID, Dict{String, Vector{String}}}() + name_to_uuid = Dict{String, UUID}() + lookup_strategy = Dict{UUID, Union{SHA1, String, Nothing, Missing}}() + + sizehint!(deps, length(manifest_d)) + sizehint!(weakdeps, length(manifest_d)) + sizehint!(extensions, length(manifest_d)) + sizehint!(name_to_uuid, length(manifest_d)) + sizehint!(lookup_strategy, length(manifest_d)) + + for (name, pkg_infos) in get_deps(manifest_d) + pkg_infos = pkg_infos::Vector{Any} + for pkg_info in pkg_infos + m_uuid = UUID(pkg_info["uuid"]::String) + + # If we have multiple packages with the same name we will overwrite things here + # but that is fine since we will only use the information in here for packages + # with unique names + names[m_uuid] = name + name_to_uuid[name] = m_uuid + + for key in ["deps", "weakdeps"] + deps_pkg = get(Vector{String}, pkg_info, key)::Union{Vector{String}, Dict{String, Any}} + d = key == "deps" ? deps : + key == "weakdeps" ? weakdeps : + error() + + # Compressed format with unique names: + if deps_pkg isa Vector{String} + d[m_uuid] = deps_pkg + # Expanded format: + else + uuids = UUID[] + for (name_dep, _dep_uuid::String) in deps_pkg + dep_uuid = UUID(_dep_uuid) + push!(uuids, dep_uuid) + names[dep_uuid] = name_dep + end + d[m_uuid] = uuids + end + end + + # Extensions + deps_pkg = get(Dict{String, Any}, pkg_info, "extensions")::Dict{String, Any} + for (ext, triggers) in deps_pkg + triggers = triggers::Union{String, Vector{String}} + if triggers isa String + triggers = [triggers] + end + deps_pkg[ext] = triggers + end + extensions[m_uuid] = deps_pkg + + # Determine strategy to find package + lookup_strat = begin + if (path = get(pkg_info, "path", nothing)::Union{String, Nothing}) !== nothing + path + elseif (git_tree_sha_str = get(pkg_info, "git-tree-sha1", nothing)::Union{String, Nothing}) !== nothing + SHA1(git_tree_sha_str) + else + nothing + end + end + lookup_strategy[m_uuid] = lookup_strat + end + end + + # No matter if the deps were stored compressed or not in the manifest, + # we internally store them expanded + deps_expanded = Dict{UUID, Vector{UUID}}() + weakdeps_expanded = Dict{UUID, Vector{UUID}}() + extensions_expanded = Dict{UUID, Dict{String, Vector{UUID}}}() + sizehint!(deps_expanded, length(deps)) + sizehint!(weakdeps_expanded, length(deps)) + sizehint!(extensions_expanded, length(deps)) + + if proj_name !== nothing + deps_expanded[proj_uuid] = filter!(!=(proj_uuid), collect(values(project_deps))) + extensions_expanded[proj_uuid] = project_extensions + path = get(project_d, "path", nothing) + entry_point = path !== nothing ? path : dirname(envpath) + lookup_strategy[proj_uuid] = entry_point + end + + for key in ["deps", "weakdeps"] + d = key == "deps" ? deps : + key == "weakdeps" ? weakdeps : + error() + d_expanded = key == "deps" ? deps_expanded : + key == "weakdeps" ? weakdeps_expanded : + error() + for (pkg, deps) in d + # dependencies was already expanded so use it directly: + if deps isa Vector{UUID} + d_expanded[pkg] = deps + for dep in deps + name_to_uuid[names[dep]] = dep + end + # find the (unique) UUID associated with the name + else + deps_pkg = UUID[] + sizehint!(deps_pkg, length(deps)) + for dep in deps + push!(deps_pkg, name_to_uuid[dep]) + end + d_expanded[pkg] = deps_pkg + end + end + end + + for (pkg, exts) in extensions + exts_expanded = Dict{String, Vector{UUID}}() + for (ext, triggers) in exts + triggers_expanded = UUID[] + sizehint!(triggers_expanded, length(triggers)) + for trigger in triggers + push!(triggers_expanded, name_to_uuid[trigger]) + end + exts_expanded[ext] = triggers_expanded + end + extensions_expanded[pkg] = exts_expanded + end + + # Everything that does not yet have a lookup_strategy is missing from the manifest + for (_, uuid) in project_deps + get!(lookup_strategy, uuid, missing) + end + + #= + # Preferences: + prefs = get(project_d, "preferences", nothing) + + # `(Julia)LocalPreferences.toml` + project_dir = dirname(envpath) + local_prefs = nothing + for name in preferences_names + toml_path = joinpath(project_dir, name) + if isfile(toml_path) + local_prefs = parsed_toml(toml_path) + break + end + end + =# + + return ExplicitEnv(envpath, project_deps, project_weakdeps, project_extras, + project_extensions, deps_expanded, weakdeps_expanded, extensions_expanded, + names, lookup_strategy, #=prefs, local_prefs=#) +end + +## PROGRESS BAR + +# using Printf +Base.@kwdef mutable struct MiniProgressBar + max::Int = 1.0 + header::String = "" + color::Symbol = :nothing + width::Int = 40 + current::Int = 0.0 + prev::Int = 0.0 + has_shown::Bool = false + time_shown::Float64 = 0.0 + percentage::Bool = true + always_reprint::Bool = false + indent::Int = 4 +end + +const PROGRESS_BAR_TIME_GRANULARITY = Ref(1 / 30.0) # 30 fps +const PROGRESS_BAR_PERCENTAGE_GRANULARITY = Ref(0.1) + +function start_progress(io::IO, _::MiniProgressBar) + ansi_disablecursor = "\e[?25l" + print(io, ansi_disablecursor) +end + +function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagereturn=true) + if p.max == 0 + perc = 0.0 + prev_perc = 0.0 + else + perc = p.current / p.max * 100 + prev_perc = p.prev / p.max * 100 + end + # Bail early if we are not updating the progress bar, + # Saves printing to the terminal + if !p.always_reprint && p.has_shown && !((perc - prev_perc) > PROGRESS_BAR_PERCENTAGE_GRANULARITY[]) + return + end + t = time() + if p.has_shown && (t - p.time_shown) < PROGRESS_BAR_TIME_GRANULARITY[] + return + end + p.time_shown = t + p.prev = p.current + p.has_shown = true + + progress_text = if false # p.percentage + # @sprintf "%2.1f %%" perc + else + string(p.current, "/", p.max) + end + termwidth = @something termwidth displaysize(io)[2] + max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width)) + n_filled = ceil(Int, max_progress_width * perc / 100) + n_left = max_progress_width - n_filled + to_print = sprint(; context=io) do io + print(io, " "^p.indent) + printstyled(io, p.header, color=p.color, bold=true) + print(io, " [") + print(io, "="^n_filled, ">") + print(io, " "^n_left, "] ", ) + print(io, progress_text) + carriagereturn && print(io, "\r") + end + # Print everything in one call + print(io, to_print) +end + +function end_progress(io, p::MiniProgressBar) + ansi_enablecursor = "\e[?25h" + ansi_clearline = "\e[2K" + print(io, ansi_enablecursor * ansi_clearline) +end + +function print_progress_bottom(io::IO) + ansi_clearline = "\e[2K" + ansi_movecol1 = "\e[1G" + ansi_moveup(n::Int) = string("\e[", n, "A") + print(io, "\e[S" * ansi_moveup(1) * ansi_clearline * ansi_movecol1) +end + + +############ +struct PkgPrecompileError <: Exception + msg::String +end +Base.showerror(io::IO, err::PkgPrecompileError) = print(io, err.msg) +# This needs a show method to make `julia> err` show nicely +Base.show(io::IO, err::PkgPrecompileError) = print(io, "PkgPrecompileError: ", err.msg) + +import Base: StaleCacheKey + +can_fancyprint(io::IO) = io isa Base.TTY && (get(ENV, "CI", nothing) != "true") + +function printpkgstyle(io, header, msg; color=:light_green) + printstyled(io, header; color, bold=true) + println(io, " ", msg) +end + + +function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false, + strict::Bool=false, warn_loaded = true, timing::Bool = false, + _from_loading::Bool=false, flags_cacheflags::Pair{Cmd, Base.CacheFlags}=(``=>Base.CacheFlags()), io::IO=stderr) + time_start = time_ns() + + env = ExplicitEnv() + + flags, cacheflags = flags_cacheflags + + # Windows sometimes hits a ReadOnlyMemoryError, so we halve the default number of tasks. Issue #2323 + # TODO: Investigate why this happens in windows and restore the full task limit + default_num_tasks = Sys.iswindows() ? div(Sys.CPU_THREADS::Int, 2) + 1 : Sys.CPU_THREADS::Int + 1 + default_num_tasks = min(default_num_tasks, 16) # limit for better stability on shared resource systems + + num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks))) + parallel_limiter = Base.Semaphore(num_tasks) + + # asking for timing disables fancy mode, as timing is shown in non-fancy mode + fancyprint = can_fancyprint(io) && !timing + + if _from_loading && !Sys.isinteractive() + io = devnull + end + + hascolor = get(io, :color, false)::Bool + color_string(cstr::String, col::Union{Int64, Symbol}) = _color_string(cstr, col, hascolor) + + direct_deps = [ + Base.PkgId(uuid, name) + for (name, uuid) in env.project_deps if !Base.in_sysimage(Base.PkgId(uuid, name)) + ] + stale_cache = Dict{StaleCacheKey, Bool}() + exts = Dict{Base.PkgId, String}() # ext -> parent + # make a flat map of each dep and its direct deps + depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}() + pkg_exts_map = Dict{Base.PkgId, Vector{Base.PkgId}}() + + for (dep, deps) in env.deps + pkg = Base.PkgId(dep, env.names[dep]) + Base.in_sysimage(pkg) && continue + deps = [Base.PkgId(x, env.names[x]) for x in deps] + depsmap[pkg] = filter!(!Base.in_sysimage, deps) + # add any extensions + pkg_exts = Dict{Base.PkgId, Vector{Base.PkgId}}() + prev_ext = nothing + for (ext_name, extdep_uuids) in env.extensions[dep] + ext_deps = Base.PkgId[] + push!(ext_deps, pkg) # depends on parent package + all_extdeps_available = true + for extdep_uuid in extdep_uuids + extdep_name = env.names[extdep_uuid] + if extdep_uuid in keys(env.deps) || Base.in_sysimage(Base.PkgId(extdep_uuid, extdep_name)) + push!(ext_deps, Base.PkgId(extdep_uuid, extdep_name)) + else + all_extdeps_available = false + break + end + end + all_extdeps_available || continue + if prev_ext isa Base.PkgId + # also make the exts depend on eachother sequentially to avoid race + push!(ext_deps, prev_ext) + end + ext_uuid = Base.uuid5(pkg.uuid, ext_name) + ext = Base.PkgId(ext_uuid, ext_name) + prev_ext = ext + filter!(!Base.in_sysimage, ext_deps) + depsmap[ext] = ext_deps + exts[ext] = pkg.name + pkg_exts[ext] = ext_deps + end + if !isempty(pkg_exts) + pkg_exts_map[pkg] = collect(keys(pkg_exts)) + end + end + + @debug "precompile: deps collected" _group=:precompile + # this loop must be run after the full depsmap has been populated + for (pkg, pkg_exts) in pkg_exts_map + # find any packages that depend on the extension(s)'s deps and replace those deps in their deps list with the extension(s), + # basically injecting the extension into the precompile order in the graph, to avoid race to precompile extensions + for (_pkg, deps) in depsmap # for each manifest dep + if !in(_pkg, keys(exts)) && pkg in deps # if not an extension and depends on pkg + append!(deps, pkg_exts) # add the package extensions to deps + filter!(!isequal(pkg), deps) # remove the pkg from deps + end + end + end + @debug "precompile: extensions collected" _group=:precompile + + # return early if no deps + if isempty(depsmap) + if isempty(pkgs) + return + elseif _from_loading + # if called from loading precompilation it may be a package from another environment stack so + # don't error and allow serial precompilation to try + # TODO: actually handle packages from other envs in the stack + return + else + error("No direct dependencies outside of the sysimage found matching $(repr([p.name for p in pkgs]))") + end + end + + # initialize signalling + started = Dict{Base.PkgId,Bool}() + was_processed = Dict{Base.PkgId,Base.Event}() + was_recompiled = Dict{Base.PkgId,Bool}() + for pkgid in keys(depsmap) + started[pkgid] = false + was_processed[pkgid] = Base.Event() + was_recompiled[pkgid] = false + end + @debug "precompile: signalling initialized" _group=:precompile + + + # find and guard against circular deps + circular_deps = Base.PkgId[] + # Three states + # !haskey -> never visited + # true -> cannot be compiled due to a cycle (or not yet determined) + # false -> not depending on a cycle + could_be_cycle = Dict{Base.PkgId, Bool}() + function scan_pkg!(pkg, dmap) + did_visit_dep = true + inpath = get!(could_be_cycle, pkg) do + did_visit_dep = false + return true + end + if did_visit_dep ? inpath : scan_deps!(pkg, dmap) + # Found a cycle. Delete this and all parents + return true + end + return false + end + function scan_deps!(pkg, dmap) + for dep in dmap[pkg] + scan_pkg!(dep, dmap) && return true + end + could_be_cycle[pkg] = false + return false + end + for pkg in keys(depsmap) + if scan_pkg!(pkg, depsmap) + push!(circular_deps, pkg) + notify(was_processed[pkg]) + end + end + if !isempty(circular_deps) + @warn """Circular dependency detected. Precompilation will be skipped for:\n $(join(string.(circular_deps), "\n "))""" + end + @debug "precompile: circular dep check done" _group=:precompile + + # if a list of packages is given, restrict to dependencies of given packages + if !isempty(pkgs) + function collect_all_deps(depsmap, dep, alldeps=Set{Base.PkgId}()) + for _dep in depsmap[dep] + if !(_dep in alldeps) + push!(alldeps, _dep) + collect_all_deps(depsmap, _dep, alldeps) + end + end + return alldeps + end + keep = Set{Base.PkgId}() + for dep in depsmap + dep_pkgid = first(dep) + if dep_pkgid.name in pkgs + push!(keep, dep_pkgid) + collect_all_deps(depsmap, dep_pkgid, keep) + end + end + for ext in keys(exts) + if issubset(collect_all_deps(depsmap, ext), keep) # if all extension deps are kept + push!(keep, ext) + end + end + filter!(d->in(first(d), keep), depsmap) + if isempty(depsmap) + if _from_loading + # if called from loading precompilation it may be a package from another environment stack so + # don't error and allow serial precompilation to try + # TODO: actually handle packages from other envs in the stack + return + else + error("No direct dependencies outside of the sysimage found matching $(repr(pkgs))") + end + end + target = join(pkgs, ", ") + else + target = "project..." + end + @debug "precompile: packages filtered" _group=:precompile + + pkg_queue = Base.PkgId[] + failed_deps = Dict{Base.PkgId, String}() + skipped_deps = Base.PkgId[] + precomperr_deps = Base.PkgId[] # packages that may succeed after a restart (i.e. loaded packages with no cache file) + + print_lock = io isa Base.LibuvStream ? io.lock::ReentrantLock : ReentrantLock() + first_started = Base.Event() + printloop_should_exit::Bool = !fancyprint # exit print loop immediately if not fancy printing + interrupted_or_done = Base.Event() + + ansi_moveup(n::Int) = string("\e[", n, "A") + ansi_movecol1 = "\e[1G" + ansi_cleartoend = "\e[0J" + ansi_cleartoendofline = "\e[0K" + ansi_enablecursor = "\e[?25h" + ansi_disablecursor = "\e[?25l" + n_done::Int = 0 + n_already_precomp::Int = 0 + n_loaded::Int = 0 + interrupted = false + + function handle_interrupt(err, in_printloop = false) + notify(interrupted_or_done) + in_printloop || wait(t_print) # wait to let the print loop cease first + if err isa InterruptException + lock(print_lock) do + println(io, " Interrupted: Exiting precompilation...") + end + interrupted = true + return true + else + return false + end + end + + std_outputs = Dict{Base.PkgId,String}() + taskwaiting = Set{Base.PkgId}() + pkgspidlocked = Dict{Base.PkgId,String}() + pkg_liveprinted = nothing + + function monitor_std(pkg, pipe; single_requested_pkg=false) + try + liveprinting = false + while !eof(pipe) + str = readline(pipe, keep=true) + if single_requested_pkg && (liveprinting || !isempty(str)) + lock(print_lock) do + if !liveprinting + printpkgstyle(io, :Info, "Given $(pkg.name) was explicitly requested, output will be shown live $ansi_cleartoendofline", + color = Base.info_color()) + liveprinting = true + pkg_liveprinted = pkg + end + print(io, ansi_cleartoendofline, str) + end + end + std_outputs[pkg] = string(get(std_outputs, pkg, ""), str) + if !in(pkg, taskwaiting) && occursin("waiting for IO to finish", str) + !fancyprint && lock(print_lock) do + println(io, pkg.name, color_string(" Waiting for background task / IO / timer.", Base.warn_color())) + end + push!(taskwaiting, pkg) + end + if !fancyprint && in(pkg, taskwaiting) + lock(print_lock) do + print(io, str) + end + end + end + catch err + err isa InterruptException || rethrow() + end + end + + ## fancy print loop + t_print = @async begin + try + wait(first_started) + (isempty(pkg_queue) || interrupted_or_done.set) && return + fancyprint && lock(print_lock) do + printpkgstyle(io, :Precompiling, target) + print(io, ansi_disablecursor) + end + t = Timer(0; interval=1/10) + anim_chars = ["◐","◓","◑","◒"] + i = 1 + last_length = 0 + bar = MiniProgressBar(; indent=2, header = "Progress", color = Base.info_color(), percentage=false, always_reprint=true) + n_total = length(depsmap) + bar.max = n_total - n_already_precomp + final_loop = false + n_print_rows = 0 + while !printloop_should_exit + lock(print_lock) do + term_size = Base.displaysize(io)::Tuple{Int,Int} + num_deps_show = term_size[1] - 3 + pkg_queue_show = if !interrupted_or_done.set && length(pkg_queue) > num_deps_show + last(pkg_queue, num_deps_show) + else + pkg_queue + end + str_ = sprint() do iostr + if i > 1 + print(iostr, ansi_cleartoend) + end + bar.current = n_done - n_already_precomp + bar.max = n_total - n_already_precomp + # when sizing to the terminal width subtract a little to give some tolerance to resizing the + # window between print cycles + termwidth = displaysize(io)[2] - 4 + if !final_loop + str = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) + print(iostr, Base._truncate_at_width_or_chars(true, str, termwidth), "\n") + end + for dep in pkg_queue_show + loaded = warn_loaded && haskey(Base.loaded_modules, dep) + _name = haskey(exts, dep) ? string(exts[dep], " → ", dep.name) : dep.name + name = dep in direct_deps ? _name : string(color_string(_name, :light_black)) + line = if dep in precomperr_deps + string(color_string(" ? ", Base.warn_color()), name) + elseif haskey(failed_deps, dep) + string(color_string(" ✗ ", Base.error_color()), name) + elseif was_recompiled[dep] + !loaded && interrupted_or_done.set && continue + loaded || @async begin # keep successful deps visible for short period + sleep(1); + filter!(!isequal(dep), pkg_queue) + end + string(color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) + elseif started[dep] + # Offset each spinner animation using the first character in the package name as the seed. + # If not offset, on larger terminal fonts it looks odd that they all sync-up + anim_char = anim_chars[(i + Int(dep.name[1])) % length(anim_chars) + 1] + anim_char_colored = dep in direct_deps ? anim_char : color_string(anim_char, :light_black) + waiting = if haskey(pkgspidlocked, dep) + who_has_lock = pkgspidlocked[dep] + color_string(" Being precompiled by $(who_has_lock)", Base.info_color()) + elseif dep in taskwaiting + color_string(" Waiting for background task / IO / timer. Interrupt to inspect", Base.warn_color()) + else + "" + end + string(" ", anim_char_colored, " ", name, waiting) + else + string(" ", name) + end + println(iostr, Base._truncate_at_width_or_chars(true, line, termwidth)) + end + end + last_length = length(pkg_queue_show) + n_print_rows = count("\n", str_) + print(io, str_) + printloop_should_exit = interrupted_or_done.set && final_loop + final_loop = interrupted_or_done.set # ensures one more loop to tidy last task after finish + i += 1 + printloop_should_exit || print(io, ansi_moveup(n_print_rows), ansi_movecol1) + end + wait(t) + end + catch err + handle_interrupt(err, true) || rethrow() + finally + fancyprint && print(io, ansi_enablecursor) + end + end + tasks = Task[] + if !_from_loading + Base.LOADING_CACHE[] = Base.LoadingCache() + end + @debug "precompile: starting precompilation loop" _group=:precompile + ## precompilation loop + for (pkg, deps) in depsmap + cachepaths = Base.find_all_in_cache_path(pkg) + sourcepath = Base.locate_package(pkg) + if sourcepath === nothing + failed_deps[pkg] = "Error: Missing source file for $(pkg)" + notify(was_processed[pkg]) + continue + end + # Heuristic for when precompilation is disabled + if occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String)) + notify(was_processed[pkg]) + continue + end + + single_requested_pkg = length(pkgs) == 1 && only(pkgs) == pkg.name + + task = @async begin + try + loaded = haskey(Base.loaded_modules, pkg) + for dep in deps # wait for deps to finish + wait(was_processed[dep]) + end + + circular = pkg in circular_deps + is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath, flags=cacheflags) + if !circular && is_stale + Base.acquire(parallel_limiter) + is_direct_dep = pkg in direct_deps + + # std monitoring + std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) + t_monitor = @async monitor_std(pkg, std_pipe; single_requested_pkg) + + _name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name + name = is_direct_dep ? _name : string(color_string(_name, :light_black)) + !fancyprint && lock(print_lock) do + isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) + end + push!(pkg_queue, pkg) + started[pkg] = true + fancyprint && notify(first_started) + if interrupted_or_done.set + notify(was_processed[pkg]) + Base.release(parallel_limiter) + return + end + try + # allows processes to wait if another process is precompiling a given package to + # a functionally identical package cache (except for preferences, which may differ) + t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg, pkgspidlocked, hascolor, cacheflags) do + Base.with_logger(Base.NullLogger()) do + # The false here means we ignore loaded modules, so precompile for a fresh session + Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags) + end + end + if ret isa Base.PrecompilableError + push!(precomperr_deps, pkg) + !fancyprint && lock(print_lock) do + println(io, _timing_string(t), color_string(" ? ", Base.warn_color()), name) + end + else + !fancyprint && lock(print_lock) do + println(io, _timing_string(t), color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) + end + was_recompiled[pkg] = true + end + loaded && (n_loaded += 1) + catch err + close(std_pipe.in) # close pipe to end the std output monitor + wait(t_monitor) + if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file")) + failed_deps[pkg] = (strict || is_direct_dep) ? string(sprint(showerror, err), "\n", strip(get(std_outputs, pkg, ""))) : "" + delete!(std_outputs, pkg) # so it's not shown as warnings, given error report + !fancyprint && lock(print_lock) do + println(io, " "^9, color_string(" ✗ ", Base.error_color()), name) + end + else + rethrow() + end + finally + isopen(std_pipe.in) && close(std_pipe.in) # close pipe to end the std output monitor + wait(t_monitor) + Base.release(parallel_limiter) + end + else + is_stale || (n_already_precomp += 1) + end + n_done += 1 + notify(was_processed[pkg]) + catch err_outer + # For debugging: + # @error "Task failed" exception=(err_outer, catch_backtrace()) + handle_interrupt(err_outer) || rethrow() + notify(was_processed[pkg]) + finally + filter!(!istaskdone, tasks) + length(tasks) == 1 && notify(interrupted_or_done) + end + end + push!(tasks, task) + end + isempty(tasks) && notify(interrupted_or_done) + try + wait(interrupted_or_done) + catch err + handle_interrupt(err) || rethrow() + finally + Base.LOADING_CACHE[] = nothing + end + notify(first_started) # in cases of no-op or !fancyprint + fancyprint && wait(t_print) + quick_exit = !all(istaskdone, tasks) || interrupted # if some not finished internal error is likely + seconds_elapsed = round(Int, (time_ns() - time_start) / 1e9) + ndeps = count(values(was_recompiled)) + if ndeps > 0 || !isempty(failed_deps) || (quick_exit && !isempty(std_outputs)) + str = sprint() do iostr + if !quick_exit + plural = ndeps == 1 ? "y" : "ies" + print(iostr, " $(ndeps) dependenc$(plural) successfully precompiled in $(seconds_elapsed) seconds") + if n_already_precomp > 0 || !isempty(circular_deps) || !isempty(skipped_deps) + n_already_precomp > 0 && (print(iostr, ". $n_already_precomp already precompiled")) + !isempty(circular_deps) && (print(iostr, ". $(length(circular_deps)) skipped due to circular dependency")) + !isempty(skipped_deps) && (print(iostr, ". $(length(skipped_deps)) skipped during auto due to previous errors")) + print(iostr, ".") + end + if n_loaded > 0 + plural1 = n_loaded == 1 ? "y" : "ies" + plural2 = n_loaded == 1 ? "a different version is" : "different versions are" + plural3 = n_loaded == 1 ? "" : "s" + print(iostr, "\n ", + color_string(string(n_loaded), Base.warn_color()), + " dependenc$(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3)" + ) + end + if !isempty(precomperr_deps) + pluralpc = length(precomperr_deps) == 1 ? "y" : "ies" + print(iostr, "\n ", + color_string(string(length(precomperr_deps)), Base.warn_color()), + " dependenc$(pluralpc) failed but may be precompilable after restarting julia" + ) + end + end + # show any stderr output, even if Pkg.precompile has been interrupted (quick_exit=true), given user may be + # interrupting a hanging precompile job with stderr output. julia#48371 + filter!(kv -> !isempty(strip(last(kv))), std_outputs) # remove empty output + if !isempty(std_outputs) + plural1 = length(std_outputs) == 1 ? "y" : "ies" + plural2 = length(std_outputs) == 1 ? "" : "s" + print(iostr, "\n ", color_string("$(length(std_outputs))", Base.warn_color()), " dependenc$(plural1) had output during precompilation:") + for (pkgid, err) in std_outputs + err = if pkgid == pkg_liveprinted + "[Output was shown above]" + else + join(split(strip(err), "\n"), color_string("\n│ ", Base.warn_color())) + end + name = haskey(exts, pkgid) ? string(exts[pkgid], " → ", pkgid.name) : pkgid.name + print(iostr, color_string("\n┌ ", Base.warn_color()), name, color_string("\n│ ", Base.warn_color()), err, color_string("\n└ ", Base.warn_color())) + end + end + end + let str=str + lock(print_lock) do + println(io, str) + end + end + quick_exit && return + err_str = "" + n_direct_errs = 0 + for (dep, err) in failed_deps + if strict || (dep in direct_deps) + err_str = string(err_str, "\n$dep\n\n$err", (n_direct_errs > 0 ? "\n" : "")) + n_direct_errs += 1 + end + end + if err_str != "" + pluralde = n_direct_errs == 1 ? "y" : "ies" + direct = strict ? "" : "direct " + err_msg = "The following $n_direct_errs $(direct)dependenc$(pluralde) failed to precompile:\n$(err_str[1:end-1])" + if internal_call # aka. auto-precompilation + if isinteractive() && !get(ENV, "CI", false) + plural1 = length(failed_deps) == 1 ? "y" : "ies" + println(io, " ", color_string("$(length(failed_deps))", Base.error_color()), " dependenc$(plural1) errored.") + println(io, " For a report of the errors see `julia> err`. To retry use `pkg> precompile`") + setglobal!(Base.MainInclude, :err, PkgPrecompileError(err_msg)) + else + # auto-precompilation shouldn't throw but if the user can't easily access the + # error messages, just show them + print(io, "\n", err_msg) + end + else + println(io) + error(err_msg) + end + end + end + nothing +end + +_timing_string(t) = string(lpad(round(t * 1e3, digits = 1), 9), " ms") + +function _color_string(cstr::String, col::Union{Int64, Symbol}, hascolor) + if hascolor + enable_ansi = get(Base.text_colors, col, Base.text_colors[:default]) + disable_ansi = get(Base.disable_text_style, col, Base.text_colors[:default]) + return string(enable_ansi, cstr, disable_ansi) + else + return cstr + end +end + +# Can be merged with `maybe_cachefile_lock` in loading? +function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg::Base.PkgId, pkgspidlocked::Dict{Base.PkgId,String}, hascolor, cacheflags) + FileWatching = Base.loaded_modules[Base.PkgId(Base.UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"), "FileWatching")] + stale_age = Base.compilecache_pidlock_stale_age + pidfile = Base.compilecache_pidfile_path(pkg) + cachefile = FileWatching.trymkpidlock(f, pidfile; stale_age) + if cachefile === false + pid, hostname, age = FileWatching.Pidfile.parse_pidfile(pidfile) + pkgspidlocked[pkg] = if isempty(hostname) || hostname == gethostname() + if pid == getpid() + "an async task in this process (pidfile: $pidfile)" + else + "another process (pid: $pid, pidfile: $pidfile)" + end + else + "another machine (hostname: $hostname, pid: $pid, pidfile: $pidfile)" + end + !fancyprint && lock(print_lock) do + println(io, " ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg])", Base.info_color(), hascolor)) + end + # wait until the lock is available + FileWatching.mkpidlock(pidfile; stale_age) do + # double-check in case the other process crashed or the lock expired + if Base.isprecompiled(pkg; ignore_loaded=true, flags=cacheflags) # don't use caches for this as the env state will have changed + return nothing # returning nothing indicates a process waited for another + else + delete!(pkgspidlocked, pkg) + return f() # precompile + end + end + end + return cachefile +end + +end diff --git a/doc/make.jl b/doc/make.jl index 8dee832bf27b7..e06fa1deb8f42 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -3,7 +3,8 @@ Base.ACTIVE_PROJECT[] = nothing empty!(LOAD_PATH) push!(LOAD_PATH, @__DIR__, "@stdlib") empty!(DEPOT_PATH) -pushfirst!(DEPOT_PATH, joinpath(@__DIR__, "deps")) +push!(DEPOT_PATH, joinpath(@__DIR__, "deps")) +push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia")) using Pkg Pkg.instantiate()