Move parallel precompilation to Base (#53403)

Parallel precompilation is more or less now required in order to use
somewhat large packages unless you want to wait an obscene long time for
it to complete. Right now, we even start a parallel precompilation on a
package load if we notice that the package you are loading is not
precompiled.

This functionally has typically been implemented in Pkg but with Pkg not
being in the sysimage it becomes a bit awkward because we then need to
load Pkg from Base. The only real reason this functionality has been
implemented in Pkg is that Pkg has some useful features for parsing
environments. Moving precompilation to Base has typically been stalled
on such an environment parser not existing in Base.

However, in #46690 I started
implemented code loading on top of a more up front environment parser
(instead of the "incremental" one that currently exists in `loading.jl`)
and we can retro fit this to be used as the basis of parallel
precompilation. At some later point code loading could be implemented on
top of it but that is for now considered future work.

This PR thus adds the environment parser from the codeloading PR and
implementes the parallel precompilation feature from Pkg on top of it
(instead of on top of the `EnvCache` in Pkg).

Some points to bring up here:

- This copy pastes the progress bar implementation in Pkg into here. It
is probably a bit excessive to use so we can simplify that
significantly.
- Parallel precompilation uses the `FileWatching` module to avoid
different processes trying to precompile the same package concurrently.
Right now, I used grab this from `Base.loaded_modules` relying on it
being in the sysimage.
- This removes the "suspended" functionality from the Pkg precompilation
which does not try to precompile packages if they have "recently" failed
which is unclear how useful it is in practice. This also requires the
Serialization stdlib and uses data structures defined in Pkg so it is
hard to keep when moving this to Base.

---------

Co-authored-by: Ian Butterworth <i.r.butterworth@gmail.com>
(cherry picked from commit 6745160)

base/Base.jl

@@ -542,6 +542,8 @@ if isdefined(Core, :Compiler) && is_primary_base_module
     Docs.loaddocs(Core.Compiler.CoreDocs.DOCS)
 end
 
+include("precompilation.jl")
+
 # finally, now make `include` point to the full version
 for m in methods(include)
     delete_method(m)

base/loading.jl

@@ -2147,8 +2147,6 @@ end
 
 require(uuidkey::PkgId) = @lock require_lock _require_prelocked(uuidkey)
 
-const REPL_PKGID = PkgId(UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL")
-
 function _require_prelocked(uuidkey::PkgId, env=nothing)
     if _require_world_age[] != typemax(UInt)
         Base.invoke_in_world(_require_world_age[], __require_prelocked, uuidkey, env)
@@ -2262,8 +2260,9 @@ function set_pkgorigin_version_path(pkg::PkgId, path::String)
     nothing
 end
 
-# A hook to allow code load to use Pkg.precompile
+# Unused
 const PKG_PRECOMPILE_HOOK = Ref{Function}()
+disable_parallel_precompile::Bool = false
 
 # Returns `nothing` or the new(ish) module
 function _require(pkg::PkgId, env=nothing)
@@ -2284,7 +2283,7 @@ function _require(pkg::PkgId, env=nothing)
         end
         set_pkgorigin_version_path(pkg, path)
 
-        pkg_precompile_attempted = false # being safe to avoid getting stuck in a Pkg.precompile loop
+        parallel_precompile_attempted = false # being safe to avoid getting stuck in a precompilepkgs loop
         reasons = Dict{String,Int}()
         # attempt to load the module file via the precompile cache locations
         if JLOptions().use_compiled_modules != 0
@@ -2314,11 +2313,13 @@ function _require(pkg::PkgId, env=nothing)
 
         if JLOptions().use_compiled_modules == 1
             if !generating_output(#=incremental=#false)
-                if !pkg_precompile_attempted && isinteractive() && isassigned(PKG_PRECOMPILE_HOOK)
-                    pkg_precompile_attempted = true
+                project = active_project()
+                if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) && project !== nothing &&
+                        isfile(project) && project_file_manifest_path(project) !== nothing
+                    parallel_precompile_attempted = true
                     unlock(require_lock)
                     try
-                        @invokelatest PKG_PRECOMPILE_HOOK[](pkg.name, _from_loading = true)
+                        Precompilation.precompilepkgs([pkg.name]; _from_loading=true)
                     finally
                         lock(require_lock)
                     end