From f2df1b41b85843120a9715449eb5ff555a4a172d Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 20 Oct 2023 02:56:22 +0200 Subject: [PATCH] Make precompile files relocatable (#49866) String replacement with `@depot` when serializing out happens with any paths that are located inside a `DEPOT_PATH` (first match wins). If no match, then we emit the absolute file path as before. Right now we only emit one token `@depot`. String replacement of `@depot` when loading happens now on a `.ji` file basis and only if all the listed include dependencies can be resolved to files located in one and the same depot on `DEPOT_PATH` (again, first match wins). If we can't resolve, then the cache is invalided with `stale_cachefile`. --- base/loading.jl | 200 ++++++++++++------ base/sysimg.jl | 5 +- doc/src/manual/modules.md | 10 +- src/precompile.c | 34 ++- src/staticdata.c | 5 +- src/staticdata_utils.c | 26 ++- test/.gitignore | 2 + test/Makefile | 26 ++- test/RelocationTestPkg1/Project.toml | 4 + .../src/RelocationTestPkg1.jl | 5 + test/RelocationTestPkg1/src/foo.txt | 0 test/RelocationTestPkg2/Project.toml | 4 + .../src/RelocationTestPkg2.jl | 6 + test/RelocationTestPkg2/src/foo.txt | 0 test/choosetests.jl | 2 +- test/precompile.jl | 16 +- test/relocatedepot.jl | 104 +++++++++ test/testenv.jl | 2 + 18 files changed, 360 insertions(+), 91 deletions(-) create mode 100644 test/RelocationTestPkg1/Project.toml create mode 100644 test/RelocationTestPkg1/src/RelocationTestPkg1.jl create mode 100644 test/RelocationTestPkg1/src/foo.txt create mode 100644 test/RelocationTestPkg2/Project.toml create mode 100644 test/RelocationTestPkg2/src/RelocationTestPkg2.jl create mode 100644 test/RelocationTestPkg2/src/foo.txt create mode 100644 test/relocatedepot.jl diff --git a/base/loading.jl b/base/loading.jl index 25fe8144b71d6..1f6bb5442540a 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -371,7 +371,7 @@ its `PkgId`, or `nothing` if it cannot be found. If only the `name` argument is provided, it searches each environment in the stack and its named direct dependencies. -There `where` argument provides the context from where to search for the +The `where` argument provides the context from where to search for the package: in this case it first checks if the name matches the context itself, otherwise it searches all recursive dependencies (from the resolved manifest of each environment) until it locates the context `where`, and from there @@ -1503,7 +1503,7 @@ function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union io = open(path, "r") try iszero(isvalid_cache_header(io)) && return ArgumentError("Invalid header in cache file $path.") - _, _, depmodnames, _, _, _, clone_targets, _ = parse_cache_header(io) + _, _, depmodnames, _, _, _, clone_targets, _ = parse_cache_header(io, path) pkgimage = !isempty(clone_targets) if pkgimage ocachepath !== nothing || return ArgumentError("Expected ocachepath to be provided") @@ -1660,9 +1660,9 @@ const include_callbacks = Any[] # used to optionally track dependencies when requiring a module: const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them -const _require_dependencies = Any[] # a list of (mod, path, mtime) tuples that are the file dependencies of the module currently being precompiled +const _require_dependencies = Any[] # a list of (mod, abspath, fsize, hash, mtime) tuples that are the file dependencies of the module currently being precompiled const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies -function _include_dependency(mod::Module, _path::AbstractString) +function _include_dependency(mod::Module, _path::AbstractString; track_content=true) prev = source_path(nothing) if prev === nothing path = abspath(_path) @@ -1671,7 +1671,15 @@ function _include_dependency(mod::Module, _path::AbstractString) end if _track_dependencies[] @lock require_lock begin - push!(_require_dependencies, (mod, path, mtime(path))) + if track_content + @assert isfile(path) "can only hash files" + # use mtime=-1.0 here so that fsize==0 && mtime==0.0 corresponds to a missing include_dependency + push!(_require_dependencies, + (mod, path, filesize(path), open(_crc32c, path, "r"), -1.0)) + else + push!(_require_dependencies, + (mod, path, UInt64(0), UInt32(0), mtime(path))) + end end end return path, prev @@ -1688,7 +1696,7 @@ This is only needed if your module depends on a path that is not used via [`incl no effect outside of compilation. """ function include_dependency(path::AbstractString) - _include_dependency(Main, path) + _include_dependency(Main, path, track_content=false) return nothing end @@ -1787,7 +1795,8 @@ function __require(into::Module, mod::Symbol) end uuidkey, env = uuidkey_env if _track_dependencies[] - push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) + path = binpack(uuidkey) + push!(_require_dependencies, (into, path, UInt64(0), UInt32(0), 0.0)) end return _require_prelocked(uuidkey, env) finally @@ -2580,14 +2589,46 @@ function isvalid_pkgimage_crc(f::IOStream, ocachefile::String) expected_crc_so == crc_so end -struct CacheHeaderIncludes - id::PkgId +mutable struct CacheHeaderIncludes + const id::PkgId filename::String - mtime::Float64 - modpath::Vector{String} # seemingly not needed in Base, but used by Revise + const fsize::UInt64 + const hash::UInt32 + const mtime::Float64 + const modpath::Vector{String} # seemingly not needed in Base, but used by Revise +end + +function replace_depot_path(path::AbstractString) + for depot in DEPOT_PATH + if startswith(path, depot) + path = replace(path, depot => "@depot") + break + end + end + return path +end + +# Find depot in DEPOT_PATH for which all @depot tags from the `includes` +# can be replaced so that they point to a file on disk each. +# Return nothing when no depot matched. +function resolve_depot(includes) + if any(includes) do inc + !startswith(inc, "@depot") + end + return :missing_depot_tag + end + for depot in DEPOT_PATH + if all(includes) do inc + isfile(replace(inc, r"^@depot" => depot)) + end + return depot + end + end + return :no_depot_found end -function parse_cache_header(f::IO) + +function parse_cache_header(f::IO, cachefile::AbstractString) flags = read(f, UInt8) modules = Vector{Pair{PkgId, UInt64}}() while true @@ -2598,7 +2639,7 @@ function parse_cache_header(f::IO) build_id = read(f, UInt64) # build UUID (mostly just a timestamp) push!(modules, PkgId(uuid, sym) => build_id) end - totbytes = read(f, Int64) # total bytes for file dependencies + preferences + totbytes = Int64(read(f, UInt64)) # total bytes for file dependencies + preferences # read the list of requirements # and split the list into include and requires statements includes = CacheHeaderIncludes[] @@ -2611,6 +2652,10 @@ function parse_cache_header(f::IO) end depname = String(read(f, n2)) totbytes -= n2 + fsize = read(f, UInt64) + totbytes -= 8 + hash = read(f, UInt32) + totbytes -= 4 mtime = read(f, Float64) totbytes -= 8 n1 = read(f, Int32) @@ -2633,7 +2678,7 @@ function parse_cache_header(f::IO) if depname[1] == '\0' push!(requires, modkey => binunpack(depname)) else - push!(includes, CacheHeaderIncludes(modkey, depname, mtime, modpath)) + push!(includes, CacheHeaderIncludes(modkey, depname, fsize, hash, mtime, modpath)) end end prefs = String[] @@ -2665,69 +2710,90 @@ function parse_cache_header(f::IO) l = read(f, Int32) clone_targets = read(f, l) - return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags + # determine path for @depot replacement from srctext files only, e.g. ignore any include_dependency files + srcfiles = srctext_files(f, srctextpos, includes) + depot = resolve_depot(srcfiles) + keepidx = Int[] + for (i, chi) in enumerate(includes) + chi.filename ∈ srcfiles && push!(keepidx, i) + end + if depot === :no_depot_found + throw(ArgumentError(""" + Failed to determine depot from srctext files in cache file $cachefile. + - Make sure you have adjusted DEPOT_PATH in case you relocated depots.""")) + elseif depot === :missing_depot_tag + @debug "Missing @depot tag for include dependencies in cache file $cachefile." + else + for inc in includes + inc.filename = replace(inc.filename, r"^@depot" => depot) + end + end + includes_srcfiles_only = includes[keepidx] + + return modules, (includes, includes_srcfiles_only, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags end -function parse_cache_header(cachefile::String; srcfiles_only::Bool=false) +function parse_cache_header(cachefile::String) io = open(cachefile, "r") try iszero(isvalid_cache_header(io)) && throw(ArgumentError("Invalid header in cache file $cachefile.")) - ret = parse_cache_header(io) - srcfiles_only || return ret - _, (includes, _), _, srctextpos, _... = ret - srcfiles = srctext_files(io, srctextpos) - delidx = Int[] - for (i, chi) in enumerate(includes) - chi.filename ∈ srcfiles || push!(delidx, i) - end - deleteat!(includes, delidx) + ret = parse_cache_header(io, cachefile) return ret finally close(io) end end -preferences_hash(f::IO) = parse_cache_header(f)[6] +preferences_hash(f::IO, cachefile::AbstractString) = parse_cache_header(f, cachefile)[6] function preferences_hash(cachefile::String) io = open(cachefile, "r") try if iszero(isvalid_cache_header(io)) throw(ArgumentError("Invalid header in cache file $cachefile.")) end - return preferences_hash(io) + return preferences_hash(io, cachefile) finally close(io) end end -function cache_dependencies(f::IO) - _, (includes, _), modules, _... = parse_cache_header(f) - return modules, map(chi -> (chi.filename, chi.mtime), includes) # return just filename and mtime +function cache_dependencies(f::IO, cachefile::AbstractString) + _, (includes, _, _), modules, _... = parse_cache_header(f, cachefile) + return modules, map(chi -> chi.filename, includes) # return just filename end function cache_dependencies(cachefile::String) io = open(cachefile, "r") try iszero(isvalid_cache_header(io)) && throw(ArgumentError("Invalid header in cache file $cachefile.")) - return cache_dependencies(io) + return cache_dependencies(io, cachefile) finally close(io) end end -function read_dependency_src(io::IO, filename::AbstractString) - srctextpos = parse_cache_header(io)[4] +function read_dependency_src(io::IO, cachefile::AbstractString, filename::AbstractString) + _, (includes, _, _), _, srctextpos, _, _, _, _ = parse_cache_header(io, cachefile) srctextpos == 0 && error("no source-text stored in cache file") seek(io, srctextpos) - return _read_dependency_src(io, filename) + return _read_dependency_src(io, filename, includes) end -function _read_dependency_src(io::IO, filename::AbstractString) +function _read_dependency_src(io::IO, filename::AbstractString, includes::Vector{CacheHeaderIncludes}=CacheHeaderIncludes[]) while !eof(io) filenamelen = read(io, Int32) filenamelen == 0 && break - fn = String(read(io, filenamelen)) + depotfn = String(read(io, filenamelen)) len = read(io, UInt64) + fn = if !startswith(depotfn, "@depot") + depotfn + else + basefn = replace(depotfn, r"^@depot" => "") + idx = findfirst(includes) do inc + endswith(inc.filename, basefn) + end + isnothing(idx) ? depotfn : includes[idx].filename + end if fn == filename return String(read(io, len)) end @@ -2740,22 +2806,22 @@ function read_dependency_src(cachefile::String, filename::AbstractString) io = open(cachefile, "r") try iszero(isvalid_cache_header(io)) && throw(ArgumentError("Invalid header in cache file $cachefile.")) - return read_dependency_src(io, filename) + return read_dependency_src(io, cachefile, filename) finally close(io) end end -function srctext_files(f::IO, srctextpos::Int64) +function srctext_files(f::IO, srctextpos::Int64, includes::Vector{CacheHeaderIncludes}) files = Set{String}() srctextpos == 0 && return files seek(f, srctextpos) while !eof(f) filenamelen = read(f, Int32) filenamelen == 0 && break - fn = String(read(f, filenamelen)) + filename = String(read(f, filenamelen)) len = read(f, UInt64) - push!(files, fn) + push!(files, filename) seek(f, position(f) + len) end return files @@ -3098,7 +3164,7 @@ end @debug "Rejecting cache file $cachefile due to it containing an invalid cache header" return true # invalid cache file end - modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags = parse_cache_header(io) + modules, (includes, _, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags = parse_cache_header(io, cachefile) if isempty(modules) return true # ignore empty file end @@ -3141,7 +3207,7 @@ end if build_id != UInt128(0) id_build = (UInt128(checksum) << 64) | id.second if id_build != build_id - @debug "Ignoring cache file $cachefile for $modkey ($((UUID(id_build)))) since it is does not provide desired build_id ($((UUID(build_id))))" + @debug "Ignoring cache file $cachefile for $modkey ($((UUID(id_build)))) since it does not provide desired build_id ($((UUID(build_id))))" return true end end @@ -3179,13 +3245,13 @@ end # check if this file is going to provide one of our concrete dependencies # or if it provides a version that conflicts with our concrete dependencies # or neither - skip_timecheck = false + skip_check = false for (req_key, req_build_id) in _concrete_dependencies build_id = get(modules, req_key, UInt64(0)) if build_id !== UInt64(0) build_id |= UInt128(checksum) << 64 if build_id === req_build_id - skip_timecheck = true + skip_check = true break end @debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))" @@ -3193,40 +3259,54 @@ end end end - # now check if this file is fresh relative to its source files - if !skip_timecheck + # now check if this file's content hash has changed relative to its source files + if !skip_check if !samefile(includes[1].filename, modpath) && !samefile(fixup_stdlib_path(includes[1].filename), modpath) @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath" return true # cache file was compiled from a different path end for (modkey, req_modkey) in requires # verify that `require(modkey, name(req_modkey))` ==> `req_modkey` - if identify_package(modkey, req_modkey.name) != req_modkey - @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed" + pkg = identify_package(modkey, req_modkey.name) + if pkg != req_modkey + @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed, expected $modkey => $pkg" return true end end for chi in includes - f, ftime_req = chi.filename, chi.mtime + f, fsize_req, hash_req, ftime_req = chi.filename, chi.fsize, chi.hash, chi.mtime if !ispath(f) _f = fixup_stdlib_path(f) if isfile(_f) && startswith(_f, Sys.STDLIB) - # mtime is changed by extraction continue end @debug "Rejecting stale cache file $cachefile because file $f does not exist" return true end - ftime = mtime(f) - is_stale = ( ftime != ftime_req ) && - ( ftime != floor(ftime_req) ) && # Issue #13606, PR #13613: compensate for Docker images rounding mtimes - ( ftime != ceil(ftime_req) ) && # PR: #47433 Compensate for CirceCI's truncating of timestamps in its caching - ( ftime != trunc(ftime_req, digits=6) ) && # Issue #20837, PR #20840: compensate for GlusterFS truncating mtimes to microseconds - ( ftime != 1.0 ) && # PR #43090: provide compatibility with Nix mtime. - !( 0 < (ftime_req - ftime) < 1e-6 ) # PR #45552: Compensate for Windows tar giving mtimes that may be incorrect by up to one microsecond - if is_stale - @debug "Rejecting stale cache file $cachefile (mtime $ftime_req) because file $f (mtime $ftime) has changed" - return true + if ftime_req >= 0.0 + # this is an include_dependency for which we only recorded the mtime + ftime = mtime(f) + is_stale = ( ftime != ftime_req ) && + ( ftime != floor(ftime_req) ) && # Issue #13606, PR #13613: compensate for Docker images rounding mtimes + ( ftime != ceil(ftime_req) ) && # PR: #47433 Compensate for CirceCI's truncating of timestamps in its caching + ( ftime != trunc(ftime_req, digits=6) ) && # Issue #20837, PR #20840: compensate for GlusterFS truncating mtimes to microseconds + ( ftime != 1.0 ) && # PR #43090: provide compatibility with Nix mtime. + !( 0 < (ftime_req - ftime) < 1e-6 ) # PR #45552: Compensate for Windows tar giving mtimes that may be incorrect by up to one microsecond + if is_stale + @debug "Rejecting stale cache file $cachefile because mtime of include_dependency $f has changed (mtime $ftime, before $ftime_req)" + return true + end + else + fsize = filesize(f) + if fsize != fsize_req + @debug "Rejecting stale cache file $cachefile because file size of $f has changed (file size $fsize, before $fsize_req)" + return true + end + hash = open(_crc32c, f, "r") + if hash != hash_req + @debug "Rejecting stale cache file $cachefile because hash of $f has changed (hash $hash, before $hash_req)" + return true + end end end end diff --git a/base/sysimg.jl b/base/sysimg.jl index bf8de0bc3f75e..a4bf21786c633 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -59,8 +59,9 @@ let print_time(stdlib, tt) end for dep in Base._require_dependencies - dep[3] == 0.0 && continue - push!(Base._included_files, dep[1:2]) + mod, path, fsize, mtime = dep[1], dep[2], dep[3], dep[5] + (fsize == 0 || mtime == 0.0) && continue + push!(Base._included_files, (mod, path)) end empty!(Base._require_dependencies) Base._track_dependencies[] = false diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index b329dbc91b923..4be08edc56f38 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -447,10 +447,12 @@ recompiled upon `using` or `import`. Dependencies are modules it imports, the Julia build, files it includes, or explicit dependencies declared by [`include_dependency(path)`](@ref) in the module file(s). -For file dependencies, a change is determined by examining whether the modification time (`mtime`) -of each file loaded by `include` or added explicitly by `include_dependency` is unchanged, or equal -to the modification time truncated to the nearest second (to accommodate systems that can't copy -mtime with sub-second accuracy). It also takes into account whether the path to the file chosen +For file dependencies loaded by `include`, a change is determined by examining whether the +file size (`fsize`) or content (condensed into a hash) is unchanged. +For file dependencies loaded by `include_dependency` a change is determined by examining whether the modification time (`mtime`) +is unchanged, or equal to the modification time truncated to the nearest second +(to accommodate systems that can't copy mtime with sub-second accuracy). +It also takes into account whether the path to the file chosen by the search logic in `require` matches the path that had created the precompile file. It also takes into account the set of dependencies already loaded into the current process and won't recompile those modules, even if their files change or disappear, in order to avoid creating incompatibilities between diff --git a/src/precompile.c b/src/precompile.c index f6266e252f609..ad33dcd9b6d65 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -36,26 +36,43 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { // char*: src text // At the end we write int32(0) as a terminal sentinel. size_t len = jl_array_len(udeps); + static jl_value_t *replace_depot_func = NULL; + if (!replace_depot_func) + replace_depot_func = jl_get_global(jl_base_module, jl_symbol("replace_depot_path")); ios_t srctext; + jl_value_t *deptuple = NULL; + JL_GC_PUSH2(&deptuple, &udeps); for (size_t i = 0; i < len; i++) { - jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); + deptuple = jl_array_ptr_ref(udeps, i); jl_value_t *depmod = jl_fieldref(deptuple, 0); // module // Dependencies declared with `include_dependency` are excluded // because these may not be Julia code (and could be huge) if (depmod != (jl_value_t*)jl_main_module) { - jl_value_t *dep = jl_fieldref(deptuple, 1); // file abspath - const char *depstr = jl_string_data(dep); - if (!depstr[0]) + jl_value_t *abspath = jl_fieldref(deptuple, 1); // file abspath + const char *abspathstr = jl_string_data(abspath); + if (!abspathstr[0]) continue; - ios_t *srctp = ios_file(&srctext, depstr, 1, 0, 0, 0); + ios_t *srctp = ios_file(&srctext, abspathstr, 1, 0, 0, 0); if (!srctp) { jl_printf(JL_STDERR, "WARNING: could not cache source text for \"%s\".\n", - jl_string_data(dep)); + abspathstr); continue; } - size_t slen = jl_string_len(dep); + + jl_value_t **replace_depot_args; + JL_GC_PUSHARGS(replace_depot_args, 2); + replace_depot_args[0] = replace_depot_func; + replace_depot_args[1] = abspath; + jl_task_t *ct = jl_current_task; + size_t last_age = ct->world_age; + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + jl_value_t *depalias = (jl_value_t*)jl_apply(replace_depot_args, 2); + ct->world_age = last_age; + JL_GC_POP(); + + size_t slen = jl_string_len(depalias); write_int32(f, slen); - ios_write(f, depstr, slen); + ios_write(f, jl_string_data(depalias), slen); posfile = ios_pos(f); write_uint64(f, 0); // placeholder for length of this file in bytes uint64_t filelen = (uint64_t) ios_copyall(f, &srctext); @@ -65,6 +82,7 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { ios_seek_end(f); } } + JL_GC_POP(); } write_int32(f, 0); // mark the end of the source text } diff --git a/src/staticdata.c b/src/staticdata.c index c684bee4c485b..69226408f711b 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -2691,8 +2691,9 @@ static void jl_write_header_for_incremental(ios_t *f, jl_array_t *worklist, jl_a write_uint8(f, jl_cache_flags()); // write description of contents (name, uuid, buildid) write_worklist_for_header(f, worklist); - // Determine unique (module, abspath, mtime) dependencies for the files defining modules in the worklist - // (see Base._require_dependencies). These get stored in `udeps` and written to the ji-file header. + // Determine unique (module, abspath, fsize, hash, mtime) dependencies for the files defining modules in the worklist + // (see Base._require_dependencies). These get stored in `udeps` and written to the ji-file header + // (abspath will be converted to a relocateable @depot path before writing, cf. Base.replace_depot_path). // Also write Preferences. // last word of the dependency list is the end of the data / start of the srctextpos *srctextpos = write_dependency_list(f, worklist, udeps); // srctextpos: position of srctext entry in header index (update later) diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index a4cbc3fd5ebc4..047042795d128 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -706,6 +706,10 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t jl_array_t *udeps = (*udepsp = deps && unique_func ? (jl_array_t*)jl_apply(uniqargs, 2) : NULL); ct->world_age = last_age; + static jl_value_t *replace_depot_func = NULL; + if (!replace_depot_func) + replace_depot_func = jl_get_global(jl_base_module, jl_symbol("replace_depot_path")); + // write a placeholder for total size so that we can quickly seek past all of the // dependencies if we don't need them initial_pos = ios_pos(s); @@ -713,11 +717,25 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t size_t i, l = udeps ? jl_array_len(udeps) : 0; for (i = 0; i < l; i++) { jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); - jl_value_t *dep = jl_fieldref(deptuple, 1); // file abspath - size_t slen = jl_string_len(dep); + jl_value_t *abspath = jl_fieldref(deptuple, 1); + + jl_value_t **replace_depot_args; + JL_GC_PUSHARGS(replace_depot_args, 2); + replace_depot_args[0] = replace_depot_func; + replace_depot_args[1] = abspath; + ct = jl_current_task; + size_t last_age = ct->world_age; + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + jl_value_t *depalias = (jl_value_t*)jl_apply(replace_depot_args, 2); + ct->world_age = last_age; + JL_GC_POP(); + + size_t slen = jl_string_len(depalias); write_int32(s, slen); - ios_write(s, jl_string_data(dep), slen); - write_float64(s, jl_unbox_float64(jl_fieldref(deptuple, 2))); // mtime + ios_write(s, jl_string_data(depalias), slen); + write_uint64(s, jl_unbox_uint64(jl_fieldref(deptuple, 2))); // fsize + write_uint32(s, jl_unbox_uint32(jl_fieldref(deptuple, 3))); // hash + write_float64(s, jl_unbox_float64(jl_fieldref(deptuple, 4))); // mtime jl_module_t *depmod = (jl_module_t*)jl_fieldref(deptuple, 0); // evaluating module jl_module_t *depmod_top = depmod; while (depmod_top->parent != jl_main_module && depmod_top->parent != depmod_top) diff --git a/test/.gitignore b/test/.gitignore index a1af9ae3d44bf..fc55a0df3a173 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -2,3 +2,5 @@ /ccalltest /ccalltest.s /libccalltest.* +/relocatedepot +/RelocationTestPkg2/src/foo.txt diff --git a/test/Makefile b/test/Makefile index 88dbe5b2b4ed6..d8605341ff9b9 100644 --- a/test/Makefile +++ b/test/Makefile @@ -14,7 +14,7 @@ unexport JULIA_BINDIR := TESTGROUPS = unicode strings compiler TESTS = all default stdlib $(TESTGROUPS) \ $(patsubst $(STDLIBDIR)/%/,%,$(dir $(wildcard $(STDLIBDIR)/*/.))) \ - $(filter-out runtests testdefs, \ + $(filter-out runtests testdefs relocatedepot, \ $(patsubst $(SRCDIR)/%.jl,%,$(wildcard $(SRCDIR)/*.jl))) \ $(foreach group,$(TESTGROUPS), \ $(patsubst $(SRCDIR)/%.jl,%,$(wildcard $(SRCDIR)/$(group)/*.jl))) @@ -34,6 +34,28 @@ $(addprefix revise-, $(TESTS)): revise-% : @cd $(SRCDIR) && \ $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) +relocatedepot: + @rm -rf $(SRCDIR)/relocatedepot + @cd $(SRCDIR) && \ + $(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) + @mkdir $(SRCDIR)/relocatedepot + @cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot + @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot + @cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot + @cd $(SRCDIR) && \ + $(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading RELOCATEDEPOT="" JULIA_DEPOT_PATH=$(SRCDIR)/relocatedepot/julia $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) + +revise-relocatedepot: revise-% : + @rm -rf $(SRCDIR)/relocatedepot + @cd $(SRCDIR) && \ + $(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) + @mkdir $(SRCDIR)/relocatedepot + @cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot + @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot + @cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot + @cd $(SRCDIR) && \ + $(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading RELOCATEDEPOT="" JULIA_DEPOT_PATH=$(SRCDIR)/relocatedepot/julia $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) + embedding: @$(MAKE) -C $(SRCDIR)/$@ check $(EMBEDDING_ARGS) @@ -47,4 +69,4 @@ clean: @$(MAKE) -C embedding $@ $(EMBEDDING_ARGS) @$(MAKE) -C gcext $@ $(GCEXT_ARGS) -.PHONY: $(TESTS) $(addprefix revise-, $(TESTS)) embedding gcext clangsa clean +.PHONY: $(TESTS) $(addprefix revise-, $(TESTS)) relocatedepot revise-relocatedepot embedding gcext clangsa clean diff --git a/test/RelocationTestPkg1/Project.toml b/test/RelocationTestPkg1/Project.toml new file mode 100644 index 0000000000000..826980207d508 --- /dev/null +++ b/test/RelocationTestPkg1/Project.toml @@ -0,0 +1,4 @@ +name = "RelocationTestPkg1" +uuid = "854e1adb-5a97-46bf-a391-1cfe05ac726d" +authors = ["flo "] +version = "0.1.0" diff --git a/test/RelocationTestPkg1/src/RelocationTestPkg1.jl b/test/RelocationTestPkg1/src/RelocationTestPkg1.jl new file mode 100644 index 0000000000000..a86543a61b3f8 --- /dev/null +++ b/test/RelocationTestPkg1/src/RelocationTestPkg1.jl @@ -0,0 +1,5 @@ +module RelocationTestPkg1 + +greet() = print("Hello World!") + +end # module RelocationTestPkg1 diff --git a/test/RelocationTestPkg1/src/foo.txt b/test/RelocationTestPkg1/src/foo.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test/RelocationTestPkg2/Project.toml b/test/RelocationTestPkg2/Project.toml new file mode 100644 index 0000000000000..68da889785215 --- /dev/null +++ b/test/RelocationTestPkg2/Project.toml @@ -0,0 +1,4 @@ +name = "RelocationTestPkg2" +uuid = "8d933983-b090-4b0b-a37e-c34793f459d1" +authors = ["flo "] +version = "0.1.0" diff --git a/test/RelocationTestPkg2/src/RelocationTestPkg2.jl b/test/RelocationTestPkg2/src/RelocationTestPkg2.jl new file mode 100644 index 0000000000000..0d8b5e15edf06 --- /dev/null +++ b/test/RelocationTestPkg2/src/RelocationTestPkg2.jl @@ -0,0 +1,6 @@ +module RelocationTestPkg2 + +include_dependency("foo.txt") +greet() = print("Hello World!") + +end # module RelocationTestPkg2 diff --git a/test/RelocationTestPkg2/src/foo.txt b/test/RelocationTestPkg2/src/foo.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test/choosetests.jl b/test/choosetests.jl index 2f77b11767dee..221a49b710d8b 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -24,7 +24,7 @@ const TESTNAMES = [ "some", "meta", "stacktraces", "docs", "gc", "misc", "threads", "stress", "binaryplatforms", "atexit", "enums", "cmdlineargs", "int", "interpreter", - "checked", "bitset", "floatfuncs", "precompile", + "checked", "bitset", "floatfuncs", "precompile", "relocatedepot", "boundscheck", "error", "ambiguous", "cartesian", "osutils", "channels", "iostream", "secretbuffer", "specificity", "reinterpretarray", "syntax", "corelogging", "missing", "asyncmap", diff --git a/test/precompile.jl b/test/precompile.jl index 671535c360625..fc4ab2490c4a8 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -381,8 +381,8 @@ precompile_test_harness(false) do dir @test string(Base.Docs.doc(Foo.Bar.bar)) == "bar function\n" @test string(Base.Docs.doc(Foo.Bar)) == "Bar module\n" - modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) - discard_module = mod_fl_mt -> (mod_fl_mt.filename, mod_fl_mt.mtime) + modules, (deps, _, requires), required_modules, _... = Base.parse_cache_header(cachefile) + discard_module = mod_fl_mt -> mod_fl_mt.filename @test modules == [ Base.PkgId(Foo) => Base.module_build_id(Foo) % UInt64 ] @test map(x -> x.filename, deps) == [ Foo_file, joinpath(dir, "foo.jl"), joinpath(dir, "bar.jl") ] @test requires == [ Base.PkgId(Foo) => Base.PkgId(string(FooBase_module)), @@ -422,7 +422,7 @@ precompile_test_harness(false) do dir @test Dict(modules) == modules_ok @test discard_module.(deps) == deps1 - modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile; srcfiles_only=true) + modules, (_, deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) @test map(x -> x.filename, deps) == [Foo_file] @test current_task()(0x01, 0x4000, 0x30031234) == 2 @@ -485,7 +485,7 @@ precompile_test_harness(false) do dir """) Nest = Base.require(Main, Nest_module) cachefile = joinpath(cachedir, "$Nest_module.ji") - modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) + modules, (deps, _, requires), required_modules, _... = Base.parse_cache_header(cachefile) @test last(deps).modpath == ["NestInner"] UsesB_module = :UsesB4b3a94a1a081a8cb @@ -507,7 +507,7 @@ precompile_test_harness(false) do dir """) UsesB = Base.require(Main, UsesB_module) cachefile = joinpath(cachedir, "$UsesB_module.ji") - modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) + modules, (deps, _, requires), required_modules, _... = Base.parse_cache_header(cachefile) id1, id2 = only(requires) @test Base.pkgorigins[id1].cachepath == cachefile @test Base.pkgorigins[id2].cachepath == joinpath(cachedir, "$B_module.ji") @@ -584,12 +584,12 @@ precompile_test_harness(false) do dir fb_uuid = Base.module_build_id(FooBar) sleep(2); touch(FooBar_file) insert!(DEPOT_PATH, 1, dir2) - @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) === true + @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc @eval using FooBar1 @test !isfile(joinpath(cachedir2, "FooBar.ji")) @test !isfile(joinpath(cachedir, "FooBar1.ji")) @test isfile(joinpath(cachedir2, "FooBar1.ji")) - @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) === true + @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc @test Base.stale_cachefile(FooBar1_file, joinpath(cachedir2, "FooBar1.ji")) isa Tsc @test fb_uuid == Base.module_build_id(FooBar) fb_uuid1 = Base.module_build_id(FooBar1) @@ -1712,7 +1712,7 @@ precompile_test_harness("PkgCacheInspector") do load_path try # isvalid_cache_header returns checksum id or zero Base.isvalid_cache_header(io) == 0 && throw(ArgumentError("Invalid header in cache file $cachefile.")) - depmodnames = Base.parse_cache_header(io)[3] + depmodnames = Base.parse_cache_header(io, cachefile)[3] Base.isvalid_file_crc(io) || throw(ArgumentError("Invalid checksum in cache file $cachefile.")) finally close(io) diff --git a/test/relocatedepot.jl b/test/relocatedepot.jl new file mode 100644 index 0000000000000..91f03d59cc168 --- /dev/null +++ b/test/relocatedepot.jl @@ -0,0 +1,104 @@ +using Test +using Logging + + +include("testenv.jl") + + +function test_harness(@nospecialize(fn)) + load_path = copy(LOAD_PATH) + depot_path = copy(DEPOT_PATH) + try + fn() + finally + copy!(LOAD_PATH, load_path) + copy!(DEPOT_PATH, depot_path) + end +end + + +if !test_relocated_depot + + @testset "precompile RelocationTestPkg1" begin + pkgname = "RelocationTestPkg1" + test_harness() do + push!(LOAD_PATH, @__DIR__) + push!(DEPOT_PATH, @__DIR__) + pkg = Base.identify_package(pkgname) + cachefiles = Base.find_all_in_cache_path(pkg) + rm.(cachefiles, force=true) + @test Base.isprecompiled(pkg) == false + Base.require(pkg) # precompile + @test Base.isprecompiled(pkg, ignore_loaded=true) == true + end + end + + @testset "precompile RelocationTestPkg2 (contains include_dependency)" begin + pkgname = "RelocationTestPkg2" + test_harness() do + push!(LOAD_PATH, @__DIR__) + push!(DEPOT_PATH, @__DIR__) + pkg = Base.identify_package(pkgname) + cachefiles = Base.find_all_in_cache_path(pkg) + rm.(cachefiles, force=true) + @test Base.isprecompiled(pkg) == false + touch(joinpath(@__DIR__, pkgname, "src", "foo.txt")) + Base.require(pkg) # precompile + @info "SERS OIDA" + @test Base.isprecompiled(pkg, ignore_loaded=true) == true + end + end + +else + + # must come before any of the load tests, because the will recompile and generate new cache files + @testset "attempt loading precompiled pkgs when depot is missing" begin + test_harness() do + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) + for pkgname in ("RelocationTestPkg1", "RelocationTestPkg2") + pkg = Base.identify_package(pkgname) + cachefile = only(Base.find_all_in_cache_path(pkg)) + @info cachefile + @test_throws ArgumentError(""" + Failed to determine depot from srctext files in cache file $cachefile. + - Make sure you have adjusted DEPOT_PATH in case you relocated depots.""") Base.isprecompiled(pkg) + end + end + end + + @testset "load stdlib from test/relocatedepot" begin + test_harness() do + push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) + # stdlib should be already precompiled + pkg = Base.identify_package("DelimitedFiles") + @test Base.isprecompiled(pkg) == true + end + end + + @testset "load RelocationTestPkg1 from test/relocatedepot" begin + pkgname = "RelocationTestPkg1" + test_harness() do + push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) + pkg = Base.identify_package(pkgname) + @test Base.isprecompiled(pkg) == true + Base.require(pkg) # re-precompile + @test Base.isprecompiled(pkg) == true + end + end + + @testset "load RelocationTestPkg2 (contains include_dependency) from test/relocatedepot" begin + pkgname = "RelocationTestPkg2" + test_harness() do + push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) + pkg = Base.identify_package(pkgname) + @test Base.isprecompiled(pkg) == false # moving depot changes mtime of include_dependency + Base.require(pkg) # re-precompile + @test Base.isprecompiled(pkg) == true + end + end + +end diff --git a/test/testenv.jl b/test/testenv.jl index 6f99291e01138..3ef1126e0e927 100644 --- a/test/testenv.jl +++ b/test/testenv.jl @@ -35,6 +35,8 @@ if !@isdefined(testenv_defined) const rr_exename = `` end + const test_relocated_depot = haskey(ENV, "RELOCATEDEPOT") + function addprocs_with_testenv(X; rr_allowed=true, kwargs...) exename = rr_allowed ? `$rr_exename $test_exename` : test_exename if X isa Integer