Use historical stdlib dependency information (#3911)

* Use historical stdlib dependency information

This makes use of the new functionality in HistoricalStdlibVersions [0]
to properly pull out the dependency information in old stdlibs.  This
fixes JuliaPackaging/HistoricalStdlibVersions.jl#9.

[0] JuliaPackaging/HistoricalStdlibVersions.jl#23

ext/REPLExt/completions.jl

@@ -159,7 +159,7 @@ function complete_add_dev(options, partial, i1, i2; hint::Bool)
     end
     comps = vcat(comps, sort(complete_remote_package(partial; hint)))
     if !isempty(partial)
-        append!(comps, filter!(startswith(partial), first.(values(Types.stdlibs()))))
+        append!(comps, filter!(startswith(partial), [info.name for info in values(Types.stdlib_infos())]))
     end
     return comps, idx, !isempty(comps)
 end

src/HistoricalStdlibs.jl

@@ -1,6 +1,17 @@
 using Base: UUID
 
-const DictStdLibs = Dict{UUID,Tuple{String,Union{VersionNumber,Nothing}}}
+struct StdlibInfo
+    name::String
+    uuid::UUID
+
+    # This can be `nothing` if it's an unregistered stdlib
+    version::Union{Nothing,VersionNumber}
+
+    deps::Vector{UUID}
+    weakdeps::Vector{UUID}
+end
+
+const DictStdLibs = Dict{UUID,StdlibInfo}
 
 # Julia standard libraries with duplicate entries removed so as to store only the
 # first release in a set of releases that all contain the same set of stdlibs.
@@ -13,32 +24,4 @@ const STDLIBS_BY_VERSION = Pair{VersionNumber, DictStdLibs}[]
 # This is a list of stdlibs that must _always_ be treated as stdlibs,
 # because they cannot be resolved in the registry; they have only ever existed within
 # the Julia stdlib source tree, and because of that, trying to resolve them will fail.
-const UNREGISTERED_STDLIBS = DictStdLibs(
-    UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => ("Base64", nothing),
-    UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => ("CRC32c", nothing),
-    UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => ("Dates", nothing),
-    UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => ("Distributed", nothing),
-    UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => ("FileWatching", nothing),
-    UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => ("Future", nothing),
-    UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => ("InteractiveUtils", nothing),
-    UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => ("LibGit2", nothing),
-    UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => ("Libdl", nothing),
-    UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => ("LinearAlgebra", nothing),
-    UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => ("Logging", nothing),
-    UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => ("Markdown", nothing),
-    UUID("a63ad114-7e13-5084-954f-fe012c677804") => ("Mmap", nothing),
-    UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => ("Pkg", nothing),
-    UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => ("Printf", nothing),
-    UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => ("Profile", nothing),
-    UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => ("REPL", nothing),
-    UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => ("Random", nothing),
-    UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => ("Serialization", nothing),
-    UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => ("SharedArrays", nothing),
-    UUID("6462fe0b-24de-5631-8697-dd941f90decc") => ("Sockets", nothing),
-    UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => ("SparseArrays", nothing),
-    UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => ("Statistics", nothing),
-    UUID("4607b0f0-06f3-5cda-b6b1-a6196a1729e9") => ("SuiteSparse", nothing),
-    UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => ("Test", nothing),
-    UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => ("UUIDs", nothing),
-    UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => ("Unicode", nothing),
-)
+const UNREGISTERED_STDLIBS = DictStdLibs()

src/Operations.jl

@@ -452,7 +452,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn
         end
     end
 
-    names = Dict{UUID, String}(uuid => name for (uuid, (name, version)) in stdlibs())
+    names = Dict{UUID, String}(uuid => info.name for (uuid, info) in stdlib_infos())
     # recursive search for packages which are tracking a path
     developed = collect_developed(env, pkgs)
     # But we only want to use information for those packages that we don't know about
@@ -514,7 +514,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn
             # Fixed packages are not returned by resolve (they already have their version set)
             pkg.version = vers[pkg.uuid]
         else
-            name = is_stdlib(uuid) ? first(stdlibs()[uuid]) : registered_name(registries, uuid)
+            name = is_stdlib(uuid) ? stdlib_infos()[uuid].name : registered_name(registries, uuid)
             push!(pkgs, PackageSpec(;name=name, uuid=uuid, version=ver))
         end
     end
@@ -576,37 +576,26 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}
             uuid in keys(fixed) && continue
             all_compat_u = get_or_make!(all_compat, uuid)
             weak_compat_u = get_or_make!(weak_compat, uuid)
-
-            uuid_is_stdlib = false
-            stdlib_name = ""
-            stdlib_version = nothing
-            if haskey(stdlibs_for_julia_version, uuid)
-                uuid_is_stdlib = true
-                stdlib_name, stdlib_version = stdlibs_for_julia_version[uuid]
-            end
+            uuid_is_stdlib = haskey(stdlibs_for_julia_version, uuid)
 
             # If we're requesting resolution of a package that is an
             # unregistered stdlib we must special-case it here.  This is further
             # complicated by the fact that we can ask this question relative to
             # a Julia version.
             if (julia_version != VERSION && is_unregistered_stdlib(uuid)) || uuid_is_stdlib
-                path = Types.stdlib_path(stdlibs_for_julia_version[uuid][1])
-                proj_file = projectfile_path(path; strict=true)
-                @assert proj_file !== nothing
-                proj = read_package(proj_file)
-
-                v = something(proj.version, VERSION)
+                # We use our historical stdlib versioning data to unpack the version, deps and weakdeps of this uuid
+                stdlib_info = stdlibs_for_julia_version[uuid]
+                v = something(stdlib_info.version, VERSION)
 
-                # TODO look at compat section for stdlibs?
                 all_compat_u_vr = get_or_make!(all_compat_u, v)
-                for (_, other_uuid) in proj.deps
+                for other_uuid in stdlib_info.deps
                     push!(uuids, other_uuid)
                     all_compat_u_vr[other_uuid] = VersionSpec()
                 end
 
-                if !isempty(proj.weakdeps)
+                if !isempty(stdlib_info.weakdeps)
                     weak_all_compat_u_vr = get_or_make!(weak_compat_u, v)
-                    for (_, other_uuid) in proj.weakdeps
+                    for other_uuid in stdlib_info.weakdeps
                         push!(uuids, other_uuid)
                         all_compat_u_vr[other_uuid] = VersionSpec()
                         push!(weak_all_compat_u_vr, other_uuid)

src/Types.jl

@@ -21,7 +21,7 @@ using SHA
 export UUID, SHA1, VersionRange, VersionSpec,
     PackageSpec, PackageEntry, EnvCache, Context, GitRepo, Context!, Manifest, Project, err_rep,
     PkgError, pkgerror,
-    has_name, has_uuid, is_stdlib, is_or_was_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, write_env, write_env_usage, parse_toml,
+    has_name, has_uuid, is_stdlib, is_or_was_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, stdlib_infos, write_env, write_env_usage, parse_toml,
     project_resolve!, project_deps_resolve!, manifest_resolve!, registry_resolve!, stdlib_resolve!, handle_repos_develop!, handle_repos_add!, ensure_resolved,
     registered_name,
     manifest_info,
@@ -480,35 +480,47 @@ function load_stdlib()
             push!(FORMER_STDLIBS_UUIDS, UUID(uuid))
             continue
         end
-        stdlib[UUID(uuid)] = (name, version)
+        deps = UUID.(values(get(project, "deps", Dict{String,Any}())))
+        weakdeps = UUID.(values(get(project, "weakdeps", Dict{String,Any}())))
+        stdlib[UUID(uuid)] = StdlibInfo(name, Base.UUID(uuid), version, deps, weakdeps)
     end
     return stdlib
 end
 
 function stdlibs()
+    # This maintains a compatible format for `stdlibs()`, but new code should always
+    # prefer `stdlib_infos` as it is more future-proofed, by returning a structure
+    # rather than a tuple of elements.
+    return Dict(uuid => (info.name, info.version) for (uuid, info) in stdlib_infos())
+end
+function stdlib_infos()
     if !isassigned(STDLIB)
         STDLIB[] = load_stdlib()
     end
     return STDLIB[]
 end
-is_stdlib(uuid::UUID) = uuid in keys(stdlibs())
+is_stdlib(uuid::UUID) = uuid in keys(stdlib_infos())
 # Includes former stdlibs
 function is_or_was_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing})
     return is_stdlib(uuid, julia_version) || uuid in FORMER_STDLIBS_UUIDS
 end
 
 
+function historical_stdlibs_check()
+    if isempty(STDLIBS_BY_VERSION)
+        pkgerror("If you want to set `julia_version`, you must first populate the `STDLIBS_BY_VERSION` global constant.  Try `using HistoricalStdlibVersions`")
+    end
+end
+
 # Find the entry in `STDLIBS_BY_VERSION`
 # that corresponds to the requested version, and use that.
 # If we can't find one, defaults to `UNREGISTERED_STDLIBS`
 function get_last_stdlibs(julia_version::VersionNumber; use_historical_for_current_version = false)
     if !use_historical_for_current_version && julia_version == VERSION
-        return stdlibs()
+        return stdlib_infos()
     end
+    historical_stdlibs_check()
     last_stdlibs = UNREGISTERED_STDLIBS
-    if isempty(STDLIBS_BY_VERSION)
-        pkgerror("If you want to set `julia_version`, you must first populate the `STDLIBS_BY_VERSION` global constant")
-    end
     for (version, stdlibs) in STDLIBS_BY_VERSION
         if VersionNumber(julia_version.major, julia_version.minor, julia_version.patch) < version
             break
@@ -520,7 +532,10 @@ end
 # If `julia_version` is set to `nothing`, that means (essentially) treat all registered
 # stdlibs as normal packages so that we get the latest versions of everything, ignoring
 # julia compat.  So we set the list of stdlibs to that of only the unregistered stdlibs.
-get_last_stdlibs(::Nothing) = UNREGISTERED_STDLIBS
+function get_last_stdlibs(::Nothing)
+    historical_stdlibs_check()
+    return UNREGISTERED_STDLIBS
+end
 
 # Allow asking if something is an stdlib for a particular version of Julia
 function is_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing})
@@ -544,10 +559,13 @@ function stdlib_version(uuid::UUID, julia_version::Union{VersionNumber,Nothing})
     if !(uuid in keys(last_stdlibs))
         return nothing
     end
-    return last_stdlibs[uuid][2]
+    return last_stdlibs[uuid].version
 end
 
-is_unregistered_stdlib(uuid::UUID) = haskey(UNREGISTERED_STDLIBS, uuid)
+function is_unregistered_stdlib(uuid::UUID)
+    historical_stdlibs_check()
+    return haskey(UNREGISTERED_STDLIBS, uuid)
+end
 
 Context!(kw_context::Vector{Pair{Symbol,Any}})::Context =
     Context!(Context(); kw_context...)
@@ -1044,13 +1062,17 @@ function stdlib_resolve!(pkgs::AbstractVector{PackageSpec})
     for pkg in pkgs
         @assert has_name(pkg) || has_uuid(pkg)
         if has_name(pkg) && !has_uuid(pkg)
-            for (uuid, (name, version)) in stdlibs()
-                name == pkg.name && (pkg.uuid = uuid)
+            for (uuid, info) in stdlib_infos()
+                if info.name == pkg.name
+                    pkg.uuid = uuid
+                end
             end
         end
         if !has_name(pkg) && has_uuid(pkg)
-            name, version = get(stdlibs(), pkg.uuid, (nothing, nothing))
-            nothing !== name && (pkg.name = name)
+            info = get(stdlib_infos(), pkg.uuid, nothing)
+            if info !== nothing
+                pkg.name = info.name
+            end
         end
     end
 end

test/Project.toml

@@ -15,4 +15,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
-HistoricalStdlibVersions = "1.2"
+HistoricalStdlibVersions = "2"

test/new.jl

@@ -2999,38 +2999,37 @@ end
 
 using Pkg.Types: is_stdlib
 @testset "is_stdlib() across versions" begin
-    append!(empty!(Pkg.Types.STDLIBS_BY_VERSION), HistoricalStdlibVersions.STDLIBS_BY_VERSION)
+    HistoricalStdlibVersions.register!()
 
     networkoptions_uuid = UUID("ca575930-c2e3-43a9-ace4-1e988b2c1908")
     pkg_uuid = UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f")
 
-    # Assume we're running on v1.6+
+    # Test NetworkOptions across multiple versions (It became an stdlib in v1.6+, and was registered)
     @test is_stdlib(networkoptions_uuid)
     @test is_stdlib(networkoptions_uuid, v"1.6")
     @test !is_stdlib(networkoptions_uuid, v"1.5")
     @test !is_stdlib(networkoptions_uuid, v"1.0.0")
     @test !is_stdlib(networkoptions_uuid, v"0.7")
     @test !is_stdlib(networkoptions_uuid, nothing)
 
-    # Pkg is an unregistered stdlib
+    # Pkg is an unregistered stdlib and has always been an stdlib
     @test is_stdlib(pkg_uuid)
     @test is_stdlib(pkg_uuid, v"1.0")
     @test is_stdlib(pkg_uuid, v"1.6")
     @test is_stdlib(pkg_uuid, v"999.999.999")
     @test is_stdlib(pkg_uuid, v"0.7")
     @test is_stdlib(pkg_uuid, nothing)
 
-    empty!(Pkg.Types.STDLIBS_BY_VERSION)
-
+    HistoricalStdlibVersions.unregister!()
     # Test that we can probe for stdlibs for the current version with no STDLIBS_BY_VERSION,
     # but that we throw a PkgError if we ask for a particular julia version.
     @test is_stdlib(networkoptions_uuid)
     @test_throws Pkg.Types.PkgError is_stdlib(networkoptions_uuid, v"1.6")
 end
 
-#=
+
 @testset "Pkg.add() with julia_version" begin
-    append!(empty!(Pkg.Types.STDLIBS_BY_VERSION), HistoricalStdlibVersions.STDLIBS_BY_VERSION)
+    HistoricalStdlibVersions.register!()
 
     # A package with artifacts that went from normal package -> stdlib
     gmp_jll_uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d"
@@ -3131,9 +3130,8 @@ end
         @test !("Pkg" in keys(Pkg.dependencies()[p7zip_jll_uuid].dependencies))
     end
 
-    empty!(Pkg.Types.STDLIBS_BY_VERSION)
+    HistoricalStdlibVersions.unregister!()
 end
-=#
 
 
 @testset "Issue #2931" begin

test/resolve.jl

@@ -705,9 +705,8 @@ end
     @test_throws ResolverError resolve_tst(deps_data, reqs_data)
 end
 
-#=
 @testset "Resolving for another version of Julia" begin
-    append!(empty!(Pkg.Types.STDLIBS_BY_VERSION), HistoricalStdlibVersions.STDLIBS_BY_VERSION)
+    HistoricalStdlibVersions.register!()
     temp_pkg_dir() do dir
         function find_by_name(versions, name)
             idx = findfirst(p -> p.name == name, versions)
@@ -750,15 +749,14 @@ end
         @test mpfr !== nothing
         @test mpfr.version.major == 4 && mpfr.version.minor == 0
     end
-    empty!(Pkg.Types.STDLIBS_BY_VERSION)
+    HistoricalStdlibVersions.unregister!()
 end
-=#
 
 @testset "Stdlib resolve smoketest" begin
     # All stdlibs should be installable and resolvable
     temp_pkg_dir() do dir
         Pkg.activate(temp=true)
-        Pkg.add(map(first, values(Pkg.Types.load_stdlib())))    # add all stdlibs
+        Pkg.add(map(x -> x.name, values(Pkg.Types.load_stdlib())))    # add all stdlibs
         iob = IOBuffer()
         Pkg.resolve(io = iob)
         str = String(take!(iob))

test/runtests.jl

@@ -20,6 +20,7 @@ if realpath(dirname(dirname(Base.pathof(Pkg)))) != realpath(dirname(@__DIR__))
 end
 
 ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0
+ENV["HISTORICAL_STDLIB_VERSIONS_AUTO_REGISTER"]="false"
 
 logdir = get(ENV, "JULIA_TEST_VERBOSE_LOGS_DIR", nothing)
 ### Send all Pkg output to a file called Pkg.log