diff --git a/NEWS.md b/NEWS.md index d88bd5b9a6d190..9942996ef46338 100644 --- a/NEWS.md +++ b/NEWS.md @@ -102,6 +102,9 @@ Standard library changes * The `Pkg.BinaryPlatforms` module has been moved into `Base` as `Base.BinaryPlatforms` and heavily reworked. Applications that want to be compatible with the old API should continue to import `Pkg.BinaryPlatforms`, however new users should use `Base.BinaryPlatforms` directly. ([#37320]) +* The `Pkg.Artifacts` module has been imported as a separate standard library. It is still available as + `Pkg.Artifacts`, however starting from Julia v1.6+, packages may import simply `Artifacts` without importing + all of `Pkg` alongside. ([#37320]) #### LinearAlgebra * New method `LinearAlgebra.issuccess(::CholeskyPivoted)` for checking whether pivoted Cholesky factorization was successful ([#36002]). diff --git a/base/sysimg.jl b/base/sysimg.jl index a2e5226baf532d..85d0ac5a76b424 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -47,6 +47,7 @@ let :Distributed, :SharedArrays, :TOML, + :Artifacts, :Pkg, :Test, :REPL, diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 3418265ca35254..fa0e4b442fa4b7 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -65,6 +65,21 @@ if Distributed !== nothing """ end +Artifacts = get(Base.loaded_modules, + Base.PkgId(Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts"), + nothing) +if Artifacts !== nothing + precompile_script *= """ + using Artifacts, Base.BinaryPlatforms + artifacts_toml = abspath($(repr(joinpath(Sys.STDLIB, "Artifacts", "test", "Artifacts.toml")))) + cd(() -> @artifact_str("c_simple"), dirname(artifacts_toml)) + artifacts = Artifacts.load_artifacts_toml(artifacts_toml) + platforms = [Artifacts.unpack_platform(e, "c_simple", artifacts_toml) for e in artifacts["c_simple"]] + best_platform = select_platform(Dict(p => triplet(p) for p in platforms)) + """ +end + + Pkg = get(Base.loaded_modules, Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg"), nothing) diff --git a/stdlib/Artifacts/Project.toml b/stdlib/Artifacts/Project.toml new file mode 100644 index 00000000000000..7251b79cea8c17 --- /dev/null +++ b/stdlib/Artifacts/Project.toml @@ -0,0 +1,8 @@ +name = "Artifacts" +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/stdlib/Artifacts/src/Artifacts.jl b/stdlib/Artifacts/src/Artifacts.jl new file mode 100644 index 00000000000000..723e3991452035 --- /dev/null +++ b/stdlib/Artifacts/src/Artifacts.jl @@ -0,0 +1,633 @@ +module Artifacts + +import Base: get, SHA1 +using Base.BinaryPlatforms, Base.TOML + +export artifact_exists, artifact_path, artifact_meta, artifact_hash, + find_artifacts_toml, @artifact_str + +""" + parse_toml(path::String) + +Uses Base.TOML to parse a TOML file +""" +function parse_toml(path::String) + p = Base.TOML.Parser() + Base.TOML.reinit!(p, read(path, String); filepath=path) + return Base.TOML.parse(p) +end + +# keep in sync with Base.project_names and Base.manifest_names +const artifact_names = ("JuliaArtifacts.toml", "Artifacts.toml") + +const ARTIFACTS_DIR_OVERRIDE = Ref{Union{String,Nothing}}(nothing) +""" + with_artifacts_directory(f::Function, artifacts_dir::String) + +Helper function to allow temporarily changing the artifact installation and search +directory. When this is set, no other directory will be searched for artifacts, and new +artifacts will be installed within this directory. Similarly, removing an artifact will +only effect the given artifact directory. To layer artifact installation locations, use +the typical Julia depot path mechanism. +""" +function with_artifacts_directory(f::Function, artifacts_dir::String) + try + ARTIFACTS_DIR_OVERRIDE[] = artifacts_dir + f() + finally + ARTIFACTS_DIR_OVERRIDE[] = nothing + end +end + +""" + artifacts_dirs(args...) + +Return a list of paths joined into all possible artifacts directories, as dictated by the +current set of depot paths and the current artifact directory override via the method +`with_artifacts_dir()`. +""" +function artifacts_dirs(args...) + if ARTIFACTS_DIR_OVERRIDE[] === nothing + return String[abspath(depot, "artifacts", args...) for depot in Base.DEPOT_PATH] + else + # If we've been given an override, use _only_ that directory. + return String[abspath(ARTIFACTS_DIR_OVERRIDE[], args...)] + end +end + +""" + ARTIFACT_OVERRIDES + +Artifact locations can be overridden by writing `Override.toml` files within the artifact +directories of Pkg depots. For example, in the default depot `~/.julia`, one may create +a `~/.julia/artifacts/Override.toml` file with the following contents: + + 78f35e74ff113f02274ce60dab6e92b4546ef806 = "/path/to/replacement" + c76f8cda85f83a06d17de6c57aabf9e294eb2537 = "fb886e813a4aed4147d5979fcdf27457d20aa35d" + + [d57dbccd-ca19-4d82-b9b8-9d660942965b] + c_simple = "/path/to/c_simple_dir" + libfoo = "fb886e813a4aed4147d5979fcdf27457d20aa35d"" + +This file defines four overrides; two which override specific artifacts identified +through their content hashes, two which override artifacts based on their bound names +within a particular package's UUID. In both cases, there are two different targets of +the override: overriding to an on-disk location through an absolutet path, and +overriding to another artifact by its content-hash. +""" +const ARTIFACT_OVERRIDES = Ref{Union{Dict{Symbol,Any},Nothing}}(nothing) +function load_overrides(;force::Bool = false) + if ARTIFACT_OVERRIDES[] !== nothing && !force + return ARTIFACT_OVERRIDES[] + end + + # We organize our artifact location overrides into two camps: + # - overrides per UUID with artifact names mapped to a new location + # - overrides per hash, mapped to a new location. + # + # Overrides per UUID/bound name are intercepted upon Artifacts.toml load, and new + # entries within the "hash" overrides are generated on-the-fly. Thus, all redirects + # mechanisticly happen through the "hash" overrides. + overrides = Dict{Symbol,Any}( + # Overrides by UUID + :UUID => Dict{Base.UUID,Dict{String,Union{String,SHA1}}}(), + + # Overrides by hash + :hash => Dict{SHA1,Union{String,SHA1}}(), + ) + + for override_file in reverse(artifacts_dirs("Overrides.toml")) + !isfile(override_file) && continue + + # Load the toml file + depot_override_dict = parse_toml(override_file) + + function parse_mapping(mapping::String, name::String) + if !isabspath(mapping) && !isempty(mapping) + mapping = tryparse(Base.SHA1, mapping) + if mapping === nothing + @error("Invalid override in '$(override_file)': entry '$(name)' must map to an absolute path or SHA1 hash!") + end + end + return mapping + end + function parse_mapping(mapping::Dict, name::String) + return Dict(k => parse_mapping(v, name) for (k, v) in mapping) + end + # Fallthrough for invalid Overrides.toml files + parse_mapping(mapping, name::String) = nothing + + for (k, mapping) in depot_override_dict + # First, parse the mapping. Is it an absolute path, a valid SHA1-hash, or neither? + mapping = parse_mapping(mapping, k) + if mapping === nothing + @error("Invalid override in '$(override_file)': failed to parse entry `$(k)`") + continue + end + + # Next, determine if this is a hash override or a UUID/name override + if isa(mapping, String) || isa(mapping, SHA1) + # if this mapping is a direct mapping (e.g. a String), store it as a hash override + local hash_str + hash = tryparse(Base.SHA1, k) + if hash === nothing + @error("Invalid override in '$(override_file)': Invalid SHA1 hash '$(k)'") + continue + end + + # If this mapping is the empty string, un-override it + if mapping == "" + delete!(overrides[:hash], hash) + else + overrides[:hash][hash] = mapping + end + elseif isa(mapping, Dict) + # Convert `k` into a uuid + uuid = tryparse(Base.UUID, k) + if uuid === nothing + @error("Invalid override in '$(override_file)': Invalid UUID '$(k)'") + continue + end + + # If this mapping is itself a dict, store it as a set of UUID/artifact name overrides + ovruuid = overrides[:UUID]::Dict{Base.UUID,Dict{String,Union{String,SHA1}}} + if !haskey(ovruuid, uuid) + ovruuid[uuid] = Dict{String,Union{String,SHA1}}() + end + + # For each name in the mapping, update appropriately + for (name, override_value) in mapping + # If the mapping for this name is the empty string, un-override it + if override_value == "" + delete!(ovruuid[uuid], name) + else + # Otherwise, store it! + ovruuid[uuid][name] = override_value + end + end + else + @error("Invalid override in '$(override_file)': unknown mapping type for '$(k)': $(typeof(mapping))") + end + end + end + + ARTIFACT_OVERRIDES[] = overrides +end + +# Helpers to map an override to an actual path +map_override_path(x::String) = x +map_override_path(x::AbstractString) = string(x) +map_override_path(x::SHA1) = artifact_path(x) +map_override_path(x::Nothing) = nothing + +""" + query_override(hash::SHA1; overrides::Dict = load_overrides()) + +Query the loaded `/artifacts/Overrides.toml` settings for artifacts that should be +redirected to a particular path or another content-hash. +""" +function query_override(hash::SHA1; overrides::Dict = load_overrides()) + return map_override_path(get(overrides[:hash], hash, nothing)) +end +function query_override(pkg::Base.UUID, artifact_name::String; overrides::Dict = load_overrides()) + if haskey(overrides[:UUID], pkg) + return map_override_path(get(overrides[:UUID][pkg], artifact_name, nothing)) + end + return nothing +end + +""" + artifact_paths(hash::SHA1; honor_overrides::Bool=true) + +Return all possible paths for an artifact given the current list of depots as returned +by `Pkg.depots()`. All, some or none of these paths may exist on disk. +""" +function artifact_paths(hash::SHA1; honor_overrides::Bool=true) + # First, check to see if we've got an override: + if honor_overrides + override = query_override(hash) + if override !== nothing + return [override] + end + end + + return artifacts_dirs(bytes2hex(hash.bytes)) +end + +""" + artifact_path(hash::SHA1; honor_overrides::Bool=true) + +Given an artifact (identified by SHA1 git tree hash), return its installation path. If +the artifact does not exist, returns the location it would be installed to. + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function artifact_path(hash::SHA1; honor_overrides::Bool=true) + # Get all possible paths (rooted in all depots) + possible_paths = artifact_paths(hash; honor_overrides=honor_overrides) + + # Find the first path that exists and return it + for p in possible_paths + if isdir(p) + return p + end + end + + # If none exist, then just return the one that would exist within `depots1()`. + return first(possible_paths) +end + +""" + artifact_exists(hash::SHA1; honor_overrides::Bool=true) + +Returns whether or not the given artifact (identified by its sha1 git tree hash) exists +on-disk. Note that it is possible that the given artifact exists in multiple locations +(e.g. within multiple depots). + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function artifact_exists(hash::SHA1; honor_overrides::Bool=true) + return any(isdir, artifact_paths(hash; honor_overrides=honor_overrides)) +end + +""" + unpack_platform(entry::Dict, name::String, artifacts_toml::String) + +Given an `entry` for the artifact named `name`, located within the file `artifacts_toml`, +returns the `Platform` object that this entry specifies. Returns `nothing` on error. +""" +function unpack_platform(entry::Dict, name::String, artifacts_toml::String)::Union{Nothing,Platform} + if !haskey(entry, "os") + @error("Invalid artifacts file at '$(artifacts_toml)': platform-specific artifact entry '$name' missing 'os' key") + return nothing + end + + if !haskey(entry, "arch") + @error("Invalid artifacts file at '$(artifacts_toml)': platform-specific artifact entrty '$name' missing 'arch' key") + return nothing + end + + # Collect all String-valued mappings in `entry` and use them as tags + tags = Dict(Symbol(k) => v for (k, v) in entry if isa(v, String)) + # Removing some known entries that shouldn't be passed through `tags` + delete!(tags, :os) + delete!(tags, :arch) + delete!(tags, Symbol("git-tree-sha1")) + return Platform(entry["arch"], entry["os"]; tags...) +end + +function pack_platform!(meta::Dict, p::AbstractPlatform) + for (k, v) in tags(p) + if v !== nothing + meta[k] = v + end + end + return meta +end + +""" + load_artifacts_toml(artifacts_toml::String; + pkg_uuid::Union{UUID,Nothing}=nothing) + +Loads an `(Julia)Artifacts.toml` file from disk. If `pkg_uuid` is set to the `UUID` of the +owning package, UUID/name overrides stored in a depot `Overrides.toml` will be resolved. +""" +function load_artifacts_toml(artifacts_toml::String; + pkg_uuid::Union{Base.UUID,Nothing} = nothing) + artifact_dict = parse_toml(artifacts_toml) + + # Process overrides for this `pkg_uuid` + process_overrides(artifact_dict, pkg_uuid) + return artifact_dict +end + +""" + process_overrides(artifact_dict::Dict, pkg_uuid::Base.UUID) + +When loading an `Artifacts.toml` file, we must check `Override.toml` files to see if any +of the artifacts within it have been overridden by UUID. If they have, we honor the +overrides by inspecting the hashes of the targeted artifacts, then overriding them to +point to the given override, punting the actual redirection off to the hash-based +override system. This does not modify the `artifact_dict` object, it merely dynamically +adds more hash-based overrides as `Artifacts.toml` files that are overridden are loaded. +""" +function process_overrides(artifact_dict::Dict, pkg_uuid::Base.UUID) + # Insert just-in-time hash overrides by looking up the names of anything we need to + # override for this UUID, and inserting new overrides for those hashes. + overrides = load_overrides() + if haskey(overrides[:UUID], pkg_uuid) + pkg_overrides = overrides[:UUID][pkg_uuid] + + for name in keys(artifact_dict) + # Skip names that we're not overriding + if !haskey(pkg_overrides, name) + continue + end + + # If we've got a platform-specific friend, override all hashes: + if isa(artifact_dict[name], Array) + for entry in artifact_dict[name] + hash = SHA1(entry["git-tree-sha1"]) + overrides[:hash][hash] = overrides[:UUID][pkg_uuid][name] + end + elseif isa(artifact_dict[name], Dict) + hash = SHA1(artifact_dict[name]["git-tree-sha1"]) + overrides[:hash][hash] = overrides[:UUID][pkg_uuid][name] + end + end + end + return artifact_dict +end + +# If someone tries to call process_overrides() with `nothing`, do exactly that +process_overrides(artifact_dict::Dict, pkg_uuid::Nothing) = nothing + +""" + artifact_meta(name::String, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Base.UUID,Nothing}=nothing) + +Get metadata about a given artifact (identified by name) stored within the given +`(Julia)Artifacts.toml` file. If the artifact is platform-specific, use `platform` to choose the +most appropriate mapping. If none is found, return `nothing`. + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function artifact_meta(name::String, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Base.UUID,Nothing}=nothing) + if !isfile(artifacts_toml) + return nothing + end + + # Parse the toml of the artifacts_toml file + artifact_dict = load_artifacts_toml(artifacts_toml; pkg_uuid=pkg_uuid) + return artifact_meta(name, artifact_dict, artifacts_toml; platform=platform) +end + +function artifact_meta(name::String, artifact_dict::Dict, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform()) + if !haskey(artifact_dict, name) + return nothing + end + meta = artifact_dict[name] + + # If it's an array, find the entry that best matches our current platform + if isa(meta, Array) + dl_dict = Dict{AbstractPlatform,Dict{String,Any}}(unpack_platform(x, name, artifacts_toml) => x for x in meta) + meta = select_platform(dl_dict, platform) + # If it's NOT a dict, complain + elseif !isa(meta, Dict) + @error("Invalid artifacts file at $(artifacts_toml): artifact '$name' malformed, must be array or dict!") + return nothing + end + + # This is such a no-no, we are going to call it out right here, right now. + if meta !== nothing && !haskey(meta, "git-tree-sha1") + @error("Invalid artifacts file at $(artifacts_toml): artifact '$name' contains no `git-tree-sha1`!") + return nothing + end + + # Return the full meta-dict. + return meta +end + +""" + artifact_hash(name::String, artifacts_toml::String; + platform::AbstractPlatform = platform_key_abi()) + +Thin wrapper around `artifact_meta()` to return the hash of the specified, platform- +collapsed artifact. Returns `nothing` if no mapping can be found. + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function artifact_hash(name::String, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Base.UUID,Nothing}=nothing) + meta = artifact_meta(name, artifacts_toml; platform=platform) + if meta === nothing + return nothing + end + + return SHA1(meta["git-tree-sha1"]) +end + +""" + find_artifacts_toml(path::String) + +Given the path to a `.jl` file, (such as the one returned by `__source__.file` in a macro +context), find the `(Julia)Artifacts.toml` that is contained within the containing project (if it +exists), otherwise return `nothing`. + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function find_artifacts_toml(path::String) + if !isdir(path) + path = dirname(path) + end + + # Run until we hit the root directory. + while dirname(path) != path + for f in artifact_names + artifacts_toml_path = joinpath(path, f) + if isfile(artifacts_toml_path) + return abspath(artifacts_toml_path) + end + end + + # Does a `(Julia)Project.toml` file exist here, in the absence of an Artifacts.toml? + # If so, stop the search as we've probably hit the top-level of this package, + # and we don't want to escape out into the larger filesystem. + for f in Base.project_names + if isfile(joinpath(path, f)) + return nothing + end + end + + # Move up a directory + path = dirname(path) + end + + # We never found anything, just return `nothing` + return nothing +end + +# We do this to avoid doing the `joinpath()` work if we don't have to, and also to +# avoid a trailing slash due to `joinpath()`'s habit of including one when the last +# argument is the empty string. +function jointail(dir, tail) + if !isempty(tail) + return joinpath(dir, tail) + else + return dir + end +end + +function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash) + if haskey(Base.module_keys, __module__) + # Process overrides for this UUID, if we know what it is + process_overrides(artifact_dict, Base.module_keys[__module__].uuid) + end + + # If the artifact exists, we're in the happy path and we can immediately + # return the path to the artifact: + for dir in artifact_paths(hash; honor_overrides=true) + if isdir(dir) + return jointail(dir, path_tail) + end + end + + # If not, we need to download it. We look up the Pkg module through `Base.loaded_modules()` + # then invoke `ensure_artifact_installed()`: + Pkg = first(filter(p-> p[1].name == "Pkg", Base.loaded_modules))[2] + return jointail(Pkg.Artifacts.ensure_artifact_installed(string(name), artifacts_toml), path_tail) +end + +""" + split_artifact_slash(name::String) + +Splits an artifact indexing string by path deliminters, isolates the first path element, +returning that and the `joinpath()` of the remaining arguments. This normalizes all path +separators to the native path separator for the current platform. Examples: + +# Examples +```jldoctest +julia> split_artifact_slash("Foo") +("Foo", "") + +julia> ret = split_artifact_slash("Foo/bar/baz.so"); + +julia> if Sys.iswindows() + ret == ("Foo", "bar\\baz.so") + else + ret == ("Foo", "bar/baz.so") + end +true + +julia> ret = split_artifact_slash("Foo\\bar\\baz.so"); + +julia> if Sys.iswindows() + ret == ("Foo", "bar\\baz.so") + else + ret == ("Foo", "bar/baz.so") + end +true +``` +""" +function split_artifact_slash(name::String) + split_name = split(name, r"(/|\\)") + if length(split_name) == 1 + return (split_name[1], "") + else + return (split_name[1], joinpath(split_name[2:end]...)) + end +end + +""" + artifact_slash_lookup(name::String, artifacts_toml::String) + +Returns `artifact_name`, `artifact_path_tail`, and `hash` by looking the results up in +the given `artifacts_toml`, first extracting the name and path tail from the given `name` +to support slash-indexing within the given artifact. +""" +function artifact_slash_lookup(name::String, artifact_dict::Dict, artifacts_toml::String) + artifact_name, artifact_path_tail = split_artifact_slash(name) + + meta = artifact_meta(artifact_name, artifact_dict, artifacts_toml) + if meta === nothing + error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'") + end + hash = SHA1(meta["git-tree-sha1"]) + return artifact_name, artifact_path_tail, hash +end + +""" + macro artifact_str(name) + +Macro that is used to automatically ensure an artifact is installed, and return its +location on-disk. Automatically looks the artifact up by name in the project's +`(Julia)Artifacts.toml` file. Throws an error on inability to install the requested +artifact. If run in the REPL, searches for the toml file starting in the current +directory, see `find_artifacts_toml()` for more. + +If `name` contains a forward or backward slash, all elements after the first slash will +be taken to be path names indexing into the artifact, allowing for an easy one-liner to +access a single file/directory within an artifact. Example: + + ffmpeg_path = @artifact"FFMPEG/bin/ffmpeg" + +!!! compat "Julia 1.3" + This macro requires at least Julia 1.3. + +!!! compat "Julia 1.6" + Slash-indexing requires at least Julia 1.6. +""" +macro artifact_str(name) + # Find Artifacts.toml file we're going to load from + srcfile = string(__source__.file) + if ((isinteractive() && startswith(srcfile, "REPL[")) || (!isinteractive() && srcfile == "none")) && !isfile(srcfile) + srcfile = pwd() + end + local artifacts_toml = find_artifacts_toml(srcfile) + if artifacts_toml === nothing + error(string( + "Cannot locate '(Julia)Artifacts.toml' file when attempting to use artifact '", + name, + "' in '", + __module__, + "'", + )) + end + + # Load Artifacts.toml at compile time, so that we don't have to use `__source__.file` + # at runtime, which gets stale if the `.ji` file is relocated. + local artifact_dict = load_artifacts_toml(artifacts_toml) + + # Invalidate calling .ji file if Artifacts.toml file changes + Base.include_dependency(artifacts_toml) + + # If `name` is a constant, we can actually load and parse the `Artifacts.toml` file now, + # saving the work from runtime. + if isa(name, AbstractString) + # To support slash-indexing, we need to split the artifact name from the path tail: + local artifact_name, artifact_path_tail, hash = artifact_slash_lookup(name, artifact_dict, artifacts_toml) + return quote + Base.invokelatest(_artifact_str, $(__module__), $(artifacts_toml), $(artifact_name), $(artifact_path_tail), $(artifact_dict), $(hash)) + end + else + return quote + local artifact_name, artifact_path_tail, hash = artifact_slash_lookup($(esc(name)), $(artifact_dict), $(artifacts_toml)) + Base.invokelatest(_artifact_str, $(__module__), $(artifacts_toml), artifact_name, artifact_path_tail, $(artifact_dict), hash) + end + end +end + +# Support `AbstractString`s, but avoid compilers needing to track backedges for callers +# of these functions in case a user defines a new type that is `<: AbstractString` +with_artifacts_directory(f::Function, artifacts_dir::AbstractString) = + with_artifacts_directory(f, string(artifacts_dir)) +query_override(pkg::Base.UUID, artifact_name::AbstractString; kwargs...) = + query_override(pkg, string(artifact_name); kwargs...) +unpack_platform(entry::Dict, name::AbstractString, artifacts_toml::AbstractString) = + unpack_platform(entry, string(name), string(artifacts_toml)) +load_artifacts_toml(artifacts_toml::AbstractString; kwargs...) = + load_artifacts_toml(string(artifacts_toml); kwargs...) +artifact_meta(name::AbstractString, artifacts_toml::AbstractString; kwargs...) = + artifact_meta(string(name), string(artifacts_toml); kwargs...) +artifact_meta(name::AbstractString, artifact_dict::Dict, artifacts_toml::AbstractString; kwargs...) = + artifact_meta(string(name), artifact_dict, string(artifacts_toml); kwargs...) +artifact_hash(name::AbstractString, artifacts_toml::AbstractString; kwargs...) = + artifact_hash(string(name), string(artifacts_toml); kwargs...) +find_artifacts_toml(path::AbstractString) = + find_artifacts_toml(string(path)) +split_artifact_slash(name::AbstractString) = + split_artifact_slash(string(name)) +artifact_slash_lookup(name::AbstractString, artifact_dict::Dict, artifacts_toml::AbstractString) = + artifact_slash_lookup(string(name), artifact_dict, string(artifacts_toml)) + +end # module Artifacts diff --git a/stdlib/Artifacts/test/Artifacts.toml b/stdlib/Artifacts/test/Artifacts.toml new file mode 100644 index 00000000000000..ee0fffb9ed92a2 --- /dev/null +++ b/stdlib/Artifacts/test/Artifacts.toml @@ -0,0 +1,138 @@ +[[c_simple]] +arch = "armv7l" +git-tree-sha1 = "0c509b3302db90a9393d6036c3ffcd14d190523d" +libc = "glibc" +os = "linux" + + [[c_simple.download]] + sha256 = "b0cfa3a2d9b5bc0632b0ee45b5d049eecbf72ed9c8cbc968b374ea995257a635" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.arm-linux-gnueabihf.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "e5a893fdac080fa0d4ae1cbd8bd67cfba5945af2" +os = "freebsd" + + [[c_simple.download]] + sha256 = "fde6e4ed00227b98e25ffdbf4e2b8b24a4e2bfa4c532c733d3626d6157e448ce" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-unknown-freebsd11.1.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "7ba74e239348ea6c060f994c083260be3abe3095" +os = "macos" + + [[c_simple.download]] + sha256 = "e88816a1492eecb4569bb24b3e52b757e59c87419dba962e99148b338369f326" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-apple-darwin14.tar.gz" + +# NOTE: We explicitly comment this out, to test porous platform support. Don't un-comment this! +#[[c_simple]] +#arch = "powerpc64le" +#git-tree-sha1 = "dc9f84891c8215f90095b619533e141179b6cc06" +#libc = "glibc" +#os = "linux" +# +# [[c_simple.download]] +# sha256 = "715af8f0405cff35feef5ad5e93836bb1bb0f93c77218bfdad411c8a4368ab4b" +# url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.powerpc64le-linux-gnu.tar.gz" + +[[c_simple]] +arch = "i686" +git-tree-sha1 = "78e282b79c16febc54a56ed244088ff92a55533f" +libc = "musl" +os = "linux" + + [[c_simple.download]] + sha256 = "900f2e55f72af0c723f9db7e9f44b1c16155010de212b430f02091dc24ff324c" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.i686-linux-musl.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "9d0075fdafe8af6430afba41fea2f32811141145" +libc = "musl" +os = "linux" + + [[c_simple.download]] + sha256 = "2769be12e00ebb0a3c7ab43b90b71bba3a6883844416457e08b880945d129689" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-linux-musl.tar.gz" + +[[c_simple]] +arch = "i686" +git-tree-sha1 = "0c890d3e6c5ee00fd06a7d418fad424159e447ce" +libc = "glibc" +os = "linux" + + [[c_simple.download]] + sha256 = "45d42cbb5cfafefeadfd46cd91445466d0e245f1f640cd4f91cdae01a654e001" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.i686-linux-gnu.tar.gz" + +[[c_simple]] +arch = "i686" +git-tree-sha1 = "956a97e2d56c0fa3b634f57e10a174dff0052ba4" +os = "windows" + + [[c_simple.download]] + sha256 = "b0214fa6f48359f2cf328b2ec2255ed975939939333db03711f63671c5d4fed9" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.i686-w64-mingw32.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "0ed63482ad1916dba12b4959d2704af4e41252da" +libc = "glibc" +os = "linux" + + [[c_simple.download]] + sha256 = "edbaf461c5c33fd7030bcd197b849396f8328648a2e04462b1bea9650f782a3b" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-linux-gnu.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "444cecb70ff39e8961dd33e230e151775d959f37" +os = "windows" + + [[c_simple.download]] + sha256 = "39b75afda9f0619f042c47bef2cdd0931a33c5c5acb7dc2977f1cd6274835c1f" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-w64-mingw32.tar.gz" + +[[c_simple]] +arch = "aarch64" +git-tree-sha1 = "efc8df78802fae852abd8e213e4b6f3f1da48125" +libc = "musl" +os = "linux" + + [[c_simple.download]] + sha256 = "53f54d76b4f9edd2a5b8c20575ef9f6a05c524a454c9126f4ecaa83a8aa72f52" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.aarch64-linux-musl.tar.gz" + +[[c_simple]] +arch = "aarch64" +git-tree-sha1 = "ca19bcae2bc6af88d6ace2648c7cc639b3cf1dfb" +libc = "glibc" +os = "linux" + + [[c_simple.download]] + sha256 = "94e303d2d779734281b60ef1880ad0e12681a2d3d6eed3a3ee3129a3efb016f7" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.aarch64-linux-gnu.tar.gz" + +[[c_simple]] +arch = "armv7l" +git-tree-sha1 = "0d8cebd76188d1bade6057b70605e553bbdfdd02" +libc = "musl" +os = "linux" + + [[c_simple.download]] + sha256 = "da1a04400ca8bcf51d2c39783e1fcc7d51fba5cf4f328d6bff2512ea5342532b" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.arm-linux-musleabihf.tar.gz" + +[socrates] +git-tree-sha1 = "43563e7631a7eafae1f9f8d9d332e3de44ad7239" +lazy = true + + [[socrates.download]] + url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.gz" + sha256 = "e65d2f13f2085f2c279830e863292312a72930fee5ba3c792b14c33ce5c5cc58" + + [[socrates.download]] + url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.bz2" + sha256 = "13fc17b97be41763b02cbb80e9d048302cec3bd3d446c2ed6e8210bddcd3ac76" \ No newline at end of file diff --git a/stdlib/Artifacts/test/runtests.jl b/stdlib/Artifacts/test/runtests.jl new file mode 100644 index 00000000000000..b81acabca31750 --- /dev/null +++ b/stdlib/Artifacts/test/runtests.jl @@ -0,0 +1,108 @@ +using Artifacts, Test, Base.BinaryPlatforms +using Artifacts: with_artifacts_directory, pack_platform!, unpack_platform + +@testset "Artifact Paths" begin + mktempdir() do tempdir + with_artifacts_directory(tempdir) do + hash = Base.SHA1("0"^40) + paths = Artifacts.artifact_paths(hash) + @test length(paths) == 1 + @test startswith(first(paths), tempdir) + + @test !artifact_exists(hash) + mkpath(first(paths)) + @test artifact_exists(hash) + @test artifact_path(hash) == first(paths) + end + end +end + +@testset "Serialization Tools" begin + # First, some basic tests + meta = Dict() + pack_platform!(meta, Platform("i686", "linux")) + @test meta["os"] == "linux" + @test meta["arch"] == "i686" + @test meta["libc"] == "glibc" + + meta = Dict() + pack_platform!(meta, Platform("armv7l", "linux"; libc="musl")) + @test meta["os"] == "linux" + @test meta["arch"] == "armv7l" + @test meta["libc"] == "musl" + + meta = Dict() + pack_platform!(meta, Platform("x86_64", "windows"; libgfortran_version=v"3")) + @test meta["os"] == "windows" + @test meta["arch"] == "x86_64" + @test meta["libgfortran_version"] == "3.0.0" + + meta = Dict() + pack_platform!(meta, Platform("aarch64", "macOS")) + @test meta == Dict("os" => "macos", "arch" => "aarch64") + + # Next, fuzz it out! Ensure that we exactly reconstruct our platforms! + platforms = Platform[] + for libgfortran_version in (v"3", v"4", v"5", nothing), + libstdcxx_version in (v"3.4.11", v"3.4.19", nothing), + cxxstring_abi in ("cxx03", "cxx11", nothing) + + for arch in ("x86_64", "i686", "aarch64", "armv7l"), + libc in ("glibc", "musl") + + push!(platforms, Platform(arch, "linux"; libc, libgfortran_version, libstdcxx_version, cxxstring_abi)) + end + push!(platforms, Platform("x86_64", "windows"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + push!(platforms, Platform("i686", "windows"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + push!(platforms, Platform("x86_64", "macOS"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + push!(platforms, Platform("aarch64", "macOS"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + push!(platforms, Platform("x86_64", "FreeBSD"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + end + + for p in platforms + meta = Dict() + pack_platform!(meta, p) + @test unpack_platform(meta, "foo", "") == p + + # Test that some things raise warnings + bad_meta = copy(meta) + delete!(bad_meta, "os") + @test_logs (:error, r"Invalid artifacts file") unpack_platform(bad_meta, "foo", "") + + bad_meta = copy(meta) + delete!(bad_meta, "arch") + @test_logs (:error, r"Invalid artifacts file") unpack_platform(bad_meta, "foo", "") + end +end + +@testset "Artifact Slash-indexing" begin + mktempdir() do tempdir + with_artifacts_directory(tempdir) do + exeext = Sys.iswindows() ? ".exe" : "" + + # simple lookup, gives us the directory for `c_simple` for the current architecture + c_simple_dir = artifact"c_simple" + @test isdir(c_simple_dir) + c_simple_exe_path = joinpath(c_simple_dir, "bin", "c_simple$(exeext)") + @test isfile(c_simple_exe_path) + + # Simple slash-indexed lookup + c_simple_bin_path = artifact"c_simple/bin" + @test isdir(c_simple_bin_path) + # Test that forward and backward slash are equivalent + @test artifact"c_simple\\bin" == artifact"c_simple/bin" + + # Dynamically-computed lookup; not done at compile-time + generate_artifact_name() = "c_simple" + c_simple_dir = @artifact_str(generate_artifact_name()) + @test isdir(c_simple_dir) + c_simple_exe_path = joinpath(c_simple_dir, "bin", "c_simple$(exeext)") + @test isfile(c_simple_exe_path) + + # Dynamically-computed slash-indexing: + generate_bin_path(pathsep) = "c_simple$(pathsep)bin$(pathsep)c_simple$(exeext)" + @test isfile(@artifact_str(generate_bin_path("/"))) + @test isfile(@artifact_str(generate_bin_path("\\"))) + end + end +end diff --git a/stdlib/Makefile b/stdlib/Makefile index ece6b58644a167..4e748b42aac198 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -14,7 +14,7 @@ VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) $(build_datarootdir)/julia/stdlib/$(VERSDIR): mkdir -p $@ -STDLIBS = Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \ +STDLIBS = Artifacts Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \ Future InteractiveUtils Libdl LibGit2 LinearAlgebra Logging \ Markdown Mmap Printf Profile Random REPL Serialization SHA \ SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs diff --git a/test/precompile.jl b/test/precompile.jl index 25fbcd36c50bbf..bbd642a625060f 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -288,7 +288,7 @@ try Dict(let m = Base.root_module(Base, s) Base.PkgId(m) => Base.module_build_id(m) end for s in - [:Base64, :CRC32c, :Dates, :DelimitedFiles, :Distributed, :FileWatching, :Markdown, + [:Artifacts, :Base64, :CRC32c, :Dates, :DelimitedFiles, :Distributed, :FileWatching, :Markdown, :Future, :Libdl, :LinearAlgebra, :Logging, :Mmap, :Printf, :Profile, :Random, :Serialization, :SharedArrays, :SparseArrays, :SuiteSparse, :Test, :Unicode, :REPL, :InteractiveUtils, :Pkg, :LibGit2, :SHA, :UUIDs, :Sockets,