diff --git a/NEWS.md b/NEWS.md index b3607e6bf173a..9942996ef4633 100644 --- a/NEWS.md +++ b/NEWS.md @@ -99,6 +99,12 @@ Standard library changes * `RegexMatch` objects can now be probed for whether a named capture group exists within it through `haskey()` ([#36717]). * For consistency `haskey(r::RegexMatch, i::Integer)` has also been added and returns if the capture group for `i` exists ([#37300]). * A new standard library `TOML` has been added for parsing and printing [TOML files](https://toml.io) ([#37034]). +* The `Pkg.BinaryPlatforms` module has been moved into `Base` as `Base.BinaryPlatforms` and heavily reworked. + Applications that want to be compatible with the old API should continue to import `Pkg.BinaryPlatforms`, + however new users should use `Base.BinaryPlatforms` directly. ([#37320]) +* The `Pkg.Artifacts` module has been imported as a separate standard library. It is still available as + `Pkg.Artifacts`, however starting from Julia v1.6+, packages may import simply `Artifacts` without importing + all of `Pkg` alongside. ([#37320]) #### LinearAlgebra * New method `LinearAlgebra.issuccess(::CholeskyPivoted)` for checking whether pivoted Cholesky factorization was successful ([#36002]). diff --git a/base/Base.jl b/base/Base.jl index 735f06ef8f0cd..057c512887c6c 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -236,6 +236,9 @@ include("weakkeydict.jl") include("logging.jl") using .CoreLogging +# BinaryPlatforms, used by Artifacts +include("binaryplatforms.jl") + # functions defined in Random function rand end function randn end diff --git a/base/binaryplatforms.jl b/base/binaryplatforms.jl new file mode 100644 index 0000000000000..7935aedb564dc --- /dev/null +++ b/base/binaryplatforms.jl @@ -0,0 +1,1010 @@ +module BinaryPlatforms + +export AbstractPlatform, Platform, HostPlatform, platform_dlext, tags, arch, os, + os_version, libc, compiler_abi, libgfortran_version, libstdcxx_version, + cxxstring_abi, parse_dl_name_version, detect_libgfortran_version, + detect_libstdcxx_version, detect_cxxstring_abi, call_abi, wordsize, triplet, + select_platform, platforms_match, platform_name +import .Libc.Libdl + +# This exists to ease compatibility with old-style Platform objects +abstract type AbstractPlatform; end + +""" + Platform + +A `Platform` represents all relevant pieces of information that a julia process may need +to know about its execution environment, such as the processor architecture, operating +system, libc implementation, etc... It is, at its heart, a key-value mapping of tags +(such as `arch`, `os`, `libc`, etc...) to values (such as `"arch" => "x86_64"`, or +`"os" => "windows"`, etc...). `Platform` objects are extensible in that the tag mapping +is open for users to add their own mappings to, as long as the mappings do not conflict +with the set of reserved tags: `arch`, `os`, `os_version`, `libc`, `call_abi`, +`libgfortran_version`, `libstdcxx_version`, `cxxstring_abi` and `julia_version`. + +Valid tags and values are composed of alphanumeric and period characters. All tags and +values will be lowercased when stored to reduce variation. + +Example: + + Platform("x86_64", "windows"; cuda = "10.1") +""" +struct Platform <: AbstractPlatform + tags::Dict{String,String} + # The "compare strategy" allows selective overriding on how a tag is compared + compare_strategies::Dict{String,Function} + + function Platform(arch::String, os::String; + validate_strict::Bool = false, + compare_strategies::Dict{String,<:Function} = Dict{String,Function}(), + kwargs...) + # A wee bit of normalization + os = lowercase(os) + arch = lowercase(arch) + if arch ∈ ("amd64",) + arch = "x86_64" + elseif arch ∈ ("i386", "i586") + arch = "i686" + elseif arch ∈ ("arm",) + arch = "armv7l" + elseif arch ∈ ("ppc64le",) + arch = "powerpc64le" + end + + tags = Dict{String,String}( + "arch" => arch, + "os" => os, + ) + for (tag, value) in kwargs + tag = lowercase(string(tag)) + if tag ∈ ("arch", "os") + throw(ArgumentError("Cannot double-pass key $(tag)")) + end + + # Drop `nothing` values; this means feature is not present or use default value. + if value === nothing + continue + end + + # Normalize things that are known to be version numbers so that comparisons are easy. + # Note that in our effort to be extremely compatible, we actually allow something that + # doesn't parse nicely into a VersionNumber to persist, but if `validate_strict` is + # set to `true`, it will cause an error later on. + if tag ∈ ("libgfortran_version", "libstdcxx_version", "os_version") + normver(x::VersionNumber) = string(x) + function normver(str::AbstractString) + v = tryparse(VersionNumber, str) + if v === nothing + # If this couldn't be parsed as a VersionNumber, return the original. + return str + end + # Otherwise, return the `string(VersionNumber(str))` version. + return normver(v) + end + value = normver(value) + end + + # I know we said only alphanumeric and dots, but let's be generous so that we can expand + # our support in the future while remaining as backwards-compatible as possible. The + # only characters that are absolutely disallowed right now are `-`, `+`, ` ` and things + # that are illegal in filenames: + nonos = raw"""+- /<>:"'\|?*""" + if any(occursin(nono, tag) for nono in nonos) + throw(ArgumentError("Invalid character in tag name \"$(tag)\"!")) + end + + # Normalize and reject nonos + value = lowercase(string(value)) + if any(occursin(nono, value) for nono in nonos) + throw(ArgumentError("Invalid character in tag value \"$(value)\"!")) + end + tags[tag] = value + end + + # Auto-map call_abi and libc where necessary: + if os == "linux" && !haskey(tags, "libc") + # Default to `glibc` on Linux + tags["libc"] = "glibc" + end + if os == "linux" && arch ∈ ("armv7l", "armv6l") && "call_abi" ∉ keys(tags) + # default `call_abi` to `eabihf` on 32-bit ARM + tags["call_abi"] = "eabihf" + end + + # If the user is asking for strict validation, do so. + if validate_strict + validate_tags(tags) + end + + # By default, we compare julia_version only against major and minor versions: + if haskey(tags, "julia_version") && !haskey(compare_strategies, "julia_version") + compare_strategies["julia_version"] = (a::String, b::String, a_comparator, b_comparator) -> begin + a = VersionNumber(a) + b = VersionNumber(b) + return a.major == b.major && a.minor == b.minor + end + end + + return new(tags, compare_strategies) + end +end + +# Other `Platform` types can override this (I'm looking at you, `AnyPlatform`) +tags(p::Platform) = p.tags +Base.haskey(p::AbstractPlatform, k::String) = haskey(tags(p), k) + +# Allow us to easily serialize Platform objects +function Base.repr(p::Platform; context=nothing) + str = string( + "Platform(", + repr(arch(p)), + ", ", + repr(os(p)), + "; ", + join(("$(k) = $(repr(v))" for (k, v) in tags(p) if k ∉ ("arch", "os")), ", "), + ")", + ) +end + +# Simple equality definition; for compatibility testing, use `platforms_match()` +Base.:(==)(a::AbstractPlatform, b::AbstractPlatform) = tags(a) == tags(b) + +const ARCHITECTURE_FLAGS = Dict( + "x86_64" => ["x86_64", "avx", "avx2", "avx512"], + "i686" => ["prescott"], + "armv7l" => ["armv7l", "neon", "vfp4"], + "armv6l" => ["generic"], + "aarch64" => ["armv8", "thunderx2", "carmel"], + "powerpc64le" => ["generic"], +) +function validate_tags(tags::Dict) + throw_invalid_key(k) = throw(ArgumentError("Key \"$(k)\" cannot have value \"$(tags[k])\"")) + # Validate `arch` + if tags["arch"] ∉ keys(ARCHITECTURE_FLAGS) + throw_invalid_key("arch") + end + # Validate `os` + if tags["os"] ∉ ("linux", "macos", "freebsd", "windows") + throw_invalid_key("os") + end + # Validate `os`/`arch` combination + throw_os_mismatch() = throw(ArgumentError("Invalid os/arch combination: $(tags["os"])/$(tags["arch"])")) + if tags["os"] == "windows" && tags["arch"] ∉ ("x86_64", "i686", "armv7l", "aarch64") + throw_os_mismatch() + end + if tags["os"] == "macos" && tags["arch"] ∉ ("x86_64", "aarch64") + throw_os_mismatch() + end + + # Validate `os`/`libc` combination + throw_libc_mismatch() = throw(ArgumentError("Invalid os/libc combination: $(tags["os"])/$(tags["libc"])")) + if tags["os"] == "linux" + # Linux always has a `libc` entry + if tags["libc"] ∉ ("glibc", "musl") + throw_libc_mismatch() + end + else + # Nothing else is allowed to have a `libc` entry + if haskey(tags, "libc") + throw_libc_mismatch() + end + end + + # Validate `os`/`arch`/`call_abi` combination + throw_call_abi_mismatch() = throw(ArgumentError("Invalid os/arch/call_abi combination: $(tags["os"])/$(tags["arch"])/$(tags["call_abi"])")) + if tags["os"] == "linux" && tags["arch"] ∈ ("armv7l", "armv6l") + # If an ARM linux has does not have `call_abi` set to something valid, be sad. + if !haskey(tags, "call_abi") || tags["call_abi"] ∉ ("eabihf", "eabi") + throw_call_abi_mismatch() + end + else + # Nothing else should have a `call_abi`. + if haskey(tags, "call_abi") + throw_call_abi_mismatch() + end + end + + # Validate `libgfortran_version` is a parsable `VersionNumber` + throw_version_number(k) = throw(ArgumentError("\"$(k)\" cannot have value \"$(tags[k])\", must be a valid VersionNumber")) + if "libgfortran_version" in keys(tags) && tryparse(VersionNumber, tags["libgfortran_version"]) === nothing + throw_version_number("libgfortran_version") + end + + # Validate `libstdcxx_version` is a parsable `VersionNumber` + if "libstdcxx_version" in keys(tags) && tryparse(VersionNumber, tags["libstdcxx_version"]) === nothing + throw_version_number("libstdcxx_version") + end + + # Validate `cxxstring_abi` is one of the two valid options: + if "cxxstring_abi" in keys(tags) && tags["cxxstring_abi"] ∉ ("cxx03", "cxx11") + throw_invalid_key("cxxstring_abi") + end + + # Validate `march` is one of our recognized microarchitectures for the architecture we're advertising + if "march" in keys(tags) && tags["march"] ∉ ARCHITECTURE_FLAGS[tags["arch"]] + throw(ArgumentError("\"march\" cannot have value \"$(tags["march"])\" for arch $(tags["arch"])")) + end + + # Validate `cuda` is a parsable `VersionNumber` + if "cuda" in keys(tags) && tryparse(VersionNumber, tags["cuda"]) === nothing + throw_version_number("cuda") + end +end + +function set_compare_strategy!(p::Platform, key::String, f::Function) + if !haskey(p.tags, key) + throw(ArgumentError("Cannot set comparison strategy for nonexistant tag $(key)!")) + end + p.compare_strategies[key] = f +end + +function get_compare_strategy(p::Platform, key::String, default = compare_default) + if !haskey(p.tags, key) + throw(ArgumentError("Cannot get comparison strategy for nonexistant tag $(key)!")) + end + return get(p.compare_strategies, key, default) +end +get_compare_strategy(p::AbstractPlatform, key::String, default = compare_default) = default + + + +""" + compare_default(a::String, b::String, a_requested::Bool, b_requested::Bool) + +Default comparison strategy that falls back to `a == b`. This only ever happens if both +`a` and `b` request this strategy, as any other strategy is preferrable to this one. +""" +function compare_default(a::String, b::String, a_requested::Bool, b_requested::Bool) + return a == b +end + +""" + compare_version_cap(a::String, b::String, a_comparator, b_comparator) + +Example comparison strategy for `set_comparison_strategy!()` that implements a version +cap for host platforms that support _up to_ a particular version number. As an example, +if an artifact is built for macOS 10.9, it can run on macOS 10.11, however if it were +built for macOS 10.12, it could not. Therefore, the host platform of macOS 10.11 has a +version cap at `v"10.11"`. + +Note that because both hosts and artifacts are represented with `Platform` objects it +is possible to call `platforms_match()` with two artifacts, a host and an artifact, an +artifact and a host, and even two hosts. We attempt to do something intelligent for all +cases, but in the case of comparing version caps between two hosts, we return `true` only +if the two host platforms are in fact identical. +""" +function compare_version_cap(a::String, b::String, a_requested::Bool, b_requested::Bool) + a = VersionNumber(a) + b = VersionNumber(b) + + # If both b and a requested, then we fall back to equality: + if a_requested && b_requested + return a == b + end + + # Otherwise, do the comparison between the the single version cap and the single version: + if a_requested + return b <= a + else + return a <= b + end +end + + + +""" + HostPlatform(p::AbstractPlatform) + +Convert a `Platform` to act like a "host"; e.g. if it has a version-bound tag such as +`"libstdcxx_version" => "3.4.26"`, it will treat that value as an upper bound, rather +than a characteristic. `Platform` objects that define artifacts generally denote the +SDK or version that the artifact was built with, but for platforms, these versions are +generally the maximal version the platform can support. The way this transformation +is implemented is to change the appropriate comparison strategies to treat these pieces +of data as bounds rather than points in any comparison. +""" +function HostPlatform(p::AbstractPlatform) + if haskey(p, "os_version") + set_compare_strategy!(p, "os_version", compare_version_cap) + end + if haskey(p, "libstdcxx_version") + set_compare_strategy!(p, "libstdcxx_version", compare_version_cap) + end + return p +end + +""" + arch(p::AbstractPlatform) + +Get the architecture for the given `Platform` object as a `String`. + +# Examples +```jldoctest +julia> arch(Platform("aarch64", "Linux")) +"aarch64" + +julia> arch(Platform("amd64", "freebsd")) +"x86_64" +``` +""" +arch(p::AbstractPlatform) = get(tags(p), "arch", nothing) + +""" + os(p::AbstractPlatform) + +Get the operating system for the given `Platform` object as a `String`. + +# Examples +```jldoctest +julia> os(Platform("armv7l", "Linux")) +"linux" + +julia> os(Platform("aarch64", "macos")) +"macos" +``` +""" +os(p::AbstractPlatform) = get(tags(p), "os", nothing) + +# As a special helper, it's sometimes useful to know the current OS at compile-time +function os() + if Sys.iswindows() + return "windows" + elseif Sys.isapple() + return "macos" + elseif Sys.isbsd() + return "freebsd" + else + return "linux" + end +end + +""" + libc(p::AbstractPlatform) + +Get the libc for the given `Platform` object as a `String`. Returns `nothing` on +platforms with no explicit `libc` choices (which is most platforms). + +# Examples +```jldoctest +julia> libc(Platform("armv7l", "Linux")) +"glibc" + +julia> libc(Platform("aarch64", "linux"; libc="musl")) +"musl" + +julia> libc(Platform("i686", "Windows")) +``` +""" +libc(p::AbstractPlatform) = get(tags(p), "libc", nothing) + +""" + call_abi(p::AbstractPlatform) + +Get the call ABI for the given `Platform` object as a `String`. Returns `nothing` on +platforms with no explicit `call_abi` choices (which is most platforms). + +# Examples +```jldoctest +julia> call_abi(Platform("armv7l", "Linux")) +"eabihf" + +julia> call_abi(Platform("x86_64", "macos")) +``` +""" +call_abi(p::AbstractPlatform) = get(tags(p), "call_abi", nothing) + +const platform_names = Dict( + "linux" => "Linux", + "macos" => "macOS", + "windows" => "Windows", + "freebsd" => "FreeBSD", + nothing => "Unknown", +) + +""" + platform_name(p::AbstractPlatform) + +Get the "platform name" of the given platform, returning e.g. "Linux" or "Windows". +""" +function platform_name(p::AbstractPlatform) + return platform_names[os(p)] +end + +function VNorNothing(d::Dict, key) + v = get(d, key, nothing) + if v === nothing + return nothing + end + return VersionNumber(v) +end + +""" + libgfortran_version(p::AbstractPlatform) + +Get the libgfortran version dictated by this `Platform` object as a `VersionNumber`, +or `nothing` if no compatibility bound is imposed. +""" +libgfortran_version(p::AbstractPlatform) = VNorNothing(tags(p), "libgfortran_version") + +""" + libstdcxx_version(p::AbstractPlatform) + +Get the libstdc++ version dictated by this `Platform` object, or `nothing` if no +compatibility bound is imposed. +""" +libstdcxx_version(p::AbstractPlatform) = VNorNothing(tags(p), "libstdcxx_version") + +""" + cxxstring_abi(p::AbstractPlatform) + +Get the c++ string ABI dictated by this `Platform` object, or `nothing` if no ABI is imposed. +""" +cxxstring_abi(p::AbstractPlatform) = get(tags(p), "cxxstring_abi", nothing) + +""" + os_version(p::AbstractPlatform) + +Get the OS version dictated by this `Platform` object, or `nothing` if no OS version is +imposed/no data is available. This is most commonly used by MacOS and FreeBSD objects +where we have high platform SDK fragmentation, and features are available only on certain +platform versions. +""" +os_version(p::AbstractPlatform) = VNorNothing(tags(p), "os_version") + +""" + wordsize(p::AbstractPlatform) + +Get the word size for the given `Platform` object. + +# Examples +```jldoctest +julia> wordsize(Platform("armv7l", "linux")) +32 + +julia> wordsize(Platform("x86_64", "macos")) +64 +``` +""" +wordsize(p::AbstractPlatform) = (arch(p) ∈ ("i686", "armv6l", "armv7l")) ? 32 : 64 + +""" + triplet(p::AbstractPlatform; exclude_tags::Vector{String}) + +Get the target triplet for the given `Platform` object as a `String`. + +# Examples +```jldoctest +julia> triplet(Platform("x86_64", "MacOS")) +"x86_64-apple-darwin" + +julia> triplet(Platform("i686", "Windows")) +"i686-w64-mingw32" + +julia> triplet(Platform("armv7l", "Linux"; libgfortran_version="3")) +"armv7l-linux-gnueabihf-libgfortran3" +``` +""" +function triplet(p::AbstractPlatform) + str = string( + arch(p), + os_str(p), + libc_str(p), + call_abi_str(p), + ) + + # Tack on optional compiler ABI flags + if libgfortran_version(p) !== nothing + str = string(str, "-libgfortran", libgfortran_version(p).major) + end + if libstdcxx_version(p) !== nothing + str = string(str, "-libstdcxx", libstdcxx_version(p).patch) + end + if cxxstring_abi(p) !== nothing + str = string(str, "-", cxxstring_abi(p)) + end + + # Tack on all extra tags + for (tag, val) in tags(p) + if tag ∈ ("os", "arch", "libc", "call_abi", "libgfortran_version", "libstdcxx_version", "cxxstring_abi") + continue + end + str = string(str, "-", tag, "+", val) + end + return str +end + +function os_str(p::AbstractPlatform) + if os(p) == "linux" + return "-linux" + elseif os(p) == "macos" + osvn = os_version(p) + if osvn !== nothing + return "-apple-darwin$(osvn.major)" + else + return "-apple-darwin" + end + elseif os(p) == "windows" + return "-w64-mingw32" + elseif os(p) == "freebsd" + osvn = os_version(p) + if osvn !== nothing + return "-unknown-freebsd$(osvn.major).$(osvn.minor)" + else + return "-unknown-freebsd" + end + else + return "-unknown" + end +end + +# Helper functions for Linux and FreeBSD libc/abi mishmashes +function libc_str(p::AbstractPlatform) + if libc(p) === nothing + return "" + elseif libc(p) === "glibc" + return "-gnu" + else + return string("-", libc(p)) + end +end +call_abi_str(p::AbstractPlatform) = (call_abi(p) === nothing) ? "" : call_abi(p) + +Sys.isapple(p::AbstractPlatform) = os(p) == "macos" +Sys.islinux(p::AbstractPlatform) = os(p) == "linux" +Sys.iswindows(p::AbstractPlatform) = os(p) == "windows" +Sys.isfreebsd(p::AbstractPlatform) = os(p) == "freebsd" +Sys.isbsd(p::AbstractPlatform) = os(p) ∈ ("freebsd", "macos") + +const arch_mapping = Dict( + "x86_64" => "(x86_|amd)64", + "i686" => "i\\d86", + "aarch64" => "(aarch64|arm64)", + "armv7l" => "arm(v7l)?", # if we just see `arm-linux-gnueabihf`, we assume it's `armv7l` + "armv6l" => "armv6l", + "powerpc64le" => "p(ower)?pc64le", +) +const os_mapping = Dict( + "macos" => "-apple-darwin[\\d\\.]*", + "freebsd" => "-(.*-)?freebsd[\\d\\.]*", + "windows" => "-w64-mingw32", + "linux" => "-(.*-)?linux", +) +const libc_mapping = Dict( + "libc_nothing" => "", + "glibc" => "-gnu", + "musl" => "-musl", +) +const call_abi_mapping = Dict( + "call_abi_nothing" => "", + "eabihf" => "eabihf", + "eabi" => "eabi", +) +const libgfortran_version_mapping = Dict( + "libgfortran_nothing" => "", + "libgfortran3" => "(-libgfortran3)|(-gcc4)", # support old-style `gccX` versioning + "libgfortran4" => "(-libgfortran4)|(-gcc7)", + "libgfortran5" => "(-libgfortran5)|(-gcc8)", +) +const libstdcxx_version_mapping = Dict{String,String}( + "libstdcxx_nothing" => "", + # This is sadly easier than parsing out the digit directly + ("libstdcxx$(idx)" => "-libstdcxx$(idx)" for idx in 18:26)..., +) +const cxxstring_abi_mapping = Dict( + "cxxstring_nothing" => "", + "cxx03" => "-cxx03", + "cxx11" => "-cxx11", +) + +""" + parse(::Type{Platform}, triplet::AbstractString) + +Parses a string platform triplet back into a `Platform` object. +""" +function Base.parse(::Type{Platform}, triplet::AbstractString; validate_strict::Bool = false) + # Helper function to collapse dictionary of mappings down into a regex of + # named capture groups joined by "|" operators + c(mapping) = string("(",join(["(?<$k>$v)" for (k, v) in mapping], "|"), ")") + + # We're going to build a mondo regex here to parse everything: + triplet_regex = Regex(string( + "^", + # First, the core triplet; arch/os/libc/call_abi + c(arch_mapping), + c(os_mapping), + c(libc_mapping), + c(call_abi_mapping), + # Next, optional things, like libgfortran/libstdcxx/cxxstring abi + c(libgfortran_version_mapping), + c(libstdcxx_version_mapping), + c(cxxstring_abi_mapping), + # Finally, the catch-all for extended tags + "(?(?:-[^-]+\\+[^-]+)*)?", + "\$", + )) + + m = match(triplet_regex, triplet) + if m !== nothing + # Helper function to find the single named field within the giant regex + # that is not `nothing` for each mapping we give it. + get_field(m, mapping) = begin + for k in keys(mapping) + if m[k] !== nothing + # Convert our sentinel `nothing` values to actual `nothing` + if endswith(k, "_nothing") + return nothing + end + # Convert libgfortran/libstdcxx version numbers + if startswith(k, "libgfortran") + return VersionNumber(parse(Int,k[12:end])) + elseif startswith(k, "libstdcxx") + return VersionNumber(3, 4, parse(Int,k[10:end])) + else + return k + end + end + end + end + + # Extract the information we're interested in: + arch = get_field(m, arch_mapping) + os = get_field(m, os_mapping) + libc = get_field(m, libc_mapping) + call_abi = get_field(m, call_abi_mapping) + libgfortran_version = get_field(m, libgfortran_version_mapping) + libstdcxx_version = get_field(m, libstdcxx_version_mapping) + cxxstring_abi = get_field(m, cxxstring_abi_mapping) + function split_tags(tagstr) + tag_fields = filter(!isempty, split(tagstr, "-")) + if isempty(tag_fields) + return Pair{String,String}[] + end + return map(v -> Symbol(v[1]) => v[2], split.(tag_fields, "+")) + end + tags = split_tags(m["tags"]) + + # Special parsing of os version number, if any exists + function extract_os_version(os_name, pattern) + m_osvn = match(pattern, m[os_name]) + if m_osvn !== nothing + return VersionNumber(m_osvn.captures[1]) + end + return nothing + end + os_version = nothing + if os == "macos" + os_version = extract_os_version("macos", r".*darwin([\d\.]+)") + end + if os == "freebsd" + os_version = extract_os_version("freebsd", r".*freebsd([\d.]+)") + end + + return Platform( + arch, os; + validate_strict, + libc, + call_abi, + libgfortran_version, + libstdcxx_version, + cxxstring_abi, + os_version, + tags..., + ) + end + throw(ArgumentError("Platform `$(triplet)` is not an officially supported platform")) +end + +function Base.tryparse(::Type{Platform}, triplet::AbstractString) + try + parse(Platform, triplet) + catch e + if isa(e, InterruptException) + rethrow(e) + end + return nothing + end +end + +""" + platform_dlext(p::AbstractPlatform = HostPlatform()) + +Return the dynamic library extension for the given platform, defaulting to the +currently running platform. E.g. returns "so" for a Linux-based platform, +"dll" for a Windows-based platform, etc... +""" +function platform_dlext(p::AbstractPlatform = HostPlatform()) + if os(p) == "windows" + return "dll" + elseif os(p) == "macos" + return "dylib" + else + return "so" + end +end + +""" + parse_dl_name_version(path::String, platform::AbstractPlatform) + +Given a path to a dynamic library, parse out what information we can +from the filename. E.g. given something like "lib/libfoo.so.3.2", +this function returns `"libfoo", v"3.2"`. If the path name is not a +valid dynamic library, this method throws an error. If no soversion +can be extracted from the filename, as in "libbar.so" this method +returns `"libbar", nothing`. +""" +function parse_dl_name_version(path::String, os::String) + # Use an extraction regex that matches the given OS + local dlregex + if os == "windows" + # On Windows, libraries look like `libnettle-6.dll` + dlregex = r"^(.*?)(?:-((?:[\.\d]+)*))?\.dll$" + elseif os == "macos" + # On OSX, libraries look like `libnettle.6.3.dylib` + dlregex = r"^(.*?)((?:\.[\d]+)*)\.dylib$" + else + # On Linux and FreeBSD, libraries look like `libnettle.so.6.3.0` + dlregex = r"^(.*?).so((?:\.[\d]+)*)$" + end + + m = match(dlregex, basename(path)) + if m === nothing + throw(ArgumentError("Invalid dynamic library path '$path'")) + end + + # Extract name and version + name = m.captures[1] + version = m.captures[2] + if version === nothing || isempty(version) + version = nothing + else + version = VersionNumber(strip(version, '.')) + end + return name, version +end + +# Adapter for `AbstractString` +function parse_dl_name_version(path::AbstractString, os::AbstractString) + return parse_dl_name_version(string(path)::String, string(os)::String) +end + +""" + detect_libgfortran_version() + +Inspects the current Julia process to determine the libgfortran version this Julia is +linked against (if any). +""" +function detect_libgfortran_version() + libgfortran_paths = filter(x -> occursin("libgfortran", x), Libdl.dllist()) + if isempty(libgfortran_paths) + # One day, I hope to not be linking against libgfortran in base Julia + return nothing + end + libgfortran_path = first(libgfortran_paths) + + name, version = parse_dl_name_version(libgfortran_path, os()) + if version === nothing + # Even though we complain about this, we allow it to continue in the hopes that + # we shall march on to a BRIGHTER TOMORROW. One in which we are not shackled + # by the constraints of libgfortran compiler ABIs upon our precious programming + # languages; one where the mistakes of yesterday are mere memories and not + # continual maintenance burdens upon the children of the dawn; one where numeric + # code may be cleanly implemented in a modern language and not bestowed onto the + # next generation by grizzled ancients, documented only with a faded yellow + # sticky note that bears a hastily-scribbled "good luck". + @warn("Unable to determine libgfortran version from '$(libgfortran_path)'") + end + return version +end + +""" + detect_libstdcxx_version() + +Inspects the currently running Julia process to find out what version of libstdc++ +it is linked against (if any). +""" +function detect_libstdcxx_version() + libstdcxx_paths = filter(x -> occursin("libstdc++", x), Libdl.dllist()) + if isempty(libstdcxx_paths) + # This can happen if we were built by clang, so we don't link against + # libstdc++ at all. + return nothing + end + + # Brute-force our way through GLIBCXX_* symbols to discover which version we're linked against + hdl = Libdl.dlopen(first(libstdcxx_paths)) + for minor_version in 26:-1:18 + if Libdl.dlsym(hdl, "GLIBCXX_3.4.$(minor_version)"; throw_error=false) !== nothing + Libdl.dlclose(hdl) + return VersionNumber("3.4.$(minor_version)") + end + end + Libdl.dlclose(hdl) + return nothing +end + +""" + detect_cxxstring_abi() + +Inspects the currently running Julia process to see what version of the C++11 string ABI +it was compiled with (this is only relevant if compiled with `g++`; `clang` has no +incompatibilities yet, bless its heart). In reality, this actually checks for symbols +within LLVM, but that is close enough for our purposes, as you can't mix configurations +between Julia and LLVM; they must match. +""" +function detect_cxxstring_abi() + # First, if we're not linked against libstdc++, then early-exit because this doesn't matter. + libstdcxx_paths = filter(x -> occursin("libstdc++", x), Libdl.dllist()) + if isempty(libstdcxx_paths) + # We were probably built by `clang`; we don't link against `libstdc++`` at all. + return nothing + end + + function open_libllvm(f::Function) + for lib_name in ("libLLVM", "LLVM", "libLLVMSupport") + hdl = Libdl.dlopen_e(lib_name) + if hdl != C_NULL + try + return f(hdl) + finally + Libdl.dlclose(hdl) + end + end + end + error("Unable to open libLLVM!") + end + + return open_libllvm() do hdl + # Check for llvm::sys::getProcessTriple(), first without cxx11 tag: + if Libdl.dlsym_e(hdl, "_ZN4llvm3sys16getProcessTripleEv") != C_NULL + return "cxx03" + elseif Libdl.dlsym_e(hdl, "_ZN4llvm3sys16getProcessTripleB5cxx11Ev") != C_NULL + return "cxx11" + else + @warn("Unable to find llvm::sys::getProcessTriple() in libLLVM!") + return nothing + end + end +end + +""" + host_triplet() + +Build host triplet out of `Sys.MACHINE` and various introspective utilities that +detect compiler ABI values such as `libgfortran_version`, `libstdcxx_version` and +`cxxstring_abi`. We do this without using any `Platform` tech as it must run before +we have much of that built. +""" +function host_triplet() + str = Sys.MACHINE + libgfortran_version = detect_libgfortran_version() + if libgfortran_version !== nothing + str = string(str, "-libgfortran", libgfortran_version.major) + end + + libstdcxx_version = detect_libstdcxx_version() + if libstdcxx_version !== nothing + str = string(str, "-libstdcxx", libstdcxx_version.patch) + end + + cxxstring_abi = detect_cxxstring_abi() + if cxxstring_abi !== nothing + str = string(str, "-", cxxstring_abi) + end + + # Add on julia_version extended tag + str = string(str, "-julia_version+", VersionNumber(VERSION.major, VERSION.minor, VERSION.patch)) + + return str +end + +# Cache the host platform value, and return it if someone asks for just `HostPlatform()`. +default_host_platform = HostPlatform(parse(Platform, host_triplet())) +""" + HostPlatform() + +Return the `Platform` object that corresponds to the current host system, with all +relevant comparison strategies set to host platform mode. This is equivalent to: + + HostPlatform(parse(Platform, Base.BinaryPlatforms.host_triplet())) +""" +function HostPlatform() + return default_host_platform::Platform +end + +""" + platforms_match(a::AbstractPlatform, b::AbstractPlatform) + +Return `true` if `a` and `b` are matching platforms, where matching is determined by +comparing all keys contained within the platform objects, and if both objects contain +entries for that key, they must match. Comparison, by default, is performed using +the `==` operator, however this can be overridden on a key-by-key basis by adding +"comparison strategies" through `set_compare_strategy!(platform, key, func)`. + +Note that as the comparison strategy is set on the `Platform` object, and not globally, +a custom comparison strategy is first looked for within the `a` object, then if none +is found, it is looked for in the `b` object. Finally, if none is found in either, the +default of `==(ak, bk)` is used. We throw an error if custom comparison strategies are +used on both `a` and `b` and they are not the same custom comparison. + +The reserved tags `os_version` and `libstdcxx_version` use this mechanism to provide +bounded version constraints, where an artifact can specify that it was built using APIs +only available in macOS `v"10.11"` and later, or an artifact can state that it requires +a libstdc++ that is at least `v"3.4.22"`, etc... +""" +function platforms_match(a::AbstractPlatform, b::AbstractPlatform) + for k in union(keys(tags(a)), keys(tags(b))) + ak = get(tags(a), k, nothing) + bk = get(tags(b), k, nothing) + + # Only continue if both `ak` and `bk` are not `nothing` + if ak === nothing || bk === nothing + continue + end + + a_comp = get_compare_strategy(a, k) + b_comp = get_compare_strategy(b, k) + + # Throw an error if `a` and `b` have both set non-default comparison strategies for `k` + # and they're not the same strategy. + if a_comp != compare_default && b_comp != compare_default && a_comp != b_comp + throw(ArgumentError("Cannot compare Platform objects with two different non-default comparison strategies for the same key \"$(k)\"")) + end + + # Select the custom comparator, if we have one. + comparator = a_comp + if b_comp != compare_default + comparator = b_comp + end + + # Call the comparator, passing in which objects requested this comparison (one, the other, or both) + # For some comparators this doesn't matter, but for non-symmetrical comparisons, it does. + if !comparator(ak, bk, a_comp == comparator, b_comp == comparator) + return false + end + end + return true +end + +function platforms_match(a::String, b::AbstractPlatform) + return platforms_match(parse(Platform, a), b) +end +function platforms_match(a::AbstractPlatform, b::String) + return platforms_match(a, parse(Platform, b)) +end +platforms_match(a::String, b::String) = platforms_match(parse(Platform, a), parse(Platform, b)) + +# Adapters for AbstractString backedge avoidance +platforms_match(a::AbstractString, b::AbstractPlatform) = platforms_match(string(a)::String, b) +platforms_match(a::AbstractPlatform, b::AbstractString) = platforms_match(a, string(b)::String) +platforms_match(a::AbstractString, b::AbstractString) = platforms_match(string(a)::String, string(b)::String) + + +""" + select_platform(download_info::Dict, platform::AbstractPlatform = HostPlatform()) + +Given a `download_info` dictionary mapping platforms to some value, choose +the value whose key best matches `platform`, returning `nothing` if no matches +can be found. + +Platform attributes such as architecture, libc, calling ABI, etc... must all +match exactly, however attributes such as compiler ABI can have wildcards +within them such as `nothing` which matches any version of GCC. +""" +function select_platform(download_info::Dict, platform::AbstractPlatform = HostPlatform()) + ps = collect(filter(p -> platforms_match(p, platform), keys(download_info))) + + if isempty(ps) + return nothing + end + + # At this point, we may have multiple possibilities. E.g. if, in the future, + # Julia can be built without a direct dependency on libgfortran, we may match + # multiple tarballs that vary only within their libgfortran ABI. To narrow it + # down, we just sort by triplet, then pick the last one. This has the effect + # of generally choosing the latest release (e.g. a `libgfortran5` tarball + # rather than a `libgfortran3` tarball) + p = last(sort(ps, by = p -> triplet(p))) + return download_info[p] +end + +end # module diff --git a/base/sysimg.jl b/base/sysimg.jl index a2e5226baf532..85d0ac5a76b42 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -47,6 +47,7 @@ let :Distributed, :SharedArrays, :TOML, + :Artifacts, :Pkg, :Test, :REPL, diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 3418265ca3525..fa0e4b442fa4b 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -65,6 +65,21 @@ if Distributed !== nothing """ end +Artifacts = get(Base.loaded_modules, + Base.PkgId(Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts"), + nothing) +if Artifacts !== nothing + precompile_script *= """ + using Artifacts, Base.BinaryPlatforms + artifacts_toml = abspath($(repr(joinpath(Sys.STDLIB, "Artifacts", "test", "Artifacts.toml")))) + cd(() -> @artifact_str("c_simple"), dirname(artifacts_toml)) + artifacts = Artifacts.load_artifacts_toml(artifacts_toml) + platforms = [Artifacts.unpack_platform(e, "c_simple", artifacts_toml) for e in artifacts["c_simple"]] + best_platform = select_platform(Dict(p => triplet(p) for p in platforms)) + """ +end + + Pkg = get(Base.loaded_modules, Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg"), nothing) diff --git a/doc/make.jl b/doc/make.jl index 164fd32cf44b9..75c55f25e1673 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -182,6 +182,7 @@ DocMeta.setdocmeta!(SuiteSparse, :DocTestSetup, :(using SparseArrays, LinearAlge DocMeta.setdocmeta!(UUIDs, :DocTestSetup, :(using UUIDs, Random), recursive=true, warn=false) DocMeta.setdocmeta!(Pkg, :DocTestSetup, :(using Pkg, Pkg.Artifacts), recursive=true, warn=false) DocMeta.setdocmeta!(Pkg.BinaryPlatforms, :DocTestSetup, :(using Pkg, Pkg.BinaryPlatforms), recursive=true, warn=false) +DocMeta.setdocmeta!(Base.BinaryPlatforms, :DocTestSetup, :(using Base.BinaryPlatforms), recursive=true, warn=false) let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS) global const buildroot = i === nothing ? (@__DIR__) : first(match(r, ARGS[i]).captures) diff --git a/stdlib/Artifacts/Project.toml b/stdlib/Artifacts/Project.toml new file mode 100644 index 0000000000000..7251b79cea8c1 --- /dev/null +++ b/stdlib/Artifacts/Project.toml @@ -0,0 +1,8 @@ +name = "Artifacts" +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/stdlib/Artifacts/src/Artifacts.jl b/stdlib/Artifacts/src/Artifacts.jl new file mode 100644 index 0000000000000..723e399145203 --- /dev/null +++ b/stdlib/Artifacts/src/Artifacts.jl @@ -0,0 +1,633 @@ +module Artifacts + +import Base: get, SHA1 +using Base.BinaryPlatforms, Base.TOML + +export artifact_exists, artifact_path, artifact_meta, artifact_hash, + find_artifacts_toml, @artifact_str + +""" + parse_toml(path::String) + +Uses Base.TOML to parse a TOML file +""" +function parse_toml(path::String) + p = Base.TOML.Parser() + Base.TOML.reinit!(p, read(path, String); filepath=path) + return Base.TOML.parse(p) +end + +# keep in sync with Base.project_names and Base.manifest_names +const artifact_names = ("JuliaArtifacts.toml", "Artifacts.toml") + +const ARTIFACTS_DIR_OVERRIDE = Ref{Union{String,Nothing}}(nothing) +""" + with_artifacts_directory(f::Function, artifacts_dir::String) + +Helper function to allow temporarily changing the artifact installation and search +directory. When this is set, no other directory will be searched for artifacts, and new +artifacts will be installed within this directory. Similarly, removing an artifact will +only effect the given artifact directory. To layer artifact installation locations, use +the typical Julia depot path mechanism. +""" +function with_artifacts_directory(f::Function, artifacts_dir::String) + try + ARTIFACTS_DIR_OVERRIDE[] = artifacts_dir + f() + finally + ARTIFACTS_DIR_OVERRIDE[] = nothing + end +end + +""" + artifacts_dirs(args...) + +Return a list of paths joined into all possible artifacts directories, as dictated by the +current set of depot paths and the current artifact directory override via the method +`with_artifacts_dir()`. +""" +function artifacts_dirs(args...) + if ARTIFACTS_DIR_OVERRIDE[] === nothing + return String[abspath(depot, "artifacts", args...) for depot in Base.DEPOT_PATH] + else + # If we've been given an override, use _only_ that directory. + return String[abspath(ARTIFACTS_DIR_OVERRIDE[], args...)] + end +end + +""" + ARTIFACT_OVERRIDES + +Artifact locations can be overridden by writing `Override.toml` files within the artifact +directories of Pkg depots. For example, in the default depot `~/.julia`, one may create +a `~/.julia/artifacts/Override.toml` file with the following contents: + + 78f35e74ff113f02274ce60dab6e92b4546ef806 = "/path/to/replacement" + c76f8cda85f83a06d17de6c57aabf9e294eb2537 = "fb886e813a4aed4147d5979fcdf27457d20aa35d" + + [d57dbccd-ca19-4d82-b9b8-9d660942965b] + c_simple = "/path/to/c_simple_dir" + libfoo = "fb886e813a4aed4147d5979fcdf27457d20aa35d"" + +This file defines four overrides; two which override specific artifacts identified +through their content hashes, two which override artifacts based on their bound names +within a particular package's UUID. In both cases, there are two different targets of +the override: overriding to an on-disk location through an absolutet path, and +overriding to another artifact by its content-hash. +""" +const ARTIFACT_OVERRIDES = Ref{Union{Dict{Symbol,Any},Nothing}}(nothing) +function load_overrides(;force::Bool = false) + if ARTIFACT_OVERRIDES[] !== nothing && !force + return ARTIFACT_OVERRIDES[] + end + + # We organize our artifact location overrides into two camps: + # - overrides per UUID with artifact names mapped to a new location + # - overrides per hash, mapped to a new location. + # + # Overrides per UUID/bound name are intercepted upon Artifacts.toml load, and new + # entries within the "hash" overrides are generated on-the-fly. Thus, all redirects + # mechanisticly happen through the "hash" overrides. + overrides = Dict{Symbol,Any}( + # Overrides by UUID + :UUID => Dict{Base.UUID,Dict{String,Union{String,SHA1}}}(), + + # Overrides by hash + :hash => Dict{SHA1,Union{String,SHA1}}(), + ) + + for override_file in reverse(artifacts_dirs("Overrides.toml")) + !isfile(override_file) && continue + + # Load the toml file + depot_override_dict = parse_toml(override_file) + + function parse_mapping(mapping::String, name::String) + if !isabspath(mapping) && !isempty(mapping) + mapping = tryparse(Base.SHA1, mapping) + if mapping === nothing + @error("Invalid override in '$(override_file)': entry '$(name)' must map to an absolute path or SHA1 hash!") + end + end + return mapping + end + function parse_mapping(mapping::Dict, name::String) + return Dict(k => parse_mapping(v, name) for (k, v) in mapping) + end + # Fallthrough for invalid Overrides.toml files + parse_mapping(mapping, name::String) = nothing + + for (k, mapping) in depot_override_dict + # First, parse the mapping. Is it an absolute path, a valid SHA1-hash, or neither? + mapping = parse_mapping(mapping, k) + if mapping === nothing + @error("Invalid override in '$(override_file)': failed to parse entry `$(k)`") + continue + end + + # Next, determine if this is a hash override or a UUID/name override + if isa(mapping, String) || isa(mapping, SHA1) + # if this mapping is a direct mapping (e.g. a String), store it as a hash override + local hash_str + hash = tryparse(Base.SHA1, k) + if hash === nothing + @error("Invalid override in '$(override_file)': Invalid SHA1 hash '$(k)'") + continue + end + + # If this mapping is the empty string, un-override it + if mapping == "" + delete!(overrides[:hash], hash) + else + overrides[:hash][hash] = mapping + end + elseif isa(mapping, Dict) + # Convert `k` into a uuid + uuid = tryparse(Base.UUID, k) + if uuid === nothing + @error("Invalid override in '$(override_file)': Invalid UUID '$(k)'") + continue + end + + # If this mapping is itself a dict, store it as a set of UUID/artifact name overrides + ovruuid = overrides[:UUID]::Dict{Base.UUID,Dict{String,Union{String,SHA1}}} + if !haskey(ovruuid, uuid) + ovruuid[uuid] = Dict{String,Union{String,SHA1}}() + end + + # For each name in the mapping, update appropriately + for (name, override_value) in mapping + # If the mapping for this name is the empty string, un-override it + if override_value == "" + delete!(ovruuid[uuid], name) + else + # Otherwise, store it! + ovruuid[uuid][name] = override_value + end + end + else + @error("Invalid override in '$(override_file)': unknown mapping type for '$(k)': $(typeof(mapping))") + end + end + end + + ARTIFACT_OVERRIDES[] = overrides +end + +# Helpers to map an override to an actual path +map_override_path(x::String) = x +map_override_path(x::AbstractString) = string(x) +map_override_path(x::SHA1) = artifact_path(x) +map_override_path(x::Nothing) = nothing + +""" + query_override(hash::SHA1; overrides::Dict = load_overrides()) + +Query the loaded `/artifacts/Overrides.toml` settings for artifacts that should be +redirected to a particular path or another content-hash. +""" +function query_override(hash::SHA1; overrides::Dict = load_overrides()) + return map_override_path(get(overrides[:hash], hash, nothing)) +end +function query_override(pkg::Base.UUID, artifact_name::String; overrides::Dict = load_overrides()) + if haskey(overrides[:UUID], pkg) + return map_override_path(get(overrides[:UUID][pkg], artifact_name, nothing)) + end + return nothing +end + +""" + artifact_paths(hash::SHA1; honor_overrides::Bool=true) + +Return all possible paths for an artifact given the current list of depots as returned +by `Pkg.depots()`. All, some or none of these paths may exist on disk. +""" +function artifact_paths(hash::SHA1; honor_overrides::Bool=true) + # First, check to see if we've got an override: + if honor_overrides + override = query_override(hash) + if override !== nothing + return [override] + end + end + + return artifacts_dirs(bytes2hex(hash.bytes)) +end + +""" + artifact_path(hash::SHA1; honor_overrides::Bool=true) + +Given an artifact (identified by SHA1 git tree hash), return its installation path. If +the artifact does not exist, returns the location it would be installed to. + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function artifact_path(hash::SHA1; honor_overrides::Bool=true) + # Get all possible paths (rooted in all depots) + possible_paths = artifact_paths(hash; honor_overrides=honor_overrides) + + # Find the first path that exists and return it + for p in possible_paths + if isdir(p) + return p + end + end + + # If none exist, then just return the one that would exist within `depots1()`. + return first(possible_paths) +end + +""" + artifact_exists(hash::SHA1; honor_overrides::Bool=true) + +Returns whether or not the given artifact (identified by its sha1 git tree hash) exists +on-disk. Note that it is possible that the given artifact exists in multiple locations +(e.g. within multiple depots). + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function artifact_exists(hash::SHA1; honor_overrides::Bool=true) + return any(isdir, artifact_paths(hash; honor_overrides=honor_overrides)) +end + +""" + unpack_platform(entry::Dict, name::String, artifacts_toml::String) + +Given an `entry` for the artifact named `name`, located within the file `artifacts_toml`, +returns the `Platform` object that this entry specifies. Returns `nothing` on error. +""" +function unpack_platform(entry::Dict, name::String, artifacts_toml::String)::Union{Nothing,Platform} + if !haskey(entry, "os") + @error("Invalid artifacts file at '$(artifacts_toml)': platform-specific artifact entry '$name' missing 'os' key") + return nothing + end + + if !haskey(entry, "arch") + @error("Invalid artifacts file at '$(artifacts_toml)': platform-specific artifact entrty '$name' missing 'arch' key") + return nothing + end + + # Collect all String-valued mappings in `entry` and use them as tags + tags = Dict(Symbol(k) => v for (k, v) in entry if isa(v, String)) + # Removing some known entries that shouldn't be passed through `tags` + delete!(tags, :os) + delete!(tags, :arch) + delete!(tags, Symbol("git-tree-sha1")) + return Platform(entry["arch"], entry["os"]; tags...) +end + +function pack_platform!(meta::Dict, p::AbstractPlatform) + for (k, v) in tags(p) + if v !== nothing + meta[k] = v + end + end + return meta +end + +""" + load_artifacts_toml(artifacts_toml::String; + pkg_uuid::Union{UUID,Nothing}=nothing) + +Loads an `(Julia)Artifacts.toml` file from disk. If `pkg_uuid` is set to the `UUID` of the +owning package, UUID/name overrides stored in a depot `Overrides.toml` will be resolved. +""" +function load_artifacts_toml(artifacts_toml::String; + pkg_uuid::Union{Base.UUID,Nothing} = nothing) + artifact_dict = parse_toml(artifacts_toml) + + # Process overrides for this `pkg_uuid` + process_overrides(artifact_dict, pkg_uuid) + return artifact_dict +end + +""" + process_overrides(artifact_dict::Dict, pkg_uuid::Base.UUID) + +When loading an `Artifacts.toml` file, we must check `Override.toml` files to see if any +of the artifacts within it have been overridden by UUID. If they have, we honor the +overrides by inspecting the hashes of the targeted artifacts, then overriding them to +point to the given override, punting the actual redirection off to the hash-based +override system. This does not modify the `artifact_dict` object, it merely dynamically +adds more hash-based overrides as `Artifacts.toml` files that are overridden are loaded. +""" +function process_overrides(artifact_dict::Dict, pkg_uuid::Base.UUID) + # Insert just-in-time hash overrides by looking up the names of anything we need to + # override for this UUID, and inserting new overrides for those hashes. + overrides = load_overrides() + if haskey(overrides[:UUID], pkg_uuid) + pkg_overrides = overrides[:UUID][pkg_uuid] + + for name in keys(artifact_dict) + # Skip names that we're not overriding + if !haskey(pkg_overrides, name) + continue + end + + # If we've got a platform-specific friend, override all hashes: + if isa(artifact_dict[name], Array) + for entry in artifact_dict[name] + hash = SHA1(entry["git-tree-sha1"]) + overrides[:hash][hash] = overrides[:UUID][pkg_uuid][name] + end + elseif isa(artifact_dict[name], Dict) + hash = SHA1(artifact_dict[name]["git-tree-sha1"]) + overrides[:hash][hash] = overrides[:UUID][pkg_uuid][name] + end + end + end + return artifact_dict +end + +# If someone tries to call process_overrides() with `nothing`, do exactly that +process_overrides(artifact_dict::Dict, pkg_uuid::Nothing) = nothing + +""" + artifact_meta(name::String, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Base.UUID,Nothing}=nothing) + +Get metadata about a given artifact (identified by name) stored within the given +`(Julia)Artifacts.toml` file. If the artifact is platform-specific, use `platform` to choose the +most appropriate mapping. If none is found, return `nothing`. + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function artifact_meta(name::String, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Base.UUID,Nothing}=nothing) + if !isfile(artifacts_toml) + return nothing + end + + # Parse the toml of the artifacts_toml file + artifact_dict = load_artifacts_toml(artifacts_toml; pkg_uuid=pkg_uuid) + return artifact_meta(name, artifact_dict, artifacts_toml; platform=platform) +end + +function artifact_meta(name::String, artifact_dict::Dict, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform()) + if !haskey(artifact_dict, name) + return nothing + end + meta = artifact_dict[name] + + # If it's an array, find the entry that best matches our current platform + if isa(meta, Array) + dl_dict = Dict{AbstractPlatform,Dict{String,Any}}(unpack_platform(x, name, artifacts_toml) => x for x in meta) + meta = select_platform(dl_dict, platform) + # If it's NOT a dict, complain + elseif !isa(meta, Dict) + @error("Invalid artifacts file at $(artifacts_toml): artifact '$name' malformed, must be array or dict!") + return nothing + end + + # This is such a no-no, we are going to call it out right here, right now. + if meta !== nothing && !haskey(meta, "git-tree-sha1") + @error("Invalid artifacts file at $(artifacts_toml): artifact '$name' contains no `git-tree-sha1`!") + return nothing + end + + # Return the full meta-dict. + return meta +end + +""" + artifact_hash(name::String, artifacts_toml::String; + platform::AbstractPlatform = platform_key_abi()) + +Thin wrapper around `artifact_meta()` to return the hash of the specified, platform- +collapsed artifact. Returns `nothing` if no mapping can be found. + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function artifact_hash(name::String, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Base.UUID,Nothing}=nothing) + meta = artifact_meta(name, artifacts_toml; platform=platform) + if meta === nothing + return nothing + end + + return SHA1(meta["git-tree-sha1"]) +end + +""" + find_artifacts_toml(path::String) + +Given the path to a `.jl` file, (such as the one returned by `__source__.file` in a macro +context), find the `(Julia)Artifacts.toml` that is contained within the containing project (if it +exists), otherwise return `nothing`. + +!!! compat "Julia 1.3" + This function requires at least Julia 1.3. +""" +function find_artifacts_toml(path::String) + if !isdir(path) + path = dirname(path) + end + + # Run until we hit the root directory. + while dirname(path) != path + for f in artifact_names + artifacts_toml_path = joinpath(path, f) + if isfile(artifacts_toml_path) + return abspath(artifacts_toml_path) + end + end + + # Does a `(Julia)Project.toml` file exist here, in the absence of an Artifacts.toml? + # If so, stop the search as we've probably hit the top-level of this package, + # and we don't want to escape out into the larger filesystem. + for f in Base.project_names + if isfile(joinpath(path, f)) + return nothing + end + end + + # Move up a directory + path = dirname(path) + end + + # We never found anything, just return `nothing` + return nothing +end + +# We do this to avoid doing the `joinpath()` work if we don't have to, and also to +# avoid a trailing slash due to `joinpath()`'s habit of including one when the last +# argument is the empty string. +function jointail(dir, tail) + if !isempty(tail) + return joinpath(dir, tail) + else + return dir + end +end + +function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash) + if haskey(Base.module_keys, __module__) + # Process overrides for this UUID, if we know what it is + process_overrides(artifact_dict, Base.module_keys[__module__].uuid) + end + + # If the artifact exists, we're in the happy path and we can immediately + # return the path to the artifact: + for dir in artifact_paths(hash; honor_overrides=true) + if isdir(dir) + return jointail(dir, path_tail) + end + end + + # If not, we need to download it. We look up the Pkg module through `Base.loaded_modules()` + # then invoke `ensure_artifact_installed()`: + Pkg = first(filter(p-> p[1].name == "Pkg", Base.loaded_modules))[2] + return jointail(Pkg.Artifacts.ensure_artifact_installed(string(name), artifacts_toml), path_tail) +end + +""" + split_artifact_slash(name::String) + +Splits an artifact indexing string by path deliminters, isolates the first path element, +returning that and the `joinpath()` of the remaining arguments. This normalizes all path +separators to the native path separator for the current platform. Examples: + +# Examples +```jldoctest +julia> split_artifact_slash("Foo") +("Foo", "") + +julia> ret = split_artifact_slash("Foo/bar/baz.so"); + +julia> if Sys.iswindows() + ret == ("Foo", "bar\\baz.so") + else + ret == ("Foo", "bar/baz.so") + end +true + +julia> ret = split_artifact_slash("Foo\\bar\\baz.so"); + +julia> if Sys.iswindows() + ret == ("Foo", "bar\\baz.so") + else + ret == ("Foo", "bar/baz.so") + end +true +``` +""" +function split_artifact_slash(name::String) + split_name = split(name, r"(/|\\)") + if length(split_name) == 1 + return (split_name[1], "") + else + return (split_name[1], joinpath(split_name[2:end]...)) + end +end + +""" + artifact_slash_lookup(name::String, artifacts_toml::String) + +Returns `artifact_name`, `artifact_path_tail`, and `hash` by looking the results up in +the given `artifacts_toml`, first extracting the name and path tail from the given `name` +to support slash-indexing within the given artifact. +""" +function artifact_slash_lookup(name::String, artifact_dict::Dict, artifacts_toml::String) + artifact_name, artifact_path_tail = split_artifact_slash(name) + + meta = artifact_meta(artifact_name, artifact_dict, artifacts_toml) + if meta === nothing + error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'") + end + hash = SHA1(meta["git-tree-sha1"]) + return artifact_name, artifact_path_tail, hash +end + +""" + macro artifact_str(name) + +Macro that is used to automatically ensure an artifact is installed, and return its +location on-disk. Automatically looks the artifact up by name in the project's +`(Julia)Artifacts.toml` file. Throws an error on inability to install the requested +artifact. If run in the REPL, searches for the toml file starting in the current +directory, see `find_artifacts_toml()` for more. + +If `name` contains a forward or backward slash, all elements after the first slash will +be taken to be path names indexing into the artifact, allowing for an easy one-liner to +access a single file/directory within an artifact. Example: + + ffmpeg_path = @artifact"FFMPEG/bin/ffmpeg" + +!!! compat "Julia 1.3" + This macro requires at least Julia 1.3. + +!!! compat "Julia 1.6" + Slash-indexing requires at least Julia 1.6. +""" +macro artifact_str(name) + # Find Artifacts.toml file we're going to load from + srcfile = string(__source__.file) + if ((isinteractive() && startswith(srcfile, "REPL[")) || (!isinteractive() && srcfile == "none")) && !isfile(srcfile) + srcfile = pwd() + end + local artifacts_toml = find_artifacts_toml(srcfile) + if artifacts_toml === nothing + error(string( + "Cannot locate '(Julia)Artifacts.toml' file when attempting to use artifact '", + name, + "' in '", + __module__, + "'", + )) + end + + # Load Artifacts.toml at compile time, so that we don't have to use `__source__.file` + # at runtime, which gets stale if the `.ji` file is relocated. + local artifact_dict = load_artifacts_toml(artifacts_toml) + + # Invalidate calling .ji file if Artifacts.toml file changes + Base.include_dependency(artifacts_toml) + + # If `name` is a constant, we can actually load and parse the `Artifacts.toml` file now, + # saving the work from runtime. + if isa(name, AbstractString) + # To support slash-indexing, we need to split the artifact name from the path tail: + local artifact_name, artifact_path_tail, hash = artifact_slash_lookup(name, artifact_dict, artifacts_toml) + return quote + Base.invokelatest(_artifact_str, $(__module__), $(artifacts_toml), $(artifact_name), $(artifact_path_tail), $(artifact_dict), $(hash)) + end + else + return quote + local artifact_name, artifact_path_tail, hash = artifact_slash_lookup($(esc(name)), $(artifact_dict), $(artifacts_toml)) + Base.invokelatest(_artifact_str, $(__module__), $(artifacts_toml), artifact_name, artifact_path_tail, $(artifact_dict), hash) + end + end +end + +# Support `AbstractString`s, but avoid compilers needing to track backedges for callers +# of these functions in case a user defines a new type that is `<: AbstractString` +with_artifacts_directory(f::Function, artifacts_dir::AbstractString) = + with_artifacts_directory(f, string(artifacts_dir)) +query_override(pkg::Base.UUID, artifact_name::AbstractString; kwargs...) = + query_override(pkg, string(artifact_name); kwargs...) +unpack_platform(entry::Dict, name::AbstractString, artifacts_toml::AbstractString) = + unpack_platform(entry, string(name), string(artifacts_toml)) +load_artifacts_toml(artifacts_toml::AbstractString; kwargs...) = + load_artifacts_toml(string(artifacts_toml); kwargs...) +artifact_meta(name::AbstractString, artifacts_toml::AbstractString; kwargs...) = + artifact_meta(string(name), string(artifacts_toml); kwargs...) +artifact_meta(name::AbstractString, artifact_dict::Dict, artifacts_toml::AbstractString; kwargs...) = + artifact_meta(string(name), artifact_dict, string(artifacts_toml); kwargs...) +artifact_hash(name::AbstractString, artifacts_toml::AbstractString; kwargs...) = + artifact_hash(string(name), string(artifacts_toml); kwargs...) +find_artifacts_toml(path::AbstractString) = + find_artifacts_toml(string(path)) +split_artifact_slash(name::AbstractString) = + split_artifact_slash(string(name)) +artifact_slash_lookup(name::AbstractString, artifact_dict::Dict, artifacts_toml::AbstractString) = + artifact_slash_lookup(string(name), artifact_dict, string(artifacts_toml)) + +end # module Artifacts diff --git a/stdlib/Artifacts/test/Artifacts.toml b/stdlib/Artifacts/test/Artifacts.toml new file mode 100644 index 0000000000000..ee0fffb9ed92a --- /dev/null +++ b/stdlib/Artifacts/test/Artifacts.toml @@ -0,0 +1,138 @@ +[[c_simple]] +arch = "armv7l" +git-tree-sha1 = "0c509b3302db90a9393d6036c3ffcd14d190523d" +libc = "glibc" +os = "linux" + + [[c_simple.download]] + sha256 = "b0cfa3a2d9b5bc0632b0ee45b5d049eecbf72ed9c8cbc968b374ea995257a635" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.arm-linux-gnueabihf.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "e5a893fdac080fa0d4ae1cbd8bd67cfba5945af2" +os = "freebsd" + + [[c_simple.download]] + sha256 = "fde6e4ed00227b98e25ffdbf4e2b8b24a4e2bfa4c532c733d3626d6157e448ce" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-unknown-freebsd11.1.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "7ba74e239348ea6c060f994c083260be3abe3095" +os = "macos" + + [[c_simple.download]] + sha256 = "e88816a1492eecb4569bb24b3e52b757e59c87419dba962e99148b338369f326" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-apple-darwin14.tar.gz" + +# NOTE: We explicitly comment this out, to test porous platform support. Don't un-comment this! +#[[c_simple]] +#arch = "powerpc64le" +#git-tree-sha1 = "dc9f84891c8215f90095b619533e141179b6cc06" +#libc = "glibc" +#os = "linux" +# +# [[c_simple.download]] +# sha256 = "715af8f0405cff35feef5ad5e93836bb1bb0f93c77218bfdad411c8a4368ab4b" +# url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.powerpc64le-linux-gnu.tar.gz" + +[[c_simple]] +arch = "i686" +git-tree-sha1 = "78e282b79c16febc54a56ed244088ff92a55533f" +libc = "musl" +os = "linux" + + [[c_simple.download]] + sha256 = "900f2e55f72af0c723f9db7e9f44b1c16155010de212b430f02091dc24ff324c" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.i686-linux-musl.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "9d0075fdafe8af6430afba41fea2f32811141145" +libc = "musl" +os = "linux" + + [[c_simple.download]] + sha256 = "2769be12e00ebb0a3c7ab43b90b71bba3a6883844416457e08b880945d129689" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-linux-musl.tar.gz" + +[[c_simple]] +arch = "i686" +git-tree-sha1 = "0c890d3e6c5ee00fd06a7d418fad424159e447ce" +libc = "glibc" +os = "linux" + + [[c_simple.download]] + sha256 = "45d42cbb5cfafefeadfd46cd91445466d0e245f1f640cd4f91cdae01a654e001" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.i686-linux-gnu.tar.gz" + +[[c_simple]] +arch = "i686" +git-tree-sha1 = "956a97e2d56c0fa3b634f57e10a174dff0052ba4" +os = "windows" + + [[c_simple.download]] + sha256 = "b0214fa6f48359f2cf328b2ec2255ed975939939333db03711f63671c5d4fed9" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.i686-w64-mingw32.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "0ed63482ad1916dba12b4959d2704af4e41252da" +libc = "glibc" +os = "linux" + + [[c_simple.download]] + sha256 = "edbaf461c5c33fd7030bcd197b849396f8328648a2e04462b1bea9650f782a3b" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-linux-gnu.tar.gz" + +[[c_simple]] +arch = "x86_64" +git-tree-sha1 = "444cecb70ff39e8961dd33e230e151775d959f37" +os = "windows" + + [[c_simple.download]] + sha256 = "39b75afda9f0619f042c47bef2cdd0931a33c5c5acb7dc2977f1cd6274835c1f" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.x86_64-w64-mingw32.tar.gz" + +[[c_simple]] +arch = "aarch64" +git-tree-sha1 = "efc8df78802fae852abd8e213e4b6f3f1da48125" +libc = "musl" +os = "linux" + + [[c_simple.download]] + sha256 = "53f54d76b4f9edd2a5b8c20575ef9f6a05c524a454c9126f4ecaa83a8aa72f52" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.aarch64-linux-musl.tar.gz" + +[[c_simple]] +arch = "aarch64" +git-tree-sha1 = "ca19bcae2bc6af88d6ace2648c7cc639b3cf1dfb" +libc = "glibc" +os = "linux" + + [[c_simple.download]] + sha256 = "94e303d2d779734281b60ef1880ad0e12681a2d3d6eed3a3ee3129a3efb016f7" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.aarch64-linux-gnu.tar.gz" + +[[c_simple]] +arch = "armv7l" +git-tree-sha1 = "0d8cebd76188d1bade6057b70605e553bbdfdd02" +libc = "musl" +os = "linux" + + [[c_simple.download]] + sha256 = "da1a04400ca8bcf51d2c39783e1fcc7d51fba5cf4f328d6bff2512ea5342532b" + url = "https://github.com/JuliaBinaryWrappers/c_simple_jll.jl/releases/download/c_simple+v1.2.3-pkgtest/c_simple.v1.2.3.arm-linux-musleabihf.tar.gz" + +[socrates] +git-tree-sha1 = "43563e7631a7eafae1f9f8d9d332e3de44ad7239" +lazy = true + + [[socrates.download]] + url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.gz" + sha256 = "e65d2f13f2085f2c279830e863292312a72930fee5ba3c792b14c33ce5c5cc58" + + [[socrates.download]] + url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.bz2" + sha256 = "13fc17b97be41763b02cbb80e9d048302cec3bd3d446c2ed6e8210bddcd3ac76" \ No newline at end of file diff --git a/stdlib/Artifacts/test/runtests.jl b/stdlib/Artifacts/test/runtests.jl new file mode 100644 index 0000000000000..b81acabca3175 --- /dev/null +++ b/stdlib/Artifacts/test/runtests.jl @@ -0,0 +1,108 @@ +using Artifacts, Test, Base.BinaryPlatforms +using Artifacts: with_artifacts_directory, pack_platform!, unpack_platform + +@testset "Artifact Paths" begin + mktempdir() do tempdir + with_artifacts_directory(tempdir) do + hash = Base.SHA1("0"^40) + paths = Artifacts.artifact_paths(hash) + @test length(paths) == 1 + @test startswith(first(paths), tempdir) + + @test !artifact_exists(hash) + mkpath(first(paths)) + @test artifact_exists(hash) + @test artifact_path(hash) == first(paths) + end + end +end + +@testset "Serialization Tools" begin + # First, some basic tests + meta = Dict() + pack_platform!(meta, Platform("i686", "linux")) + @test meta["os"] == "linux" + @test meta["arch"] == "i686" + @test meta["libc"] == "glibc" + + meta = Dict() + pack_platform!(meta, Platform("armv7l", "linux"; libc="musl")) + @test meta["os"] == "linux" + @test meta["arch"] == "armv7l" + @test meta["libc"] == "musl" + + meta = Dict() + pack_platform!(meta, Platform("x86_64", "windows"; libgfortran_version=v"3")) + @test meta["os"] == "windows" + @test meta["arch"] == "x86_64" + @test meta["libgfortran_version"] == "3.0.0" + + meta = Dict() + pack_platform!(meta, Platform("aarch64", "macOS")) + @test meta == Dict("os" => "macos", "arch" => "aarch64") + + # Next, fuzz it out! Ensure that we exactly reconstruct our platforms! + platforms = Platform[] + for libgfortran_version in (v"3", v"4", v"5", nothing), + libstdcxx_version in (v"3.4.11", v"3.4.19", nothing), + cxxstring_abi in ("cxx03", "cxx11", nothing) + + for arch in ("x86_64", "i686", "aarch64", "armv7l"), + libc in ("glibc", "musl") + + push!(platforms, Platform(arch, "linux"; libc, libgfortran_version, libstdcxx_version, cxxstring_abi)) + end + push!(platforms, Platform("x86_64", "windows"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + push!(platforms, Platform("i686", "windows"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + push!(platforms, Platform("x86_64", "macOS"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + push!(platforms, Platform("aarch64", "macOS"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + push!(platforms, Platform("x86_64", "FreeBSD"; libgfortran_version, libstdcxx_version, cxxstring_abi)) + end + + for p in platforms + meta = Dict() + pack_platform!(meta, p) + @test unpack_platform(meta, "foo", "") == p + + # Test that some things raise warnings + bad_meta = copy(meta) + delete!(bad_meta, "os") + @test_logs (:error, r"Invalid artifacts file") unpack_platform(bad_meta, "foo", "") + + bad_meta = copy(meta) + delete!(bad_meta, "arch") + @test_logs (:error, r"Invalid artifacts file") unpack_platform(bad_meta, "foo", "") + end +end + +@testset "Artifact Slash-indexing" begin + mktempdir() do tempdir + with_artifacts_directory(tempdir) do + exeext = Sys.iswindows() ? ".exe" : "" + + # simple lookup, gives us the directory for `c_simple` for the current architecture + c_simple_dir = artifact"c_simple" + @test isdir(c_simple_dir) + c_simple_exe_path = joinpath(c_simple_dir, "bin", "c_simple$(exeext)") + @test isfile(c_simple_exe_path) + + # Simple slash-indexed lookup + c_simple_bin_path = artifact"c_simple/bin" + @test isdir(c_simple_bin_path) + # Test that forward and backward slash are equivalent + @test artifact"c_simple\\bin" == artifact"c_simple/bin" + + # Dynamically-computed lookup; not done at compile-time + generate_artifact_name() = "c_simple" + c_simple_dir = @artifact_str(generate_artifact_name()) + @test isdir(c_simple_dir) + c_simple_exe_path = joinpath(c_simple_dir, "bin", "c_simple$(exeext)") + @test isfile(c_simple_exe_path) + + # Dynamically-computed slash-indexing: + generate_bin_path(pathsep) = "c_simple$(pathsep)bin$(pathsep)c_simple$(exeext)" + @test isfile(@artifact_str(generate_bin_path("/"))) + @test isfile(@artifact_str(generate_bin_path("\\"))) + end + end +end diff --git a/stdlib/Makefile b/stdlib/Makefile index ece6b58644a16..4e748b42aac19 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -14,7 +14,7 @@ VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) $(build_datarootdir)/julia/stdlib/$(VERSDIR): mkdir -p $@ -STDLIBS = Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \ +STDLIBS = Artifacts Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \ Future InteractiveUtils Libdl LibGit2 LinearAlgebra Logging \ Markdown Mmap Printf Profile Random REPL Serialization SHA \ SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs diff --git a/test/binaryplatforms.jl b/test/binaryplatforms.jl new file mode 100644 index 0000000000000..bfb28a82a2a5c --- /dev/null +++ b/test/binaryplatforms.jl @@ -0,0 +1,357 @@ +using Test, Base.BinaryPlatforms + +# Helper constructor to create a Platform with `validate_strict` set to `true`. +P(args...; kwargs...) = Platform(args...; validate_strict=true, kwargs...) + +# Ensure the platform type constructors are well behaved +@testset "Platform validation" begin + for os_name in ("Linux", "linux", "FREEBSD", "wiNDows", "MacOS") + p = P("x86_64", os_name) + @test isa(p, Platform) + @test os(p) == lowercase(os_name) + end + for arch_name in ("x86_64", "ARMv7l", "armv6l", "I686", "pOWerpc64le") + p = P(arch_name, "linux") + @test isa(p, Platform) + @test arch(p) == lowercase(arch_name) + end + # Test some normalization + @test arch(P("amd64", "freebsd")) == "x86_64" + @test arch(P("ARM", "linux")) == "armv7l" + @test arch(P("ppc64le", "linux")) == "powerpc64le" + + # test some error cases + @test_throws ArgumentError P("x64_86", "linux") + @test_throws ArgumentError P("x86_64", "not_an_os") + @test_throws ArgumentError P("x86_64", "linux"; libc="crazy_libc") + @test_throws ArgumentError P("x86_64", "linux"; libc="glibc", call_abi="crazy_abi") + @test_throws ArgumentError P("x86_64", "linux"; libc="glibc", call_abi="eabihf") + @test_throws ArgumentError P("i686", "linux"; libc="musl", call_abi="eabi") + @test_throws ArgumentError P("arm", "linux"; call_abi="") + @test_throws ArgumentError P("armv7l", "linux"; call_abi="kekeke") + @test_throws ArgumentError P("armv6l", "linux"; call_abi="kekeke") + @test_throws ArgumentError P("armv6l", "linux"; libgfortran_version="lel") + @test_throws ArgumentError P("x86_64", "linux"; cxxstring_abi="lel") + @test_throws ArgumentError P("x86_64", "windows"; libstdcxx_version="lel") + @test_throws ArgumentError P("i686", "macos") + @test_throws ArgumentError P("x86_64", "macos"; libc="glibc") + @test_throws ArgumentError P("x86_64", "macos"; call_abi="eabihf") + @test_throws ArgumentError P("powerpc64le", "windows") + @test_throws ArgumentError P("x86_64", "windows"; call_abi="eabihf") + @test_throws ArgumentError P("x86_64", "freebsd"; libc="crazy_libc") + @test_throws ArgumentError P("x86_64", "freebsd"; call_abi="crazy_abi") + @test_throws ArgumentError P("x86_64", "freebsd"; call_abi="eabihf") + @test_throws ArgumentError P("x86_64", "linux"; arch="i686") + @test_throws ArgumentError P("x86_64", "linux"; ARCH="i686") + @test_throws ArgumentError P("x86_64", "linux"; os="windows") +end + +@testset "Platform properties" begin + # Test that `platform_name()` works + platforms = ("Linux", "macOS", "Windows", "FreeBSD") + for platform in platforms + @test platform_name(P("x86_64", platform)) == platform + end + + # Test `arch()` + arch_names = ("x86_64", "i686", "powerpc64le", "armv7l", "armv6l", "aarch64") + for arch_name in arch_names + @test arch(P(arch_name, "linux")) == arch_name + end + + # Test that if we aren't using strict validation, we can actually use new names too: + @test arch(Platform("jpu", "linux")) == "jpu" + + # platform_dlext() + @test platform_dlext(P("x86_64", "linux")) == "so" + @test platform_dlext(P("armv7l", "windows")) == "dll" + @test platform_dlext(P("x86_64", "freebsd")) == "so" + @test platform_dlext(P("aarch64", "macos")) == "dylib" + @test platform_dlext() == platform_dlext(HostPlatform()) + + # wordsize() + @test wordsize(P("i686", "linux")) == wordsize(P("armv7l", "windows")) == 32 + @test wordsize(P("aarch64", "macos")) == wordsize(P("x86_64", "freebsd")) == 64 + @test wordsize(P("x86_64", "windows")) == wordsize(P("powerpc64le", "linux")) == 64 + + # call_abi() + for platform in platforms + @test call_abi(P("x86_64", platform)) === nothing + end + @test call_abi(P("armv7l", "linux")) == "eabihf" + @test call_abi(P("armv7l", "linux"; call_abi="eabihf")) == "eabihf" + @test call_abi(P("armv7l", "linux"; call_abi="eabi")) == "eabi" + @test call_abi(P("armv6l", "linux")) == "eabihf" + # Test that we can at least set an `eabi` call ABI, not that Julia actually supports it... + @test call_abi(P("armv7l", "linux"; call_abi="eabi")) == "eabi" + + # Test some different OS's and libc/call ABIs + @test triplet(P("i686", "Windows")) == "i686-w64-mingw32" + @test triplet(P("x86_64", "linux"; libc="musl")) == "x86_64-linux-musl" + @test triplet(P("armv7l", "linux"; libc="musl")) == "armv7l-linux-musleabihf" + @test triplet(P("armv6l", "linux"; libc="musl", call_abi="eabihf")) == "armv6l-linux-musleabihf" + @test triplet(P("armv6l", "linux"; call_abi="eabi")) == "armv6l-linux-gnueabi" + @test triplet(P("x86_64", "linux")) == "x86_64-linux-gnu" + @test triplet(P("armv6l", "linux")) == "armv6l-linux-gnueabihf" + @test triplet(P("x86_64", "macos")) == "x86_64-apple-darwin" + @test triplet(P("x86_64", "freebsd")) == "x86_64-unknown-freebsd" + @test triplet(P("i686", "freebsd")) == "i686-unknown-freebsd" + + # Now test libgfortran/cxxstring ABIs + @test triplet(P("x86_64", "linux"; libgfortran_version=v"3", cxxstring_abi="cxx11")) == "x86_64-linux-gnu-libgfortran3-cxx11" + @test triplet(P("armv7l", "linux"; libc="musl", cxxstring_abi="cxx03")) == "armv7l-linux-musleabihf-cxx03" + + # Test tags() + t = tags(P("x86_64", "linux")) + @test all(haskey.(Ref(t), ("arch", "os", "libc"))) + @test haskey(tags(P("x86_64", "linux"; customtag="foo")), "customtag") + @test tags(HostPlatform())["julia_version"] == string(VERSION.major, ".", VERSION.minor, ".", VERSION.patch) +end + +@testset "Triplet parsing" begin + # Make sure the Platform() with explicit triplet works + R(str) = parse(Platform, str; validate_strict=true) + @test R("x86_64-linux-gnu") == P("x86_64", "linux") + @test R("x86_64-linux-musl") == P("x86_64", "linux"; libc="musl") + @test R("i686-unknown-linux-gnu") == P("i686", "linux") + @test R("x86_64-apple-darwin") == P("x86_64", "macos") + @test R("x86_64-apple-darwin14") == P("x86_64", "macos"; os_version="14") + @test R("x86_64-apple-darwin17.0.0") == P("x86_64", "macos"; os_version="17") + @test R("armv7l-pc-linux-gnueabihf") == P("armv7l", "linux") + @test R("armv7l-linux-musleabihf") == P("armv7l", "linux"; libc="musl") + @test R("armv6l-linux-gnueabi") == P("armv6l", "linux"; call_abi="eabi") + # Test that the short name "arm" goes to `armv7l` + @test R("arm-linux-gnueabihf") == P("armv7l", "linux") + @test R("aarch64-unknown-linux-gnu") == P("aarch64", "linux") + @test R("powerpc64le-linux-gnu") == P("powerpc64le", "linux") + @test R("ppc64le-linux-gnu") == P("powerpc64le", "linux") + @test R("x86_64-w64-mingw32") == P("x86_64", "windows") + @test R("i686-w64-mingw32") == P("i686", "windows") + + # FreeBSD has lots of arch names that don't match elsewhere + @test R("x86_64-unknown-freebsd11.1") == P("x86_64", "freebsd"; os_version=v"11.1") + @test R("i686-unknown-freebsd11.1") == P("i686", "freebsd"; os_version=v"11.1") + @test R("amd64-unknown-freebsd12.0") == P("x86_64", "freebsd"; os_version=v"12.0") + @test R("i386-unknown-freebsd10.3") == P("i686", "freebsd"; os_version=v"10.3") + @test R("aarch64-apple-darwin18.7") == P("aarch64", "macos"; os_version=v"18.7") + @test R("arm64-apple-darwin20") == P("aarch64", "macos"; os_version=v"20") + + # Test inclusion of ABI stuff, both old-style and new-style + @test R("x86_64-linux-gnu-gcc7") == P("x86_64", "linux"; libgfortran_version=v"4") + @test R("x86_64-linux-gnu-gcc4-cxx11") == P("x86_64", "linux"; libgfortran_version=v"3", cxxstring_abi="cxx11") + @test R("x86_64-linux-gnu-cxx11") == P("x86_64", "linux"; cxxstring_abi="cxx11") + @test R("x86_64-linux-gnu-libgfortran3-cxx03") == P("x86_64", "linux"; libgfortran_version=v"3", cxxstring_abi="cxx03") + @test R("x86_64-linux-gnu-libstdcxx26") == P("x86_64", "linux"; libstdcxx_version=v"3.4.26") + + @test_throws ArgumentError R("totally FREEFORM text!!1!!!1!") + @test_throws ArgumentError R("invalid-triplet-here") + @test parse(Platform, "aarch64-linux-gnueabihf") == Platform("aarch64", "linux"; call_abi="eabihf") + @test_throws ArgumentError R("aarch64-linux-gnueabihf") + @test_throws ArgumentError R("x86_64-w32-mingw64") + + # Test extended attributes + @test R("x86_64-linux-gnu-march+avx2") == P("x86_64", "linux"; march="avx2") + @test R("x86_64-linux-gnu-march+x86_64-cuda+10.1") == P("x86_64", "linux"; march="x86_64", cuda="10.1") + @test_throws ArgumentError R("x86_64-linux-gnu-march+generic";) + @test_throws ArgumentError R("x86_64-linux-gnu-cuda+version") + + # Round-trip our little homie through `triplet()` + @test parse(Platform, triplet(HostPlatform())) == HostPlatform() + + # Also test round-tripping through `repr()`: + @test eval(Meta.parse(repr(HostPlatform()))) == HostPlatform() +end + +@testset "platforms_match()" begin + # Just do a quick combinatorial sweep for completeness' sake for platform matching + linux = P("x86_64", "linux") + for libgfortran_version in (nothing, v"3", v"5"), + libstdcxx_version in (nothing, v"3.4.18", v"3.4.26"), + cxxstring_abi in (nothing, :cxx03, :cxx11) + + p = P("x86_64", "linux"; libgfortran_version, libstdcxx_version, cxxstring_abi) + @test platforms_match(linux, p) + @test platforms_match(p, linux) + + # Also test auto-string-parsing + @test platforms_match(triplet(linux), p) + @test platforms_match(linux, triplet(p)) + end + + # Test that Julia version is matched only on major.minor by default + @test platforms_match(P("x86_64", "linux"; julia_version=v"1.5.0"), + P("x86_64", "linux"; julia_version=v"1.5.1")) + @test !platforms_match(P("x86_64", "linux"; julia_version=v"1.5.0"), + P("x86_64", "linux"; julia_version=v"1.6.0")) + + # Ensure many of these things do NOT match + @test !platforms_match(linux, P("i686", "linux")) + @test !platforms_match(linux, P("x86_64", "windows")) + @test !platforms_match(linux, P("x86_64", "macos")) + + # Make some explicitly non-matching compiler ABI platforms + host = P("x86_64", "linux"; libgfortran_version=v"5", cxxstring_abi="cxx11") + for arch in ("x86_64", "i686", "aarch64", "armv6l", "armv7l", "powerpc64le"), + kwargs in ((:libgfortran_version => v"3",), (:cxxstring_abi => "cxx03",), + (:libgfortran_version => v"4", :cxxstring_abi => "cxx11"), + (:libgfortran_version => v"3", :cxxstring_abi => "cxx03")) + a = P(arch, "linux"; libgfortran_version=v"5", cxxstring_abi="cxx11") + b = P(arch, "linux"; kwargs...) + @test !platforms_match(a, b) + end + + # Test version bounds with HostPlatform() + host = HostPlatform(P("x86_64", "macos"; os_version="14", libstdcxx_version=v"3.4.26")) + @test platforms_match(host, P("x86_64", "macos")) + @test platforms_match(host, P("x86_64", "macos"; os_version="14")) + @test platforms_match(host, P("x86_64", "macos"; os_version="13")) + @test !platforms_match(host, P("x86_64", "macos"; os_version="15")) + @test platforms_match(host, P("x86_64", "macos"; libstdcxx_version="3.4.18")) + @test platforms_match(host, P("x86_64", "macos"; os_version=v"10", libstdcxx_version="3.4.18")) + @test !platforms_match(host, P("x86_64", "macos"; os_version=v"10", libstdcxx_version="3.4.27")) + @test !platforms_match(host, P("x86_64", "macos"; os_version=v"14", libstdcxx_version=v"4")) +end + +@testset "DL name/version parsing" begin + # Make sure our version parsing code is working + @test parse_dl_name_version("libgfortran.so", "linux") == ("libgfortran", nothing) + @test parse_dl_name_version("libgfortran.so.3", "linux") == ("libgfortran", v"3") + @test parse_dl_name_version("libgfortran.so.3.4", "linux") == ("libgfortran", v"3.4") + @test_throws ArgumentError parse_dl_name_version("libgfortran.so.3.4a", "linux") + @test_throws ArgumentError parse_dl_name_version("libgfortran", "linux") + @test parse_dl_name_version("libgfortran.so", "freebsd") == ("libgfortran", nothing) + @test parse_dl_name_version("libgfortran.so.3", "freebsd") == ("libgfortran", v"3") + @test parse_dl_name_version("libgfortran.so.3.4", "freebsd") == ("libgfortran", v"3.4") + @test_throws ArgumentError parse_dl_name_version("libgfortran.so.3.4a", "freebsd") + @test_throws ArgumentError parse_dl_name_version("libgfortran", "freebsd") + @test parse_dl_name_version("libgfortran.dylib", "macos") == ("libgfortran", nothing) + @test parse_dl_name_version("libgfortran.3.dylib", "macos") == ("libgfortran", v"3") + @test parse_dl_name_version("libgfortran.3.4.dylib", "macos") == ("libgfortran", v"3.4") + @test parse_dl_name_version("libgfortran.3.4a.dylib", "macos") == ("libgfortran.3.4a", nothing) + @test_throws ArgumentError parse_dl_name_version("libgfortran", "macos") + @test parse_dl_name_version("libgfortran.dll", "windows") == ("libgfortran", nothing) + @test parse_dl_name_version("libgfortran-3.dll", "windows") == ("libgfortran", v"3") + @test parse_dl_name_version("libgfortran-3.4.dll", "windows") == ("libgfortran", v"3.4") + @test parse_dl_name_version("libgfortran-3.4a.dll", "windows") == ("libgfortran-3.4a", nothing) + @test_throws ArgumentError parse_dl_name_version("libgfortran", "windows") +end + +@testset "Sys.is* overloading" begin + # Test that we can indeed ask if something is linux or windows, etc... + @test Sys.islinux(P("aarch64", "linux")) + @test !Sys.islinux(P("x86_64", "windows")) + @test Sys.iswindows(P("i686", "windows")) + @test !Sys.iswindows(P("powerpc64le", "linux")) + @test Sys.isapple(P("x86_64", "macos")) + @test !Sys.isapple(P("armv7l", "windows")) + @test Sys.isbsd(P("aarch64", "macos")) + @test Sys.isbsd(P("x86_64", "freebsd")) + @test !Sys.isbsd(P("x86_64", "linux"; libc="musl")) +end + +@testset "Compiler ABI detection" begin + # Let's check and ensure that we can autodetect the currently-running Julia process + @test detect_libgfortran_version() !== nothing + + # We run these to get coverage, but we can't test anything, because we could be built + # with `clang`, which wouldn't have any `libstdc++` constraints at all + detect_libstdcxx_version() + detect_cxxstring_abi() +end + +@testset "select_platform" begin + platforms = Dict( + # Typical binning test + P("x86_64", "linux"; libgfortran_version=v"3") => "linux4", + P("x86_64", "linux"; libgfortran_version=v"4") => "linux7", + P("x86_64", "linux"; libgfortran_version=v"5") => "linux8", + + # Ambiguity test + P("aarch64", "linux"; libgfortran_version=v"3") => "linux4", + P("aarch64", "linux"; libgfortran_version=v"3", libstdcxx_version=v"3.4.18") => "linux5", + + # OS test + P("x86_64", "macos"; libgfortran_version=v"3") => "mac4", + P("x86_64", "windows"; cxxstring_abi=:cxx11) => "win", + ) + + @test select_platform(platforms, P("x86_64", "linux")) == "linux8" + @test select_platform(platforms, P("x86_64", "linux"; libgfortran_version=v"4")) == "linux7" + + # Ambiguity test + @test select_platform(platforms, P("aarch64", "linux")) == "linux5" + @test select_platform(platforms, P("aarch64", "linux"; libgfortran_version=v"3")) == "linux5" + @test select_platform(platforms, P("aarch64", "linux"; libgfortran_version=v"4")) === nothing + + @test select_platform(platforms, P("x86_64", "macos")) == "mac4" + @test select_platform(platforms, P("x86_64", "macos"; libgfortran_version=v"4")) === nothing + + @test select_platform(platforms, P("x86_64", "windows"; cxxstring_abi="cxx11")) == "win" + @test select_platform(platforms, P("x86_64", "windows"; cxxstring_abi="cxx03")) === nothing + + # Sorry, Alex. ;) + @test select_platform(platforms, P("x86_64", "freebsd")) === nothing +end + +@testset "Custom comparators" begin + # We're going to define here some custom comparators for Platform objects to ensure they work. + # First, a symmetric one, which doesn't care which `Platform` object requested this comparison: + function matches_oddness(a::String, b::String, a_requested::Bool, b_requested::Bool) + return (parse(Int, a) % 2) == (parse(Int, b) % 2) + end + + comp_strat = Dict("vally" => matches_oddness) + + # First, test that these two do not match, because it's using equality to test the `vally` tag + a = Platform("x86_64", "linux"; vally="2") + b = Platform("x86_64", "linux"; vally="4") + @test !platforms_match(a, b) + + # Now, test that setting one or both `Platform`'s to use the `matches_oddness()` comparator works: + ac = Platform("x86_64", "linux"; vally="2", compare_strategies=comp_strat) + bc = Platform("x86_64", "linux"; vally="4", compare_strategies=comp_strat) + @test platforms_match(ac, b) + @test platforms_match(a, bc) + @test platforms_match(ac, bc) + + # Test that even with the comparison strat, we don't match if they're not both even: + bfc = Platform("x86_64", "linux"; vally="3", compare_strategies=comp_strat) + @test !platforms_match(ac, bfc) + + + # Next, an asymmetric comparison strategy. We'll create a "less than or equal to" constraint + # that uses the `{a,b}_requested` paramters to determine which number represents the limit. + function less_than_constraint(a::String, b::String, a_requested::Bool, b_requested::Bool) + a = parse(Int, a) + b = parse(Int, b) + if a_requested && !b_requested + return b < a + end + if b_requested && !a_requested + return a < b + end + # If two constraints have been requested, return true if they are the same constraint. + return a == b + end + + comp_strat = Dict("vally" => less_than_constraint) + a = Platform("x86_64", "linux"; vally="2") + b = Platform("x86_64", "linux"; vally="4") + + ac = Platform("x86_64", "linux"; vally="2", compare_strategies=comp_strat) + bc = Platform("x86_64", "linux"; vally="4", compare_strategies=comp_strat) + + # Vanilla comparison doesn't work + @test !platforms_match(a, b) + + # a and bc match, but not ac and b. Also test reciprocity. + @test platforms_match(a, bc) + @test platforms_match(bc, a) + @test !platforms_match(ac, b) + @test !platforms_match(b, ac) + + # ac and bc do not match, but ac and ac do + @test !platforms_match(ac, bc) + @test platforms_match(ac, ac) + @test platforms_match(bc, bc) +end \ No newline at end of file diff --git a/test/choosetests.jl b/test/choosetests.jl index 8e8b4ae87ed6d..4f8cae9e4828a 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -49,7 +49,7 @@ function choosetests(choices = []) "euler", "show", "client", "errorshow", "sets", "goto", "llvmcall", "llvmcall2", "ryu", "some", "meta", "stacktraces", "docs", - "misc", "threads", "stress", + "misc", "threads", "stress", "binaryplatforms", "enums", "cmdlineargs", "int", "interpreter", "checked", "bitset", "floatfuncs", "precompile", "boundscheck", "error", "ambiguous", "cartesian", "osutils", diff --git a/test/precompile.jl b/test/precompile.jl index 25fbcd36c50bb..bbd642a625060 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -288,7 +288,7 @@ try Dict(let m = Base.root_module(Base, s) Base.PkgId(m) => Base.module_build_id(m) end for s in - [:Base64, :CRC32c, :Dates, :DelimitedFiles, :Distributed, :FileWatching, :Markdown, + [:Artifacts, :Base64, :CRC32c, :Dates, :DelimitedFiles, :Distributed, :FileWatching, :Markdown, :Future, :Libdl, :LinearAlgebra, :Logging, :Mmap, :Printf, :Profile, :Random, :Serialization, :SharedArrays, :SparseArrays, :SuiteSparse, :Test, :Unicode, :REPL, :InteractiveUtils, :Pkg, :LibGit2, :SHA, :UUIDs, :Sockets,