From 1d18679b385c8dc344c1d52f31f9c97b9ac7b568 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 2 Oct 2020 09:29:58 +0000 Subject: [PATCH] Rework Preferences loading framework Implements the `Preferences` loading framework as outlined in [0]. The most drastic change is that the list of compile-time preferences is no longer sequestered within its own dictionary, but is instead autodetected at compile-time and communicated back to the compiler. This list of compile-time preferences is now embedded as an array of strings that the loader must load, then index into the preferences dictionary with that list to check the preferences hash. [0]: https://github.com/JuliaLang/julia/issues/37791#issuecomment-701696972 --- Project.toml | 3 +- README.md | 73 +++-- src/Preferences.jl | 229 ++++++++++++- src/common.jl | 163 ---------- src/utils.jl | 29 ++ test/UsesPreferences/.gitignore | 1 + test/UsesPreferences/Manifest.toml | 24 ++ test/UsesPreferences/src/UsesPreferences.jl | 41 +-- test/UsesPreferences/test/runtests.jl | 36 --- test/runtests.jl | 341 +++++++++++--------- 10 files changed, 535 insertions(+), 405 deletions(-) delete mode 100644 src/common.jl create mode 100644 src/utils.jl create mode 100644 test/UsesPreferences/.gitignore create mode 100644 test/UsesPreferences/Manifest.toml delete mode 100644 test/UsesPreferences/test/runtests.jl diff --git a/Project.toml b/Project.toml index 2ea13cb..b047172 100644 --- a/Project.toml +++ b/Project.toml @@ -7,11 +7,10 @@ version = "1.0.0" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" [extras] -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "Pkg"] +test = ["Test"] [compat] julia = "1.6" diff --git a/README.md b/README.md index 8fed29c..0e2a9e7 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,66 @@ # Preferences -`Preferences` supports embedding a simple `Dict` of metadata for a package on a per-project basis. -These preferences allow for packages to set simple, persistent pieces of data, and optionally trigger recompilation of the package when the preferences change, to allow for customization of package behavior at compile-time. +The `Preferences` package provides a convenient, integrated way for packages to store configuration switches to persistent TOML files, and use those pieces of information at both run time and compile time. +This enables the user to modify the behavior of a package, and have that choice reflected in everything from run time algorithm choice to code generation at compile time. +Preferences are stored as TOML dictionaries and are, by default, stored within a `(Julia)LocalPreferences.toml` file next to the currently-active project. +If a preference is "exported", it is instead stored within the `(Julia)Project.toml` instead. +The intention is to allow shared projects to contain shared preferences, while allowing for users themselves to override those preferences with their own settings in the `LocalPreferences.toml` file, which should be `.gitignore`d as the name implies. -## API Overview +Preferences can be set with depot-wide defaults; if package `Foo` is installed within your global environment and it has preferences set, these preferences will apply as long as your global environment is part of your [`LOAD_PATH`](https://docs.julialang.org/en/v1/manual/code-loading/#Environment-stacks). +Preferences in environments higher up in the environment stack get overridden by the more proximal entries in the load path, ending with the currently active project. +This allows depot-wide preference defaults to exist, with active projects able to merge or even completely overwrite these inherited preferences. +See the docstring for `set_preferences!()` for the full details of how to set preferences to allow or disallow merging. -`Preferences` are used primarily through the `@load_preferences`, `@save_preferences` and `@modify_preferences` macros. -These macros will auto-detect the UUID of the calling package, throwing an error if the calling module does not belong to a package. -The function forms can be used to load, save or modify preferences belonging to another package. +Preferences that are accessed during compilation are automatically marked as compile-time preferences, and any change recorded to these preferences will cause the Julia compiler to recompile any cached precompilation `.ji` files for that module. +This allows preferences to be used to influence code generation. +When your package sets a compile-time preference, it is usually best to suggest to the user that they should restart Julia, to allow recompilation to occur. -Example usage: +## API + +Preferences use is very simple; it is all based around two functions (which each have convenience macros): `@set_preferences!()` and `@load_preference()`. + +* `@load_preference(key, default = nothing)`: This loads a preference named `key` for the current package. If no such preference is found, it returns `default`. + +* `@set_preferences!(pairs...)`: This allows setting multiple preferences at once as pairs. + +To illustrate the usage, we show a toy module, taken directly from this package's tests: ```julia -using Preferences +module UsesPreferences + +function set_backend(new_backend::String) + if !(new_backend in ("OpenCL", "CUDA", "jlFPGA")) + throw(ArgumentError("Invalid backend: \"$(new_backend)\"")) + end -function get_preferred_backend() - prefs = @load_preferences() - return get(prefs, "backend", "native") + # Set it in our runtime values, as well as saving it to disk + @set_preferences!("backend" => new_backend) + @info("New backend set; restart your Julia session for this change to take effect!") end -function set_backend(new_backend) - @modify_preferences!() do prefs - prefs["backend"] = new_backend +const backend = @load_preference("backend", "OpenCL") + +# An example that helps us to prove that things are happening at compile-time +function do_computation() + @static if backend == "OpenCL" + return "OpenCL is the best!" + elseif backend == "CUDA" + return "CUDA; so fast, so fresh!" + elseif backend == "jlFPGA" + return "The Future is Now, jlFPGA online!" + else + return nothing end end -``` -Preferences are stored within the first `Project.toml` that represents an environment that contains the given UUID, even as a transitive dependency. -If no project that contains the given UUID is found, the preference is recorded in the `Project.toml` file of the currently-active project. -The initial state for preferences is an empty dictionary, package authors that wish to have a default value set for their preferences should use the `get(prefs, key, default)` pattern as shown in the code example above. -## Compile-Time Preferences +# A non-compiletime preference +function set_username(username::String) + @set_preferences!("username" => username) +end +function get_username() + return @load_preference("username") +end -If a preference must be known at compile-time, (and hence changing it should invalidate your package's precompiled `.ji` file) access of it should be done through the `Preferences.CompileTime` module. -The exact same API is exposed, but the preferences will be stored within a separate dictionary from normal `Preferences`, and any change made to these preferences will cause your package to be recompiled the next time it is loaded. -Packages that wish to use purely compile-time preferences can simply `using Preferences.CompileTime`, mixed usage will require compile-time usage to access functions and macros via `CompileTime.@load_preferences()`, etc... +end # module UsesPreferences +``` diff --git a/src/Preferences.jl b/src/Preferences.jl index cfd8c25..2442bd7 100644 --- a/src/Preferences.jl +++ b/src/Preferences.jl @@ -1,21 +1,226 @@ module Preferences +using TOML +using Base: UUID, TOMLCache + +export load_preference, @load_preference, + set_preferences!, @set_preferences! + +include("utils.jl") + +""" + load_preference(uuid_or_module, key, default = nothing) + +Load a particular preference from the `Preferences.toml` file, shallowly merging keys +as it walks the hierarchy of load paths, loading preferences from all environments that +list the given UUID as a direct dependency. + +Most users should use the `@load_preference` convenience macro which auto-determines the +calling `Module`. +""" +function load_preference(uuid::UUID, key::String, default = nothing) + # Re-use definition in `base/loading.jl` so as to not repeat code. + d = Base.get_preferences(uuid) + if currently_compiling() + Base.record_compiletime_preference(uuid, key) + end + # Drop any nested `__clear__` keys: + function drop_clears(data::Dict) + delete!(data, "__clear__") + for (k, v) in data + if isa(v, Dict) + drop_clears(v) + end + end + return data + end + drop_clears(x) = x + + return drop_clears(get(d, key, default)) +end +function load_preference(m::Module, key::String, default = nothing) + return load_preference(get_uuid(m), key, default) +end + """ - CompileTime + @load_preference(key) -This module provides bindings for setting/getting preferences that can be used at compile -time and will cause your `.ji` file to be invalidated when they are changed. +Convenience macro to call `load_preference()` for the current package. """ -module CompileTime -const PREFS_KEY = "compile-preferences" -include("common.jl") -end # module CompileTime +macro load_preference(key, default = nothing) + return quote + load_preference($(esc(get_uuid(__module__))), $(esc(key)), $(esc(default))) + end +end -# Export `CompileTime` but don't `using` it -export CompileTime -# Second copy of code for non-compiletime preferences -const PREFS_KEY = "preferences" -include("common.jl") +""" + process_sentinel_values!(prefs::Dict) + +Recursively search for preference values that end in `nothing` or `missing` leaves, +which we handle specially, see the `set_preferences!()` docstring for more detail. +""" +function process_sentinel_values!(prefs::Dict) + # Need to widen `prefs` so that when we try to assign to `__clear__` below, + # we don't error due to a too-narrow type on `prefs` + prefs = Base._typeddict(prefs, Dict{String,Vector{String}}()) + + clear_keys = get(prefs, "__clear__", String[]) + for k in collect(keys(prefs)) + if prefs[k] === nothing + # If this should add `k` to the `__clear__` list, do so, then remove `k` + push!(clear_keys, k) + delete!(prefs, k) + elseif prefs[k] === missing + # If this should clear out the mapping for `k`, do so, and drop it from `clear_keys` + delete!(prefs, k) + filter!(x -> x != k, clear_keys) + elseif isa(prefs[k], Dict) + # Recurse for nested dictionaries + prefs[k] = process_sentinel_values!(prefs[k]) + end + end + # Store the updated list of clear_keys + if !isempty(clear_keys) + prefs["__clear__"] = collect(Set(clear_keys)) + else + delete!(prefs, "__clear__") + end + return prefs +end + +# See the `set_preferences!()` docstring below for more details +function set_preferences!(target_toml::String, pkg_name::String, pairs::Pair{String,<:Any}...; force::Bool = false) + # Load the old preferences in first, as we'll merge ours into whatever currently exists + d = Dict{String,Any}() + if isfile(target_toml) + d = Base.parsed_toml(target_toml) + end + prefs = d + if endswith(target_toml, "Project.toml") + if !haskey(prefs, "preferences") + prefs["preferences"] = Dict{String,Any}() + end + # If this is a `(Julia)Project.toml` file, we squirrel everything away under the + # "preferences" key, while for a `Preferences.toml` file it sits at top-level. + prefs = prefs["preferences"] + end + # Index into our package name + if !haskey(prefs, pkg_name) + prefs[pkg_name] = Dict{String,Any}() + end + # Set each preference, erroring unless `force` is set to `true` + for (k, v) in pairs + if !force && haskey(prefs[pkg_name], k) && (v === missing || prefs[pkg_name][k] != v) + throw(ArgumentError("Cannot set preference '$(k)' to '$(v)' for $(pkg_name) in $(target_toml): preference already set to '$(prefs[pkg_name][k])'!")) + end + prefs[pkg_name][k] = v + + # Recursively scan for `nothing` and `missing` values that we need to handle specially + prefs[pkg_name] = process_sentinel_values!(prefs[pkg_name]) + end + open(target_toml, "w") do io + TOML.print(io, d, sorted=true) + end + return nothing +end + +""" + set_preferences!(uuid_or_module, prefs::Pair{String,Any}...; export_prefs=false, force=false) + +Sets a series of preferences for the given UUID/Module, identified by the pairs passed in +as `prefs`. Preferences are loaded from `Project.toml` and `LocalPreferences.toml` files +on the load path, merging values together into a cohesive view, with preferences taking +precedence in `LOAD_PATH` order, just as package resolution does. Preferences stored in +`Project.toml` files are considered "exported", as they are easily shared across package +installs, whereas the `LocalPreferences.toml` file is meant to represent local +preferences that are not typically shared. `LocalPreferences.toml` settings override +`Project.toml` settings where appropriate. + +After running `set_preferences!(uuid, "key" => value)`, a future invocation of +`load_preference(uuid, "key")` will generally result in `value`, with the exception of +the merging performed by `load_preference()` due to inheritance of preferences from +elements higher up in the `load_path()`. To control this inheritance, there are two +special values that can be passed to `set_preferences!()`: `nothing` and `missing`. + +* Passing `missing` as the value causes all mappings of the associated key to be removed + from the current level of `LocalPreferences.toml` settings, allowing preferences set + higher in the chain of preferences to pass through. Use this value when you want to + clear your settings but still inherit any higher settings for this key. + +* Passing `nothing` as the value causes all mappings of the associated key to be removed + from the current level of `LocalPreferences.toml` settings and blocks preferences set + higher in the chain of preferences from passing through. Internally, this adds the + preference key to a `__clear__` list in the `LocalPreferences.toml` file, that will + prevent any preferences from leaking through from higher environments. + +Note that the behavior of `missing` and `nothing` is both similar (they both clear the +current settings) and diametrically opposed (one allows inheritance of preferences, the +other does not). They can also be composed with a normal `set_preferences!()` call: + +```julia +@set_preferences!("compiler_options" => nothing) +@set_preferences!("compiler_options" => Dict("CXXFLAGS" => "-g", LDFLAGS => "-ljulia")) +``` + +The above snippet first clears the `"compiler_options"` key of any inheriting influence, +then sets a preference option, which guarantees that future loading of that preference +will be exactly what was saved here. If we wanted to re-enable inheritance from higher +up in the chain, we could do the same but passing `missing` first. + +The `export_prefs` option determines whether the preferences being set should be stored +within `LocalPreferences.toml` or `Project.toml`. +""" +function set_preferences!(u::UUID, prefs::Pair{String,<:Any}...; export_prefs=false, kwargs...) + # Find the first `Project.toml` that has this UUID as a direct dependency + project_toml, pkg_name = find_first_project_with_uuid(u) + if project_toml === nothing && pkg_name === nothing + # If we couldn't find one, we're going to use `active_project()` + project_toml = Base.active_project() + + # And we're going to need to add this UUID as an "extras" dependency: + # We're going to assume you want to name this this dependency in the + # same way as it's been loaded: + pkg_uuid_matches = filter(d -> d.uuid == u, keys(Base.loaded_modules)) + if isempty(pkg_uuid_matches) + error("Cannot set preferences of an unknown package that is not loaded!") + end + pkg_name = first(pkg_uuid_matches).name + + # Read in the project, add the deps, write it back out! + project = Base.parsed_toml(project_toml) + if !haskey(project, "extras") + project["extras"] = Dict{String,Any}() + end + project["extras"][pkg_name] = string(u) + open(project_toml, "w") do io + TOML.print(io, project; sorted=true) + end + end + + # Finally, save the preferences out to either `Project.toml` or `Preferences.toml` + # keyed under that `pkg_name`: + target_toml = project_toml + if !export_prefs + target_toml = joinpath(dirname(project_toml), "LocalPreferences.toml") + end + return set_preferences!(target_toml, pkg_name, prefs...; kwargs...) +end +function set_preferences!(m::Module, prefs::Pair{String,<:Any}...; kwargs...) + return set_preferences!(get_uuid(m), prefs...; kwargs...) +end + +""" + @set_preferences!(prefs...) + +Convenience macro to call `set_preferences!()` for the current package. Defaults to +setting `force=true`, since a package should have full control over itself, but not +so for setting the preferences in other packages, pending private dependencies. +""" +macro set_preferences!(prefs...) + return quote + set_preferences!($(esc(get_uuid(__module__))), $(esc(prefs...)), force=true) + end +end end # module Preferences diff --git a/src/common.jl b/src/common.jl deleted file mode 100644 index 06d67e1..0000000 --- a/src/common.jl +++ /dev/null @@ -1,163 +0,0 @@ -using TOML -using Base: UUID, TOMLCache - -export load_preferences, @load_preferences, - save_preferences!, @save_preferences!, - modify_preferences!, @modify_preferences!, - clear_preferences!, @clear_preferences! - -# Helper function to get the UUID of a module, throwing an error if it can't. -function get_uuid(m::Module) - uuid = Base.PkgId(m).uuid - if uuid === nothing - throw(ArgumentError("Module does not correspond to a loaded package!")) - end - return uuid -end - - -""" - load_preferences(uuid_or_module) - -Load the preferences for the given package, returning them as a `Dict`. Most users -should use the `@load_preferences()` macro which auto-determines the calling `Module`. - -Preferences can be stored in `Project.toml` files that are higher up in the chain of -environments in the LOAD_PATH, the first environment that contains the given UUID (even -as a transitive dependency) will be the one that is searched in for preferences. -""" -function load_preferences(uuid::UUID, toml_cache::TOMLCache = TOMLCache()) - # Re-use definition in `base/loading.jl` so as to not repeat code. - return Base.get_preferences(uuid, toml_cache; prefs_key=PREFS_KEY) -end -load_preferences(m::Module) = load_preferences(get_uuid(m)) - - -""" - save_preferences!(uuid_or_module, prefs::Dict) - -Save the preferences for the given package. Most users should use the -`@save_preferences!()` macro which auto-determines the calling `Module`. See also the -`modify_preferences!()` function (and the associated `@modifiy_preferences!()` macro) for -easy load/modify/save workflows. The same `Project.toml` file that is loaded from in -`load_preferences()` will be the one that these preferences are stored to, falling back -to the currently-active project if no previous mapping is found. -""" -function save_preferences!(uuid::UUID, prefs::Dict) - # Save to Project.toml - proj_path = something(Base.get_preferences_project_path(uuid), Base.active_project()) - mkpath(dirname(proj_path)) - project = Dict{String,Any}() - if isfile(proj_path) - project = TOML.parsefile(proj_path) - end - if !haskey(project, PREFS_KEY) - project[PREFS_KEY] = Dict{String,Any}() - end - if !isa(project[PREFS_KEY], Dict) - error("$(proj_path) has conflicting `$(PREFS_KEY)` entry type: Not a Dict!") - end - project[PREFS_KEY][string(uuid)] = prefs - open(proj_path, "w") do io - TOML.print(io, project, sorted=true) - end - return nothing -end -function save_preferences!(m::Module, prefs::Dict) - return save_preferences!(get_uuid(m), prefs) -end - - -""" - modify_preferences!(f::Function, uuid::UUID) - modify_preferences!(f::Function, m::Module) - -Supports `do`-block modification of preferences. Loads the preferences, passes them to a -user function, then writes the modified `Dict` back to the preferences file. Example: - -```julia -modify_preferences!(@__MODULE__) do prefs - prefs["key"] = "value" -end -``` - -This function returns the full preferences object. Most users should use the -`@modify_preferences!()` macro which auto-determines the calling `Module`. -""" -function modify_preferences!(f::Function, uuid::UUID) - prefs = load_preferences(uuid) - f(prefs) - save_preferences!(uuid, prefs) - return prefs -end -modify_preferences!(f::Function, m::Module) = modify_preferences!(f, get_uuid(m)) - - -""" - clear_preferences!(uuid::UUID) - clear_preferences!(m::Module) - -Convenience method to remove all preferences for the given package. Most users should -use the `@clear_preferences!()` macro, which auto-determines the calling `Module`. -""" -function clear_preferences!(uuid::UUID) - # Clear the project preferences key, if it exists - proj_path = Base.get_preferences_project_path(uuid) - if proj_path !== nothing && isfile(proj_path) - project = TOML.parsefile(proj_path) - if haskey(project, PREFS_KEY) && isa(project[PREFS_KEY], Dict) - delete!(project[PREFS_KEY], string(uuid)) - open(proj_path, "w") do io - TOML.print(io, project, sorted=true) - end - end - end -end - - -""" - @load_preferences() - -Convenience macro to call `load_preferences()` for the current package. -""" -macro load_preferences() - return quote - load_preferences($(esc(get_uuid(__module__)))) - end -end - - -""" - @save_preferences!(prefs) - -Convenience macro to call `save_preferences!()` for the current package. -""" -macro save_preferences!(prefs) - return quote - save_preferences!($(esc(get_uuid(__module__))), $(esc(prefs))) - end -end - - -""" - @modify_preferences!(func) - -Convenience macro to call `modify_preferences!()` for the current package. -""" -macro modify_preferences!(func) - return quote - modify_preferences!($(esc(func)), $(esc(get_uuid(__module__)))) - end -end - - -""" - @clear_preferences!() - -Convenience macro to call `clear_preferences!()` for the current package. -""" -macro clear_preferences!() - return quote - preferences!($(esc(get_uuid(__module__)))) - end -end diff --git a/src/utils.jl b/src/utils.jl new file mode 100644 index 0000000..c8299d4 --- /dev/null +++ b/src/utils.jl @@ -0,0 +1,29 @@ +# Helper function to detect if we're currently compiling +currently_compiling() = ccall(:jl_generating_output, Cint, ()) != 0 + +# Helper function to get the UUID of a module, throwing an error if it can't. +function get_uuid(m::Module) + uuid = Base.PkgId(m).uuid + if uuid === nothing + throw(ArgumentError("Module $(m) does not correspond to a loaded package!")) + end + return uuid +end + +function find_first_project_with_uuid(uuid::UUID) + # Find first element in `Base.load_path()` that contains this UUID + # This code should look similar to the search in `Base.get_preferences()` + for env in Base.load_path() + project_toml = Base.env_project_file(env) + if !isa(project_toml, String) + continue + end + + # Check to see if this project has a name mapping + pkg_name = Base.get_uuid_name(project_toml, uuid) + if pkg_name !== nothing + return (project_toml, pkg_name) + end + end + return (nothing, nothing) +end diff --git a/test/UsesPreferences/.gitignore b/test/UsesPreferences/.gitignore new file mode 100644 index 0000000..0af85c2 --- /dev/null +++ b/test/UsesPreferences/.gitignore @@ -0,0 +1 @@ +LocalPreferences.toml diff --git a/test/UsesPreferences/Manifest.toml b/test/UsesPreferences/Manifest.toml new file mode 100644 index 0000000..a4a81c1 --- /dev/null +++ b/test/UsesPreferences/Manifest.toml @@ -0,0 +1,24 @@ +# This file is machine-generated - editing it directly is not advised + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[Preferences]] +deps = ["TOML"] +path = "../.." +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.0.0" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[TOML]] +deps = ["Dates"] +git-tree-sha1 = "d0ac7eaad0fb9f6ba023a1d743edca974ae637c4" +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.0" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/test/UsesPreferences/src/UsesPreferences.jl b/test/UsesPreferences/src/UsesPreferences.jl index 93157a0..1feba91 100644 --- a/test/UsesPreferences/src/UsesPreferences.jl +++ b/test/UsesPreferences/src/UsesPreferences.jl @@ -1,33 +1,38 @@ module UsesPreferences -using Preferences.CompileTime +using Preferences -# This will get initialized in __init__() -backend = Ref{String}() - -function set_backend(new_backend::AbstractString) +function set_backend(new_backend::String) if !(new_backend in ("OpenCL", "CUDA", "jlFPGA")) throw(ArgumentError("Invalid backend: \"$(new_backend)\"")) end # Set it in our runtime values, as well as saving it to disk - backend[] = new_backend - @modify_preferences!() do prefs - prefs["backend"] = new_backend - end + @set_preferences!("backend" => new_backend) + @info("New backend set; restart your Julia session for this change to take effect!") end -function get_backend() - return backend[] +const backend = @load_preference("backend", "OpenCL") + +# An example that helps us to prove that things are happening at compile-time +function do_computation() + @static if backend == "OpenCL" + return "OpenCL is the best!" + elseif backend == "CUDA" + return "CUDA; so fast, so fresh!" + elseif backend == "jlFPGA" + return "The Future is Now, jlFPGA online!" + else + return nothing + end end -function __init__() - @modify_preferences!() do prefs - prefs["initialized"] = "true" - # If it's never been set before, default it to OpenCL - prefs["backend"] = get(prefs, "backend", "OpenCL") - backend[] = prefs["backend"] - end +# A non-compiletime preference +function set_username(username::String) + @set_preferences!("username" => username) +end +function get_username() + return @load_preference("username") end end # module UsesPreferences diff --git a/test/UsesPreferences/test/runtests.jl b/test/UsesPreferences/test/runtests.jl deleted file mode 100644 index d27f3d9..0000000 --- a/test/UsesPreferences/test/runtests.jl +++ /dev/null @@ -1,36 +0,0 @@ -using UsesPreferences, Test - -# We know we want to use compile-time preferences, so only bring the -# compile-time bindings in. We still need to access `Preferences` -# bindings though, so `import` that without bringing the bindings themselves in. -import Preferences -using Preferences.CompileTime - -# Get the UUID for UsesPreferences -up_uuid = Preferences.get_uuid(UsesPreferences) - -prefs = load_preferences(up_uuid) -@test haskey(prefs, "backend") -@test prefs["backend"] == "OpenCL" -@test UsesPreferences.get_backend() == "OpenCL" - -UsesPreferences.set_backend("CUDA") -prefs = load_preferences(up_uuid) -@test haskey(prefs, "backend") -@test prefs["backend"] == "CUDA" -@test UsesPreferences.get_backend() == "CUDA" - -# sorry, AMD -@test_throws ArgumentError UsesPreferences.set_backend("ROCm") -prefs = load_preferences(up_uuid) -@test haskey(prefs, "backend") -@test prefs["backend"] == "CUDA" -@test UsesPreferences.get_backend() == "CUDA" - -clear_preferences!(up_uuid) -prefs = load_preferences(up_uuid) -@test !haskey(prefs, "backend") -@test UsesPreferences.get_backend() == "CUDA" - -# And finally, save something back so that the parent process can read it: -UsesPreferences.set_backend("jlFPGA") diff --git a/test/runtests.jl b/test/runtests.jl index e17d07e..82bacb0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,9 +1,9 @@ using Base: UUID -using Preferences, Test, TOML, Pkg +using Preferences, Test, TOML -function activate(f::Function, project::String) +function activate(f::Function, env_dir::String) saved_active_project = Base.ACTIVE_PROJECT[] - Base.ACTIVE_PROJECT[] = project + Base.ACTIVE_PROJECT[] = env_dir try f() finally @@ -11,21 +11,13 @@ function activate(f::Function, project::String) end end -function with_temp_project(f::Function) - mktempdir() do dir - activate(dir) do - f(dir) - end - end -end - -function with_temp_depot_and_project(f::Function) +function with_temp_depot(f::Function) mktempdir() do dir saved_depot_path = copy(Base.DEPOT_PATH) empty!(Base.DEPOT_PATH) push!(Base.DEPOT_PATH, dir) try - with_temp_project(f) + f() finally empty!(Base.DEPOT_PATH) append!(Base.DEPOT_PATH, saved_depot_path) @@ -33,160 +25,205 @@ function with_temp_depot_and_project(f::Function) end end -# Some useful constants -up_uuid = UUID(TOML.parsefile(joinpath(@__DIR__, "UsesPreferences", "Project.toml"))["uuid"]) -up_path = joinpath(@__DIR__, "UsesPreferences") - -# Silence Pkg output: -Pkg.DEFAULT_IO[] = IOBuffer() - -@testset "Preferences" begin - # Create a temporary package, store some preferences within it. - with_temp_project() do project_dir - Pkg.develop(path=up_path) - @test isempty(load_preferences(up_uuid)) - modify_preferences!(up_uuid) do prefs - prefs["foo"] = "bar" - prefs["baz"] = Dict("qux" => "spoon") +function activate_and_run(project_dir::String, code::String; env::Dict = Dict()) + mktempdir() do dir + open(joinpath(dir, "test_code.jl"), "w") do io + write(io, code) end - prefs = load_preferences(up_uuid) - @test haskey(prefs, "foo") - @test prefs["foo"] == "bar" - @test prefs["baz"]["qux"] == "spoon" - - project_path = joinpath(project_dir, "Project.toml") - @test isfile(project_path) - proj = TOML.parsefile(project_path) - @test haskey(proj, "preferences") - @test isa(proj["preferences"], Dict) - @test haskey(proj["preferences"], string(up_uuid)) - @test isa(proj["preferences"][string(up_uuid)], Dict) - @test proj["preferences"][string(up_uuid)]["foo"] == "bar" - @test isa(proj["preferences"][string(up_uuid)]["baz"], Dict) - @test proj["preferences"][string(up_uuid)]["baz"]["qux"] == "spoon" - - clear_preferences!(up_uuid) - proj = TOML.parsefile(project_path) - @test !haskey(proj, "preferences") - @test isempty(load_preferences(up_uuid)) + out = Pipe() + cmd = setenv(`$(Base.julia_cmd()) --project=$(project_dir) $(dir)/test_code.jl`, + env..., "JULIA_DEPOT_PATH" => Base.DEPOT_PATH[1]) + p = run(pipeline(cmd, stdout=out, stderr=out); wait=false) + close(out.in) + wait(p) + output = String(read(out)) + if !success(p) + println(output) + end + @test success(p) + return output end end -@testset "CompileTime" begin - # Create a temporary package, store some preferences within it. - with_temp_project() do project_dir - # Add UsesPreferences as a package to this project so that the preferences are visible - Pkg.develop(path=up_path) - CompileTime.save_preferences!(up_uuid, Dict("foo" => "bar")) - - project_path = joinpath(project_dir, "Project.toml") - @test isfile(project_path) - proj = TOML.parsefile(project_path) - @test haskey(proj, "compile-preferences") - @test isa(proj["compile-preferences"], Dict) - @test haskey(proj["compile-preferences"], string(up_uuid)) - @test isa(proj["compile-preferences"][string(up_uuid)], Dict) - @test proj["compile-preferences"][string(up_uuid)]["foo"] == "bar" - - prefs = CompileTime.modify_preferences!(up_uuid) do prefs - prefs["foo"] = "baz" - prefs["spoon"] = [Dict("qux" => "idk")] - end - @test prefs == CompileTime.load_preferences(up_uuid) +# Some useful constants +up_uuid = UUID(TOML.parsefile(joinpath(@__DIR__, "UsesPreferences", "Project.toml"))["uuid"]) +up_path = joinpath(@__DIR__, "UsesPreferences") - CompileTime.clear_preferences!(up_uuid) - proj = TOML.parsefile(project_path) - @test !haskey(proj, "compile-preferences") +@testset "Preferences" begin + # Ensure there is no LocalPreferences.toml file in UsesPreferences: + local_prefs_toml = joinpath(up_path, "LocalPreferences.toml") + rm(local_prefs_toml; force=true) + with_temp_depot() do + # Start with the default test of the backend being un-set, we just get the default + activate_and_run(up_path, """ + using UsesPreferences, Test, Preferences + using Base: UUID + @test load_preference($(repr(up_uuid)), "backend") === nothing + @test UsesPreferences.backend == "OpenCL" + """) + + # Next, change a setting + activate_and_run(up_path, """ + using UsesPreferences + UsesPreferences.set_backend("CUDA") + """) + + # Ensure that's showing up in LocalPreferences.toml: + prefs = TOML.parsefile(local_prefs_toml) + @test haskey(prefs, "UsesPreferences") + @test prefs["UsesPreferences"]["backend"] == "CUDA" + + # Now show that it forces recompilation + did_precompile(output) = occursin("Precompiling UsesPreferences [$(string(up_uuid))]", output) + cuda_test = """ + using UsesPreferences, Test + @test UsesPreferences.backend == "CUDA" + """ + output = activate_and_run(up_path, cuda_test; env=Dict("JULIA_DEBUG" => "loading")) + @test did_precompile(output) + + # Show that it does not force a recompile the second time + output = activate_and_run(up_path, cuda_test; env=Dict("JULIA_DEBUG" => "loading")) + @test !did_precompile(output) + + # Test non-compiletime preferences a bit + activate_and_run(up_path, """ + using UsesPreferences, Test, Preferences + using Base: UUID + @test load_preference($(repr(up_uuid)), "username") === nothing + @test UsesPreferences.get_username() === nothing + UsesPreferences.set_username("giordano") + @test UsesPreferences.get_username() == "giordano" + """) + + # This does not cause a recompilation, and we can also get the username back again: + username_test = """ + using UsesPreferences, Test + @test UsesPreferences.get_username() == "giordano" + """ + output = activate_and_run(up_path, username_test; env=Dict("JULIA_DEBUG" => "loading")) + @test !did_precompile(output) end +end - # Do a test with stacked environments - mktempdir() do outer_env - # Set preferences for the package within the outer env - activate(outer_env) do - CompileTime.save_preferences!(up_uuid, Dict("foo" => "outer")) - end - - OLD_LOAD_PATH = deepcopy(Base.LOAD_PATH) - try - empty!(Base.LOAD_PATH) - append!(Base.LOAD_PATH, ["@", outer_env, "@stdlib"]) - - with_temp_project() do project_dir - CompileTime.save_preferences!(up_uuid, Dict("foo" => "inner")) - - # Ensure that an initial load finds none of these, since the Package is not added anywhere: - @test isempty(CompileTime.load_preferences(up_uuid)) - - # add it to the inner project, ensure that we get "inner" as the "foo" value: - Pkg.develop(path=up_path) - prefs = CompileTime.load_preferences(up_uuid) - @test haskey(prefs, "foo") - @test prefs["foo"] == "inner" - - # Remove it from the inner project, add it to the outer project, ensure we get "outer" - Pkg.rm("UsesPreferences") - activate(outer_env) do - Pkg.develop(path=up_path) - end - prefs = CompileTime.load_preferences(up_uuid) - @test haskey(prefs, "foo") - @test prefs["foo"] == "outer" +# Load UsesPreferences, as we need it loaded for some set/get trickery below +activate(up_path) do + eval(:(using UsesPreferences)) +end +@testset "Inheritance" begin + # Ensure there is no LocalPreferences.toml file in UsesPreferences: + local_prefs_toml = joinpath(up_path, "LocalPreferences.toml") + rm(local_prefs_toml; force=true) + with_temp_depot() do + mktempdir() do env_dir + # We're going to create a higher environment + push!(Base.LOAD_PATH, env_dir) + + # We're going to add `UsesPreferences` to this environment + open(joinpath(env_dir, "Project.toml"), "w") do io + TOML.print(io, Dict( + "deps" => Dict( + "UsesPreferences" => string(up_uuid), + ) + )) end - finally - empty!(Base.LOAD_PATH) - append!(Base.LOAD_PATH, OLD_LOAD_PATH) - end - end - - # Do a test within a package to ensure that we can use the macros - with_temp_project() do project_dir - Pkg.develop(path=up_path) - # Run UsesPreferences tests manually, so that they can run in the explicitly-given project - test_script = joinpath(@__DIR__, "UsesPreferences", "test", "runtests.jl") - run(`$(Base.julia_cmd()) --project=$(project_dir) $(test_script)`) + # We're going to write out some Preferences for UP in the higher environment + activate(env_dir) do + set_preferences!(up_uuid, "location" => "outer_public"; export_prefs=true) + # Verify that this is stored in the environment's Project.toml file + proj = Base.parsed_toml(joinpath(env_dir, "Project.toml")) + @test haskey(proj, "preferences") + @test haskey(proj["preferences"], "UsesPreferences") + @test proj["preferences"]["UsesPreferences"]["location"] == "outer_public" + @test load_preference(up_uuid, "location") == "outer_public" + + # Add preferences to the outer env's `LocalPreferences.toml` + set_preferences!(up_uuid, "location" => "outer_local") + prefs = Base.parsed_toml(joinpath(env_dir, "LocalPreferences.toml")) + @test haskey(prefs, "UsesPreferences") + @test prefs["UsesPreferences"]["location"] == "outer_local" + @test load_preference(up_uuid, "location") == "outer_local" + end - # Load the preferences, ensure we see the `jlFPGA` backend: - prefs = CompileTime.load_preferences(up_uuid) - @test haskey(prefs, "backend") - @test prefs["backend"] == "jlFPGA" - end + # Ensure that we can load the preferences the same even if we exit the `activate()` + @test load_preference(up_uuid, "location") == "outer_local" + + # Next, we're going to create a lower environment, add some preferences there, and ensure + # the inheritance works properly. + activate(up_path) do + # Ensure that activating this other path doesn't change anything + @test load_preference(up_uuid, "location") == "outer_local" + + # Set a local preference in this location, which should be the first location on the load path + set_preferences!(up_uuid, "location" => "inner_local") + prefs = Base.parsed_toml(joinpath(up_path, "LocalPreferences.toml")) + @test haskey(prefs, "UsesPreferences") + @test prefs["UsesPreferences"]["location"] == "inner_local" + @test load_preference(up_uuid, "location") == "inner_local" + end - # Run another test, this time setting up a whole new depot so that compilation caching can be checked: - with_temp_depot_and_project() do project_dir - Pkg.develop(path=up_path) - - # Helper function to run a sub-julia process and ensure that it either does or does not precompile. - function did_precompile() - out = Pipe() - cmd = setenv(`$(Base.julia_cmd()) -i --project=$(project_dir) -e 'using UsesPreferences; exit(0)'`, - "JULIA_DEPOT_PATH" => Base.DEPOT_PATH[1], "JULIA_DEBUG" => "loading") - run(pipeline(cmd, stdout=out, stderr=out)) - close(out.in) - # To debug failures, print this out and scan for precompilation messsages - output = String(read(out)) - return occursin("Precompiling UsesPreferences [$(string(up_uuid))]", output) - end + # Let's add some complex objects, test that recursive merging works, and that + # the special meaning of `nothing` and `missing` works + activate(env_dir) do + set_preferences!(up_uuid, "nested" => Dict( + "nested2" => Dict("a" => 1, "b" => 2), + "leaf" => "hello", + ); export_prefs=true) + set_preferences!(up_uuid, "nested" => Dict( + "nested2" => Dict("b" => 3)), + ) + + nested = load_preference(up_uuid, "nested") + @test isa(nested, Dict) && haskey(nested, "nested2") + @test nested["nested2"]["a"] == 1 + @test nested["nested2"]["b"] == 3 + @test nested["leaf"] == "hello" + end - # Initially, we must precompile, of course, because no preferences are set. - @test did_precompile() - # Next, we recompile, because the preferences have been altered - @test did_precompile() - # Finally, we no longer have to recompile. - @test !did_precompile() + # Add another layer in the inner environment + activate(up_path) do + set_preferences!(up_uuid, "nested" => Dict( + "nested2" => Dict("a" => "foo"), + "leaf" => "world", + )) + nested = load_preference(up_uuid, "nested") + @test isa(nested, Dict) && haskey(nested, "nested2") + @test nested["nested2"]["a"] == "foo" + @test nested["nested2"]["b"] == 3 + @test nested["leaf"] == "world" + end - # Modify the preferences, ensure that causes precompilation and then that too shall pass. - prefs = CompileTime.modify_preferences!(up_uuid) do prefs - prefs["backend"] = "something new" - end - @test did_precompile() - @test !did_precompile() + # Set the local setting of the upper environment to `missing`; this causes it to + # pass through and `b` will suddenly equal `2`: + activate(env_dir) do + # Test that trying to over-set a preference in another package fails unless we force it + @test_throws ArgumentError set_preferences!(up_uuid, "nested" => nothing) + set_preferences!(up_uuid, "nested" => Dict( + "nested2" => missing, + "leaf" => nothing, + ); force=true) + nested = load_preference(up_uuid, "nested") + @test isa(nested, Dict) && haskey(nested, "nested2") + @test nested["nested2"]["a"] == 1 + @test nested["nested2"]["b"] == 2 + + # Let's check that the `__clear__` keys are what we expect: + prefs = Base.parsed_toml(joinpath(env_dir, "LocalPreferences.toml")) + @test prefs["UsesPreferences"]["nested"]["__clear__"] == ["leaf"] + @test !haskey(prefs["UsesPreferences"]["nested"], "leaf") + @test !haskey(nested, "leaf") + end - # Finally, switch it back, and ensure that this does not cause precompilation - prefs = CompileTime.modify_preferences!(up_uuid) do prefs - prefs["backend"] = "OpenCL" + # Show that it cascades down to the lower levels as well + activate(up_path) do + nested = load_preference(up_uuid, "nested") + @test isa(nested, Dict) && haskey(nested, "nested2") + @test nested["nested2"]["a"] == "foo" + @test nested["nested2"]["b"] == 2 + @test nested["leaf"] == "world" + end end - @test !did_precompile() end end