From 8d93a5692c2b52f4c3b05560aeb37c169a664a91 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Tue, 30 Nov 2021 19:25:27 +0100 Subject: [PATCH] Propagate package environment to local workers, fixes #28781, closes #42089. --- NEWS.md | 3 + stdlib/Distributed/src/cluster.jl | 1 + stdlib/Distributed/src/managers.jl | 21 +++- stdlib/Distributed/test/distributed_exec.jl | 108 ++++++++++++++++++++ 4 files changed, 132 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 98fc1af270914..3cad647ce4fbb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -129,6 +129,9 @@ Standard library changes #### Tar #### Distributed +* The package environment (active project, `LOAD_PATH`, `DEPOT_PATH`) are now propagated + when adding *local* workers (e.g. with `addprocs(N::Int)` or through the `--procs=N` + command line flag). ([#43270]). #### UUIDs diff --git a/stdlib/Distributed/src/cluster.jl b/stdlib/Distributed/src/cluster.jl index cea8258f36939..2bed7bef86740 100644 --- a/stdlib/Distributed/src/cluster.jl +++ b/stdlib/Distributed/src/cluster.jl @@ -533,6 +533,7 @@ default_addprocs_params() = Dict{Symbol,Any}( :dir => pwd(), :exename => joinpath(Sys.BINDIR::String, julia_exename()), :exeflags => ``, + :env => [], :enable_threaded_blas => false, :lazy => true) diff --git a/stdlib/Distributed/src/managers.jl b/stdlib/Distributed/src/managers.jl index 91a27aa95cb98..83046d6171ccd 100644 --- a/stdlib/Distributed/src/managers.jl +++ b/stdlib/Distributed/src/managers.jl @@ -462,10 +462,29 @@ function launch(manager::LocalManager, params::Dict, launched::Array, c::Conditi exename = params[:exename] exeflags = params[:exeflags] bind_to = manager.restrict ? `127.0.0.1` : `$(LPROC.bind_addr)` + env = Dict{String,String}(params[:env]) + + # JULIA_(LOAD|DEPOT)_PATH are used to populate (LOAD|DEPOT)_PATH on startup, + # but since (LOAD|DEPOT)_PATH might have changed they are re-serialized here. + # Users can opt-out of this by passing `env = ...` to addprocs(...). + pathsep = Sys.iswindows() ? ";" : ":" + if get(env, "JULIA_LOAD_PATH", nothing) === nothing + env["JULIA_LOAD_PATH"] = join(LOAD_PATH, pathsep) + end + if get(env, "JULIA_DEPOT_PATH", nothing) === nothing + env["JULIA_DEPOT_PATH"] = join(DEPOT_PATH, pathsep) + end + # Set the active project on workers using JULIA_PROJECT. + # Users can opt-out of this by (i) passing `env = ...` or (ii) passing + # `--project=...` as `exeflags` to addprocs(...). + project = Base.ACTIVE_PROJECT[] + if project !== nothing && get(env, "JULIA_PROJECT", nothing) === nothing + env["JULIA_PROJECT"] = project + end for i in 1:manager.np cmd = `$(julia_cmd(exename)) $exeflags --bind-to $bind_to --worker` - io = open(detach(setenv(cmd, dir=dir)), "r+") + io = open(detach(setenv(addenv(cmd, env), dir=dir)), "r+") write_cookie(io) wconfig = WorkerConfig() diff --git a/stdlib/Distributed/test/distributed_exec.jl b/stdlib/Distributed/test/distributed_exec.jl index 78182e876f459..29ab69c469eda 100644 --- a/stdlib/Distributed/test/distributed_exec.jl +++ b/stdlib/Distributed/test/distributed_exec.jl @@ -1726,6 +1726,114 @@ for p in procs() @test @fetchfrom(p, i27429) == 27429 end +# Propagation of package environments for local workers (#28781) +let julia = `$(Base.julia_cmd()) --startup-file=no`; mktempdir() do tmp + project = mkdir(joinpath(tmp, "project")) + depots = [mkdir(joinpath(tmp, "depot1")), mkdir(joinpath(tmp, "depot2"))] + load_path = [mkdir(joinpath(tmp, "load_path")), "@stdlib", "@"] + pathsep = Sys.iswindows() ? ";" : ":" + env = Dict( + "JULIA_DEPOT_PATH" => join(depots, pathsep), + "JULIA_LOAD_PATH" => join(load_path, pathsep), + ) + setupcode = """ + using Distributed, Test + @everywhere begin + depot_path() = DEPOT_PATH + load_path() = LOAD_PATH + active_project() = Base.ACTIVE_PROJECT[] + end + """ + testcode = setupcode * """ + for w in workers() + @test remotecall_fetch(depot_path, w) == DEPOT_PATH + @test remotecall_fetch(load_path, w) == LOAD_PATH + @test remotecall_fetch(Base.load_path, w) == Base.load_path() + @test remotecall_fetch(active_project, w) == Base.ACTIVE_PROJECT[] + @test remotecall_fetch(Base.active_project, w) == Base.active_project() + end + """ + # No active project + extracode = """ + for w in workers() + @test remotecall_fetch(active_project, w) === Base.ACTIVE_PROJECT[] === nothing + end + """ + cmd = setenv(`$(julia) -p1 -e $(testcode * extracode)`, env) + @test success(cmd) + # --project + extracode = """ + for w in workers() + @test remotecall_fetch(active_project, w) == Base.ACTIVE_PROJECT[] == + $(repr(project)) + end + """ + cmd = setenv(`$(julia) --project=$(project) -p1 -e $(testcode * extracode)`, env) + @test success(cmd) + # JULIA_PROJECT + cmd = setenv(`$(julia) -p1 -e $(testcode * extracode)`, + (env["JULIA_PROJECT"] = project; env)) + @test success(cmd) + # Pkg.activate(...) + activateish = """ + Base.ACTIVE_PROJECT[] = $(repr(project)) + using Distributed + addprocs(1) + """ + cmd = setenv(`$(julia) -e $(activateish * testcode * extracode)`, env) + @test success(cmd) + # JULIA_(LOAD|DEPOT)_PATH + shufflecode = """ + d = reverse(DEPOT_PATH) + append!(empty!(DEPOT_PATH), d) + l = reverse(LOAD_PATH) + append!(empty!(LOAD_PATH), l) + """ + addcode = """ + using Distributed + addprocs(1) # after shuffling + """ + extracode = """ + for w in workers() + @test remotecall_fetch(load_path, w) == $(repr(reverse(load_path))) + @test remotecall_fetch(depot_path, w) == $(repr(reverse(depots))) + end + """ + cmd = setenv(`$(julia) -e $(shufflecode * addcode * testcode * extracode)`, env) + @test success(cmd) + # Mismatch when shuffling after proc addition + failcode = shufflecode * setupcode * """ + for w in workers() + @test remotecall_fetch(load_path, w) == reverse(LOAD_PATH) == $(repr(load_path)) + @test remotecall_fetch(depot_path, w) == reverse(DEPOT_PATH) == $(repr(depots)) + end + """ + cmd = setenv(`$(julia) -p1 -e $(failcode)`, env) + @test success(cmd) + # Passing env or exeflags to addprocs(...) to override defaults + envcode = """ + using Distributed + tmp = $(repr(tmp)) + project = try; mkdir(joinpath(tmp, "project2")); catch e; end + env = Dict( + "JULIA_LOAD_PATH" => LOAD_PATH[1], + "JULIA_DEPOT_PATH" => DEPOT_PATH[1], + ) + addprocs(1; env = env, exeflags = `--project=\$(project)) + env["JULIA_PROJECT"] = project + addprocs(1; env = env) + """ * setupcode * """ + for w in workers() + @test remotecall_fetch(depot_path, w) == [DEPOT_PATH[1]] + @test remotecall_fetch(load_path, w) == [LOAD_PATH[1]] + @test remotecall_fetch(active_project, w) == project + @test remotecall_fetch(Base.active_project, w) == joinpath(project, "Project.toml") + end + """ + cmd = setenv(`$(julia) -e $(envcode)`, env) + run(cmd) +end end + include("splitrange.jl") # Run topology tests last after removing all workers, since a given