Skip to content

Commit

Permalink
Merge pull request #12405 from JuliaLang/amitm/tweak_pmap_topology_tests
Browse files Browse the repository at this point in the history 
move pmap tests back into conditional exec, fix topology tests
  • Loading branch information
@amitmurthy
amitmurthy committed Jul 31, 2015
2 parents 69232e0 + 8a4b005 commit 863758c
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 55 deletions.
85 changes: 38 additions & 47 deletions test/parallel.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,6 @@ catch ex
@test collect(1:5) == sort(map(x->parse(Int, x), errors))
end


try
remotecall_fetch(id_other, ()->throw(ErrorException("foobar")))
catch e
Expand All @@ -304,59 +303,46 @@ catch e
@test e.captured.ex.msg == "foobar"
end

# pmap tests
# needs at least 4 processors dedicated to the below tests
nprocs() < 5 && remotecall_fetch(1, ()->addprocs(5-nprocs()))
s = "a"*"bcdefghijklmnopqrstuvwxyz"^100;
ups = "A"*"BCDEFGHIJKLMNOPQRSTUVWXYZ"^100;
@test ups == bytestring(UInt8[UInt8(c) for c in pmap(x->uppercase(x), s)])
@test ups == bytestring(UInt8[UInt8(c) for c in pmap(x->uppercase(Char(x)), s.data)])

DoFullTest = Bool(parse(Int,(get(ENV, "JULIA_TESTFULL", "0"))))

# retry, on error exit
res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : uppercase(x), s; err_retry=true, err_stop=true);
# Commenting out the below test for now since it depends on how the workers are scheduled on presumably already
# overloaded systems in CI.
#@test (length(res) < length(ups))
@test isa(res[1], Exception)

# no retry, on error exit
res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : uppercase(x), s; err_retry=false, err_stop=true);
#@test (length(res) < length(ups))
@test isa(res[1], Exception)

# retry, on error continue
res = pmap(x->iseven(myid()) ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : uppercase(x), s; err_retry=true, err_stop=false);
@test length(res) == length(ups)
@test ups == bytestring(UInt8[UInt8(c) for c in res])

# no retry, on error continue
res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : uppercase(x), s; err_retry=false, err_stop=false);
@test length(res) == length(ups)
@test isa(res[1], Exception)


# The below block of tests are usually run only on local development systems, since:
# - tests which print errors
# - addprocs tests are memory intensive
# - ssh addprocs requires sshd to be running locally with passwordless login enabled.
# The test block is enabled by defining env JULIA_TESTFULL=1

if DoFullTest
println("Testing exception printing on remote worker from a `remote_do` call")
println("Please ensure the remote error and backtrace is displayed on screen")

Base.remote_do(id_other, ()->throw(ErrorException("TESTING EXCEPTION ON REMOTE DO. PLEASE IGNORE")))
sleep(0.5) # Give some time for the above error to be printed

# Topology tests need to run externally since
# a given cluster at any time can only support a single topology and the
# current session is already running in parallel under the default
# topology.
# They also print errors to screen
print("\n\nTopology tests. \n")
DoFullTest = Bool(parse(Int,(get(ENV, "JULIA_TESTFULL", "0"))))

if DoFullTest
# pmap tests
# needs at least 4 processors dedicated to the below tests
ppids = remotecall_fetch(1, ()->addprocs(4))
s = "abcdefghijklmnopqrstuvwxyz";
ups = uppercase(s);
@test ups == bytestring(UInt8[UInt8(c) for c in pmap(x->uppercase(x), s)])
@test ups == bytestring(UInt8[UInt8(c) for c in pmap(x->uppercase(Char(x)), s.data)])

# retry, on error exit
res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=true, err_stop=true, pids=ppids);
@test (length(res) < length(ups))
@test isa(res[1], Exception)

# no retry, on error exit
res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=false, err_stop=true, pids=ppids);
@test (length(res) < length(ups))
@test isa(res[1], Exception)

# retry, on error continue
res = pmap(x->iseven(myid()) ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=true, err_stop=false, pids=ppids);
@test length(res) == length(ups)
@test ups == bytestring(UInt8[UInt8(c) for c in res])

# no retry, on error continue
res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=false, err_stop=false, pids=ppids);
@test length(res) == length(ups)
@test isa(res[1], Exception)

# Topology tests need to run externally since a given cluster at any
# time can only support a single topology and the current session
# is already running in parallel under the default topology.
script = joinpath(dirname(@__FILE__), "topology.jl")
cmd = `$(joinpath(JULIA_HOME,Base.julia_exename())) $script`

Expand All @@ -367,6 +353,11 @@ if DoFullTest
error("Topology tests failed : $cmd")
end

println("Testing exception printing on remote worker from a `remote_do` call")
println("Please ensure the remote error and backtrace is displayed on screen")

Base.remote_do(id_other, ()->throw(ErrorException("TESTING EXCEPTION ON REMOTE DO. PLEASE IGNORE")))
sleep(0.5) # Give some time for the above error to be printed

@unix_only begin
function test_n_remove_pids(new_pids)
Expand Down
18 changes: 10 additions & 8 deletions test/topology.jl
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
include("testdefs.jl")
addprocs(4; topology="master_slave")
v=remotecall_fetch(2, ()->remotecall_fetch(3,myid))
@test isa(v, Exception)
@test_throws RemoteException remotecall_fetch(2, ()->remotecall_fetch(3,myid))

function test_worker_counts()
# check if the nprocs/nworkers/workers are the same on the remaining workers
Expand All @@ -26,9 +25,12 @@ end

remove_workers_and_test()

# a hack just to test the "custom" topology.
# connect even pids to other even pids, odd to odd.
function Base.launch(manager::Base.LocalManager, params::Dict, launched::Array, c::Condition)
type TopoTestManager <: ClusterManager
np::Integer
end

function Base.launch(manager::TopoTestManager, params::Dict, launched::Array, c::Condition)
dir = params[:dir]
exename = params[:exename]
exeflags = params[:exeflags]
Expand All @@ -46,16 +48,17 @@ function Base.launch(manager::Base.LocalManager, params::Dict, launched::Array,

notify(c)
end

const map_pid_ident=Dict()
function Base.manage(manager::Base.LocalManager, id::Integer, config::WorkerConfig, op::Symbol)
function Base.manage(manager::TopoTestManager, id::Integer, config::WorkerConfig, op::Symbol)
if op == :register
map_pid_ident[id] = get(config.ident)
elseif op == :interrupt
kill(get(config.process), 2)
end
end

addprocs(8; topology="custom")
addprocs(TopoTestManager(8); topology="custom")

while true
if any(x->get(map_pid_ident, x, 0)==0, workers())
Expand All @@ -72,8 +75,7 @@ for p1 in workers()
if (iseven(i1) && iseven(i2)) || (isodd(i1) && isodd(i2))
@test p2 == remotecall_fetch(p1, p->remotecall_fetch(p,myid), p2)
else
v1 = remotecall_fetch(p1, p->remotecall_fetch(p,myid), p2)
@test isa(v1, Exception)
@test_throws RemoteException remotecall_fetch(p1, p->remotecall_fetch(p,myid), p2)
end
end
end
Expand Down

0 comments on commit 863758c

Please sign in to comment.