Erroring when inside a thread in a distributed process causes MethodError

[samuel-massinon]

Running this on Julia 1.3, 1.4, and a 1.6 Nightly from last week

using Distributed
addprocs(2)
@everywhere using Base.Threads

pmap(1:2) do i
    @threads for i in 1:10
        error()
    end
end

gets this error

➜ julia
               _
   _       _ _(_)_     |  Documentation: https://docs.julialang.org
  (_)     | (_) (_)    |
   _ _   _| |_  __ _   |  Type "?" for help, "]?" for Pkg help.
  | | | | | | |/ _` |  |
  | | |_| | | | (_| |  |  Version 1.4.0 (2020-03-21)
 _/ |\__'_|_|_|\__'_|  |  Official https://julialang.org/ release
|__/                   |

julia> versioninfo()
Julia Version 1.4.0
Commit b8e9a9ecc6 (2020-03-21 16:36 UTC)
Platform Info:
  OS: Linux (x86_64-pc-linux-gnu)
  CPU: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-8.0.1 (ORCJIT, skylake)

julia> using Distributed

julia> addprocs(2)
2-element Array{Int64,1}:
 2
 3

julia> @everywhere using Base.Threads

julia> pmap(1:2) do i
           @threads for i in 1:10
               error()
           end
       end

ERROR: On worker 2:
TaskFailedException:

wait at ./task.jl:267
macro expansion at ./threadingconstructs.jl:69 [inlined]
#3 at ./REPL[5]:2
#104 at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Distributed/src/process_messages.jl:294
run_work_thunk at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Distributed/src/process_messages.jl:79
macro expansion at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Distributed/src/process_messages.jl:294 [inlined]
#103 at ./task.jl:358
Stacktrace:
 [1] (::Base.var"#726#728")(::Task) at ./asyncmap.jl:178
 [2] foreach(::Base.var"#726#728", ::Array{Any,1}) at ./abstractarray.jl:1919
 [3] maptwice(::Function, ::Channel{Any}, ::Array{Any,1}, ::UnitRange{Int64}) at ./asyncmap.jl:178
 [4] wrap_n_exec_twice(::Channel{Any}, ::Array{Any,1}, ::Distributed.var"#204#207"{WorkerPool}, ::Function, ::UnitRange{Int64}) at ./asyncmap.jl:154
 [5] async_usemap(::Distributed.var"#188#190"{Distributed.var"#188#189#191"{WorkerPool,var"#3#4"}}, ::UnitRange{Int64}; ntasks::Function, batch_size::Nothing) at ./asyncmap.jl:103
 [6] #asyncmap#710 at ./asyncmap.jl:81 [inlined]
 [7] pmap(::Function, ::WorkerPool, ::UnitRange{Int64}; distributed::Bool, batch_size::Int64, on_error::Nothing, retry_delays::Array{Any,1}, retry_check::Nothing) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Distributed/src/pmap.jl:126
 [8] pmap(::Function, ::WorkerPool, ::UnitRange{Int64}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Distributed/src/pmap.jl:101
 [9] pmap(::Function, ::UnitRange{Int64}; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Distributed/src/pmap.jl:156
 [10] pmap(::Function, ::UnitRange{Int64}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Distributed/src/pmap.jl:156
 [11] top-level scope at REPL[5]:1
SYSTEM (REPL): showing an error caused an error
ERROR: MethodError: no method matching show_backtrace(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::Nothing)
Closest candidates are:
  show_backtrace(::IO, ::Array{Any,1}) at errorshow.jl:637
  show_backtrace(::IO, ::Array{T,1} where T) at errorshow.jl:608
Stacktrace:
 [1] showerror(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::ErrorException, ::Nothing; backtrace::Bool) at ./errorshow.jl:79
 [2] showerror(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::ErrorException, ::Nothing) at ./errorshow.jl:74
 [3] showerror(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::TaskFailedException) at ./task.jl:76
 [4] (::Base.var"#649#650"{TaskFailedException})(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}) at ./errorshow.jl:76
 [5] with_output_color(::Function, ::Symbol, ::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}; bold::Bool) at ./util.jl:385
 [6] with_output_color(::Function, ::Symbol, ::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}) at ./util.jl:383
 [7] showerror(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::TaskFailedException, ::Array{Any,1}; backtrace::Bool) at ./errorshow.jl:75
 [8] showerror(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::CapturedException) at ./task.jl:25
 [9] showerror(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::RemoteException) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Distributed/src/process_messages.jl:57
 [10] (::Base.var"#649#650"{RemoteException})(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}) at ./errorshow.jl:76
 [11] with_output_color(::Function, ::Symbol, ::IOContext{REPL.Terminals.TTYTerminal}; bold::Bool) at ./util.jl:385
 [12] with_output_color(::Function, ::Symbol, ::IOContext{REPL.Terminals.TTYTerminal}) at ./util.jl:383
 [13] showerror(::IOContext{REPL.Terminals.TTYTerminal}, ::RemoteException, ::Array{Base.StackTraces.StackFrame,1}; backtrace::Bool) at ./errorshow.jl:75
 [14] show_exception_stack(::IOContext{REPL.Terminals.TTYTerminal}, ::Array{Any,1}) at ./errorshow.jl:705
 [15] display_error(::IOContext{REPL.Terminals.TTYTerminal}, ::Array{Any,1}) at ./client.jl:104
 [16] #invokelatest#1 at ./essentials.jl:712 [inlined]
 [17] invokelatest at ./essentials.jl:711 [inlined]
 [18] print_response(::IO, ::Any, ::Bool, ::Bool, ::Any) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/REPL/src/REPL.jl:156
 [19] print_response(::REPL.AbstractREPL, ::Any, ::Bool, ::Bool) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/REPL/src/REPL.jl:146
 [20] (::REPL.var"#do_respond#38"{Bool,REPL.var"#48#57"{REPL.LineEditREPL,REPL.REPLHistoryProvider},REPL.LineEditREPL,REPL.LineEdit.Prompt})(::Any, ::Any, ::Any) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/REPL/src/REPL.jl:729
 [21] (::REPL.var"#53#62")(::Any, ::Any, ::Vararg{Any,N} where N) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/REPL/src/REPL.jl:997
 [22] #invokelatest#1 at ./essentials.jl:712 [inlined]
 [23] invokelatest at ./essentials.jl:711 [inlined]
 [24] (::REPL.LineEdit.var"#22#23"{REPL.var"#53#62",String})(::Any, ::Any) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/REPL/src/LineEdit.jl:1364
 [25] prompt!(::REPL.Terminals.TextTerminal, ::REPL.LineEdit.ModalInterface, ::REPL.LineEdit.MIState) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/REPL/src/LineEdit.jl:2446
 [26] run_interface(::REPL.Terminals.TextTerminal, ::REPL.LineEdit.ModalInterface, ::REPL.LineEdit.MIState) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/REPL/src/LineEdit.jl:2349
 [27] run_frontend(::REPL.LineEditREPL, ::REPL.REPLBackendRef) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/REPL/src/REPL.jl:1055
 [28] run_repl(::REPL.AbstractREPL, ::Any) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/REPL/src/REPL.jl:206
 [29] (::Base.var"#764#766"{Bool,Bool,Bool,Bool})(::Module) at ./client.jl:383
 [30] #invokelatest#1 at ./essentials.jl:712 [inlined]
 [31] invokelatest at ./essentials.jl:711 [inlined]
 [32] run_main_repl(::Bool, ::Bool, ::Bool, ::Bool, ::Bool) at ./client.jl:367
 [33] exec_options(::Base.JLOptions) at ./client.jl:305
 [34] _start() at ./client.jl:484

julia> 

The error

ERROR: MethodError: no method matching show_backtrace(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::Nothing)

with the following stack trace eats up a lot of the logs and makes it harder to parse the real issue.

Found the same problem here JunoLab/Juno.jl#572

[JeffBezanson]

Looks like this comes from trying to serialize a TaskFailedException.