Stack Overflow in type intersection

[Keno]

@YingboMa reports a stackoverflow error in type intersection with the following stack trace:

#0  jl_type_intersection2 (t1=0x7f0699e81790, t2=0x7f06d62d7c80 <jl_system_image_data+49560000>, isect=0x7ffe1f8bd198, isect2=0x7ffe1f8bd108)
    at /buildworker/worker/package_linux64/build/src/gf.c:1555
#1  0x00007f06e0e486d5 in ml_matches (mt=<optimized out>, type=type@entry=0x7f0698166050, lim=lim@entry=3, include_ambiguous=include_ambiguous@entry=0, 
    intersections=intersections@entry=1, world=world@entry=29411, cache_result=<optimized out>, min_valid=<optimized out>, max_valid=<optimized out>, 
    has_ambiguity=<optimized out>, offs=0) at /buildworker/worker/package_linux64/build/src/gf.c:2875
#2  0x00007f06e0e4c28d in jl_matching_methods (types=0x7f0698166050, lim=3, include_ambiguous=0, world=29411, min_valid=0x7ffe1f8bd2c0, max_valid=0x7ffe1f8bd2b0, 
    ambig=0x7ffe1f8bd2a8) at /buildworker/worker/package_linux64/build/src/gf.c:1839
#3  0x00007f06d28e239d in _methods_by_ftype () at reflection.jl:815
#4  #findall#208 () at compiler/methodtable.jl:55
#5  julia_findall##kw_11037 () at compiler/methodtable.jl:52
#6  0x00007f06d28e24e6 in jfptr_findallYY.YY.kw_11038.clone_1 () from /home/yingboma/bin/julia-8987f7ac6d/lib/julia/sys.so
#7  0x00007f06e0e4cdda in _jl_invoke (world=4177, mfunc=<optimized out>, nargs=4, args=0x7ffe1f8bd468, F=0x7f06d3a5bc70 <jl_system_image_data+7109040>)
    at /buildworker/worker/package_linux64/build/src/gf.c:2192
#8  jl_apply_generic (F=<optimized out>, args=0x7ffe1f8bd468, nargs=<optimized out>) at /buildworker/worker/package_linux64/build/src/gf.c:2374
#9  0x00007f06d28a4926 in julia_#210_10804 () at compiler/methodtable.jl:65
#10 0x00007f06d28a4567 in get! () at iddict.jl:161
#11 #findall#209 () at compiler/methodtable.jl:64
#12 julia_findall##kw_11032 () at compiler/methodtable.jl:63
#13 0x00007f06d28a4636 in jfptr_findallYY.YY.kw_11033.clone_1 () at compiler/methodtable.jl:65
#14 0x00007f06e0e4cdda in _jl_invoke (world=4177, mfunc=<optimized out>, nargs=4, args=0x7ffe1f8bd6e0, F=0x7f06d3a5bc70 <jl_system_image_data+7109040>)
    at /buildworker/worker/package_linux64/build/src/gf.c:2192
#15 jl_apply_generic (F=<optimized out>, args=0x7ffe1f8bd6e0, nargs=<optimized out>) at /buildworker/worker/package_linux64/build/src/gf.c:2374
#16 0x00007f06d2d62ebf in julia_abstract_call_gf_by_type_10992 () at compiler/abstractinterpretation.jl:71
#17 0x00007f06d2d5f34b in julia_abstract_call_known_11157 () at compiler/abstractinterpretation.jl:1003
#18 0x00007f06d288fb66 in julia_abstract_call_11359 () at compiler/abstractinterpretation.jl:1026
#19 0x00007f06d2b3cce0 in julia_abstract_call_11371 () at compiler/abstractinterpretation.jl:1010
#20 0x00007f06d2eca527 in julia_abstract_eval_statement_11528 () at compiler/abstractinterpretation.jl:1132
...

the the types in question are:

X1 = Tuple{typeof(ldiv!), LU, Base.ReshapedArray{T1, T2, T3, T4}} where T4 where T3<:(AbstractArray) where T2 where T1
X2 = Tuple{typeof(LinearAlgebra.ldiv!), LinearAlgebra.LU{T, XX} where XX<:(Union{DenseArray{T, 2}, Base.ReinterpretArray{T, 2, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T}, Base.ReshapedArray{T, 2, A, MI} where MI<:Tuple{Vararg{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}, N} where N} where A<:Union{Base.ReinterpretArray{T, N, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T}, Base.SubArray{T, 2, A, I, L} where L where I<:Tuple{Vararg{Union{Int64, Base.AbstractRange{Int64}, Base.AbstractCartesianIndex{N} where N, Base.ReshapedArray{T, N, A, Tuple{}} where A<:(Base.AbstractUnitRange{T} where T) where N where T}, N} where N} where A<:Union{Base.ReinterpretArray{T, N, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, Base.ReshapedArray{T, N, A, MI} where MI<:Tuple{Vararg{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}, N} where N} where A<:Union{Base.ReinterpretArray{T, N, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, DenseArray{T, N} where N where T}} where T), Union{DenseArray{T, 1}, DenseArray{T, 2}, Base.ReinterpretArray{T, 1, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T}, Base.ReinterpretArray{T, 2, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T}, Base.ReshapedArray{T, 1, A, MI} where MI<:Tuple{Vararg{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}, N} where N} where A<:Union{Base.ReinterpretArray{T, N, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T}, Base.ReshapedArray{T, 2, A, MI} where MI<:Tuple{Vararg{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}, N} where N} where A<:Union{Base.ReinterpretArray{T, N, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T}, Base.SubArray{T, 1, A, I, L} where L where I<:Tuple{Vararg{Union{Int64, Base.AbstractRange{Int64}, Base.AbstractCartesianIndex{N} where N, Base.ReshapedArray{T, N, A, Tuple{}} where A<:(Base.AbstractUnitRange{T} where T) where N where T}, N} where N} where A<:Union{Base.ReinterpretArray{T, N, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, Base.ReshapedArray{T, N, A, MI} where MI<:Tuple{Vararg{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}, N} where N} where A<:Union{Base.ReinterpretArray{T, N, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, DenseArray{T, N} where N where T}, Base.SubArray{T, 2, A, I, L} where L where I<:Tuple{Vararg{Union{Int64, Base.AbstractRange{Int64}, Base.AbstractCartesianIndex{N} where N, Base.ReshapedArray{T, N, A, Tuple{}} where A<:(Base.AbstractUnitRange{T} where T) where N where T}, N} where N} where A<:Union{Base.ReinterpretArray{T, N, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, Base.ReshapedArray{T, N, A, MI} where MI<:Tuple{Vararg{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}, N} where N} where A<:Union{Base.ReinterpretArray{T, N, S, A, IsReshaped} where S where IsReshaped where A<:Union{Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, Base.SubArray{T, N, A, I, true} where I<:Union{Tuple{Vararg{Real, N} where N}, Tuple{Base.AbstractUnitRange{T} where T, Vararg{Any, N} where N}} where A<:(DenseArray{T, N} where N where T) where N where T, DenseArray{T, N} where N where T} where N where T, DenseArray{T, N} where N where T}}} where T<:Union{Base.Complex{Float32}, Base.Complex{Float64}, Float32, Float64}

julia> typeintersect(X1, X2)
ERROR: StackOverflowError:
Stacktrace:
 [1] typeintersect(a::Any, b::Any)
   @ Base ./reflection.jl:563
 [2] top-level scope
   @ REPL[19]:1

[oxinabox]

I have been seeing this with LayeredLayouts.jl
though I have not isolated it more

[Sacha0]

On the off chance these stack overflows are related, ref. #22787 (comment).

[chriscoey]

and possibly: #37685