From addb20c22886e442435503f8661c9570ebaad922 Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Fri, 30 Oct 2020 16:48:15 -0400
Subject: [PATCH] fix #22787, another bounds circularity in type intersection

---
 src/subtype.c   | 23 +++++++++++++++++++++--
 test/subtype.jl | 11 +++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/src/subtype.c b/src/subtype.c
index 34590492c11e2..260df0a5b3756 100644
--- a/src/subtype.c
+++ b/src/subtype.c
@@ -2865,6 +2865,25 @@ static jl_value_t *intersect_type_type(jl_value_t *x, jl_value_t *y, jl_stenv_t
     */
 }
 
+// cmp <= 0: is x already <= y in this environment
+// cmp >= 0: is x already >= y in this environment
+static int compareto_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e, int cmp)
+{
+    if (x == (jl_value_t*)y)
+        return 1;
+    if (!jl_is_typevar(x))
+        return 0;
+    jl_varbinding_t *xv = lookup(e, (jl_tvar_t*)x);
+    if (xv == NULL)
+        return 0;
+    int ans = 1;
+    if (cmp <= 0)
+        ans &= compareto_var(xv->ub, y, e, cmp);
+    if (cmp >= 0)
+        ans &= compareto_var(xv->lb, y, e, cmp);
+    return ans;
+}
+
 // `param` means we are currently looking at a parameter of a type constructor
 // (as opposed to being outside any type constructor, or comparing variable bounds).
 // this is used to record the positions where type variables occur for the
@@ -2936,9 +2955,9 @@ static jl_value_t *intersect(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int pa
                 else
                     lb = simple_join(xlb, ylb);
                 if (yy) {
-                    if (lb != y)
+                    if (!compareto_var(lb, (jl_tvar_t*)y, e, -1))
                         yy->lb = lb;
-                    if (ub != y)
+                    if (!compareto_var(ub, (jl_tvar_t*)y, e,  1))
                         yy->ub = ub;
                     assert(yy->ub != y);
                     assert(yy->lb != y);
diff --git a/test/subtype.jl b/test/subtype.jl
index c987167a57712..eb6bd27ba138b 100644
--- a/test/subtype.jl
+++ b/test/subtype.jl
@@ -1786,3 +1786,14 @@ let X1 = Tuple{AlmostLU, Vector{T}} where T,
     @test I >: actual
     @test_broken I == actual
 end
+
+# issue #22787
+# for now check that these don't stack overflow
+let
+    t = typeintersect(Tuple{Type{Q}, Q, Ref{Q}} where Q<:Ref,
+                      Tuple{Type{S}, Union{Ref{S}, Ref{R}}, R} where R where S)
+    @test_broken t != Union{}
+    t = typeintersect(Tuple{Type{T}, T, Ref{T}} where T,
+                      Tuple{Type{S}, Ref{S}, S} where S)
+    @test_broken t != Union{}
+end