From 50a3a3e32b2c17b2dc011bc66760a8e8f5dac305 Mon Sep 17 00:00:00 2001
From: Martin Holters <martin.holters@hsu-hh.de>
Date: Thu, 17 May 2018 11:04:05 +0200
Subject: [PATCH] Avoid over-pessimization in apply_type_tfunc

---
 base/compiler/tfuncs.jl   | 26 +++++++++++++++++++-------
 test/compiler/compiler.jl |  9 +++++++++
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl
index 0b80173f0b8df0..2681de4e3d6a73 100644
--- a/base/compiler/tfuncs.jl
+++ b/base/compiler/tfuncs.jl
@@ -724,20 +724,34 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
     elseif isconstType(headtypetype)
         headtype = headtypetype.parameters[1]
     else
-        return Any
+        return Type
     end
     largs = length(args)
     if headtype === Union
         largs == 0 && return Const(Bottom)
-        largs == 1 && return args[1]
+        hasnonType = false
         for i = 1:largs
             ai = args[i]
-            if !isa(ai, Const) || !isa(ai.val, Type)
+            if isa(ai, Const)
+                if !isa(ai.val, Type)
+                    if isa(ai.val, TypeVar) || isa(ai.val, PartialTypeVar)
+                        return hasnonType = true
+                    else
+                        return Union{}
+                    end
+                end
+            else
                 if !isType(ai)
-                    return Any
+                    if !isa(ai, Type) || typeintersect(ai, Type) != Union{}
+                        hasnonType = true
+                    else
+                        return Union{}
+                    end
                 end
             end
         end
+        largs == 1 && return isa(args[1], Type) ? typeintersect(args[1], Type) : Type
+        hasnonType && return Type
         ty = Union{}
         allconst = true
         for i = 1:largs
@@ -754,8 +768,7 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
     end
     istuple = (headtype == Tuple)
     if !istuple && !isa(headtype, UnionAll)
-        # TODO: return `Bottom` for trying to apply a non-UnionAll
-        return Any
+        return Union{}
     end
     uncertain = false
     canconst = true
@@ -773,7 +786,6 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
             canconst = false
             push!(tparams, ai.tv)
         else
-            # TODO: return `Bottom` for trying to apply a non-UnionAll
             uncertain = true
             # These blocks improve type info but make compilation a bit slower.
             # XXX
diff --git a/test/compiler/compiler.jl b/test/compiler/compiler.jl
index f8435898714fb1..a310a712ae134f 100644
--- a/test/compiler/compiler.jl
+++ b/test/compiler/compiler.jl
@@ -1583,3 +1583,12 @@ end
 
 # Equivalence of Const(T.instance) and T for singleton types
 @test Const(nothing) ⊑ Nothing && Nothing ⊑ Const(nothing)
+
+# Don't pessimize apply_type to anything worse than Type and yield Bottom for invalid Unions
+@test Core.Compiler.return_type(Core.apply_type, Tuple{Type{Union}}) == Type{Union{}}
+@test Core.Compiler.return_type(Core.apply_type, Tuple{Type{Union},Any}) == Type
+@test Core.Compiler.return_type(Core.apply_type, Tuple{Type{Union},Any,Any}) == Type
+@test Core.Compiler.return_type(Core.apply_type, Tuple{Type{Union},Int}) == Union{}
+@test Core.Compiler.return_type(Core.apply_type, Tuple{Type{Union},Any,Int}) == Union{}
+@test Core.Compiler.return_type(Core.apply_type, Tuple{Any}) == Type
+@test Core.Compiler.return_type(Core.apply_type, Tuple{Any,Any}) == Type