diff --git a/base/summarysize.jl b/base/summarysize.jl
index 8c0a38d5116484..1aa8a7803e0624 100644
--- a/base/summarysize.jl
+++ b/base/summarysize.jl
@@ -134,12 +134,13 @@ function (ss::SummarySize)(obj::Array)
     if !haskey(ss.seen, datakey)
         ss.seen[datakey] = true
         dsize = Core.sizeof(obj)
-        if isbitsunion(eltype(obj))
+        T = eltype(obj)
+        if isbitsunion(T)
             # add 1 union selector byte for each element
             dsize += length(obj)
         end
         size += dsize
-        if !isempty(obj) && !Base.allocatedinline(eltype(obj))
+        if !isempty(obj) && (!Base.allocatedinline(T) || (T isa DataType && !Base.datatype_pointerfree(T)))
             push!(ss.frontier_x, obj)
             push!(ss.frontier_i, 1)
         end
diff --git a/test/misc.jl b/test/misc.jl
index 37c5397b4948f4..e765dc9279b86d 100644
--- a/test/misc.jl
+++ b/test/misc.jl
@@ -302,6 +302,11 @@ let vec = vcat(missing, ones(100000))
     @test length(unique(summarysize(vec) for i = 1:20)) == 1
 end
 
+# issue #40773
+let s = Set(1:100)
+    @test summarysize([s]) > summarysize(s)
+end
+
 # issue #13021
 let ex = try
     Main.x13021 = 0