From bc4e7a22edf1c0992ca3780a70d3b3123c656b70 Mon Sep 17 00:00:00 2001
From: Kesler <4ck@mac101489.ornl.gov>
Date: Thu, 13 Jun 2019 18:03:56 -0400
Subject: [PATCH 1/2] Changed saving format to save array length as item in the
 sparse dict when writing arrays. Uses the lengthkey while reading the array
 to allocate the neccessary length of the array. Backwards compatible with
 existing BSON dumps.

---
 src/extensions.jl | 38 +++++++++++++++++++++++++++++---------
 src/read.jl       | 15 ++++++++++-----
 src/write.jl      |  2 +-
 test/runtests.jl  | 25 +++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 15 deletions(-)

diff --git a/src/extensions.jl b/src/extensions.jl
index cd61f91..d83d9dc 100644
--- a/src/extensions.jl
+++ b/src/extensions.jl
@@ -67,18 +67,38 @@ lower(x::Vector{UInt8}) = x
 reinterpret_(::Type{T}, x) where T =
     T[_x for _x in reinterpret(T, x)]
 
-function lower(x::Array)
-  ndims(x) == 1 && !isbitstype(eltype(x)) && return Any[x...]
-  BSONDict(:tag => "array", :type => eltype(x), :size => Any[size(x)...],
-           :data => isbitstype(eltype(x)) ? reinterpret_(UInt8, reshape(x, :)) : Any[x...])
+
+function collect_any(xs)
+  ys = Vector{Any}(undef, length(xs))
+  for i = 1:length(xs)
+    isassigned(xs, i) && (ys[i] = xs[i])
+  end
+  return ys
+end
+
+
+function lower(x::Array{T,N}) where {T,N}
+  isone(N) && !isbitstype(T) && return collect_any(x)
+  BSONDict(:tag => "array", :type => T, :size => Any[size(x)...],
+           :data => isbitstype(T) ? reinterpret_(UInt8, reshape(x, :)) : collect_any(x))
 end
 
+
 tags[:array] = d ->
-  isbitstype(d[:type]) ?
-    sizeof(d[:type]) == 0 ?
-      fill(d[:type](), d[:size]...) :
-      reshape(reinterpret_(d[:type], d[:data]), d[:size]...) :
-    Array{d[:type]}(reshape(d[:data], d[:size]...))
+if isbitstype(d[:type])
+    if sizeof(d[:type]) == 0
+      fill(d[:type](), d[:size]...)
+    else
+      reshape(reinterpret_(d[:type], d[:data]), d[:size]...)
+    end
+  else
+    a = Array{d[:type]}(undef, d[:size]...)
+    for i in 1:length(d[:data])
+      isassigned(d[:data], i) && (a[i] = d[:data][i])
+    end
+    a
+  end
+
 
 # Structs
 
diff --git a/src/read.jl b/src/read.jl
index bd3177c..982bdda 100644
--- a/src/read.jl
+++ b/src/read.jl
@@ -35,12 +35,17 @@ function parse_array(io::IO)::BSONArray
 
   while (tag = read(io, BSONType)) ≠ eof
     # Note that arrays are dicts with the index as the key
-    while read(io, UInt8) != 0x00
-      nothing
-    end
-    push!(ps, parse_tag(io::IO, tag))
+    # The first index in dict is "length" => length(x)
+      index_or_length = parse_cstr(io)
+      val = parse_tag(io::IO, tag)
+      if index_or_length == "length"
+        resize!(ps, Int(val))
+      else
+        i = Base.parse(Int, index_or_length) + 1
+        resize!(ps, i)
+        ps[i] = val
+      end
   end
-
   ps
 end
 
diff --git a/src/write.jl b/src/write.jl
index 97d1d04..1ec0c1e 100644
--- a/src/write.jl
+++ b/src/write.jl
@@ -35,7 +35,7 @@ end
 
 bson_primitive(io::IO, doc::BSONDict) = bson_doc(io, doc)
 bson_primitive(io::IO, x::BSONArray) =
-  bson_doc(io, [Base.string(i-1) => v for (i, v) in enumerate(x)])
+  bson_doc(io, vcat(["length" => length(x)],[Base.string(i-1) => x[i] for i = 1:length(x) if isassigned(x, i)]))
 
 # Lowering
 
diff --git a/test/runtests.jl b/test/runtests.jl
index eadc6bd..13641d1 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -2,6 +2,29 @@ using BSON
 using Test
 
 roundtrip_equal(x) = BSON.roundtrip(x) == x
+function roundtrip_equal(x::AbstractArray)
+  result = BSON.roundtrip(x)
+  all(eachindex(result, x)) do I
+    if !isassigned(x, I)
+      !isassigned(result, I)
+    else
+      x[I] == result[I]
+    end
+  end
+end
+
+function roundtrip_equal(x::Dict{Any,Array{Any,2}})
+  result = BSON.roundtrip(x)
+  keys(result) == keys(x)
+  all(keys(x)) do key
+    if isassigned(x[key])
+      x[key] == result[key]
+    else
+      isassigned(x[key]) == isassigned(result[key])
+    end
+  end
+end
+
 
 mutable struct Foo
   x
@@ -33,6 +56,7 @@ end
   @test roundtrip_equal(Array)
   @test roundtrip_equal([1,2,3])
   @test roundtrip_equal(rand(2,3))
+  @test roundtrip_equal(Array{Real}(undef, 2,3))
   @test roundtrip_equal(Array{Real}(rand(2,3)))
   @test roundtrip_equal(1+2im)
   @test roundtrip_equal(Nothing[])
@@ -41,6 +65,7 @@ end
   @test roundtrip_equal(fill(S(), (1,3)))
   @test roundtrip_equal(Set([1,2,3]))
   @test roundtrip_equal(Dict("a"=>1))
+  @test roundtrip_equal(Dict("a"=>Array{Real}(undef, 2,3)))
   @test roundtrip_equal(T(()))
 end
 

From 153f477b19ea372d491dc16e15d5df641d4bb376 Mon Sep 17 00:00:00 2001
From: Codyk12 <codac@live.com>
Date: Thu, 13 Jun 2019 21:08:27 -0400
Subject: [PATCH 2/2] Added test case and check for undef in applychildren
 function

---
 src/BSON.jl      | 1 +
 src/read.jl      | 5 +++--
 test/runtests.jl | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/BSON.jl b/src/BSON.jl
index 9d7cbd0..6da655f 100644
--- a/src/BSON.jl
+++ b/src/BSON.jl
@@ -25,6 +25,7 @@ end
 
 function applychildren!(f::Function, x::BSONArray)::BSONArray
   for i = 1:length(x)
+    isassigned(x, i) || continue
     x[i] = f(x[i])
   end
   return x
diff --git a/src/read.jl b/src/read.jl
index 982bdda..3ec1583 100644
--- a/src/read.jl
+++ b/src/read.jl
@@ -32,7 +32,7 @@ end
 function parse_array(io::IO)::BSONArray
   len = read(io, Int32)
   ps = BSONArray()
-
+  len_set = false
   while (tag = read(io, BSONType)) ≠ eof
     # Note that arrays are dicts with the index as the key
     # The first index in dict is "length" => length(x)
@@ -40,9 +40,10 @@ function parse_array(io::IO)::BSONArray
       val = parse_tag(io::IO, tag)
       if index_or_length == "length"
         resize!(ps, Int(val))
+        len_set = true
       else
         i = Base.parse(Int, index_or_length) + 1
-        resize!(ps, i)
+        len_set || resize!(ps, i)
         ps[i] = val
       end
   end
diff --git a/test/runtests.jl b/test/runtests.jl
index 13641d1..a70e994 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -13,7 +13,7 @@ function roundtrip_equal(x::AbstractArray)
   end
 end
 
-function roundtrip_equal(x::Dict{Any,Array{Any,2}})
+function roundtrip_equal(x::Dict{String,Array{Real,2}})
   result = BSON.roundtrip(x)
   keys(result) == keys(x)
   all(keys(x)) do key
@@ -57,6 +57,7 @@ end
   @test roundtrip_equal([1,2,3])
   @test roundtrip_equal(rand(2,3))
   @test roundtrip_equal(Array{Real}(undef, 2,3))
+  @test roundtrip_equal(Array{String}(undef, 2,3))
   @test roundtrip_equal(Array{Real}(rand(2,3)))
   @test roundtrip_equal(1+2im)
   @test roundtrip_equal(Nothing[])