more 1.0 fixes

.travis.yml

@@ -4,12 +4,14 @@ os:
   - linux
   - osx
 julia:
+  - 0.7
+  - 1.0
   - nightly
 notifications:
   email: false
 # uncomment the following lines to override the default test script
 script:
   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-  - julia -e 'Pkg.clone(pwd()); Pkg.build("IndexedTables"); Pkg.test("IndexedTables"; coverage=true)'
+  - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("IndexedTables"); Pkg.test("IndexedTables"; coverage=true)'
 after_success:
   - julia -e 'cd(Pkg.dir("IndexedTables")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder()); Codecov.submit(Codecov.process_folder())'

src/IndexedTables.jl

@@ -7,9 +7,6 @@ import Base:
     ==, broadcast, empty!, copy, similar, sum, merge, merge!, mapslices,
     permutedims, sort, sort!, iterate
 
-using Serialization
-import Serialization:  serialize, deserialize
-
 export NDSparse, flush!, aggregate!, aggregate_vec, where, pairs, convertdim, columns, column, rows,
     itable, update!, aggregate, reducedim_vec, dimlabels, collect_columns
 

src/collect.jl

@@ -2,13 +2,8 @@ import DataValues: DataValue
 
 _is_subtype(::Type{S}, ::Type{T}) where {S, T} = promote_type(S, T) == T
 
-Base.@pure function dataarrayof(T)
-    if T<:DataValue
-        DataValueArray{T.parameters[1],1}
-    else
-        Vector{T}
-    end
-end
+dataarrayof(::Type{<:DataValue{T}}, len) where {T} = DataValueArray{T,1}(len)
+dataarrayof(::Type{T}, len) where {T} = Vector{T}(undef, len)
 
 """
 `collect_columns(itr)`
@@ -185,7 +180,7 @@ function widencolumns(dest, i, el::S, ::Type{T}) where{S <: Tup, T<:Tup}
         idx = findall(collect(!(s <: t) for (s, t) in zip(sp, tp)))
         new = dest
         for l in idx
-            newcol = dataarrayof(promote_type(sp[l], tp[l]))(undef, length(dest))
+            newcol = dataarrayof(promote_type(sp[l], tp[l]), length(dest))
             copyto!(newcol, 1, column(dest, l), 1, i-1)
             new = setcol(new, l, newcol)
         end
@@ -194,7 +189,7 @@ function widencolumns(dest, i, el::S, ::Type{T}) where{S <: Tup, T<:Tup}
 end
 
 function widencolumns(dest, i, el::S, ::Type{T}) where{S, T}
-    new = dataarrayof(promote_type(S, T))(length(dest))
+    new = dataarrayof(promote_type(S, T), length(dest))
     copyto!(new, 1, dest, 1, i-1)
     new
 end

src/columns.jl

@@ -6,8 +6,6 @@ export Columns, colnames, ncols, ColDict, insertafter!, insertbefore!, @cols, se
 export map_rows
 export All, Not, Between, Keys
 
-import Serialization: serialize, deserialize
-
 """
 A type that stores an array of tuples as a tuple of arrays.
 
@@ -356,36 +354,6 @@ function Base.vcat(c::Columns{<:Pair}, cs::Columns{<:Pair}...)
             vcat(c.columns.second, (x.columns.second for x in cs)...))
 end
 
-
-abstract type SerializedColumns end
-
-function serialize(s::AbstractSerializer, c::Columns)
-    Base.Serializer.serialize_type(s, SerializedColumns)
-    serialize(s, eltype(c) <: NamedTuple)
-    serialize(s, isa(c.columns, NamedTuple))
-    serialize(s, fieldnames(typeof(c.columns)))
-    for col in c.columns
-        serialize(s, col)
-    end
-end
-
-function deserialize(s::AbstractSerializer, ::Type{SerializedColumns})
-    Dnamed = deserialize(s)
-    Cnamed = deserialize(s)
-    fn = deserialize(s)
-    cols = Any[ deserialize(s) for i = 1:length(fn) ]
-    if Cnamed
-        c = Columns(cols..., names = fn)
-        if !Dnamed
-            dt = eltypes(typeof((c.columns...,)))
-            c = Columns{dt,typeof(c.columns)}(c.columns)
-        end
-    else
-        c = Columns(cols...)
-    end
-    return c
-end
-
 # fused indexing operations
 # these can be implemented for custom vector types like PooledVector where
 # you can get big speedups by doing indexing and an operation in one step.

src/flatten.jl

@@ -10,7 +10,7 @@ function dedup_names(ns)
         end
     end
 
-    repeated = filter((k,v) -> v > 1, count)
+    repeated = filter(((k,v),) -> v > 1, count)
     for k in keys(repeated)
         repeated[k] = 0
     end
@@ -212,7 +212,7 @@ x  a  b
 """
 function flatten(t::NextTable, col=length(columns(t)); pkey=nothing)
     vecvec = rows(t, col)
-    method_exists(start, (eltype(vecvec),)) || return t
+    hasmethod(iterate, (eltype(vecvec),)) || return t
     everythingbut = excludecols(t, col)
 
     order_others = Int[colindex(t, everythingbut)...]

src/indexing.jl

@@ -154,7 +154,7 @@ function _setindex!(d::NDSparse{T,D}, rhs::AbstractArray, idxs) where {T,D}
     I = d.index
     data = d.data
     ll = length(I)
-    p = collect(Iterators.product(idxs...)) # FIXME: remove collect
+    p = product(idxs...) #collect(Iterators.product(idxs...)) # FIXME: remove collect
     elem = iterate(p)
     elem === nothing && return d
     R, s = elem

src/join.jl

@@ -453,18 +453,18 @@ function Base.join(f, left::Dataset, right::Dataset;
 
     if !isempty(lnull_idx) && lout !== nothing
         lnulls = zeros(Bool, length(lout))
-        lnulls[lnull_idx] = true
+        lnulls[lnull_idx] .= true
         lout = if lout isa Columns
             Columns(map(lout.columns) do col
                         if col isa DataValueArray
-                            col.isnull[lnull_idx] = true
+                            col.isna[lnull_idx] .= true
                         else
                             DataValueArray(col, lnulls)
                         end
                     end)
         else
             if lout isa DataValueArray
-                lout.isnull[lnull_idx] = true
+                lout.isna[lnull_idx] .= true
             else
                 DataValueArray(lout, lnulls)
             end
@@ -474,18 +474,18 @@ function Base.join(f, left::Dataset, right::Dataset;
 
     if !isempty(rnull_idx) && rout !== nothing
         rnulls = zeros(Bool, length(rout))
-        rnulls[rnull_idx] = true
+        rnulls[rnull_idx] .= true
         rout = if rout isa Columns
             Columns(map(rout.columns) do col
                         if col isa DataValueArray
-                            col.isnull[rnull_idx] = true
+                            col.isna[rnull_idx] .= true
                         else
                             DataValueArray(col, rnulls)
                         end
                     end)
         else
             if rout isa DataValueArray
-                rout.isnull[rnull_idx] = true
+                rout.isna[rnull_idx] .= true
             else
                 DataValueArray(rout, rnulls)
             end

src/ndsparse.jl

@@ -173,7 +173,7 @@ function ndsparse(::Val{:serial}, ks::Tup, vs::Union{Tup, AbstractVector};
    #                   intersect(colnames(I), colnames(d))))
    #    error("All column names, including index and data columns, must be distinct")
    #end
-    if fieldcount(eltype(d)) !== 0
+    if !isconcretetype(eltype(d)) || fieldcount(eltype(d)) !== 0
         length(I) == length(d) ||
             error("index and data must have the same number of elements")
     end
@@ -268,7 +268,7 @@ pkeynames(t::NDSparse) = (dimlabels(t)...,)
 function valuenames(t::NDSparse)
     if isa(values(t), Columns)
         T = eltype(values(t))
-        ((ndims(t) + (1:fieldcount(eltype(values(t)))))...,)
+        ((ndims(t) .+ (1:fieldcount(eltype(values(t)))))...,)
     else
         ndims(t) + 1
     end
@@ -422,21 +422,6 @@ function showmeta(io, t::NDSparse, cnames)
     end
 end
 
-abstract type SerializedNDSparse end
-
-function serialize(s::AbstractSerializer, x::NDSparse)
-    flush!(x)
-    Base.Serializer.serialize_type(s, SerializedNDSparse)
-    serialize(s, x.index)
-    serialize(s, x.data)
-end
-
-function deserialize(s::AbstractSerializer, ::Type{SerializedNDSparse})
-    I = deserialize(s)
-    d = deserialize(s)
-    NDSparse(I, d, presorted=true)
-end
-
 @noinline convert(::Type{NDSparse}, @nospecialize(ks), @nospecialize(vs); kwargs...) = ndsparse(ks, vs; kwargs...)
 @noinline convert(T::Type{NDSparse}, c::Columns{<:Pair}; kwargs...) = convert(T, c.columns.first, c.columns.second; kwargs...)
 

src/reshape.jl

@@ -34,7 +34,7 @@ function stack(t::D, by = pkeynames(t); select = isa(t, NDSparse) ? valuenames(t
     valuecols = columns(t, select)
     valuecol = [valuecol[i] for i in 1:length(t) for valuecol in valuecols]
 
-    labels = fieldnames(valuecols)
+    labels = fieldnames(typeof(valuecols))
     labelcol = [label for i in 1:length(t) for label in labels]
 
     bycols = map(arg -> repeat(arg, inner = length(valuecols)), columns(t, by))
@@ -46,7 +46,7 @@ function unstack(::Type{D}, ::Type{T}, key, val, cols::AbstractVector{S}) where
     for (i, el) in enumerate(val)
         for j in el
             k, v = j
-            isnull(columns(dest_val, S(k))[i]) || error("Repeated values with same label are not allowed")
+            isna(columns(dest_val, S(k))[i]) || error("Repeated values with same label are not allowed")
             columns(dest_val, S(k))[i] = v
         end
     end

src/selection.jl

@@ -284,9 +284,9 @@ function _nonna(t::Union{Columns, NextTable}, by=(colnames(t)...,))
        #if Missing <: eltype(x)
        #    y = Array{nonmissing(eltype(x))}(undef, length(x))
        #    y[indxs] = x[indxs]
-        filt_by_col!(!isnull, x, indxs)
+        filt_by_col!(!isna, x, indxs)
         if isa(x, Array{<:DataValue})
-            y = Array{eltype(eltype(x))}(length(x))
+            y = Array{eltype(eltype(x))}(undef, length(x))
             y[indxs] = map(get, x[indxs])
             x = y
         elseif isa(x, DataValueArray)

src/utils.jl

@@ -158,7 +158,7 @@ end
 function append_n!(X, val, n)
     l = length(X)
     resize!(X, l+n)
-    for i in (1:n)+l
+    for i in (1:n) .+ l
         @inbounds X[i] = val
     end
     X
@@ -348,11 +348,16 @@ end
 compact_mem(x) = x
 compact_mem(x::StringArray{String}) = convert(StringArray{WeakRefString{UInt8}}, x)
 
-nonmissing(::Type{Union{Missing, T}}) where T = T
+#nonmissing(::Type{Union{Missing, T}}) where T = T
 
 function getsubfields(n::NamedTuple, fields)
     fns = fieldnames(typeof(n))
     ts = fieldtypes(typeof(n))
     NamedTuple{Tuple(fns[f] for f in fields), Tuple{map(f->ts[f], fields)...}}(Tuple(n[f] for f in fields))
 end
 getsubfields(t::Tuple, fields) = t[fields]
+
+# lexicographic order product iterator
+
+product(itr) = itr
+product(itrs...) = Base.Generator(reverse, Iterators.product(reverse(itrs)...))

test/test_collect.jl

@@ -95,9 +95,9 @@ end
     tuple_itr = (exp(i) for i in itr)
     @test collect_columns(tuple_itr) == Float64[]
 
-    t = collect_columns((a = i,) for i in (1, missing, 3))
-    @test Missing <: eltype(column(t, 1))
-    @test isequal(column(t, 1), [1, missing, 3])
+    t = collect_columns((a = i,) for i in (1, DataValue{Int}(), 3))
+    @test columns(t, 1) isa DataValueArray
+    @test isequal(columns(t, 1), DataValueArray([1, DataValue{Int}(), 3]))
 end
 
 @testset "collectpairs" begin
@@ -126,8 +126,8 @@ end
     @test collect_columns(v) == Columns(Columns((a = Int[],))=>Columns((b = String[],)))
     @test eltype(collect_columns(v)) == Pair{NamedTuple{(:a,), Tuple{Int}}, NamedTuple{(:b,), Tuple{String}}}
 
-    t = table(collect_columns((b=1,) => (a = i,) for i in (2, missing, 3)))
-    @test isequal(t, table((b = [1,1,1], a = [2, missing, 3]), pkey = :b))
+    t = table(collect_columns((b = 1,) => (a = i,) for i in (2, DataValue{Int}(), 3)))
+    @test t == table((b = [1,1,1], a = [2, DataValue{Int}(), 3]), pkey = :b)
 end
 
 @testset "issubtype" begin