DataFrames 0.11+ compat via IndirectArrays

.travis.yml

@@ -10,6 +10,9 @@ matrix:
     - julia: nightly
 notifications:
   email: false
+matrix:
+  allow_failures:
+  - julia: nightly
 script:
   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
   - julia -e 'Pkg.clone(pwd()); Pkg.build("Gadfly");'

REQUIRE

@@ -1,10 +1,10 @@
-julia 0.5
+julia 0.6
 Colors 0.3.4
 Compat 0.18.0
 Compose 0.5.2
 Contour 0.1.1
 CoupledFields
-DataFrames 0.4.2 0.11.0
+DataFrames 0.11.4
 DataArrays
 DataStructures
 Distributions
@@ -16,3 +16,10 @@ Loess
 Showoff 0.0.3
 StatsBase
 Juno
+IndirectArrays 0.4.1
+Missings
+# Gadfly doesn't use WeakRefString directly but because of a bug in Julia 0.6.2, Gadfly
+# will trigger a ambiguity error between WeakRefString methods. A temporary method was
+# added in version WeakRefStrings 0.4.3 to work around this. When Gadfly requires at least
+# Julia 0.6.3, this dependency can therefore be dropped.
+WeakRefStrings 0.4.3

docs/src/lib/scales/scale_x_discrete.md

@@ -19,7 +19,7 @@ easiest way to get that data to plot appropriately.
     should map a value in `x` to a string giving its label.
   * `levels`: If non-nothing, give values for the scale. Order will be respected
     and anything in the data that's not respresented in `levels` will be set to
-    `NA`.
+    `missing`.
   * `order`: If non-nothing, give a vector of integers giving a permutation of
     the values pool of the data.
 

docs/src/lib/scales/scale_y_discrete.md

@@ -19,7 +19,7 @@ easiest way to get that data to plot appropriately.
     should map a value in `x` to a string giving its label.
   * `levels`: If non-nothing, give values for the scale. Order will be respected
     and anything in the data that's not respresented in `levels` will be set to
-    `NA`.
+    `missing`.
   * `order`: If non-nothing, give a vector of integers giving a permutation of
     the values pool of the data.
 

src/Gadfly.jl

@@ -10,6 +10,8 @@ using DataFrames
 using DataStructures
 using JSON
 using Showoff
+using IndirectArrays
+using CategoricalArrays
 
 import IterTools
 import IterTools: distinct, drop, chain
@@ -1221,8 +1223,9 @@ end
 
 const CategoricalType = Union{AbstractString, Bool, Symbol}
 
-classify_data{N, T <: CategoricalType}(data::AbstractArray{T, N}) = :categorical
+classify_data{N, T <: Union{CategoricalType,Missing}}(data::AbstractArray{T, N})        = :categorical
 classify_data{N, T <: Union{Base.Callable,Measure,Colorant}}(data::AbstractArray{T, N}) = :functional
+classify_data(data::CategoricalArray) = :categorical
 classify_data{T <: Base.Callable}(data::T) = :functional
 classify_data(data::AbstractArray) = :numerical
 classify_data(data::Distribution) = :distribution

src/aesthetics.jl

@@ -1,8 +1,8 @@
 const NumericalOrCategoricalAesthetic =
-    Union{(Void), Vector, DataArray, PooledDataArray}
+    Union{(Void), Vector, DataArray, IndirectArray}
 
 const CategoricalAesthetic =
-    Union{(Void), PooledDataArray}
+    Union{(Void), IndirectArray}
 
 const NumericalAesthetic =
     Union{(Void), Matrix, Vector, DataArray}
@@ -235,6 +235,7 @@ function concat(aess::Aesthetics...)
                              mv == nothing ? mu :
                                  max(mu, mv))
             else
+                cat_aes_var!(getfield(cataes, var), getfield(aes, var))
                 setfield!(cataes, var,
                           cat_aes_var!(getfield(cataes, var), getfield(aes, var)))
             end
@@ -287,20 +288,6 @@ function cat_aes_var!{T, U}(a::AbstractArray{T}, b::AbstractArray{U})
     return ab
 end
 
-function cat_aes_var!{T}(xs::PooledDataVector{T}, ys::PooledDataVector{T})
-    newpool = T[x for x in union(Set(xs.pool), Set(ys.pool))]
-    newdata = vcat(T[x for x in xs], T[y for y in ys])
-    PooledDataArray(newdata, newpool, [false for _ in newdata])
-end
-
-function cat_aes_var!{T, U}(xs::PooledDataVector{T}, ys::PooledDataVector{U})
-    V = promote_type(T, U)
-    newpool = V[x for x in union(Set(xs.pool), Set(ys.pool))]
-    newdata = vcat(V[x for x in xs], V[y for y in ys])
-    PooledDataArray(newdata, newpool, [false for _ in newdata])
-end
-
-
 # Summarizing aesthetics
 
 # Produce a matrix of Aesthetic or Data objects partitioning the original
@@ -333,10 +320,10 @@ function by_xy_group{T <: Union{Data, Aesthetics}}(aes::T, xgroup, ygroup,
 
     xgroup === nothing && ygroup === nothing && return aes_grid
 
-    make_pooled_data_array{T,U,V}(::Type{PooledDataArray{T,U,V}}, arr::AbstractArray) =
-            PooledDataArray(convert(Array{T}, arr))
-    make_pooled_data_array{T,U,V}(::Type{PooledDataArray{T,U,V}},
-            arr::PooledDataArray{T,U,V}) = arr
+    make_pooled_array{T,R,N,RA}(::Type{IndirectArray{T,R,N,RA}}, arr::AbstractArray) =
+            IndirectArray(convert(Array{T}, arr))
+    make_pooled_array{T,R,N,RA}(::Type{IndirectArray{T,R,N,RA}},
+            arr::IndirectArray{T,R,N,RA}) = arr
 
     for var in fieldnames(T)
         # Skipped aesthetics. Don't try to partition aesthetics for which it
@@ -366,9 +353,9 @@ function by_xy_group{T <: Union{Data, Aesthetics}}(aes::T, xgroup, ygroup,
             end
 
             for i in 1:n, j in 1:m
-                if typeof(vals) <: PooledDataArray
+                if typeof(vals) <: IndirectArray
                     setfield!(aes_grid[i, j], var,
-                              make_pooled_data_array(typeof(vals), staging[i, j]))
+                              make_pooled_array(typeof(vals), staging[i, j]))
                 else
                     if !applicable(convert, typeof(vals), staging[i, j])
                         T2 = eltype(vals)

src/geom/bar.jl

@@ -76,8 +76,8 @@ function render_stacked_bar(geom::BarGeometry,
                                      aes::Gadfly.Aesthetics,
                                      orientation::Symbol)
     # preserve factor orders of pooled data arrays
-    if isa(aes.color, PooledDataArray)
-        idxs = sortperm(aes.color.refs, rev=true)
+    if isa(aes.color, IndirectArray)
+        idxs = sortperm(aes.color.index, rev=true)
     else
         idxs = 1:length(aes.color)
     end
@@ -152,8 +152,8 @@ function render_dodged_bar(geom::BarGeometry,
                                     aes::Gadfly.Aesthetics,
                                     orientation::Symbol)
     # preserve factor orders of pooled data arrays
-    if isa(aes.color, PooledDataArray)
-        idxs = sortperm(aes.color.refs, rev=true)
+    if isa(aes.color, IndirectArray)
+        idxs = sortperm(aes.color.index, rev=true)
     else
         idxs = 1:length(aes.color)
     end

src/geom/beeswarm.jl

@@ -16,7 +16,7 @@ function render(geom::BeeswarmGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthet
     Gadfly.assert_aesthetics_equal_length("Geom.point", aes,
                                           element_aesthetics(geom)...)
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGBA{Float32}[theme.default_color])
     default_aes.size = Measure[theme.point_size]
     aes = inherit(aes, default_aes)
     padding = 1.0mm

src/geom/boxplot.jl

@@ -27,7 +27,7 @@ function render(geom::BoxplotGeometry, theme::Gadfly.Theme, aes::Gadfly.Aestheti
                                      :upper_hinge, :upper_fence, :outliers)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGB{Float32}[theme.default_color])
     default_aes.x = Float64[0.5]
     aes = inherit(aes, default_aes)
 

src/geom/errorbar.jl

@@ -54,7 +54,7 @@ function render(geom::YErrorBarGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthe
                                           element_aesthetics(geom)...)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGB{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
     caplen = theme.errorbar_cap_length/2
     ttc, teb, tbc = subtags(geom.tag, :top_cap, :error_bar, :bottom_cap)
@@ -95,7 +95,7 @@ function render(geom::XErrorBarGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthe
 
     colored = aes.color != nothing
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGB{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
     caplen = theme.errorbar_cap_length/2
     tlc, teb, trc = subtags(geom.tag, :left_cap, :error_bar, :right_cap)

src/geom/hexbin.jl

@@ -16,7 +16,7 @@ element_aesthetics(geom::HexagonalBinGeometry) = [:x, :y, :xsize, :ysize, :color
 
 function render(geom::HexagonalBinGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGB{Float32}[theme.default_color])
     default_aes.xsize = [1.0]
     default_aes.ysize = [1.0]
     aes = inherit(aes, default_aes)
@@ -25,7 +25,7 @@ function render(geom::HexagonalBinGeometry, theme::Gadfly.Theme, aes::Gadfly.Aes
     Gadfly.assert_aesthetics_equal_length("Geom.hexbin", aes, :x, :y)
 
     n = length(aes.x)
-    visibility = Bool[!isna(c) for c in takestrict(cycle(aes.color), n)]
+    visibility = Bool[!ismissing(c) for c in takestrict(cycle(aes.color), n)]
     xs = aes.x[visibility]
     ys = aes.y[visibility]
     xsizes = collect(eltype(aes.xsize), takestrict(cycle(aes.xsize), n))[visibility]

src/geom/line.jl

@@ -58,7 +58,7 @@ function render(geom::LineGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
                                           element_aesthetics(geom)...)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGBA{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
 
     ctx = context(order=geom.order)
@@ -132,7 +132,7 @@ function render(geom::LineGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
                       svgclass("geometry"))
 
     elseif length(aes.color) == 1 &&
-            !(isa(aes.color, PooledDataArray) && length(levels(aes.color)) > 1)
+            !(isa(aes.color, IndirectArray) && count(!ismissing, aes.color.values) > 1)
         T = (Tuple{eltype(aes.x), eltype(aes.y)})
         points = T[(x, y) for (x, y) in zip(aes.x, aes.y)]
         geom.preserve_order || sort!(points, by=first)

src/geom/point.jl

@@ -25,7 +25,7 @@ function render(geom::PointGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics
 
     default_aes = Gadfly.Aesthetics()
     default_aes.shape = Function[Shape.circle]
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGBA{Float32}[theme.default_color])
     default_aes.size = Measure[theme.point_size]
     aes = inherit(aes, default_aes)
 

src/geom/polygon.jl

@@ -38,7 +38,7 @@ function render(geom::PolygonGeometry, theme::Gadfly.Theme,
     Gadfly.assert_aesthetics_defined("Geom.polygon", aes, :x, :y)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGBA{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
 
     ctx = context(order=geom.order)
@@ -66,7 +66,7 @@ function render(geom::PolygonGeometry, theme::Gadfly.Theme,
             compose!(ctx, fill(nothing), stroke(cs))
         end
     elseif length(aes.color) == 1 &&
-            !(isa(aes.color, PooledDataArray) && length(levels(aes.color)) > 1)
+            !(isa(aes.color, IndirectArray) && length(filter(!ismissing, aes.color.values)) > 1)
         compose!(ctx, Compose.polygon(polygon_points(aes.x, aes.y, geom.preserve_order), geom.tag))
         if geom.fill
             compose!(ctx, fill(aes.color[1]),

src/geom/rectbin.jl

@@ -42,7 +42,7 @@ element_aesthetics(::RectangularBinGeometry) =
 function render(geom::RectangularBinGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGBA{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
 
     Gadfly.assert_aesthetics_defined("RectangularBinGeometry", aes, :xmin, :xmax, :ymin, :ymax)

src/geom/ribbon.jl

@@ -17,13 +17,13 @@ function render(geom::RibbonGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetic
                                           element_aesthetics(geom)...)
 
     default_aes = Gadfly.Aesthetics()
-    default_aes.color = PooledDataArray(RGB{Float32}[theme.default_color])
+    default_aes.color = IndirectArray(RGB{Float32}[theme.default_color])
     aes = inherit(aes, default_aes)
 
     aes_x, aes_ymin, aes_ymax = concretize(aes.x, aes.ymin, aes.ymax)
 
     if length(aes.color) == 1 &&
-        !(isa(aes.color, PooledDataArray) && length(levels(aes.color)) > 1)
+        !(isa(aes.color, IndirectArray) && length(filter(!ismissing, aes.color.values)) > 1)
         max_points = collect(zip(aes_x, aes_ymax))
         sort!(max_points, by=first)
 

src/geometry.jl

@@ -8,6 +8,7 @@ using DataStructures
 using Distributions
 using Gadfly
 using Measures
+using IndirectArrays
 
 import Compose.combine # Prevent DataFrame.combine from taking over.
 import Gadfly: render, layers, element_aesthetics, inherit, escape_id,

src/misc.jl

@@ -1,20 +1,7 @@
 #Is this usable data?
-isconcrete{T<:Number}(x::T) = !isna(x) && isfinite(x)
+isconcrete{T<:Number}(x::T) = !ismissing(x) && isfinite(x)
 isconcrete(x::(Irrational)) = true
-isconcrete(x) = !isna(x)
-
-hasna(xs) = false
-
-function hasna(xs::AbstractDataArray)
-    for x in xs
-        if isna(x)
-            return true
-        end
-    end
-    return false
-end
-
-
+isconcrete(x) = !ismissing(x)
 
 function isallconcrete(xs)
     ans = true
@@ -221,18 +208,6 @@ function concrete_minmax{T<:Real}(xs::IterTools.Chain, xmin::T, xmax::T)
 end
 
 
-
-function nonzero_length(xs)
-    n = 0
-    for x in xs
-        if x != 0
-            n += 1
-        end
-    end
-    n
-end
-
-
 # Create a new object of type T from a with missing values (i.e., those set to
 # nothing) inherited from b.
 function inherit{T}(a::T, b::T)
@@ -355,16 +330,6 @@ end
 
 svg_color_class_from_label(label::AbstractString) = @sprintf("color_%s", escape_id(label))
 
-
-
-"""
-A faster map function for PooledDataVector
-"""
-function pooled_map(T::Type, f::Function, xs::PooledDataVector)
-    newpool = T[f(x) for x in xs.pool]
-    return T[newpool[i] for i in xs.refs]
-end
-
 using Base.Dates
 
 # Arbitrarily order colors