From eb0e591de110e61bdb37d94f0b0c8179c9393f9c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 17 Feb 2016 15:41:56 -0500 Subject: [PATCH 1/3] make `map` and `collect` more general and uniform by adding iterator traits --- base/abstractarray.jl | 150 +-------------------------------------- base/array.jl | 99 +++++++++++++++++++++----- base/dict.jl | 3 + base/generator.jl | 39 +++++++++- base/iterator.jl | 46 ++++++++---- test/abstractarray.jl | 14 +--- test/arrayops.jl | 2 +- test/datafmt.jl | 2 +- test/unicode/utf8proc.jl | 2 +- 9 files changed, 163 insertions(+), 194 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 7fd228855bd3d..f826793617f56 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1087,34 +1087,6 @@ foreach(f) = (f(); nothing) foreach(f, itr) = (for x in itr; f(x); end; nothing) foreach(f, itrs...) = (for z in zip(itrs...); f(z...); end; nothing) -# generic map on any iterator -function map(f, iters...) - result = [] - len = length(iters) - states = [start(iters[idx]) for idx in 1:len] - nxtvals = cell(len) - cont = true - for idx in 1:len - if done(iters[idx], states[idx]) - cont = false - break - end - end - while cont - for idx in 1:len - nxtvals[idx],states[idx] = next(iters[idx], states[idx]) - end - push!(result, f(nxtvals...)) - for idx in 1:len - if done(iters[idx], states[idx]) - cont = false - break - end - end - end - result -end - ## map over arrays ## ## transform any set of dimensions @@ -1175,39 +1147,6 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector) return R end - -# using promote_type -function promote_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray) - # map to dest array, checking the type of each result. if a result does not - # match, do a type promotion and re-dispatch. - for i = offs:length(A) - @inbounds Ai = A[i] - el = f(Ai) - S = typeof(el) - if S === T || S <: T - @inbounds dest[i] = el::T - else - R = promote_type(T, S) - if R !== T - new = similar(dest, R) - copy!(new,1, dest,1, i-1) - new[i] = el - return promote_to!(f, i+1, new, A) - end - @inbounds dest[i] = el - end - end - return dest -end - -function map_promote(f, A::AbstractArray) - if isempty(A); return similar(A, Bottom); end - first = f(A[1]) - dest = similar(A, typeof(first)) - dest[1] = first - return promote_to!(f, 2, dest, A) -end - # These are needed because map(eltype, As) is not inferrable promote_eltype_op(::Any) = (@_pure_meta; Bottom) promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T)) @@ -1226,39 +1165,7 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray) return dest end -function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A) - # map to dest array, checking the type of each result. if a result does not - # match, widen the result type and re-dispatch. - i = offs - while !done(A, st) - @inbounds Ai, st = next(A, st) - el = f(Ai) - S = typeof(el) - if S === T || S <: T - @inbounds dest[i] = el::T - i += 1 - else - R = typejoin(T, S) - new = similar(dest, R) - copy!(new,1, dest,1, i-1) - @inbounds new[i] = el - return map_to!(f, i+1, st, new, A) - end - end - return dest -end - -function map(f, A::AbstractArray) - if isempty(A) - return isa(f,Type) ? similar(A,f) : similar(A) - end - st = start(A) - A1, st = next(A, st) - first = f(A1) - dest = similar(A, typeof(first)) - dest[1] = first - return map_to!(f, 2, st, dest, A) -end +map{F}(f::F, A::AbstractArray) = collect(Generator(f,A)) ## 2 argument function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray) @@ -1268,34 +1175,6 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray) return dest end -function map_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray, B::AbstractArray) - for i = offs:length(A) - @inbounds Ai, Bi = A[i], B[i] - el = f(Ai, Bi) - S = typeof(el) - if (S !== T) && !(S <: T) - R = typejoin(T, S) - new = similar(dest, R) - copy!(new,1, dest,1, i-1) - @inbounds new[i] = el - return map_to!(f, i+1, new, A, B) - end - @inbounds dest[i] = el::T - end - return dest -end - -function map(f, A::AbstractArray, B::AbstractArray) - shp = promote_shape(size(A),size(B)) - if prod(shp) == 0 - return similar(A, promote_type(eltype(A),eltype(B)), shp) - end - first = f(A[1], B[1]) - dest = similar(A, typeof(first), shp) - dest[1] = first - return map_to!(f, 2, dest, A, B) -end - ## N argument ith_all(i, ::Tuple{}) = () @@ -1311,32 +1190,9 @@ end map!{F}(f::F, dest::AbstractArray, As::AbstractArray...) = map_n!(f, dest, As) -function map_to_n!{T,F}(f::F, offs, dest::AbstractArray{T}, As) - for i = offs:length(As[1]) - el = f(ith_all(i, As)...) - S = typeof(el) - if (S !== T) && !(S <: T) - R = typejoin(T, S) - new = similar(dest, R) - copy!(new,1, dest,1, i-1) - @inbounds new[i] = el - return map_to_n!(f, i+1, new, As) - end - @inbounds dest[i] = el::T - end - return dest -end +spread(f) = (args)->f(args...) -function map(f, As::AbstractArray...) - shape = mapreduce(size, promote_shape, As) - if prod(shape) == 0 - return similar(As[1], promote_eltype(As...), shape) - end - first = f(map(a->a[1], As)...) - dest = similar(As[1], typeof(first), shape) - dest[1] = first - return map_to_n!(f, 2, dest, As) -end +map(f, iters...) = collect(Generator(spread(f),zip(iters...))) # multi-item push!, unshift! (built on top of type-specific 1-item version) # (note: must not cause a dispatch loop when 1-item case is not defined) diff --git a/base/array.jl b/base/array.jl index 89596506a1206..ad20b5a96cd5a 100644 --- a/base/array.jl +++ b/base/array.jl @@ -204,30 +204,93 @@ promote_rule{T,n,S}(::Type{Array{T,n}}, ::Type{Array{S,n}}) = Array{promote_type Return an array of type `Array{element_type,1}` of all items in a collection. """ -function collect{T}(::Type{T}, itr) - if applicable(length, itr) - # when length() isn't defined this branch might pollute the - # type of the other. - a = Array(T,length(itr)::Integer) - i = 0 - for x in itr - a[i+=1] = x - end - else - a = Array(T,0) - for x in itr - push!(a,x) - end - end - return a -end +collect{T}(::Type{T}, itr) = _collect_t(T, itr, iteratorsize(itr)) """ collect(collection) Return an array of all items in a collection. For associative collections, returns Pair{KeyType, ValType}. """ -collect(itr) = collect(eltype(itr), itr) +collect(itr) = _collect(itr, iteratoreltype(itr), iteratorsize(itr)) + +_collect(itr, ::HasEltype, isz) = _collect_t(eltype(itr), itr, isz) + +_collect_t(T::Type, itr, ::HasLength) = copy!(Array(T,Int(length(itr)::Integer)), itr) +_collect_t(T::Type, itr, ::HasShape) = copy!(Array(T,convert(Dims,size(itr))), itr) +function _collect_t(T::Type, itr, ::SizeUnknown) + a = Array(T,0) + for x in itr + push!(a,x) + end + return a +end + +_collect(itr, ::EltypeUnknown, ::HasLength) = _collect_shaped(itr, (Int(length(itr)),)) +_collect(itr, ::EltypeUnknown, ::HasShape) = _collect_shaped(itr, convert(Dims,size(itr))) + +_default_container(itr, elty, sz) = Array(elty, sz) +_default_container(itr::AbstractArray, elty, sz) = similar(itr, elty, sz) + +function collect_to!{T}(itr, offs, st, dest::AbstractArray{T}) + # collect to dest array, checking the type of each result. if a result does not + # match, widen the result type and re-dispatch. + i = offs + while !done(itr, st) + el, st = next(itr, st) + S = typeof(el) + if S === T || S <: T + @inbounds dest[i] = el::T + i += 1 + else + R = typejoin(T, S) + new = similar(dest, R) + copy!(new,1, dest,1, i-1) + @inbounds new[i] = el + return collect_to!(itr, i+1, st, new) + end + end + return dest +end + +function _collect_shaped(itr, sz) + if prod(sz) == 0 + return _default_container(itr, Union{}, sz) + end + st = start(itr) + v1, st = next(itr, st) + dest = _default_container(itr, typeof(v1), sz) + dest[1] = v1 + return collect_to!(itr, 2, st, dest) +end + +function grow_to!{T}(itr, st, dest::AbstractArray{T}) + while !done(itr, st) + el, st = next(itr, st) + S = typeof(el) + if S === T || S <: T + push!(dest, el::T) + else + R = typejoin(T, S) + n = length(dest) + new = similar(dest, R, n+1) + copy!(new,1, dest,1, n) + @inbounds new[n+1] = el + return grow_to!(itr, st, new) + end + end + return dest +end + +function _collect(itr, ::EltypeUnknown, ::SizeUnknown) + st = start(itr) + if done(itr,st) + return _default_container(itr, Union{}, 0) + end + v1, st = next(itr, st) + dest = _default_container(itr, typeof(v1), 1) + dest[1] = v1 + return grow_to!(itr, st, dest) +end ## Iteration ## start(A::Array) = 1 diff --git a/base/dict.jl b/base/dict.jl index c3d4ee24a8745..80ef8a2c4a45a 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -4,6 +4,9 @@ abstract Associative{K,V} +iteratorsize{T<:Associative}(::Type{T}) = HasLength() +iteratoreltype{T<:Associative}(::Type{T}) = HasEltype() + const secret_table_token = :__c782dbf1cf4d6a2e5e3865d7e95634f2e09b5902__ haskey(d::Associative, k) = in(k,keys(d)) diff --git a/base/generator.jl b/base/generator.jl index 34da38023886b..cce7238c83601 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -11,6 +11,8 @@ immutable Generator{I,F} iter::I end +Generator{T,I}(::Type{T}, iter::I) = Generator{I,Type{T}}(T, iter) + start(g::Generator) = start(g.iter) done(g::Generator, s) = done(g.iter, s) function next(g::Generator, s) @@ -18,4 +20,39 @@ function next(g::Generator, s) g.f(v), s2 end -collect(g::Generator) = map(g.f, g.iter) +## iterator traits + +abstract IteratorSize +immutable SizeUnknown <: IteratorSize end +immutable HasLength <: IteratorSize end +immutable HasShape <: IteratorSize end + +iteratorsize(x) = iteratorsize(typeof(x)) +iteratorsize(::Type) = SizeUnknown() + +and_iteratorsize{T}(isz::T, ::T) = isz +and_iteratorsize(::HasLength, ::HasShape) = HasLength() +and_iteratorsize(::HasShape, ::HasLength) = HasLength() +and_iteratorsize(a, b) = SizeUnknown() + +abstract IteratorEltype +immutable EltypeUnknown <: IteratorEltype end +immutable HasEltype <: IteratorEltype end + +iteratoreltype(x) = iteratoreltype(typeof(x)) +iteratoreltype(::Type) = EltypeUnknown() + +and_iteratoreltype{T}(iel::T, ::T) = iel +and_iteratoreltype(a, b) = EltypeUnknown() + +iteratorsize{T<:AbstractArray}(::Type{T}) = HasShape() +iteratorsize{T<:AbstractString}(::Type{T}) = HasLength() +iteratorsize{T<:Tuple}(::Type{T}) = HasLength() +iteratorsize{I,F}(::Type{Generator{I,F}}) = iteratorsize(I) +length(g::Generator) = length(g.iter) +size(g::Generator) = size(g.iter) + +iteratoreltype{T<:AbstractArray}(::Type{T}) = HasEltype() +iteratoreltype{T<:AbstractString}(::Type{T}) = HasEltype() +iteratoreltype{I,T}(::Type{Generator{I,Type{T}}}) = HasEltype() +eltype{I,T}(::Type{Generator{I,Type{T}}}) = T diff --git a/base/iterator.jl b/base/iterator.jl index 0b526aae32cbf..de4bfa34085e3 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -10,6 +10,7 @@ end enumerate(itr) = Enumerate(itr) length(e::Enumerate) = length(e.itr) +size(e::Enumerate) = size(e.itr) start(e::Enumerate) = (1, start(e.itr)) function next(e::Enumerate, state) n = next(e.itr,state[2]) @@ -19,6 +20,9 @@ done(e::Enumerate, state) = done(e.itr, state[2]) eltype{I}(::Type{Enumerate{I}}) = Tuple{Int, eltype(I)} +iteratorsize{I}(::Type{Enumerate{I}}) = iteratorsize(I) +iteratoreltype{I}(::Type{Enumerate{I}}) = iteratoreltype(I) + # zip abstract AbstractZipIterator @@ -28,6 +32,7 @@ immutable Zip1{I} <: AbstractZipIterator end zip(a) = Zip1(a) length(z::Zip1) = length(z.a) +size(z::Zip1) = size(z.a) eltype{I}(::Type{Zip1{I}}) = Tuple{eltype(I)} start(z::Zip1) = (start(z.a),) function next(z::Zip1, st) @@ -36,12 +41,16 @@ function next(z::Zip1, st) end done(z::Zip1, st) = done(z.a,st[1]) +iteratorsize{I}(::Type{Zip1{I}}) = iteratorsize(I) +iteratoreltype{I}(::Type{Zip1{I}}) = iteratoreltype(I) + immutable Zip2{I1, I2} <: AbstractZipIterator a::I1 b::I2 end zip(a, b) = Zip2(a, b) length(z::Zip2) = min(length(z.a), length(z.b)) +size(z::Zip2) = promote_shape(size(z.a), size(z.b)) eltype{I1,I2}(::Type{Zip2{I1,I2}}) = Tuple{eltype(I1), eltype(I2)} start(z::Zip2) = (start(z.a), start(z.b)) function next(z::Zip2, st) @@ -51,12 +60,16 @@ function next(z::Zip2, st) end done(z::Zip2, st) = done(z.a,st[1]) | done(z.b,st[2]) +iteratorsize{I1,I2}(::Type{Zip2{I1,I2}}) = and_iteratorsize(iteratorsize(I1),iteratorsize(I2)) +iteratoreltype{I1,I2}(::Type{Zip2{I1,I2}}) = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2)) + immutable Zip{I, Z<:AbstractZipIterator} <: AbstractZipIterator a::I z::Z end zip(a, b, c...) = Zip(a, zip(b, c...)) length(z::Zip) = min(length(z.a), length(z.z)) +size(z::Zip) = promote_shape(size(z.a), size(z.z)) tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{} function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T}) @_pure_meta @@ -71,6 +84,9 @@ function next(z::Zip, st) end done(z::Zip, st) = done(z.a,st[1]) | done(z.z,st[2]) +iteratorsize{I1,I2}(::Type{Zip{I1,I2}}) = and_iteratorsize(iteratorsize(I1),iteratorsize(I2)) +iteratoreltype{I1,I2}(::Type{Zip{I1,I2}}) = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2)) + # filter immutable Filter{F,I} @@ -108,6 +124,7 @@ end done(f::Filter, s) = s[1] eltype{I}(::Type{Filter{I}}) = eltype(I) +iteratoreltype{F,I}(::Type{Filter{F,I}}) = iteratoreltype(I) # Rest -- iterate starting at the given state @@ -122,6 +139,7 @@ next(i::Rest, st) = next(i.itr, st) done(i::Rest, st) = done(i.itr, st) eltype{I}(::Type{Rest{I}}) = eltype(I) +iteratoreltype{I,S}(::Type{Rest{I,S}}) = iteratoreltype(I) # Count -- infinite counting @@ -134,6 +152,7 @@ countfrom(start::Number) = Count(start, one(start)) countfrom() = Count(1, 1) eltype{S}(::Type{Count{S}}) = S +iteratoreltype{I<:Count}(::Type{I}) = HasEltype() start(it::Count) = it.start next(it::Count, state) = (state, state + it.step) @@ -148,6 +167,7 @@ end take(xs, n::Int) = Take(xs, n) eltype{I}(::Type{Take{I}}) = eltype(I) +iteratoreltype{I}(::Type{Take{I}}) = iteratoreltype(I) start(it::Take) = (it.n, start(it.xs)) @@ -171,6 +191,7 @@ end drop(xs, n::Int) = Drop(xs, n) eltype{I}(::Type{Drop{I}}) = eltype(I) +iteratoreltype{I}(::Type{Drop{I}}) = iteratoreltype(I) function start(it::Drop) xs_state = start(it.xs) @@ -195,6 +216,7 @@ end cycle(xs) = Cycle(xs) eltype{I}(::Type{Cycle{I}}) = eltype(I) +iteratoreltype{I}(::Type{Cycle{I}}) = iteratoreltype(I) function start(it::Cycle) s = start(it.xs) @@ -219,6 +241,7 @@ immutable Repeated{O} end repeated(x) = Repeated(x) eltype{O}(::Type{Repeated{O}}) = O +iteratoreltype{O}(::Type{Repeated{O}}) = HasEltype() start(it::Repeated) = nothing next(it::Repeated, state) = (it.x, nothing) done(it::Repeated, state) = false @@ -253,7 +276,9 @@ changes the fastest. Example: product(a) = Zip1(a) product(a, b) = Prod2(a, b) eltype{I1,I2}(::Type{Prod2{I1,I2}}) = Tuple{eltype(I1), eltype(I2)} +iteratoreltype{I1,I2}(::Type{Prod2{I1,I2}}) = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2)) length(p::AbstractProdIterator) = length(p.a)*length(p.b) +iteratorsize{I1,I2}(::Type{Prod2{I1,I2}}) = prod_iteratorsize(iteratorsize(I1),iteratorsize(I2)) function start(p::AbstractProdIterator) s1, s2 = start(p.a), start(p.b) @@ -287,12 +312,17 @@ end product(a, b, c...) = Prod(a, product(b, c...)) eltype{I1,I2}(::Type{Prod{I1,I2}}) = tuple_type_cons(eltype(I1), eltype(I2)) +iteratoreltype{I1,I2}(::Type{Prod{I1,I2}}) = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2)) +iteratorsize{I1,I2}(::Type{Prod{I1,I2}}) = prod_iteratorsize(iteratorsize(I1),iteratorsize(I2)) @inline function next{I1,I2}(p::Prod{I1,I2}, st) x = prod_next(p, st) ((x[1][1],x[1][2]...), x[2]) end +prod_iteratorsize(::Union{HasLength,HasShape}, ::Union{HasLength,HasShape}) = HasLength() +prod_iteratorsize(a, b) = SizeUnknown() + _size(p::Prod2) = (length(p.a), length(p.b)) _size(p::Prod) = (length(p.a), _size(p.b)...) @@ -325,19 +355,7 @@ next(i::IteratorND, s) = next(i.iter, s) size(i::IteratorND) = i.dims length(i::IteratorND) = prod(size(i)) ndims{I,N}(::IteratorND{I,N}) = N +iteratorsize{T<:IteratorND}(::Type{T}) = HasShape() eltype{I}(::IteratorND{I}) = eltype(I) - -collect(i::IteratorND) = copy!(Array(eltype(i),size(i)), i) - -function collect{I<:IteratorND}(g::Generator{I}) - sz = size(g.iter) - if length(g.iter) == 0 - return Array(Union{}, sz) - end - st = start(g) - first, st = next(g, st) - dest = Array(typeof(first), sz) - dest[1] = first - return map_to!(g.f, 2, st, dest, g.iter) -end +iteratoreltype{I}(::Type{IteratorND{I}}) = iteratoreltype(I) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 363ad822840d1..28d22b31554a4 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -411,23 +411,16 @@ function test_map(::Type{TestAbstractArray}) A = Array(Int, 10) B = Float64[1:10...] C = Float64[1:10...] - @test Base.map_to!(f, 1, A, B, C) == Real[ 2 * i for i in 1:10 ] - @test map(f, Int[], Float64[]) == Float64[] + @test map(f, convert(Vector{Int},B), C) == Float64[ 2 * i for i in 1:10 ] + @test map(f, Int[], Float64[]) == Union{}[] # AbstractArray map for N-arg case f(x, y, z) = x + y + z D = Float64[1:10...] @test map!(f, A, B, C, D) == Int[ 3 * i for i in 1:10 ] - @test Base.map_to_n!(f, 1, A, (B, C, D)) == Real[ 3 * i for i in 1:10 ] @test map(f, B, C, D) == Float64[ 3 * i for i in 1:10 ] - @test map(f, Int[], Int[], Complex{Int}[]) == Number[] -end - -function test_map_promote(::Type{TestAbstractArray}) - A = [1:10...] - f(x) = iseven(x) ? 1.0 : 1 - @test Base.map_promote(f, A) == fill(1.0, 10) + @test map(f, Int[], Int[], Complex{Int}[]) == Union{}[] end function test_UInt_indexing(::Type{TestAbstractArray}) @@ -492,7 +485,6 @@ test_get(TestAbstractArray) test_cat(TestAbstractArray) test_ind2sub(TestAbstractArray) test_map(TestAbstractArray) -test_map_promote(TestAbstractArray) test_UInt_indexing(TestAbstractArray) test_vcat_depwarn(TestAbstractArray) test_13315(TestAbstractArray) diff --git a/test/arrayops.jl b/test/arrayops.jl index edff1668b653d..a3537207778a6 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1459,7 +1459,7 @@ let A = zeros(Int, 2, 2), B = zeros(Float64, 2, 2) end # issue #14482 -@inferred Base.map_to!(Int8, 1, 1, Int8[0], Int[0]) +@inferred map(Int8, Int[0]) # make sure @inbounds isn't used too much type OOB_Functor{T}; a::T; end diff --git a/test/datafmt.jl b/test/datafmt.jl index 8e7cba730e8ce..fe32b81f6408e 100644 --- a/test/datafmt.jl +++ b/test/datafmt.jl @@ -259,7 +259,7 @@ for writefunc in ((io,x) -> writemime(io, "text/csv", x), let x = ["foo", "bar"], io = IOBuffer() writefunc(io, x) seek(io, 0) - @test collect(readcsv(io)) == x + @test vec(readcsv(io)) == x end end diff --git a/test/unicode/utf8proc.jl b/test/unicode/utf8proc.jl index 4f979c347b721..5cca4da65c1b6 100644 --- a/test/unicode/utf8proc.jl +++ b/test/unicode/utf8proc.jl @@ -244,7 +244,7 @@ let grphtest = (("b\u0300lahβlahb\u0302láh", ["b\u0300","l","a","h", @test typeof(first(graphemes(s_))) == SubString{typeof(s_)} end grph = collect(graphemes(s_)) - @test eltype(grph) == SubString{typeof(s_)} + @test isempty(grph) || eltype(grph) == SubString{typeof(s_)} @test grph == g_ @test length(graphemes(s_)) == length(grph) end From 6b90b4d8adbc0a4592b77fbdb7a0ac6229efbd36 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 1 Mar 2016 16:42:07 -0500 Subject: [PATCH 2/3] more iterator trait declarations make HasEltype and HasLength the default --- base/channels.jl | 2 ++ base/dict.jl | 3 --- base/generator.jl | 10 ++++------ base/io.jl | 2 ++ base/iterator.jl | 10 ++++++++-- base/libgit2/reference.jl | 2 ++ base/libgit2/walker.jl | 2 ++ base/multidimensional.jl | 11 +++++++---- base/regex.jl | 1 + base/task.jl | 2 ++ test/simdloop.jl | 8 ++++---- test/unicode/utf8proc.jl | 2 +- 12 files changed, 35 insertions(+), 20 deletions(-) diff --git a/base/channels.jl b/base/channels.jl index 34778b1585b3a..2362b5742a900 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -95,3 +95,5 @@ function done(c::Channel, state::Ref) end end next{T}(c::Channel{T}, state) = (v=get(state[]); state[]=nothing; (v, state)) + +iteratorsize{C<:Channel}(::Type{C}) = SizeUnknown() diff --git a/base/dict.jl b/base/dict.jl index 80ef8a2c4a45a..c3d4ee24a8745 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -4,9 +4,6 @@ abstract Associative{K,V} -iteratorsize{T<:Associative}(::Type{T}) = HasLength() -iteratoreltype{T<:Associative}(::Type{T}) = HasEltype() - const secret_table_token = :__c782dbf1cf4d6a2e5e3865d7e95634f2e09b5902__ haskey(d::Associative, k) = in(k,keys(d)) diff --git a/base/generator.jl b/base/generator.jl index cce7238c83601..efc867a08f5ea 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -26,9 +26,10 @@ abstract IteratorSize immutable SizeUnknown <: IteratorSize end immutable HasLength <: IteratorSize end immutable HasShape <: IteratorSize end +immutable IsInfinite <: IteratorSize end iteratorsize(x) = iteratorsize(typeof(x)) -iteratorsize(::Type) = SizeUnknown() +iteratorsize(::Type) = HasLength() # HasLength is the default and_iteratorsize{T}(isz::T, ::T) = isz and_iteratorsize(::HasLength, ::HasShape) = HasLength() @@ -40,19 +41,16 @@ immutable EltypeUnknown <: IteratorEltype end immutable HasEltype <: IteratorEltype end iteratoreltype(x) = iteratoreltype(typeof(x)) -iteratoreltype(::Type) = EltypeUnknown() +iteratoreltype(::Type) = HasEltype() # HasEltype is the default and_iteratoreltype{T}(iel::T, ::T) = iel and_iteratoreltype(a, b) = EltypeUnknown() iteratorsize{T<:AbstractArray}(::Type{T}) = HasShape() -iteratorsize{T<:AbstractString}(::Type{T}) = HasLength() -iteratorsize{T<:Tuple}(::Type{T}) = HasLength() iteratorsize{I,F}(::Type{Generator{I,F}}) = iteratorsize(I) length(g::Generator) = length(g.iter) size(g::Generator) = size(g.iter) -iteratoreltype{T<:AbstractArray}(::Type{T}) = HasEltype() -iteratoreltype{T<:AbstractString}(::Type{T}) = HasEltype() +iteratoreltype{I,T}(::Type{Generator{I,T}}) = EltypeUnknown() iteratoreltype{I,T}(::Type{Generator{I,Type{T}}}) = HasEltype() eltype{I,T}(::Type{Generator{I,Type{T}}}) = T diff --git a/base/io.jl b/base/io.jl index fd7db0f1daaef..2ea4533fa4b39 100644 --- a/base/io.jl +++ b/base/io.jl @@ -378,6 +378,8 @@ eltype(::Type{EachLine}) = ByteString readlines(s=STDIN) = collect(eachline(s)) +iteratorsize(::Type{EachLine}) = SizeUnknown() + # IOStream Marking # Note that these functions expect that io.mark exists for # the concrete IO type. This may not be true for IO types diff --git a/base/iterator.jl b/base/iterator.jl index de4bfa34085e3..348f92d2e3db6 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -125,6 +125,7 @@ done(f::Filter, s) = s[1] eltype{I}(::Type{Filter{I}}) = eltype(I) iteratoreltype{F,I}(::Type{Filter{F,I}}) = iteratoreltype(I) +iteratorsize{T<:Filter}(::Type{T}) = SizeUnknown() # Rest -- iterate starting at the given state @@ -140,6 +141,7 @@ done(i::Rest, st) = done(i.itr, st) eltype{I}(::Type{Rest{I}}) = eltype(I) iteratoreltype{I,S}(::Type{Rest{I,S}}) = iteratoreltype(I) +iteratorsize{T<:Rest}(::Type{T}) = SizeUnknown() # Count -- infinite counting @@ -152,12 +154,13 @@ countfrom(start::Number) = Count(start, one(start)) countfrom() = Count(1, 1) eltype{S}(::Type{Count{S}}) = S -iteratoreltype{I<:Count}(::Type{I}) = HasEltype() start(it::Count) = it.start next(it::Count, state) = (state, state + it.step) done(it::Count, state) = false +iteratorsize{S}(::Type{Count{S}}) = IsInfinite() + # Take -- iterate through the first n elements immutable Take{I} @@ -168,6 +171,7 @@ take(xs, n::Int) = Take(xs, n) eltype{I}(::Type{Take{I}}) = eltype(I) iteratoreltype{I}(::Type{Take{I}}) = iteratoreltype(I) +iteratorsize{T<:Take}(::Type{T}) = SizeUnknown() # TODO start(it::Take) = (it.n, start(it.xs)) @@ -192,6 +196,7 @@ drop(xs, n::Int) = Drop(xs, n) eltype{I}(::Type{Drop{I}}) = eltype(I) iteratoreltype{I}(::Type{Drop{I}}) = iteratoreltype(I) +iteratorsize{T<:Drop}(::Type{T}) = SizeUnknown() # TODO function start(it::Drop) xs_state = start(it.xs) @@ -217,6 +222,7 @@ cycle(xs) = Cycle(xs) eltype{I}(::Type{Cycle{I}}) = eltype(I) iteratoreltype{I}(::Type{Cycle{I}}) = iteratoreltype(I) +iteratorsize{I}(::Type{Cycle{I}}) = IsInfinite() function start(it::Cycle) s = start(it.xs) @@ -241,7 +247,7 @@ immutable Repeated{O} end repeated(x) = Repeated(x) eltype{O}(::Type{Repeated{O}}) = O -iteratoreltype{O}(::Type{Repeated{O}}) = HasEltype() +iteratorsize{O}(::Type{Repeated{O}}) = IsInfinite() start(it::Repeated) = nothing next(it::Repeated, state) = (it.x, nothing) done(it::Repeated, state) = false diff --git a/base/libgit2/reference.jl b/base/libgit2/reference.jl index 38081e3634492..23119c1478baa 100644 --- a/base/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -181,6 +181,8 @@ function Base.next(bi::GitBranchIter, state) return (state[1:2], (GitReference(ref_ptr_ptr[]), btype[1], false)) end +Base.iteratorsize(::Type{GitBranchIter}) = Base.SizeUnknown() + function Base.map(f::Function, bi::GitBranchIter) res = nothing s = start(bi) diff --git a/base/libgit2/walker.jl b/base/libgit2/walker.jl index 9bf2fa93529df..1d564c3c406db 100644 --- a/base/libgit2/walker.jl +++ b/base/libgit2/walker.jl @@ -25,6 +25,8 @@ function Base.next(w::GitRevWalker, state) return (state[1], (id_ptr[], false)) end +Base.iteratorsize(::Type{GitRevWalker}) = Base.SizeUnknown() + function push_head!(w::GitRevWalker) @check ccall((:git_revwalk_push_head, :libgit2), Cint, (Ptr{Void},), w.ptr) return w diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 7ba32048f70b6..37fc21a182d77 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -3,7 +3,7 @@ ### Multidimensional iterators module IteratorsMD -import Base: eltype, length, start, done, next, last, getindex, setindex!, linearindexing, min, max, eachindex, ndims +import Base: eltype, length, size, start, done, next, last, getindex, setindex!, linearindexing, min, max, eachindex, ndims, iteratorsize importall ..Base.Operators import Base: simd_outer_range, simd_inner_length, simd_index, @generated import Base: @nref, @ncall, @nif, @nexprs, LinearFast, LinearSlow, to_index, AbstractCartesianIndex @@ -103,6 +103,7 @@ end end eltype{I}(::Type{CartesianRange{I}}) = I +iteratorsize{I}(::Type{CartesianRange{I}}) = Base.HasShape() @generated function start{I<:CartesianIndex}(iter::CartesianRange{I}) N = length(I) @@ -135,13 +136,15 @@ start{I<:CartesianIndex{0}}(iter::CartesianRange{I}) = false next{I<:CartesianIndex{0}}(iter::CartesianRange{I}, state) = iter.start, true done{I<:CartesianIndex{0}}(iter::CartesianRange{I}, state) = state -@generated function length{I<:CartesianIndex}(iter::CartesianRange{I}) +@generated function size{I<:CartesianIndex}(iter::CartesianRange{I}) N = length(I) - N == 0 && return 1 + N == 0 && return () args = [:(iter.stop[$i]-iter.start[$i]+1) for i=1:N] - Expr(:call,:*,args...) + Expr(:tuple,args...) end +length(iter::CartesianRange) = prod(size(iter)) + last(iter::CartesianRange) = iter.stop @generated function simd_outer_range{I}(iter::CartesianRange{I}) diff --git a/base/regex.jl b/base/regex.jl index 5898349717c85..711df7a207f08 100644 --- a/base/regex.jl +++ b/base/regex.jl @@ -327,6 +327,7 @@ compile(itr::RegexMatchIterator) = (compile(itr.regex); itr) eltype(::Type{RegexMatchIterator}) = RegexMatch start(itr::RegexMatchIterator) = match(itr.regex, itr.string, 1, UInt32(0)) done(itr::RegexMatchIterator, prev_match) = (prev_match === nothing) +iteratorsize(::Type{RegexMatchIterator}) = SizeUnknown() # Assumes prev_match is not nothing function next(itr::RegexMatchIterator, prev_match) diff --git a/base/task.jl b/base/task.jl index a0e2422f84d01..095d89b31950a 100644 --- a/base/task.jl +++ b/base/task.jl @@ -253,6 +253,8 @@ function done(t::Task, val) istaskdone(t) end next(t::Task, val) = (t.result, nothing) +iteratorsize(::Type{Task}) = SizeUnknown() +iteratoreltype(::Type{Task}) = EltypeUnknown() isempty(::Task) = error("isempty not defined for Tasks") diff --git a/test/simdloop.jl b/test/simdloop.jl index 782c4e91f9cd6..771383aedf419 100644 --- a/test/simdloop.jl +++ b/test/simdloop.jl @@ -104,17 +104,17 @@ end crng = CartesianRange(CartesianIndex{4}(2,0,1,3), CartesianIndex{4}(4,1,1,5)) indexes = simd_cartesian_range!(Array(eltype(crng), 0), crng) -@test indexes == collect(crng) +@test indexes == vec(collect(crng)) crng = CartesianRange(CartesianIndex{2}(-1,1), CartesianIndex{2}(1,3)) indexes = simd_cartesian_range!(Array(eltype(crng), 0), crng) -@test indexes == collect(crng) +@test indexes == vec(collect(crng)) crng = CartesianRange(CartesianIndex{2}(-1,1), CartesianIndex{2}(-1,3)) indexes = simd_cartesian_range!(Array(eltype(crng), 0), crng) -@test indexes == collect(crng) +@test indexes == vec(collect(crng)) crng = CartesianRange(CartesianIndex{1}(2), CartesianIndex{1}(4)) @@ -124,7 +124,7 @@ indexes = simd_cartesian_range!(Array(eltype(crng), 0), crng) crng = CartesianRange(CartesianIndex{0}(), CartesianIndex{0}()) indexes = simd_cartesian_range!(Array(eltype(crng), 0), crng) -@test indexes == collect(crng) +@test indexes == vec(collect(crng)) # @simd with array as "range" # issue #13869 diff --git a/test/unicode/utf8proc.jl b/test/unicode/utf8proc.jl index 5cca4da65c1b6..4f979c347b721 100644 --- a/test/unicode/utf8proc.jl +++ b/test/unicode/utf8proc.jl @@ -244,7 +244,7 @@ let grphtest = (("b\u0300lahβlahb\u0302láh", ["b\u0300","l","a","h", @test typeof(first(graphemes(s_))) == SubString{typeof(s_)} end grph = collect(graphemes(s_)) - @test isempty(grph) || eltype(grph) == SubString{typeof(s_)} + @test eltype(grph) == SubString{typeof(s_)} @test grph == g_ @test length(graphemes(s_)) == length(grph) end From c915d34b1bc5339b89aa6ca1f9af1c8f030c3bea Mon Sep 17 00:00:00 2001 From: mschauer Date: Tue, 8 Mar 2016 16:56:05 +0100 Subject: [PATCH 3/3] zips, products and takes with infinite components --- base/iterator.jl | 22 +++++++++++++++++----- test/functional.jl | 9 +++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/base/iterator.jl b/base/iterator.jl index 348f92d2e3db6..b877f4f03b844 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -27,6 +27,15 @@ iteratoreltype{I}(::Type{Enumerate{I}}) = iteratoreltype(I) abstract AbstractZipIterator +zip_iteratorsize(a, b) = and_iteratorsize(a,b) # as `and_iteratorsize` but inherit `Union{HasLength,IsInfinite}` of the shorter iterator +zip_iteratorsize(::HasLength, ::IsInfinite) = HasLength() +zip_iteratorsize(::HasShape, ::IsInfinite) = HasLength() +zip_iteratorsize(a::IsInfinite, b) = zip_iteratorsize(b,a) +_min_length(a, b, ::IsInfinite, ::IsInfinite) = min(length(a),length(b)) # inherit behaviour, error +_min_length(a, b, A, ::IsInfinite) = length(a) +_min_length(a, b, ::IsInfinite, B) = length(b) +_min_length(a, b, A, B) = min(length(a),length(b)) + immutable Zip1{I} <: AbstractZipIterator a::I end @@ -49,7 +58,7 @@ immutable Zip2{I1, I2} <: AbstractZipIterator b::I2 end zip(a, b) = Zip2(a, b) -length(z::Zip2) = min(length(z.a), length(z.b)) +length(z::Zip2) = _min_length(z.a, z.b, iteratorsize(z.a), iteratorsize(z.b)) size(z::Zip2) = promote_shape(size(z.a), size(z.b)) eltype{I1,I2}(::Type{Zip2{I1,I2}}) = Tuple{eltype(I1), eltype(I2)} start(z::Zip2) = (start(z.a), start(z.b)) @@ -60,7 +69,7 @@ function next(z::Zip2, st) end done(z::Zip2, st) = done(z.a,st[1]) | done(z.b,st[2]) -iteratorsize{I1,I2}(::Type{Zip2{I1,I2}}) = and_iteratorsize(iteratorsize(I1),iteratorsize(I2)) +iteratorsize{I1,I2}(::Type{Zip2{I1,I2}}) = zip_iteratorsize(iteratorsize(I1),iteratorsize(I2)) iteratoreltype{I1,I2}(::Type{Zip2{I1,I2}}) = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2)) immutable Zip{I, Z<:AbstractZipIterator} <: AbstractZipIterator @@ -68,7 +77,7 @@ immutable Zip{I, Z<:AbstractZipIterator} <: AbstractZipIterator z::Z end zip(a, b, c...) = Zip(a, zip(b, c...)) -length(z::Zip) = min(length(z.a), length(z.z)) +length(z::Zip) = _min_length(z.a, z.z, iteratorsize(z.a), iteratorsize(z.z)) size(z::Zip) = promote_shape(size(z.a), size(z.z)) tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{} function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T}) @@ -84,7 +93,7 @@ function next(z::Zip, st) end done(z::Zip, st) = done(z.a,st[1]) | done(z.z,st[2]) -iteratorsize{I1,I2}(::Type{Zip{I1,I2}}) = and_iteratorsize(iteratorsize(I1),iteratorsize(I2)) +iteratorsize{I1,I2}(::Type{Zip{I1,I2}}) = zip_iteratorsize(iteratorsize(I1),iteratorsize(I2)) iteratoreltype{I1,I2}(::Type{Zip{I1,I2}}) = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2)) # filter @@ -141,7 +150,9 @@ done(i::Rest, st) = done(i.itr, st) eltype{I}(::Type{Rest{I}}) = eltype(I) iteratoreltype{I,S}(::Type{Rest{I,S}}) = iteratoreltype(I) -iteratorsize{T<:Rest}(::Type{T}) = SizeUnknown() +rest_iteratorsize(a) = SizeUnknown() +rest_iteratorsize(::IsInfinite) = IsInfinite() +iteratorsize{I,S}(::Type{Rest{I,S}}) = rest_iteratorsize(iteratorsize(I)) # Count -- infinite counting @@ -327,6 +338,7 @@ iteratorsize{I1,I2}(::Type{Prod{I1,I2}}) = prod_iteratorsize(iteratorsize(I1),it end prod_iteratorsize(::Union{HasLength,HasShape}, ::Union{HasLength,HasShape}) = HasLength() +prod_iteratorsize(a, ::IsInfinite) = IsInfinite() # products can have an infinite last iterator (which moves slowest) prod_iteratorsize(a, b) = SizeUnknown() _size(p::Prod2) = (length(p.a), length(p.b)) diff --git a/test/functional.jl b/test/functional.jl index 826e4e8d2792c..842163cacd695 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -76,6 +76,11 @@ let zeb = IOBuffer("1\n2\n3\n4\n5\n"), @test res == [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')] end +@test length(zip(cycle(1:3), 1:7)) == 7 +@test length(zip(cycle(1:3), 1:7, cycle(1:3))) == 7 +@test length(zip(1:3,Base.product(1:7,cycle(1:3)))) == 3 +@test length(zip(1:3,Base.product(1:7,cycle(1:3)),8)) == 1 + # rest # ---- let s = "hello" @@ -83,6 +88,8 @@ let s = "hello" @test collect(rest(s, st)) == ['e','l','l','o'] end +@test_throws MethodError collect(rest(countfrom(1), 5)) + # countfrom # --------- @@ -155,6 +162,7 @@ let i = 0 end end + # product # ------- @@ -165,6 +173,7 @@ end @test collect(Base.product(1:2,3:4)) == [(1,3),(2,3),(1,4),(2,4)] @test isempty(collect(Base.product(1:0,1:2))) @test length(Base.product(1:2,1:10,4:6)) == 60 +@test Base.iteratorsize(Base.product(1:2, countfrom(1))) == Base.IsInfinite() # foreach let