From c59b8b184e10805a23ebc1e0702e0199147afbb1 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Wed, 18 Oct 2017 17:23:54 -0400 Subject: [PATCH 1/2] bugfix in collect(A) for zero-dimensional array --- base/array.jl | 2 +- test/abstractarray.jl | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/base/array.jl b/base/array.jl index 3b7096fdd254b..c4abdf899b377 100644 --- a/base/array.jl +++ b/base/array.jl @@ -620,7 +620,7 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown) return a end -_collect_indices(::Tuple{}, A) = copy!(Vector{eltype(A)}(), A) +_collect_indices(::Tuple{}, A) = copy!(Array{eltype(A)}(), A) _collect_indices(indsA::Tuple{Vararg{OneTo}}, A) = copy!(Array{eltype(A)}(length.(indsA)), A) function _collect_indices(indsA, A) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index a1d698d2ac933..6e852a90c5b22 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -872,3 +872,8 @@ end copy!(Array{T,n}(size(a)), a) @test isa(similar(Dict(:a=>1, :b=>2.0), Pair{Union{},Union{}}), Dict{Union{}, Union{}}) end + +@testset "zero-dimensional copy" begin + Z = Array{Int}(); Z[] = 17 + @test Z == collect(Z) == copy(Z) +end From 6f4987e12a860eadefc3e2b2ccdc29f718e5d796 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Tue, 17 Oct 2017 17:24:31 -0400 Subject: [PATCH 2/2] add Iterators.reverse and Iterators.Reverse type for reverse-order iteration --- NEWS.md | 3 ++ base/array.jl | 3 +- base/iterators.jl | 74 ++++++++++++++++++++++++++++++++++-- base/multidimensional.jl | 28 ++++++++++++++ base/strings/basic.jl | 8 ++++ base/strings/types.jl | 1 + doc/src/manual/interfaces.md | 19 +++++++++ doc/src/stdlib/iterators.md | 2 + test/iterators.jl | 18 +++++++++ 9 files changed, 152 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index b261a65620b2d..b35b254ad2e65 100644 --- a/NEWS.md +++ b/NEWS.md @@ -274,6 +274,9 @@ Library improvements For example, `x^-1` is now essentially a synonym for `inv(x)`, and works in a type-stable way even if `typeof(x) != typeof(inv(x))` ([#24240]). + * New `Iterators.reverse(itr)` for reverse-order iteration ([#24187]). Iterator + types `T` can implement `start` etc. for `Iterators.Reverse{T}` to support this. + * The functions `nextind` and `prevind` now accept `nchar` argument that indicates the number of characters to move ([#23805]). diff --git a/base/array.jl b/base/array.jl index c4abdf899b377..8a002399ec862 100644 --- a/base/array.jl +++ b/base/array.jl @@ -1450,7 +1450,8 @@ end """ reverse(v [, start=1 [, stop=length(v) ]] ) -Return a copy of `v` reversed from start to stop. +Return a copy of `v` reversed from start to stop. See also [`Iterators.reverse`](@ref) +for reverse-order iteration without making a copy. # Examples ```jldoctest diff --git a/base/iterators.jl b/base/iterators.jl index 6d006070c7717..1b537e3f6beae 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -5,10 +5,10 @@ Methods for working with Iterators. """ module Iterators -import Base: start, done, next, isempty, length, size, eltype, iteratorsize, iteratoreltype, indices, ndims, pairs +import Base: start, done, next, isempty, length, size, eltype, iteratorsize, iteratoreltype, indices, ndims, pairs, last, first using Base: tail, tuple_type_head, tuple_type_tail, tuple_type_cons, SizeUnknown, HasLength, HasShape, - IsInfinite, EltypeUnknown, HasEltype, OneTo, @propagate_inbounds + IsInfinite, EltypeUnknown, HasEltype, OneTo, @propagate_inbounds, Generator, AbstractRange export enumerate, zip, rest, countfrom, take, drop, cycle, repeated, product, flatten, partition @@ -30,6 +30,52 @@ and_iteratorsize(a, b) = SizeUnknown() and_iteratoreltype(iel::T, ::T) where {T} = iel and_iteratoreltype(a, b) = EltypeUnknown() +## Reverse-order iteration for arrays and other collections. Collections +## should implement start/next/done etcetera if possible/practical. +""" + Iterators.reverse(itr) + +Given an iterator `itr`, then `reverse(itr)` is an iterator over the +same collection but in the reverse order. + +This iterator is "lazy" in that it does not make a copy of the collection in +order to reverse it; see [`Base.reverse`](@ref) for an eager implementation. + +Not all iterator types `T` support reverse-order iteration. If `T` +doesn't, then iterating over `Iterators.reverse(itr::T)` will throw a [`MethodError`](@ref) +because of the missing [`start`](@ref), [`next`](@ref), and [`done`](@ref) +methods for `Iterators.Reverse{T}`. (To implement these methods, the original iterator +`itr::T` can be obtained from `r = Iterators.reverse(itr)` by `r.itr`.) +""" +reverse(itr) = Reverse(itr) + +struct Reverse{T} + itr::T +end +eltype(r::Reverse) = eltype(r.itr) +length(r::Reverse) = length(r.itr) +size(r::Reverse) = size(r.itr) +iteratorsize(r::Reverse) = iteratorsize(r.itr) +iteratoreltype(r::Reverse) = iteratoreltype(r.itr) +last(r::Reverse) = first(r.itr) # the first shall be last +first(r::Reverse) = last(r.itr) # and the last shall be first + +# reverse-order array iterators: assumes more-specialized Reverse for eachindex +@inline start(A::Reverse{<:AbstractArray}) = (itr = reverse(eachindex(A.itr)); (itr, start(itr))) +@propagate_inbounds next(A::Reverse{<:AbstractArray}, i) = ((idx, s) = next(i[1], i[2]); (A.itr[idx], (i[1], s))) +@propagate_inbounds done(A::Reverse{<:AbstractArray}, i) = done(i[1], i[2]) + +reverse(R::AbstractRange) = Base.reverse(R) # copying ranges is cheap +reverse(G::Generator) = Generator(G.f, reverse(G.iter)) +reverse(r::Reverse) = r.itr +reverse(x::Union{Number,Char}) = x +reverse(p::Pair) = Base.reverse(p) # copying pairs is cheap + +start(r::Reverse{<:Tuple}) = length(r.itr) +done(r::Reverse{<:Tuple}, i::Int) = i < 1 +next(r::Reverse{<:Tuple}, i::Int) = (r.itr[i], i-1) + + # enumerate struct Enumerate{I} @@ -75,6 +121,16 @@ eltype(::Type{Enumerate{I}}) where {I} = Tuple{Int, eltype(I)} iteratorsize(::Type{Enumerate{I}}) where {I} = iteratorsize(I) iteratoreltype(::Type{Enumerate{I}}) where {I} = iteratoreltype(I) +@inline function start(r::Reverse{<:Enumerate}) + ri = reverse(r.itr.itr) + return (length(ri), ri, start(ri)) +end +@inline function next(r::Reverse{<:Enumerate}, state) + n = next(state[2],state[3]) + (state[1],n[1]), (state[1]-1,state[2],n[2]) +end +@inline done(r::Reverse{<:Enumerate}, state) = state[1] < 1 + struct IndexValue{I,A<:AbstractArray} data::A itr::I @@ -147,6 +203,8 @@ eltype(::Type{IndexValue{I,A}}) where {I,A} = Pair{eltype(I), eltype(A)} iteratorsize(::Type{IndexValue{I}}) where {I} = iteratorsize(I) iteratoreltype(::Type{IndexValue{I}}) where {I} = iteratoreltype(I) +reverse(v::IndexValue) = IndexValue(v.data, reverse(v.itr)) + # zip abstract type AbstractZipIterator end @@ -246,6 +304,10 @@ end iteratorsize(::Type{Zip{I1,I2}}) where {I1,I2} = zip_iteratorsize(iteratorsize(I1),iteratorsize(I2)) iteratoreltype(::Type{Zip{I1,I2}}) where {I1,I2} = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2)) +reverse(z::Zip1) = Zip1(reverse(z.a)) +reverse(z::Zip2) = Zip2(reverse(z.a), reverse(z.b)) +reverse(z::Zip) = Zip(reverse(z.a), reverse(z.z)) + # filter struct Filter{F,I} @@ -313,6 +375,8 @@ eltype(::Type{Filter{F,I}}) where {F,I} = eltype(I) iteratoreltype(::Type{Filter{F,I}}) where {F,I} = iteratoreltype(I) iteratorsize(::Type{<:Filter}) = SizeUnknown() +reverse(f::Filter) = Filter(f.flt, reverse(f.itr)) + # Rest -- iterate starting at the given state struct Rest{I,S} @@ -346,7 +410,6 @@ rest_iteratorsize(a) = SizeUnknown() rest_iteratorsize(::IsInfinite) = IsInfinite() iteratorsize(::Type{Rest{I,S}}) where {I,S} = rest_iteratorsize(iteratorsize(I)) - # Count -- infinite counting struct Count{S<:Number} @@ -539,6 +602,7 @@ end done(it::Cycle, state) = state[2] +reverse(it::Cycle) = Cycle(reverse(it.xs)) # Repeated - repeat an object infinitely many times @@ -576,6 +640,7 @@ done(it::Repeated, state) = false iteratorsize(::Type{<:Repeated}) = IsInfinite() iteratoreltype(::Type{<:Repeated}) = HasEltype() +reverse(it::Union{Repeated,Take{<:Repeated}}) = it # Product -- cartesian product of iterators struct ProductIterator{T<:Tuple} @@ -706,6 +771,8 @@ function _prod_next(iterators, states, nvalues) end end +reverse(p::ProductIterator) = ProductIterator(map(reverse, p.iterators)) + # flatten an iterator of iterators struct Flatten{I} @@ -781,6 +848,7 @@ end return done(f.it, s) && done(inner, s2) end +reverse(f::Flatten) = Flatten(reverse(itr) for itr in reverse(f.it)) """ partition(collection, n) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index cc5f4349d573c..7559c91010f84 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -9,6 +9,7 @@ module IteratorsMD import Base: +, -, * import Base: simd_outer_range, simd_inner_length, simd_index using Base: IndexLinear, IndexCartesian, AbstractCartesianIndex, fill_to_length, tail + using Base.Iterators: Reverse export CartesianIndex, CartesianRange @@ -314,6 +315,33 @@ module IteratorsMD i, j = split(R.indices, V) CartesianRange(i), CartesianRange(j) end + + # reversed CartesianRange iteration + @inline function start(r::Reverse{<:CartesianRange}) + iterfirst, iterlast = last(r.itr), first(r.itr) + if any(map(<, iterfirst.I, iterlast.I)) + return iterlast-1 + end + iterfirst + end + @inline function next(r::Reverse{<:CartesianRange}, state) + state, CartesianIndex(dec(state.I, last(r.itr).I, first(r.itr).I)) + end + # decrement & carry + @inline dec(::Tuple{}, ::Tuple{}, ::Tuple{}) = () + @inline dec(state::Tuple{Int}, start::Tuple{Int}, stop::Tuple{Int}) = (state[1]-1,) + @inline function dec(state, start, stop) + if state[1] > stop[1] + return (state[1]-1,tail(state)...) + end + newtail = dec(tail(state), tail(start), tail(stop)) + (start[1], newtail...) + end + @inline done(r::Reverse{<:CartesianRange}, state) = state.I[end] < first(r.itr.indices[end]) + # 0-d cartesian ranges are special-cased to iterate once and only once + start(iter::Reverse{<:CartesianRange{0}}) = false + next(iter::Reverse{<:CartesianRange{0}}, state) = CartesianIndex(), true + done(iter::Reverse{<:CartesianRange{0}}, state) = state end # IteratorsMD diff --git a/base/strings/basic.jl b/base/strings/basic.jl index c04e56eb9a6ac..6dd8ffddc3566 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -638,3 +638,11 @@ function last(str::AbstractString, nchar::Integer) end str[prevind(str, e, nchar-1):e] end + +# reverse-order iteration for strings and indices thereof +start(r::Iterators.Reverse{<:AbstractString}) = endof(r.itr) +done(r::Iterators.Reverse{<:AbstractString}, i) = i < start(r.itr) +next(r::Iterators.Reverse{<:AbstractString}, i) = (r.itr[i], prevind(r.itr, i)) +start(r::Iterators.Reverse{<:EachStringIndex}) = endof(r.itr.s) +done(r::Iterators.Reverse{<:EachStringIndex}, i) = i < start(r.itr.s) +next(r::Iterators.Reverse{<:EachStringIndex}, i) = (i, prevind(r.itr.s, i)) diff --git a/base/strings/types.jl b/base/strings/types.jl index 925c95d6d45b1..d4fd4af39f053 100644 --- a/base/strings/types.jl +++ b/base/strings/types.jl @@ -133,6 +133,7 @@ main utility is for reversed-order string processing, especially for reversed regular-expression searches. See also [`reverseind`](@ref) to convert indices in `s` to indices in `reverse(s)` and vice-versa, and [`graphemes`](@ref) to operate on user-visible "characters" (graphemes) rather than codepoints. +See also [`Iterators.reverse`](@ref) for reverse-order iteration without making a copy. # Examples ```jldoctest diff --git a/doc/src/manual/interfaces.md b/doc/src/manual/interfaces.md index 4bee25b025040..7e19c255df667 100644 --- a/doc/src/manual/interfaces.md +++ b/doc/src/manual/interfaces.md @@ -136,6 +136,25 @@ define an informal interface that enable many fancier behaviors. In some cases, to additionally specialize those extra behaviors when they know a more efficient algorithm can be used in their specific case. +It is also often useful to allow iteration over a collection in *reverse order* +by iterating over [`Iterators.reverse(iterator)`](@ref). To actually support +reverse-order iteration, however, an iterator +type `T` needs to implement `start`, `next`, and `done` methods for `Iterators.Reverse{T}`. +(Given `r::Iterators.Reverse{T}`, the underling iterator of type `T` is `r.itr`.) +In our `Squares` example, we would implement `Iterators.Reverse{Squares}` methods: + +```jldoctest squaretype +julia> Base.start(rS::Iterators.Reverse{Squares}) = rS.itr.count + +julia> Base.next(::Iterators.Reverse{Squares}, state) = (state*state, state-1) + +julia> Base.done(::Iterators.Reverse{Squares}, state) = state < 1 + +julia> collect(Iterators.reverse(Squares(10)))' # transposed to save space +1×10 RowVector{Int64,Array{Int64,1}}: + 100 81 64 49 36 25 16 9 4 1 +``` + ## Indexing | Methods to implement | Brief description | diff --git a/doc/src/stdlib/iterators.md b/doc/src/stdlib/iterators.md index a2b1aae2b4bea..7fa782ef2669f 100644 --- a/doc/src/stdlib/iterators.md +++ b/doc/src/stdlib/iterators.md @@ -12,4 +12,6 @@ Base.Iterators.repeated Base.Iterators.product Base.Iterators.flatten Base.Iterators.partition +Base.Iterators.filter +Base.Iterators.reverse ``` diff --git a/test/iterators.jl b/test/iterators.jl index b53b4e8ca1400..9e09d85c8b823 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -429,3 +429,21 @@ end @test length(arr) == 0 @test eltype(arr) == Int end + +@testset "reverse iterators" begin + squash(A) = reshape(A, length(A)) + Z = Array{Int}(); Z[] = 17 # zero-dimensional test case + for itr in (2:10, "∀ϵ>0", 1:0, "", (2,3,5,7,11), [2,3,5,7,11], rand(5,6), Z, 3, true, 'x', 4=>5, + eachindex("∀ϵ>0"), view(Z), view(rand(5,6),2:4,2:6), (x^2 for x in 1:10), + Iterators.Filter(isodd, 1:10), flatten((1:10, 50:60)), enumerate("foo"), + pairs(50:60), zip(1:10,21:30,51:60), product(1:3, 10:12), repeated(3.14159, 5)) + @test squash(collect(Iterators.reverse(itr))) == reverse(squash(collect(itr))) + end + @test collect(take(Iterators.reverse(cycle(1:3)), 7)) == collect(take(cycle(3:-1:1), 7)) + let r = repeated(3.14159) + @test Iterators.reverse(r) === r + end + let t = (2,3,5,7,11) + @test Iterators.reverse(Iterators.reverse(t)) === t + end +end