From 154a4692dc20df264e8550c8bc4c0fb6ed3d7ca1 Mon Sep 17 00:00:00 2001 From: Sam O'Connor Date: Sun, 17 Jan 2016 10:07:46 +1100 Subject: [PATCH 1/7] Fix uv_fs_read() EOF handling. --- base/filesystem.jl | 11 +++++++++-- src/builtins.c | 8 ++++++++ src/jl_uv.c | 9 ++++++--- src/julia.h | 1 + src/sys.c | 9 +-------- test/file.jl | 33 +++++++++++++++++++++++++++++++++ 6 files changed, 58 insertions(+), 13 deletions(-) diff --git a/base/filesystem.jl b/base/filesystem.jl index a6f032ee95844..7f3c9fd36e35d 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -158,19 +158,26 @@ function read!(f::File, a::Vector{UInt8}, nel=length(a)) end ret = ccall(:jl_fs_read, Int32, (Int32, Ptr{Void}, Csize_t), f.handle, a, nel) + if ret < nel + throw(EOFError()) + end uv_error("read",ret) return a end nb_available(f::File) = filesize(f) - position(f) +eof(f::File) = nb_available(f) == 0 + function readbytes!(f::File, b::Array{UInt8}, nb=length(b)) nr = min(nb, nb_available(f)) if length(b) < nr resize!(b, nr) end - read!(f, b, nr) - return nr + ret = ccall(:jl_fs_read, Int32, (Int32, Ptr{Void}, Csize_t), + f.handle, b, nr) + uv_error("read",ret) + return ret end read(io::File) = read!(io, Array(UInt8, nb_available(io))) read(io::File, nb::Integer) = read!(io, Array(UInt8, min(nb, nb_available(io)))) diff --git a/src/builtins.c b/src/builtins.c index 50ed9b0a85114..b79a2ac1e2b1b 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -182,6 +182,14 @@ JL_DLLEXPORT void JL_NORETURN jl_bounds_error_ints(jl_value_t *v, size_t *idxs, jl_throw(jl_new_struct((jl_datatype_t*)jl_boundserror_type, v, t)); } +JL_DLLEXPORT void JL_NORETURN jl_eof_error(void) +{ + jl_datatype_t *eof_error = + (jl_datatype_t*)jl_get_global(jl_base_module, jl_symbol("EOFError")); + assert(eof_error != NULL); + jl_exceptionf(eof_error, ""); +} + JL_CALLABLE(jl_f_throw) { JL_NARGS(throw, 1, 1); diff --git a/src/jl_uv.c b/src/jl_uv.c index 344356d5a1645..e31d5da8d434a 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -339,9 +339,12 @@ JL_DLLEXPORT int jl_fs_read_byte(int handle) buf[0].len = 1; int ret = uv_fs_read(jl_io_loop, &req, handle, buf, 1, -1, NULL); uv_fs_req_cleanup(&req); - if (ret == -1) - return ret; - return (int)c; + switch (ret) { + case -1: return ret; + case 0: jl_eof_error(); + case 1: return (int)c; + default: assert(0); + } } JL_DLLEXPORT int jl_fs_close(int handle) diff --git a/src/julia.h b/src/julia.h index 2815e4ad52b43..de49880d2fd5b 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1250,6 +1250,7 @@ JL_DLLEXPORT void JL_NORETURN jl_bounds_error_tuple_int(jl_value_t **v, size_t nv, size_t i); JL_DLLEXPORT void JL_NORETURN jl_bounds_error_unboxed_int(void *v, jl_value_t *vt, size_t i); JL_DLLEXPORT void JL_NORETURN jl_bounds_error_ints(jl_value_t *v, size_t *idxs, size_t nidxs); +JL_DLLEXPORT void JL_NORETURN jl_eof_error(void); JL_DLLEXPORT jl_value_t *jl_exception_occurred(void); JL_DLLEXPORT void jl_exception_clear(void); diff --git a/src/sys.c b/src/sys.c index d426e4cbc9a73..270c32858edce 100644 --- a/src/sys.c +++ b/src/sys.c @@ -308,13 +308,6 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim) return (jl_value_t*)a; } -static void JL_NORETURN throw_eof_error(void) -{ - jl_datatype_t *eof_error = (jl_datatype_t*)jl_get_global(jl_base_module, jl_symbol("EOFError")); - assert(eof_error != NULL); - jl_exceptionf(eof_error, ""); -} - JL_DLLEXPORT uint64_t jl_ios_get_nbyte_int(ios_t *s, const size_t n) { assert(n <= 8); @@ -323,7 +316,7 @@ JL_DLLEXPORT uint64_t jl_ios_get_nbyte_int(ios_t *s, const size_t n) space = (size_t)(s->size - s->bpos); ret = ios_readprep(s, n); if (space == ret && ret < n) - throw_eof_error(); + jl_eof_error(); } while(ret < n); uint64_t x = 0; uint8_t *buf = (uint8_t*)&s->buf[s->bpos]; diff --git a/test/file.jl b/test/file.jl index 5771c5f80b54f..a28e0e888b46e 100644 --- a/test/file.jl +++ b/test/file.jl @@ -1115,3 +1115,36 @@ end @test copy(DevNull) === DevNull @test eof(DevNull) @test print(DevNull, "go to /dev/null") === nothing + +# Filesystem.File +tmpdir = mktempdir() do d + f = joinpath(d, "test.txt") + open(io->write(io, "123"), f, "w") + f1 = open(f) + f2 = Base.Filesystem.open(f, Base.Filesystem.JL_O_RDONLY) + @test read(f1, UInt8) == read(f2, UInt8) + @test read(f1, UInt8) == read(f2, UInt8) + @test read(f1, UInt8) == read(f2, UInt8) + @test_throws EOFError read(f1, UInt8) + @test_throws EOFError read(f2, UInt8) + close(f1) + close(f2) + + a = UInt8[0,0,0] + f1 = open(f) + f2 = Base.Filesystem.open(f, Base.Filesystem.JL_O_RDONLY) + @test read!(f1, a) == read!(f2, a) + @test_throws EOFError read!(f1, a) + @test_throws EOFError read!(f2, a) + close(f1) + close(f2) + + a = UInt8[0,0,0,0] + f1 = open(f) + f2 = Base.Filesystem.open(f, Base.Filesystem.JL_O_RDONLY) + @test_throws EOFError read!(f1, a) + @test_throws EOFError read!(f2, a) + close(f1) + close(f2) +end + From 33503109cbf523af6961dd19484a81e8137e606c Mon Sep 17 00:00:00 2001 From: Sam O'Connor Date: Sun, 17 Jan 2016 15:06:26 +1100 Subject: [PATCH 2/7] New test/read.jl Add eof(::File), seek(::File) etc --- base/filesystem.jl | 19 +++++ test/choosetests.jl | 2 +- test/read.jl | 177 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 test/read.jl diff --git a/base/filesystem.jl b/base/filesystem.jl index 7f3c9fd36e35d..5516bdb89dcad 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -40,6 +40,7 @@ export File, import Base: uvtype, uvhandle, eventloop, fd, position, stat, close, write, read, read!, isopen, show, + seek, seekend, skip, eof, check_open, _sizeof_uv_fs, uv_error, UVError include("path.jl") @@ -186,6 +187,24 @@ const SEEK_SET = Int32(0) const SEEK_CUR = Int32(1) const SEEK_END = Int32(2) +function seek(f::File, n::Integer) + ret = ccall(:jl_lseek, Int64, (Int32, Int64, Int32), f.handle, n, SEEK_SET) + systemerror("seek", ret == -1) + return f +end + +function seekend(f::File) + ret = ccall(:jl_lseek, Int64, (Int32, Int64, Int32), f.handle, 0, SEEK_END) + systemerror("seekend", ret == -1) + return f +end + +function skip(f::File, n::Integer) + ret = ccall(:jl_lseek, Int64, (Int32, Int64, Int32), f.handle, n, SEEK_CUR) + systemerror("skip", ret == -1) + return f +end + function position(f::File) check_open(f) ret = ccall(:jl_lseek, Int64, (Int32, Int64, Int32), f.handle, 0, SEEK_CUR) diff --git a/test/choosetests.jl b/test/choosetests.jl index b9346a96758ee..aa6fd49205707 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -23,7 +23,7 @@ function choosetests(choices = []) "bitarray", "copy", "math", "fastmath", "functional", "operators", "path", "ccall", "parse", "loading", "bigint", "sorting", "statistics", "spawn", "backtrace", - "priorityqueue", "file", "mmap", "version", "resolve", + "priorityqueue", "file", "read", "mmap", "version", "resolve", "pollfd", "mpfr", "broadcast", "complex", "socket", "floatapprox", "datafmt", "reflection", "regex", "float16", "combinatorics", "sysinfo", "rounding", "ranges", "mod2pi", diff --git a/test/read.jl b/test/read.jl new file mode 100644 index 0000000000000..8923d955ab625 --- /dev/null +++ b/test/read.jl @@ -0,0 +1,177 @@ +mktempdir() do dir + +tasks = [] + +# Create test file... +filename = joinpath(dir, "file.txt") +text = "C1,C2\n1,2\na,b\n" +open(io-> write(io, text), filename, "w") + +# List of IO producers... +l = Vector{Tuple{AbstractString,Function}}() + + +# File +io = ()->Base.Filesystem.open(filename, Base.Filesystem.JL_O_RDONLY) +s = io() +@test isa(s, IO) +@test isa(s, Base.Filesystem.File) +close(s) +push!(l, ("File", io)) + + +# IOStream +io = ()->open(filename) +s = io() +@test isa(s, IO) +@test isa(s, IOStream) +close(s) +push!(l, ("IOStream", io)) + + +# IOBuffer +io = ()->IOBuffer(text) +s = io() +@test isa(s, IO) +@test isa(s, IOBuffer) +close(s) +push!(l, ("IOBuffer", io)) + +@windows ? nothing : begin + +# PipeEndpoint +socketname = joinpath(dir, "socket") +io = ()-> begin + c = Base.Condition() + tsk = @async begin + con = listen(socketname) + Base.notify(c) + sock = accept(con) + write(sock,text) + close(con) + close(sock) + end + push!(tasks, tsk) + wait(c) + connect(socketname) +end +s = io() +@test isa(s, IO) +@test isa(s, Base.PipeEndpoint) +close(s) +for tsk in tasks + wait(tsk) +end +push!(l, ("PipeEndpoint", io)) + + +# Pipe +io = () -> open(`echo -n $text`)[1] +s = io() +@test isa(s, IO) +@test isa(s, Pipe) +close(s) +push!(l, ("Pipe", io)) + +end + +open_streams = [] + +for (name, f) in l + + io = ()->(s=f(); push!(open_streams, s); s) + + #println("$name readall...") + @test readall(io()) == text + @test readall(io()) == readall(filename) + + #println("$name read...") + @test readbytes(io()) == Vector{UInt8}(text) + @test readbytes(io()) == open(readbytes,filename) + @test read(io(), UInt8) == read(IOBuffer(text), UInt8) + @test read(io(), UInt8) == open(io->read(io, UInt8), filename) + @test read(io(), Int) == read(IOBuffer(text), Int) + @test read(io(), Int) == open(io->read(io,Int),filename) + s1 = io() + s2 = IOBuffer(text) + @test read(s1, UInt32, 2) == read(s2, UInt32, 2) + @test !eof(s1) + @test read(s1, UInt8, 5) == read(s2, UInt8, 5) + @test !eof(s1) + @test read(s1, UInt8, 1) == read(s2, UInt8, 1) + @test eof(s1) + @test_throws EOFError read(s1, UInt8) + @test eof(s1) + close(s1) + close(s2) + + #println("$name readuntil...") + @test readuntil(io(), '\n') == open(io->readuntil(io,'\n'),filename) + @test readuntil(io(), "\n") == open(io->readuntil(io,"\n"),filename) + @test readuntil(io(), ',') == open(io->readuntil(io,','),filename) + + #println("$name eof...") + n = length(text) - 1 + @test read!(io(), Vector{UInt8}(n)) == + read!(IOBuffer(text), Vector{UInt8}(n)) + @test (s = io(); read!(s, Vector{UInt8}(n)); !eof(s)) + n = length(text) + @test read!(io(), Vector{UInt8}(n)) == + read!(IOBuffer(text), Vector{UInt8}(n)) + @test (s = io(); read!(s, Vector{UInt8}(n)); eof(s)) + n = length(text) + 1 + @test_throws EOFError read!(io(), Vector{UInt8}(n)) + @test_throws EOFError read!(io(), Vector{UInt8}(n)) + + #println("$name read!...") + for n = 1:length(text) + @test read!(io(), Vector{UInt8}(n)) == + read!(IOBuffer(text), Vector{UInt8}(n)) + @test read!(io(), Vector{UInt8}(n)) == + open(io->read!(io, Vector{UInt8}(n)), filename) + end + @test_throws EOFError read!(io(), Vector{UInt8}(length(text)+1)) + + #println("$name readline...") + @test readline(io()) == readline(IOBuffer(text)) + @test readline(io()) == open(readline,filename) + + #println("$name readlines...") + @test readlines(io()) == readlines(IOBuffer(text)) + @test readlines(io()) == open(readlines,filename) + @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) + + #println("$name countlines...") + @test countlines(io()) == countlines(IOBuffer(text)) + + #println("$name readcsv...") + @test readcsv(io()) == readcsv(IOBuffer(text)) + + if !(typeof(io()) in [Base.PipeEndpoint, Pipe]) + + #println("$name position...") + @test (s = io(); read!(s, Vector{UInt8}(4)); position(s)) == 4 + + #println("$name seek...") + for n = 0:length(text)-1 + @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) + end + #println("$name skip...") + for n = 0:length(text)-1 + @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) + @test readlines(skip(io(), n)) == readlines(skip(IOBuffer(text), n)) + end + #println("$name seekend...") + @test readall(seekend(io())) == "" + end +end + +for s in open_streams + try close(s) end +end + +for tsk in tasks + wait(tsk) +end + +end From 1a3b7ec4c454ef837641889ac85a66f95802d064 Mon Sep 17 00:00:00 2001 From: Sam O'Connor Date: Tue, 19 Jan 2016 11:33:18 +1100 Subject: [PATCH 3/7] Fix read!(::LibuvStream) and readbytes!(::LibuvStream) to be consisitent with other ::IOs See test case --- base/stream.jl | 23 +++---- test/read.jl | 176 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 134 insertions(+), 65 deletions(-) diff --git a/base/stream.jl b/base/stream.jl index ee59da6c00faa..d49da0e39c936 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -894,12 +894,13 @@ function stop_reading(stream::LibuvStream) end end -function readbytes!(s::LibuvStream, b::AbstractArray{UInt8}, nb=length(b)) - wait_readnb(s, nb) - nr = nb_available(s) - resize!(b, nr) # shrink to just contain input data if was resized - read!(s.buffer, b) - return nr +function read!(s::LibuvStream, b::Vector{UInt8}) + nb = length(b) + r = readbytes!(s, b, nb) + if r < nb + throw(EOFError()) + end + return b end function read(stream::LibuvStream) @@ -907,19 +908,18 @@ function read(stream::LibuvStream) return takebuf_array(stream.buffer) end -function read!(s::LibuvStream, a::Array{UInt8, 1}) - nb = length(a) +function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb = length(a)) sbuf = s.buffer @assert sbuf.seekable == false @assert sbuf.maxsize >= nb if nb_available(sbuf) >= nb - return read!(sbuf, a) + return readbytes!(sbuf, a, nb) end if nb <= SZ_UNBUFFERED_IO # Under this limit we are OK with copying the array from the stream's buffer wait_readnb(s, nb) - read!(sbuf, a) + r = readbytes!(sbuf, a, nb) else try stop_reading(s) # Just playing it safe, since we are going to switch buffers. @@ -928,6 +928,7 @@ function read!(s::LibuvStream, a::Array{UInt8, 1}) s.buffer = newbuf write(newbuf, sbuf) wait_readnb(s, nb) + r = nb_available(newbuf) finally s.buffer = sbuf if !isempty(s.readnotify.waitq) @@ -935,7 +936,7 @@ function read!(s::LibuvStream, a::Array{UInt8, 1}) end end end - return a + return r end function read(this::LibuvStream, ::Type{UInt8}) diff --git a/test/read.jl b/test/read.jl index 8923d955ab625..4df6be00f5c86 100644 --- a/test/read.jl +++ b/test/read.jl @@ -5,15 +5,17 @@ tasks = [] # Create test file... filename = joinpath(dir, "file.txt") text = "C1,C2\n1,2\na,b\n" -open(io-> write(io, text), filename, "w") # List of IO producers... l = Vector{Tuple{AbstractString,Function}}() # File -io = ()->Base.Filesystem.open(filename, Base.Filesystem.JL_O_RDONLY) -s = io() +io = (text) -> begin + open(io-> write(io, text), filename, "w") + Base.Filesystem.open(filename, Base.Filesystem.JL_O_RDONLY) +end +s = io(text) @test isa(s, IO) @test isa(s, Base.Filesystem.File) close(s) @@ -21,8 +23,11 @@ push!(l, ("File", io)) # IOStream -io = ()->open(filename) -s = io() +io = (text) -> begin + open(io-> write(io, text), filename, "w") + open(filename) +end +s = io(text) @test isa(s, IO) @test isa(s, IOStream) close(s) @@ -30,44 +35,69 @@ push!(l, ("IOStream", io)) # IOBuffer -io = ()->IOBuffer(text) -s = io() +io = (text)->IOBuffer(text) +s = io(text) @test isa(s, IO) @test isa(s, IOBuffer) close(s) push!(l, ("IOBuffer", io)) + +# TCPSocket + +# PR#14627 +Base.connect!(sock::TCPSocket, addr::Base.InetAddr) = Base.connect!(sock, addr.host, addr.port) + +addr = Base.InetAddr(ip"127.0.0.1", 4444) +io = (text) -> begin + c = Condition() + tsk = @async begin + srv = listen(addr) + notify(c) + sock = accept(srv) + write(sock,text) + close(sock) + close(srv) + end + push!(tasks, tsk) + wait(c) + connect(addr) +end +s = io(text) +@test isa(s, IO) +@test isa(s, TCPSocket) +close(s) +push!(l, ("TCPSocket", io)) + + @windows ? nothing : begin # PipeEndpoint socketname = joinpath(dir, "socket") -io = ()-> begin - c = Base.Condition() +io = (text)-> begin + c = Condition() tsk = @async begin con = listen(socketname) - Base.notify(c) + notify(c) sock = accept(con) - write(sock,text) - close(con) + try write(sock,text) end close(sock) + close(con) end push!(tasks, tsk) wait(c) connect(socketname) end -s = io() +s = io(text) @test isa(s, IO) @test isa(s, Base.PipeEndpoint) close(s) -for tsk in tasks - wait(tsk) -end push!(l, ("PipeEndpoint", io)) # Pipe -io = () -> open(`echo -n $text`)[1] -s = io() +io = (text) -> open(`echo -n $text`)[1] +s = io(text) @test isa(s, IO) @test isa(s, Pipe) close(s) @@ -76,16 +106,26 @@ push!(l, ("Pipe", io)) end open_streams = [] +function cleanup() + for s in open_streams + try close(s) end + end + for tsk in tasks + wait(tsk) + end +end + +verbose = false for (name, f) in l - io = ()->(s=f(); push!(open_streams, s); s) + io = ()->(s=f(text); push!(open_streams, s); s) - #println("$name readall...") + verbose && println("$name readall...") @test readall(io()) == text @test readall(io()) == readall(filename) - #println("$name read...") + verbose && println("$name read...") @test readbytes(io()) == Vector{UInt8}(text) @test readbytes(io()) == open(readbytes,filename) @test read(io(), UInt8) == read(IOBuffer(text), UInt8) @@ -105,12 +145,12 @@ for (name, f) in l close(s1) close(s2) - #println("$name readuntil...") + verbose && println("$name readuntil...") @test readuntil(io(), '\n') == open(io->readuntil(io,'\n'),filename) @test readuntil(io(), "\n") == open(io->readuntil(io,"\n"),filename) @test readuntil(io(), ',') == open(io->readuntil(io,','),filename) - #println("$name eof...") + verbose && println("$name eof...") n = length(text) - 1 @test read!(io(), Vector{UInt8}(n)) == read!(IOBuffer(text), Vector{UInt8}(n)) @@ -123,55 +163,83 @@ for (name, f) in l @test_throws EOFError read!(io(), Vector{UInt8}(n)) @test_throws EOFError read!(io(), Vector{UInt8}(n)) - #println("$name read!...") - for n = 1:length(text) - @test read!(io(), Vector{UInt8}(n)) == - read!(IOBuffer(text), Vector{UInt8}(n)) - @test read!(io(), Vector{UInt8}(n)) == - open(io->read!(io, Vector{UInt8}(n)), filename) - end - @test_throws EOFError read!(io(), Vector{UInt8}(length(text)+1)) + old_text = text + + for text in [ + old_text, + UTF8String(['A' + i % 52 for i in 1:(div(Base.SZ_UNBUFFERED_IO,2))]), + UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO -1)]), + UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO )]), + UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO +1)]), + UTF8String(['A' + i % 52 for i in 1:(7 + Base.SZ_UNBUFFERED_IO *3)]) + ] + + verbose && println("$name readbytes!...") + l = length(text) + for n = [1, 2, l-2, l-1, l, l+1, l+2] + a1 = Vector{UInt8}(n); + a2 = Vector{UInt8}(n) + s1 = io() + s2 = IOBuffer(text) + n1 = readbytes!(s1, a1) + n2 = readbytes!(s2, a2) + @test n1 == n2 + @test length(a1) == length(a2) + @test a1[1:n1] == a2[1:n2] + @test n <= length(text) || eof(s1) + @test n <= length(text) || eof(s2) + cleanup() + end - #println("$name readline...") - @test readline(io()) == readline(IOBuffer(text)) - @test readline(io()) == open(readline,filename) + verbose && println("$name read!...") + l = length(text) + for n = [1, 2, l-2, l-1, l] + @test read!(io(), Vector{UInt8}(n)) == + read!(IOBuffer(text), Vector{UInt8}(n)) + cleanup() + end + @test_throws EOFError read!(io(), Vector{UInt8}(length(text)+1)) - #println("$name readlines...") - @test readlines(io()) == readlines(IOBuffer(text)) - @test readlines(io()) == open(readlines,filename) - @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) + cleanup() - #println("$name countlines...") - @test countlines(io()) == countlines(IOBuffer(text)) + verbose && println("$name readline...") + @test readline(io()) == readline(IOBuffer(text)) - #println("$name readcsv...") - @test readcsv(io()) == readcsv(IOBuffer(text)) + verbose && println("$name readlines...") + @test readlines(io()) == readlines(IOBuffer(text)) + @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) - if !(typeof(io()) in [Base.PipeEndpoint, Pipe]) + verbose && println("$name countlines...") + @test countlines(io()) == countlines(IOBuffer(text)) - #println("$name position...") + verbose && println("$name readcsv...") + @test readcsv(io()) == readcsv(IOBuffer(text)) + end + + text = old_text + + + if !(typeof(io()) in [Base.PipeEndpoint, Pipe, TCPSocket]) + + verbose && println("$name position...") @test (s = io(); read!(s, Vector{UInt8}(4)); position(s)) == 4 - #println("$name seek...") + verbose && println("$name seek...") for n = 0:length(text)-1 @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) + cleanup() end - #println("$name skip...") + verbose && println("$name skip...") for n = 0:length(text)-1 @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) @test readlines(skip(io(), n)) == readlines(skip(IOBuffer(text), n)) + cleanup() end - #println("$name seekend...") + verbose && println("$name seekend...") @test readall(seekend(io())) == "" end -end - -for s in open_streams - try close(s) end -end -for tsk in tasks - wait(tsk) + cleanup() end end From f290e2e5b92e7c54cdc8d0a015ad182f02f8219c Mon Sep 17 00:00:00 2001 From: Sam O'Connor Date: Wed, 20 Jan 2016 08:50:50 +1100 Subject: [PATCH 4/7] more tests for readall and readbytes --- test/read.jl | 113 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 45 deletions(-) diff --git a/test/read.jl b/test/read.jl index 4df6be00f5c86..c26158d92f115 100644 --- a/test/read.jl +++ b/test/read.jl @@ -43,25 +43,32 @@ close(s) push!(l, ("IOBuffer", io)) -# TCPSocket +function run_test_server(srv, text) + push!(tasks, @async begin + try + sock = accept(srv) + try + write(sock,text) + catch e + if typeof(e) != Base.UVError + rethrow(e) + end + finally + close(sock) + end + finally + close(srv) + end + end) + yield() +end -# PR#14627 -Base.connect!(sock::TCPSocket, addr::Base.InetAddr) = Base.connect!(sock, addr.host, addr.port) -addr = Base.InetAddr(ip"127.0.0.1", 4444) +# TCPSocket io = (text) -> begin - c = Condition() - tsk = @async begin - srv = listen(addr) - notify(c) - sock = accept(srv) - write(sock,text) - close(sock) - close(srv) - end - push!(tasks, tsk) - wait(c) - connect(addr) + port, srv = listenany(rand(2000:4000)) + run_test_server(srv, text) + connect(port) end s = io(text) @test isa(s, IO) @@ -70,22 +77,13 @@ close(s) push!(l, ("TCPSocket", io)) -@windows ? nothing : begin - # PipeEndpoint -socketname = joinpath(dir, "socket") -io = (text)-> begin - c = Condition() - tsk = @async begin - con = listen(socketname) - notify(c) - sock = accept(con) - try write(sock,text) end - close(sock) - close(con) - end - push!(tasks, tsk) - wait(c) +io = (text) -> begin + a = "\\\\.\\pipe\\uv-test-$(randstring(6))" + b = joinpath(dir, "socket-$(randstring(6))") + socketname = @windows ? a : b + srv = listen(socketname) + run_test_server(srv, text) connect(socketname) end s = io(text) @@ -95,8 +93,18 @@ close(s) push!(l, ("PipeEndpoint", io)) +@windows ? nothing : begin + +# See "could not spawn `type 'C:\Users\appveyor\AppData\Local\Temp\1\jul3516.tmp\file.txt'`" +#https://ci.appveyor.com/project/StefanKarpinski/julia/build/1.0.12733/job/hpwjs4hmf03vs5ag#L1244 + # Pipe -io = (text) -> open(`echo -n $text`)[1] +io = (text) -> begin + open(io->write(io, text), filename, "w") + open(`$(@windows ? "type" : "cat") $filename`)[1] +# Was open(`echo -n $text`)[1] +# See https://github.com/JuliaLang/julia/issues/14747 +end s = io(text) @test isa(s, IO) @test isa(s, Pipe) @@ -105,33 +113,33 @@ push!(l, ("Pipe", io)) end + open_streams = [] function cleanup() for s in open_streams try close(s) end end + empty!(open_streams) for tsk in tasks wait(tsk) end + empty!(tasks) end verbose = false +cleanup() + for (name, f) in l io = ()->(s=f(text); push!(open_streams, s); s) - verbose && println("$name readall...") - @test readall(io()) == text - @test readall(io()) == readall(filename) - verbose && println("$name read...") - @test readbytes(io()) == Vector{UInt8}(text) - @test readbytes(io()) == open(readbytes,filename) @test read(io(), UInt8) == read(IOBuffer(text), UInt8) @test read(io(), UInt8) == open(io->read(io, UInt8), filename) @test read(io(), Int) == read(IOBuffer(text), Int) @test read(io(), Int) == open(io->read(io,Int),filename) + cleanup() s1 = io() s2 = IOBuffer(text) @test read(s1, UInt32, 2) == read(s2, UInt32, 2) @@ -145,11 +153,6 @@ for (name, f) in l close(s1) close(s2) - verbose && println("$name readuntil...") - @test readuntil(io(), '\n') == open(io->readuntil(io,'\n'),filename) - @test readuntil(io(), "\n") == open(io->readuntil(io,"\n"),filename) - @test readuntil(io(), ',') == open(io->readuntil(io,','),filename) - verbose && println("$name eof...") n = length(text) - 1 @test read!(io(), Vector{UInt8}(n)) == @@ -170,10 +173,17 @@ for (name, f) in l UTF8String(['A' + i % 52 for i in 1:(div(Base.SZ_UNBUFFERED_IO,2))]), UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO -1)]), UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO )]), - UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO +1)]), - UTF8String(['A' + i % 52 for i in 1:(7 + Base.SZ_UNBUFFERED_IO *3)]) + UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO +1)]) ] + verbose && println("$name readall...") + @test readall(io()) == text + cleanup() + + verbose && println("$name readbytes...") + @test readbytes(io()) == Vector{UInt8}(text) + cleanup() + verbose && println("$name readbytes!...") l = length(text) for n = [1, 2, l-2, l-1, l, l+1, l+2] @@ -202,18 +212,31 @@ for (name, f) in l cleanup() + verbose && println("$name readuntil...") + @test readuntil(io(), '\n') == readuntil(IOBuffer(text),'\n') + cleanup() + @test readuntil(io(), "\n") == readuntil(IOBuffer(text),"\n") + cleanup() + @test readuntil(io(), ',') == readuntil(IOBuffer(text),',') + cleanup() + verbose && println("$name readline...") @test readline(io()) == readline(IOBuffer(text)) + cleanup() verbose && println("$name readlines...") @test readlines(io()) == readlines(IOBuffer(text)) + cleanup() @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) + cleanup() verbose && println("$name countlines...") @test countlines(io()) == countlines(IOBuffer(text)) + cleanup() verbose && println("$name readcsv...") @test readcsv(io()) == readcsv(IOBuffer(text)) + cleanup() end text = old_text From 0ce7a52eb399c02e735c55451d9b21668a05cd22 Mon Sep 17 00:00:00 2001 From: Sam O'Connor Date: Thu, 21 Jan 2016 11:25:43 +1100 Subject: [PATCH 5/7] update per #14660, add tests for filename-as-1st-arg read methods --- base/io.jl | 5 +++- test/read.jl | 83 ++++++++++++++++++++++++++++++++++------------------ 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/base/io.jl b/base/io.jl index eaf6f472e8f0e..53123506d0d2b 100644 --- a/base/io.jl +++ b/base/io.jl @@ -348,7 +348,10 @@ type EachLine EachLine(stream, ondone) = new(stream, ondone) end eachline(stream::IO) = EachLine(stream) -eachline(filename::AbstractString) = EachLine(open(filename), close) +function eachline(filename::AbstractString) + s = open(filename) + EachLine(s, ()->close(s)) +end start(itr::EachLine) = nothing function done(itr::EachLine, nada) diff --git a/test/read.jl b/test/read.jl index c26158d92f115..3cc46f8bdc87c 100644 --- a/test/read.jl +++ b/test/read.jl @@ -12,7 +12,7 @@ l = Vector{Tuple{AbstractString,Function}}() # File io = (text) -> begin - open(io-> write(io, text), filename, "w") + write(filename, text) Base.Filesystem.open(filename, Base.Filesystem.JL_O_RDONLY) end s = io(text) @@ -24,7 +24,7 @@ push!(l, ("File", io)) # IOStream io = (text) -> begin - open(io-> write(io, text), filename, "w") + write(filename, text) open(filename) end s = io(text) @@ -100,7 +100,7 @@ push!(l, ("PipeEndpoint", io)) # Pipe io = (text) -> begin - open(io->write(io, text), filename, "w") + write(filename, text) open(`$(@windows ? "type" : "cat") $filename`)[1] # Was open(`echo -n $text`)[1] # See https://github.com/JuliaLang/julia/issues/14747 @@ -128,18 +128,20 @@ end verbose = false -cleanup() + cleanup() for (name, f) in l io = ()->(s=f(text); push!(open_streams, s); s) + write(filename, text) + verbose && println("$name read...") @test read(io(), UInt8) == read(IOBuffer(text), UInt8) - @test read(io(), UInt8) == open(io->read(io, UInt8), filename) + @test read(io(), UInt8) == read(filename, UInt8) @test read(io(), Int) == read(IOBuffer(text), Int) - @test read(io(), Int) == open(io->read(io,Int),filename) - cleanup() + @test read(io(), Int) == read(filename,Int) + cleanup() s1 = io() s2 = IOBuffer(text) @test read(s1, UInt32, 2) == read(s2, UInt32, 2) @@ -176,13 +178,19 @@ for (name, f) in l UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO +1)]) ] - verbose && println("$name readall...") - @test readall(io()) == text - cleanup() + write(filename, text) + + verbose && println("$name readstring...") + @test readstring(io()) == text + cleanup() + @test readstring(io()) == readstring(filename) + cleanup() - verbose && println("$name readbytes...") - @test readbytes(io()) == Vector{UInt8}(text) - cleanup() + verbose && println("$name read...") + @test read(io()) == Vector{UInt8}(text) + cleanup() + @test read(io()) == read(filename) + cleanup() verbose && println("$name readbytes!...") l = length(text) @@ -198,7 +206,7 @@ for (name, f) in l @test a1[1:n1] == a2[1:n2] @test n <= length(text) || eof(s1) @test n <= length(text) || eof(s2) - cleanup() + cleanup() end verbose && println("$name read!...") @@ -206,41 +214,58 @@ for (name, f) in l for n = [1, 2, l-2, l-1, l] @test read!(io(), Vector{UInt8}(n)) == read!(IOBuffer(text), Vector{UInt8}(n)) - cleanup() + cleanup() + @test read!(io(), Vector{UInt8}(n)) == + read!(filename, Vector{UInt8}(n)) + cleanup() end @test_throws EOFError read!(io(), Vector{UInt8}(length(text)+1)) - cleanup() + cleanup() verbose && println("$name readuntil...") @test readuntil(io(), '\n') == readuntil(IOBuffer(text),'\n') - cleanup() + cleanup() + @test readuntil(io(), '\n') == readuntil(filename,'\n') + cleanup() @test readuntil(io(), "\n") == readuntil(IOBuffer(text),"\n") - cleanup() + cleanup() + @test readuntil(io(), "\n") == readuntil(filename,"\n") + cleanup() @test readuntil(io(), ',') == readuntil(IOBuffer(text),',') - cleanup() + cleanup() + @test readuntil(io(), ',') == readuntil(filename,',') + cleanup() verbose && println("$name readline...") @test readline(io()) == readline(IOBuffer(text)) - cleanup() + cleanup() + @test readline(io()) == readline(filename) + cleanup() verbose && println("$name readlines...") @test readlines(io()) == readlines(IOBuffer(text)) - cleanup() + cleanup() + @test readlines(io()) == readlines(filename) + cleanup() @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) - cleanup() + cleanup() + @test collect(eachline(io())) == collect(eachline(filename)) + cleanup() verbose && println("$name countlines...") @test countlines(io()) == countlines(IOBuffer(text)) - cleanup() + cleanup() verbose && println("$name readcsv...") @test readcsv(io()) == readcsv(IOBuffer(text)) - cleanup() + cleanup() + @test readcsv(io()) == readcsv(filename) + cleanup() end text = old_text - + write(filename, text) if !(typeof(io()) in [Base.PipeEndpoint, Pipe, TCPSocket]) @@ -250,19 +275,19 @@ for (name, f) in l verbose && println("$name seek...") for n = 0:length(text)-1 @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) - cleanup() + cleanup() end verbose && println("$name skip...") for n = 0:length(text)-1 @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) @test readlines(skip(io(), n)) == readlines(skip(IOBuffer(text), n)) - cleanup() + cleanup() end verbose && println("$name seekend...") - @test readall(seekend(io())) == "" + @test readstring(seekend(io())) == "" end - cleanup() + cleanup() end end From ab32dfb4304684df8b08e21c4a87784b05e55369 Mon Sep 17 00:00:00 2001 From: Sam O'Connor Date: Thu, 21 Jan 2016 12:45:09 +1100 Subject: [PATCH 6/7] write(::IO, ::IO) tests --- base/filesystem.jl | 3 ++- base/iobuffer.jl | 1 + base/iostream.jl | 2 ++ test/read.jl | 16 ++++++++++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/base/filesystem.jl b/base/filesystem.jl index 5516bdb89dcad..29a534b6d51dd 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -39,7 +39,7 @@ export File, S_IROTH, S_IWOTH, S_IXOTH, S_IRWXO import Base: uvtype, uvhandle, eventloop, fd, position, stat, close, - write, read, read!, isopen, show, + write, read, readavailable, read!, isopen, show, seek, seekend, skip, eof, check_open, _sizeof_uv_fs, uv_error, UVError @@ -181,6 +181,7 @@ function readbytes!(f::File, b::Array{UInt8}, nb=length(b)) return ret end read(io::File) = read!(io, Array(UInt8, nb_available(io))) +readavailable(io::File) = read(io) read(io::File, nb::Integer) = read!(io, Array(UInt8, min(nb, nb_available(io)))) const SEEK_SET = Int32(0) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index 1434e2fab5d76..2d31a9ed23098 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -337,6 +337,7 @@ function readbytes!(io::AbstractIOBuffer, b::Array{UInt8}, nb=length(b)) return nr end read(io::AbstractIOBuffer) = read!(io, Array(UInt8, nb_available(io))) +readavailable(io::AbstractIOBuffer) = read(io) read(io::AbstractIOBuffer, nb::Integer) = read!(io, Array(UInt8, min(nb, nb_available(io)))) function search(buf::IOBuffer, delim::UInt8) diff --git a/base/iostream.jl b/base/iostream.jl index 3e3aa61ca003a..0168db9fc5d01 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -158,6 +158,8 @@ end # num bytes available without blocking nb_available(s::IOStream) = ccall(:jl_nb_available, Int32, (Ptr{Void},), s.ios) +readavailable(s::IOStream) = read!(s, Vector{UInt8}(nb_available(s))) + function read(s::IOStream, ::Type{UInt8}) b = ccall(:ios_getc, Cint, (Ptr{Void},), s.ios) if b == -1 diff --git a/test/read.jl b/test/read.jl index 3cc46f8bdc87c..a7a274c139aa9 100644 --- a/test/read.jl +++ b/test/read.jl @@ -287,6 +287,22 @@ for (name, f) in l @test readstring(seekend(io())) == "" end + + verbose && println("$name write(::IOStream, ...)") + to = open("$filename.to", "w") + write(to, io()) + close(to) + @test readstring("$filename.to") == text + cleanup() + verbose && println("$name write(filename, ...)") + write("$filename.to", io()) + @test readstring("$filename.to") == text + cleanup() + + verbose && println("$name write(::IOBuffer, ...)") + to = IOBuffer(Vector{UInt8}(copy(text)), false, true) + write(to, io()) + @test takebuf_string(to) == text cleanup() end From 98744deedebd5aa6ffdc4cacd72f3b43377c8350 Mon Sep 17 00:00:00 2001 From: Sam O'Connor Date: Thu, 21 Jan 2016 13:57:08 +1100 Subject: [PATCH 7/7] Disable test of read*() from open(::Command). https://github.com/JuliaLang/julia/issues/14747 clean up (hopefully) redundant celanup() calls in test/read.jl (left over from debugging hanging CI) --- test/read.jl | 62 +++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/test/read.jl b/test/read.jl index a7a274c139aa9..8af6ab9e0e1c8 100644 --- a/test/read.jl +++ b/test/read.jl @@ -93,8 +93,12 @@ close(s) push!(l, ("PipeEndpoint", io)) +#FIXME See https://github.com/JuliaLang/julia/issues/14747 +# Reading from open(::Command) seems to deadlock on Linux/Travis +#= @windows ? nothing : begin +# Windows type command not working? # See "could not spawn `type 'C:\Users\appveyor\AppData\Local\Temp\1\jul3516.tmp\file.txt'`" #https://ci.appveyor.com/project/StefanKarpinski/julia/build/1.0.12733/job/hpwjs4hmf03vs5ag#L1244 @@ -112,6 +116,7 @@ close(s) push!(l, ("Pipe", io)) end +=# open_streams = [] @@ -126,9 +131,9 @@ function cleanup() empty!(tasks) end + verbose = false - cleanup() for (name, f) in l @@ -141,7 +146,6 @@ for (name, f) in l @test read(io(), UInt8) == read(filename, UInt8) @test read(io(), Int) == read(IOBuffer(text), Int) @test read(io(), Int) == read(filename,Int) - cleanup() s1 = io() s2 = IOBuffer(text) @test read(s1, UInt32, 2) == read(s2, UInt32, 2) @@ -169,6 +173,7 @@ for (name, f) in l @test_throws EOFError read!(io(), Vector{UInt8}(n)) old_text = text + cleanup() for text in [ old_text, @@ -182,15 +187,17 @@ for (name, f) in l verbose && println("$name readstring...") @test readstring(io()) == text - cleanup() + @test readstring(io()) == readstring(filename) - cleanup() + verbose && println("$name read...") @test read(io()) == Vector{UInt8}(text) - cleanup() + @test read(io()) == read(filename) - cleanup() + + cleanup() + verbose && println("$name readbytes!...") l = length(text) @@ -206,7 +213,8 @@ for (name, f) in l @test a1[1:n1] == a2[1:n2] @test n <= length(text) || eof(s1) @test n <= length(text) || eof(s2) - cleanup() + + cleanup() end verbose && println("$name read!...") @@ -214,54 +222,44 @@ for (name, f) in l for n = [1, 2, l-2, l-1, l] @test read!(io(), Vector{UInt8}(n)) == read!(IOBuffer(text), Vector{UInt8}(n)) - cleanup() @test read!(io(), Vector{UInt8}(n)) == read!(filename, Vector{UInt8}(n)) - cleanup() + + cleanup() end @test_throws EOFError read!(io(), Vector{UInt8}(length(text)+1)) - cleanup() verbose && println("$name readuntil...") @test readuntil(io(), '\n') == readuntil(IOBuffer(text),'\n') - cleanup() @test readuntil(io(), '\n') == readuntil(filename,'\n') - cleanup() @test readuntil(io(), "\n") == readuntil(IOBuffer(text),"\n") - cleanup() @test readuntil(io(), "\n") == readuntil(filename,"\n") - cleanup() - @test readuntil(io(), ',') == readuntil(IOBuffer(text),',') - cleanup() - @test readuntil(io(), ',') == readuntil(filename,',') - cleanup() + @test readuntil(io(), ',') == readuntil(IOBuffer(text),',') + @test readuntil(io(), ',') == readuntil(filename,',') + + cleanup() verbose && println("$name readline...") @test readline(io()) == readline(IOBuffer(text)) - cleanup() @test readline(io()) == readline(filename) - cleanup() verbose && println("$name readlines...") @test readlines(io()) == readlines(IOBuffer(text)) - cleanup() @test readlines(io()) == readlines(filename) - cleanup() @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) - cleanup() @test collect(eachline(io())) == collect(eachline(filename)) - cleanup() + + cleanup() verbose && println("$name countlines...") @test countlines(io()) == countlines(IOBuffer(text)) - cleanup() verbose && println("$name readcsv...") @test readcsv(io()) == readcsv(IOBuffer(text)) - cleanup() @test readcsv(io()) == readcsv(filename) - cleanup() + + cleanup() end text = old_text @@ -275,13 +273,13 @@ for (name, f) in l verbose && println("$name seek...") for n = 0:length(text)-1 @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) - cleanup() + cleanup() end verbose && println("$name skip...") for n = 0:length(text)-1 @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) @test readlines(skip(io(), n)) == readlines(skip(IOBuffer(text), n)) - cleanup() + cleanup() end verbose && println("$name seekend...") @test readstring(seekend(io())) == "" @@ -293,17 +291,17 @@ for (name, f) in l write(to, io()) close(to) @test readstring("$filename.to") == text - cleanup() + verbose && println("$name write(filename, ...)") write("$filename.to", io()) @test readstring("$filename.to") == text - cleanup() verbose && println("$name write(::IOBuffer, ...)") to = IOBuffer(Vector{UInt8}(copy(text)), false, true) write(to, io()) @test takebuf_string(to) == text - cleanup() + + cleanup() end end