diff --git a/base/filesystem.jl b/base/filesystem.jl index a6f032ee95844..29a534b6d51dd 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -39,7 +39,8 @@ export File, S_IROTH, S_IWOTH, S_IXOTH, S_IRWXO import Base: uvtype, uvhandle, eventloop, fd, position, stat, close, - write, read, read!, isopen, show, + write, read, readavailable, read!, isopen, show, + seek, seekend, skip, eof, check_open, _sizeof_uv_fs, uv_error, UVError include("path.jl") @@ -158,27 +159,53 @@ function read!(f::File, a::Vector{UInt8}, nel=length(a)) end ret = ccall(:jl_fs_read, Int32, (Int32, Ptr{Void}, Csize_t), f.handle, a, nel) + if ret < nel + throw(EOFError()) + end uv_error("read",ret) return a end nb_available(f::File) = filesize(f) - position(f) +eof(f::File) = nb_available(f) == 0 + function readbytes!(f::File, b::Array{UInt8}, nb=length(b)) nr = min(nb, nb_available(f)) if length(b) < nr resize!(b, nr) end - read!(f, b, nr) - return nr + ret = ccall(:jl_fs_read, Int32, (Int32, Ptr{Void}, Csize_t), + f.handle, b, nr) + uv_error("read",ret) + return ret end read(io::File) = read!(io, Array(UInt8, nb_available(io))) +readavailable(io::File) = read(io) read(io::File, nb::Integer) = read!(io, Array(UInt8, min(nb, nb_available(io)))) const SEEK_SET = Int32(0) const SEEK_CUR = Int32(1) const SEEK_END = Int32(2) +function seek(f::File, n::Integer) + ret = ccall(:jl_lseek, Int64, (Int32, Int64, Int32), f.handle, n, SEEK_SET) + systemerror("seek", ret == -1) + return f +end + +function seekend(f::File) + ret = ccall(:jl_lseek, Int64, (Int32, Int64, Int32), f.handle, 0, SEEK_END) + systemerror("seekend", ret == -1) + return f +end + +function skip(f::File, n::Integer) + ret = ccall(:jl_lseek, Int64, (Int32, Int64, Int32), f.handle, n, SEEK_CUR) + systemerror("skip", ret == -1) + return f +end + function position(f::File) check_open(f) ret = ccall(:jl_lseek, Int64, (Int32, Int64, Int32), f.handle, 0, SEEK_CUR) diff --git a/base/io.jl b/base/io.jl index eaf6f472e8f0e..53123506d0d2b 100644 --- a/base/io.jl +++ b/base/io.jl @@ -348,7 +348,10 @@ type EachLine EachLine(stream, ondone) = new(stream, ondone) end eachline(stream::IO) = EachLine(stream) -eachline(filename::AbstractString) = EachLine(open(filename), close) +function eachline(filename::AbstractString) + s = open(filename) + EachLine(s, ()->close(s)) +end start(itr::EachLine) = nothing function done(itr::EachLine, nada) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index 1434e2fab5d76..2d31a9ed23098 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -337,6 +337,7 @@ function readbytes!(io::AbstractIOBuffer, b::Array{UInt8}, nb=length(b)) return nr end read(io::AbstractIOBuffer) = read!(io, Array(UInt8, nb_available(io))) +readavailable(io::AbstractIOBuffer) = read(io) read(io::AbstractIOBuffer, nb::Integer) = read!(io, Array(UInt8, min(nb, nb_available(io)))) function search(buf::IOBuffer, delim::UInt8) diff --git a/base/iostream.jl b/base/iostream.jl index 3e3aa61ca003a..0168db9fc5d01 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -158,6 +158,8 @@ end # num bytes available without blocking nb_available(s::IOStream) = ccall(:jl_nb_available, Int32, (Ptr{Void},), s.ios) +readavailable(s::IOStream) = read!(s, Vector{UInt8}(nb_available(s))) + function read(s::IOStream, ::Type{UInt8}) b = ccall(:ios_getc, Cint, (Ptr{Void},), s.ios) if b == -1 diff --git a/base/stream.jl b/base/stream.jl index ee59da6c00faa..d49da0e39c936 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -894,12 +894,13 @@ function stop_reading(stream::LibuvStream) end end -function readbytes!(s::LibuvStream, b::AbstractArray{UInt8}, nb=length(b)) - wait_readnb(s, nb) - nr = nb_available(s) - resize!(b, nr) # shrink to just contain input data if was resized - read!(s.buffer, b) - return nr +function read!(s::LibuvStream, b::Vector{UInt8}) + nb = length(b) + r = readbytes!(s, b, nb) + if r < nb + throw(EOFError()) + end + return b end function read(stream::LibuvStream) @@ -907,19 +908,18 @@ function read(stream::LibuvStream) return takebuf_array(stream.buffer) end -function read!(s::LibuvStream, a::Array{UInt8, 1}) - nb = length(a) +function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb = length(a)) sbuf = s.buffer @assert sbuf.seekable == false @assert sbuf.maxsize >= nb if nb_available(sbuf) >= nb - return read!(sbuf, a) + return readbytes!(sbuf, a, nb) end if nb <= SZ_UNBUFFERED_IO # Under this limit we are OK with copying the array from the stream's buffer wait_readnb(s, nb) - read!(sbuf, a) + r = readbytes!(sbuf, a, nb) else try stop_reading(s) # Just playing it safe, since we are going to switch buffers. @@ -928,6 +928,7 @@ function read!(s::LibuvStream, a::Array{UInt8, 1}) s.buffer = newbuf write(newbuf, sbuf) wait_readnb(s, nb) + r = nb_available(newbuf) finally s.buffer = sbuf if !isempty(s.readnotify.waitq) @@ -935,7 +936,7 @@ function read!(s::LibuvStream, a::Array{UInt8, 1}) end end end - return a + return r end function read(this::LibuvStream, ::Type{UInt8}) diff --git a/src/builtins.c b/src/builtins.c index 50ed9b0a85114..b79a2ac1e2b1b 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -182,6 +182,14 @@ JL_DLLEXPORT void JL_NORETURN jl_bounds_error_ints(jl_value_t *v, size_t *idxs, jl_throw(jl_new_struct((jl_datatype_t*)jl_boundserror_type, v, t)); } +JL_DLLEXPORT void JL_NORETURN jl_eof_error(void) +{ + jl_datatype_t *eof_error = + (jl_datatype_t*)jl_get_global(jl_base_module, jl_symbol("EOFError")); + assert(eof_error != NULL); + jl_exceptionf(eof_error, ""); +} + JL_CALLABLE(jl_f_throw) { JL_NARGS(throw, 1, 1); diff --git a/src/jl_uv.c b/src/jl_uv.c index 344356d5a1645..e31d5da8d434a 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -339,9 +339,12 @@ JL_DLLEXPORT int jl_fs_read_byte(int handle) buf[0].len = 1; int ret = uv_fs_read(jl_io_loop, &req, handle, buf, 1, -1, NULL); uv_fs_req_cleanup(&req); - if (ret == -1) - return ret; - return (int)c; + switch (ret) { + case -1: return ret; + case 0: jl_eof_error(); + case 1: return (int)c; + default: assert(0); + } } JL_DLLEXPORT int jl_fs_close(int handle) diff --git a/src/julia.h b/src/julia.h index 2815e4ad52b43..de49880d2fd5b 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1250,6 +1250,7 @@ JL_DLLEXPORT void JL_NORETURN jl_bounds_error_tuple_int(jl_value_t **v, size_t nv, size_t i); JL_DLLEXPORT void JL_NORETURN jl_bounds_error_unboxed_int(void *v, jl_value_t *vt, size_t i); JL_DLLEXPORT void JL_NORETURN jl_bounds_error_ints(jl_value_t *v, size_t *idxs, size_t nidxs); +JL_DLLEXPORT void JL_NORETURN jl_eof_error(void); JL_DLLEXPORT jl_value_t *jl_exception_occurred(void); JL_DLLEXPORT void jl_exception_clear(void); diff --git a/src/sys.c b/src/sys.c index d426e4cbc9a73..270c32858edce 100644 --- a/src/sys.c +++ b/src/sys.c @@ -308,13 +308,6 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim) return (jl_value_t*)a; } -static void JL_NORETURN throw_eof_error(void) -{ - jl_datatype_t *eof_error = (jl_datatype_t*)jl_get_global(jl_base_module, jl_symbol("EOFError")); - assert(eof_error != NULL); - jl_exceptionf(eof_error, ""); -} - JL_DLLEXPORT uint64_t jl_ios_get_nbyte_int(ios_t *s, const size_t n) { assert(n <= 8); @@ -323,7 +316,7 @@ JL_DLLEXPORT uint64_t jl_ios_get_nbyte_int(ios_t *s, const size_t n) space = (size_t)(s->size - s->bpos); ret = ios_readprep(s, n); if (space == ret && ret < n) - throw_eof_error(); + jl_eof_error(); } while(ret < n); uint64_t x = 0; uint8_t *buf = (uint8_t*)&s->buf[s->bpos]; diff --git a/test/choosetests.jl b/test/choosetests.jl index b9346a96758ee..aa6fd49205707 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -23,7 +23,7 @@ function choosetests(choices = []) "bitarray", "copy", "math", "fastmath", "functional", "operators", "path", "ccall", "parse", "loading", "bigint", "sorting", "statistics", "spawn", "backtrace", - "priorityqueue", "file", "mmap", "version", "resolve", + "priorityqueue", "file", "read", "mmap", "version", "resolve", "pollfd", "mpfr", "broadcast", "complex", "socket", "floatapprox", "datafmt", "reflection", "regex", "float16", "combinatorics", "sysinfo", "rounding", "ranges", "mod2pi", diff --git a/test/file.jl b/test/file.jl index 5771c5f80b54f..a28e0e888b46e 100644 --- a/test/file.jl +++ b/test/file.jl @@ -1115,3 +1115,36 @@ end @test copy(DevNull) === DevNull @test eof(DevNull) @test print(DevNull, "go to /dev/null") === nothing + +# Filesystem.File +tmpdir = mktempdir() do d + f = joinpath(d, "test.txt") + open(io->write(io, "123"), f, "w") + f1 = open(f) + f2 = Base.Filesystem.open(f, Base.Filesystem.JL_O_RDONLY) + @test read(f1, UInt8) == read(f2, UInt8) + @test read(f1, UInt8) == read(f2, UInt8) + @test read(f1, UInt8) == read(f2, UInt8) + @test_throws EOFError read(f1, UInt8) + @test_throws EOFError read(f2, UInt8) + close(f1) + close(f2) + + a = UInt8[0,0,0] + f1 = open(f) + f2 = Base.Filesystem.open(f, Base.Filesystem.JL_O_RDONLY) + @test read!(f1, a) == read!(f2, a) + @test_throws EOFError read!(f1, a) + @test_throws EOFError read!(f2, a) + close(f1) + close(f2) + + a = UInt8[0,0,0,0] + f1 = open(f) + f2 = Base.Filesystem.open(f, Base.Filesystem.JL_O_RDONLY) + @test_throws EOFError read!(f1, a) + @test_throws EOFError read!(f2, a) + close(f1) + close(f2) +end + diff --git a/test/read.jl b/test/read.jl new file mode 100644 index 0000000000000..8af6ab9e0e1c8 --- /dev/null +++ b/test/read.jl @@ -0,0 +1,307 @@ +mktempdir() do dir + +tasks = [] + +# Create test file... +filename = joinpath(dir, "file.txt") +text = "C1,C2\n1,2\na,b\n" + +# List of IO producers... +l = Vector{Tuple{AbstractString,Function}}() + + +# File +io = (text) -> begin + write(filename, text) + Base.Filesystem.open(filename, Base.Filesystem.JL_O_RDONLY) +end +s = io(text) +@test isa(s, IO) +@test isa(s, Base.Filesystem.File) +close(s) +push!(l, ("File", io)) + + +# IOStream +io = (text) -> begin + write(filename, text) + open(filename) +end +s = io(text) +@test isa(s, IO) +@test isa(s, IOStream) +close(s) +push!(l, ("IOStream", io)) + + +# IOBuffer +io = (text)->IOBuffer(text) +s = io(text) +@test isa(s, IO) +@test isa(s, IOBuffer) +close(s) +push!(l, ("IOBuffer", io)) + + +function run_test_server(srv, text) + push!(tasks, @async begin + try + sock = accept(srv) + try + write(sock,text) + catch e + if typeof(e) != Base.UVError + rethrow(e) + end + finally + close(sock) + end + finally + close(srv) + end + end) + yield() +end + + +# TCPSocket +io = (text) -> begin + port, srv = listenany(rand(2000:4000)) + run_test_server(srv, text) + connect(port) +end +s = io(text) +@test isa(s, IO) +@test isa(s, TCPSocket) +close(s) +push!(l, ("TCPSocket", io)) + + +# PipeEndpoint +io = (text) -> begin + a = "\\\\.\\pipe\\uv-test-$(randstring(6))" + b = joinpath(dir, "socket-$(randstring(6))") + socketname = @windows ? a : b + srv = listen(socketname) + run_test_server(srv, text) + connect(socketname) +end +s = io(text) +@test isa(s, IO) +@test isa(s, Base.PipeEndpoint) +close(s) +push!(l, ("PipeEndpoint", io)) + + +#FIXME See https://github.com/JuliaLang/julia/issues/14747 +# Reading from open(::Command) seems to deadlock on Linux/Travis +#= +@windows ? nothing : begin + +# Windows type command not working? +# See "could not spawn `type 'C:\Users\appveyor\AppData\Local\Temp\1\jul3516.tmp\file.txt'`" +#https://ci.appveyor.com/project/StefanKarpinski/julia/build/1.0.12733/job/hpwjs4hmf03vs5ag#L1244 + +# Pipe +io = (text) -> begin + write(filename, text) + open(`$(@windows ? "type" : "cat") $filename`)[1] +# Was open(`echo -n $text`)[1] +# See https://github.com/JuliaLang/julia/issues/14747 +end +s = io(text) +@test isa(s, IO) +@test isa(s, Pipe) +close(s) +push!(l, ("Pipe", io)) + +end +=# + + +open_streams = [] +function cleanup() + for s in open_streams + try close(s) end + end + empty!(open_streams) + for tsk in tasks + wait(tsk) + end + empty!(tasks) +end + + +verbose = false + + +for (name, f) in l + + io = ()->(s=f(text); push!(open_streams, s); s) + + write(filename, text) + + verbose && println("$name read...") + @test read(io(), UInt8) == read(IOBuffer(text), UInt8) + @test read(io(), UInt8) == read(filename, UInt8) + @test read(io(), Int) == read(IOBuffer(text), Int) + @test read(io(), Int) == read(filename,Int) + s1 = io() + s2 = IOBuffer(text) + @test read(s1, UInt32, 2) == read(s2, UInt32, 2) + @test !eof(s1) + @test read(s1, UInt8, 5) == read(s2, UInt8, 5) + @test !eof(s1) + @test read(s1, UInt8, 1) == read(s2, UInt8, 1) + @test eof(s1) + @test_throws EOFError read(s1, UInt8) + @test eof(s1) + close(s1) + close(s2) + + verbose && println("$name eof...") + n = length(text) - 1 + @test read!(io(), Vector{UInt8}(n)) == + read!(IOBuffer(text), Vector{UInt8}(n)) + @test (s = io(); read!(s, Vector{UInt8}(n)); !eof(s)) + n = length(text) + @test read!(io(), Vector{UInt8}(n)) == + read!(IOBuffer(text), Vector{UInt8}(n)) + @test (s = io(); read!(s, Vector{UInt8}(n)); eof(s)) + n = length(text) + 1 + @test_throws EOFError read!(io(), Vector{UInt8}(n)) + @test_throws EOFError read!(io(), Vector{UInt8}(n)) + + old_text = text + cleanup() + + for text in [ + old_text, + UTF8String(['A' + i % 52 for i in 1:(div(Base.SZ_UNBUFFERED_IO,2))]), + UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO -1)]), + UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO )]), + UTF8String(['A' + i % 52 for i in 1:( Base.SZ_UNBUFFERED_IO +1)]) + ] + + write(filename, text) + + verbose && println("$name readstring...") + @test readstring(io()) == text + + @test readstring(io()) == readstring(filename) + + + verbose && println("$name read...") + @test read(io()) == Vector{UInt8}(text) + + @test read(io()) == read(filename) + + cleanup() + + + verbose && println("$name readbytes!...") + l = length(text) + for n = [1, 2, l-2, l-1, l, l+1, l+2] + a1 = Vector{UInt8}(n); + a2 = Vector{UInt8}(n) + s1 = io() + s2 = IOBuffer(text) + n1 = readbytes!(s1, a1) + n2 = readbytes!(s2, a2) + @test n1 == n2 + @test length(a1) == length(a2) + @test a1[1:n1] == a2[1:n2] + @test n <= length(text) || eof(s1) + @test n <= length(text) || eof(s2) + + cleanup() + end + + verbose && println("$name read!...") + l = length(text) + for n = [1, 2, l-2, l-1, l] + @test read!(io(), Vector{UInt8}(n)) == + read!(IOBuffer(text), Vector{UInt8}(n)) + @test read!(io(), Vector{UInt8}(n)) == + read!(filename, Vector{UInt8}(n)) + + cleanup() + end + @test_throws EOFError read!(io(), Vector{UInt8}(length(text)+1)) + + + verbose && println("$name readuntil...") + @test readuntil(io(), '\n') == readuntil(IOBuffer(text),'\n') + @test readuntil(io(), '\n') == readuntil(filename,'\n') + @test readuntil(io(), "\n") == readuntil(IOBuffer(text),"\n") + @test readuntil(io(), "\n") == readuntil(filename,"\n") + @test readuntil(io(), ',') == readuntil(IOBuffer(text),',') + @test readuntil(io(), ',') == readuntil(filename,',') + + cleanup() + + verbose && println("$name readline...") + @test readline(io()) == readline(IOBuffer(text)) + @test readline(io()) == readline(filename) + + verbose && println("$name readlines...") + @test readlines(io()) == readlines(IOBuffer(text)) + @test readlines(io()) == readlines(filename) + @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) + @test collect(eachline(io())) == collect(eachline(filename)) + + cleanup() + + verbose && println("$name countlines...") + @test countlines(io()) == countlines(IOBuffer(text)) + + verbose && println("$name readcsv...") + @test readcsv(io()) == readcsv(IOBuffer(text)) + @test readcsv(io()) == readcsv(filename) + + cleanup() + end + + text = old_text + write(filename, text) + + if !(typeof(io()) in [Base.PipeEndpoint, Pipe, TCPSocket]) + + verbose && println("$name position...") + @test (s = io(); read!(s, Vector{UInt8}(4)); position(s)) == 4 + + verbose && println("$name seek...") + for n = 0:length(text)-1 + @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) + cleanup() + end + verbose && println("$name skip...") + for n = 0:length(text)-1 + @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) + @test readlines(skip(io(), n)) == readlines(skip(IOBuffer(text), n)) + cleanup() + end + verbose && println("$name seekend...") + @test readstring(seekend(io())) == "" + end + + + verbose && println("$name write(::IOStream, ...)") + to = open("$filename.to", "w") + write(to, io()) + close(to) + @test readstring("$filename.to") == text + + verbose && println("$name write(filename, ...)") + write("$filename.to", io()) + @test readstring("$filename.to") == text + + verbose && println("$name write(::IOBuffer, ...)") + to = IOBuffer(Vector{UInt8}(copy(text)), false, true) + write(to, io()) + @test takebuf_string(to) == text + + cleanup() +end + +end