diff --git a/src/AWS4AuthRequest.jl b/src/AWS4AuthRequest.jl index 25c91b071..0829c3f42 100644 --- a/src/AWS4AuthRequest.jl +++ b/src/AWS4AuthRequest.jl @@ -8,7 +8,6 @@ using ..URIs using ..Pairs: getkv, setkv, rmkv import ..@debug, ..DEBUG_LEVEL - """ request(AWS4AuthLayer, ::URI, ::Request, body) -> HTTP.Response @@ -19,7 +18,6 @@ Add a [AWS Signature Version 4](http://docs.aws.amazon.com/general/latest/gr/sig Credentials are read from environment variables `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN`. """ - abstract type AWS4AuthLayer{Next <: Layer} <: Layer end export AWS4AuthLayer @@ -38,7 +36,6 @@ function request(::Type{AWS4AuthLayer{Next}}, return request(Next, url, req, body; kw...) end - function sign_aws4!(method::String, url::URI, headers::Headers, @@ -53,7 +50,6 @@ function sign_aws4!(method::String, aws_session_token::String=get(ENV, "AWS_SESSION_TOKEN", ""), kw...) - # ISO8601 date/time strings for time of request... date = Dates.format(t, dateformat"yyyymmdd") datetime = Dates.format(t, dateformat"yyyymmddTHHMMSS\Z") @@ -128,7 +124,6 @@ credentials = NamedTuple() Load Credentials from [AWS CLI ~/.aws/credentials file] (http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html). """ - function dot_aws_credentials()::NamedTuple global credentials diff --git a/src/BasicAuthRequest.jl b/src/BasicAuthRequest.jl index 28ec7e5f9..46f129315 100644 --- a/src/BasicAuthRequest.jl +++ b/src/BasicAuthRequest.jl @@ -7,13 +7,11 @@ using ..URIs using ..Pairs: getkv, setkv import ..@debug, ..DEBUG_LEVEL - """ request(BasicAuthLayer, method, ::URI, headers, body) -> HTTP.Response Add `Authorization: Basic` header using credentials from url userinfo. """ - abstract type BasicAuthLayer{Next <: Layer} <: Layer end export BasicAuthLayer diff --git a/src/CanonicalizeRequest.jl b/src/CanonicalizeRequest.jl index afc9ce5b3..278893c91 100644 --- a/src/CanonicalizeRequest.jl +++ b/src/CanonicalizeRequest.jl @@ -2,15 +2,13 @@ module CanonicalizeRequest import ..Layer, ..request using ..Messages -using ..Strings.tocameldash - +using ..Strings: tocameldash """ request(CanonicalizeLayer, method, ::URI, headers, body) -> HTTP.Response Rewrite request and response headers in Canonical-Camel-Dash-Format. """ - abstract type CanonicalizeLayer{Next <: Layer} <: Layer end export CanonicalizeLayer @@ -26,8 +24,6 @@ function request(::Type{CanonicalizeLayer{Next}}, return res end - -canonicalizeheaders(h::T) where T = T([tocameldash(k) => v for (k,v) in h]) - +canonicalizeheaders(h::T) where {T} = T([tocameldash(k) => v for (k,v) in h]) end # module CanonicalizeRequest diff --git a/src/ConnectionPool.jl b/src/ConnectionPool.jl index 5e434d631..43212f514 100644 --- a/src/ConnectionPool.jl +++ b/src/ConnectionPool.jl @@ -22,7 +22,6 @@ When the `request` function has read the Response Message it calls `closeread` to signal that the `Connection` can be reused for reading. """ - module ConnectionPool export Connection, Transaction, @@ -35,7 +34,6 @@ import ..@debug, ..@debugshow, ..DEBUG_LEVEL, ..taskid import ..@require, ..precondition_error, ..@ensure, ..postcondition_error using MbedTLS: SSLConfig, SSLContext, setup!, associate!, hostname!, handshake! - const default_connection_limit = 8 const default_pipeline_limit = 16 const nolimit = typemax(Int) @@ -44,7 +42,6 @@ const nobytes = view(UInt8[], 1:0) byteview(bytes::ByteView) = bytes byteview(bytes)::ByteView = view(bytes, 1:length(bytes)) - """ Connection{T <: IO} @@ -67,7 +64,6 @@ Fields: - `readdone`, signal that `readcount` was incremented. - `timestamp`, time data was last recieved. """ - mutable struct Connection{T <: IO} host::String port::String @@ -86,7 +82,6 @@ mutable struct Connection{T <: IO} timestamp::Float64 end - """ A single pipelined HTTP Request/Response transaction`. @@ -94,13 +89,11 @@ Fields: - `c`, the shared [`Connection`](@ref) used for this `Transaction`. - `sequence::Int`, identifies this `Transaction` among the others that share `c`. """ - struct Transaction{T <: IO} <: IO c::Connection{T} sequence::Int end - Connection(host::AbstractString, port::AbstractString, pipeline_limit::Int, io::T) where T <: IO = Connection{T}(host, port, pipeline_limit, @@ -122,10 +115,8 @@ function client_transaction(c) return t end - getrawstream(t::Transaction) = t.c.io - inactiveseconds(t::Transaction) = inactiveseconds(t.c) function inactiveseconds(c::Connection)::Float64 @@ -135,7 +126,6 @@ function inactiveseconds(c::Connection)::Float64 return time() - c.timestamp end - Base.unsafe_write(t::Transaction, p::Ptr{UInt8}, n::UInt) = unsafe_write(t.c.io, p, n) @@ -157,12 +147,10 @@ bytesavailable(t::Transaction) = bytesavailable(t.c) bytesavailable(c::Connection) = !isempty(c.excess) ? length(c.excess) : bytesavailable(c.io) - Base.isreadable(t::Transaction) = t.c.readbusy && t.c.readcount == t.sequence Base.iswritable(t::Transaction) = t.c.writebusy && t.c.writecount == t.sequence - function Base.read(t::Transaction, nb::Integer)::ByteView bytes = readavailable(t) l = length(bytes) @@ -173,7 +161,6 @@ function Base.read(t::Transaction, nb::Integer)::ByteView return bytes end - function Base.readavailable(t::Transaction)::ByteView @require isreadable(t) if !isempty(t.c.excess) @@ -188,14 +175,12 @@ function Base.readavailable(t::Transaction)::ByteView return bytes end - """ unread!(::Transaction, bytes) Push bytes back into a connection's `excess` buffer (to be returned by the next read). """ - function IOExtras.unread!(t::Transaction, bytes::ByteView) @require isreadable(t) @require !isempty(bytes) @@ -204,13 +189,11 @@ function IOExtras.unread!(t::Transaction, bytes::ByteView) return end - """ startwrite(::Transaction) Wait for prior pending writes to complete. """ - function IOExtras.startwrite(t::Transaction) @require !iswritable(t) ;t.c.writecount != t.sequence && @debug 1 "ā³ Wait write: $t" @@ -222,13 +205,11 @@ function IOExtras.startwrite(t::Transaction) return end - """ closewrite(::Transaction) Signal that an entire Request Message has been written to the `Transaction`. """ - function IOExtras.closewrite(t::Transaction) @require iswritable(t) @@ -241,13 +222,11 @@ function IOExtras.closewrite(t::Transaction) return end - """ startread(::Transaction) Wait for prior pending reads to complete. """ - function IOExtras.startread(t::Transaction) @require !isreadable(t) ;t.c.readcount != t.sequence && @debug 1 "ā³ Wait read: $t" @@ -260,7 +239,6 @@ function IOExtras.startread(t::Transaction) return end - """ closeread(::Transaction) @@ -268,7 +246,6 @@ Signal that an entire Response Message has been read from the `Transaction`. Increment `readcount` and wake up tasks waiting in `startread`. """ - function IOExtras.closeread(t::Transaction) @require isreadable(t) @@ -301,13 +278,11 @@ function Base.close(c::Connection) return end - """ purge(::Connection) Remove unread data from a `Connection`. """ - function purge(c::Connection) @require !isopen(c.io) while !eof(c.io) @@ -317,7 +292,6 @@ function purge(c::Connection) @ensure bytesavailable(c) == 0 end - """ The `pool` is a collection of open `Connection`s. The `request` function calls `getconnection` to retrieve a connection from the @@ -327,7 +301,6 @@ for writing (to send the next Request). When the `request` function has read the Response Message it calls `closeread` to signal that the `Connection` can be reused for reading. """ - const pool = Vector{Connection}() const poollock = ReentrantLock() const poolcondition = Condition() @@ -337,7 +310,6 @@ const poolcondition = Condition() Close all connections in `pool`. """ - function closeall() lock(poollock) @@ -350,13 +322,11 @@ function closeall() return end - """ findwritable(type, host, port) -> Vector{Connection} Find `Connections` in the `pool` that are ready for writing. """ - function findwritable(T::Type, host::AbstractString, port::AbstractString, @@ -373,14 +343,12 @@ function findwritable(T::Type, isopen(c.io)), pool) end - """ findoverused(type, host, port, reuse_limit) -> Vector{Connection} Find `Connections` in the `pool` that are over the reuse limit and have no more active readers. """ - function findoverused(T::Type, host::AbstractString, port::AbstractString, @@ -394,13 +362,11 @@ function findoverused(T::Type, isopen(c.io)), pool) end - """ findall(type, host, port) -> Vector{Connection} Find all `Connections` in the `pool` for `host` and `port`. """ - function findall(T::Type, host::AbstractString, port::AbstractString, @@ -413,7 +379,6 @@ function findall(T::Type, isopen(c.io)), pool) end - """ purge() @@ -424,14 +389,12 @@ function purge() deleteat!(pool, map(isdeletable, pool)) end - """ getconnection(type, host, port) -> Connection Find a reusable `Connection` in the `pool`, or create a new `Connection` if required. """ - function getconnection(::Type{Transaction{T}}, host::AbstractString, port::AbstractString; @@ -488,7 +451,6 @@ function getconnection(::Type{Transaction{T}}, end end - function getconnection(::Type{TCPSocket}, host::AbstractString, port::AbstractString; @@ -499,7 +461,6 @@ function getconnection(::Type{TCPSocket}, connect(getaddrinfo(host), p) end - const nosslconfig = SSLConfig() default_sslconfig = nothing noverify_sslconfig = nothing @@ -535,7 +496,6 @@ function getconnection(::Type{SSLContext}, return io end - function Base.show(io::IO, c::Connection) nwaiting = bytesavailable(tcpsocket(c.io)) print( @@ -555,7 +515,6 @@ end Base.show(io::IO, t::Transaction) = print(io, "T$(rpad(t.sequence,2)) ", t.c) - function tcpstatus(c::Connection) s = Base.uv_status_string(tcpsocket(c.io)) if s == "connecting" return "šŸ”œšŸ”—" diff --git a/src/ConnectionRequest.jl b/src/ConnectionRequest.jl index 2752c7816..4ec29f15e 100644 --- a/src/ConnectionRequest.jl +++ b/src/ConnectionRequest.jl @@ -8,7 +8,6 @@ using ..ConnectionPool using MbedTLS: SSLContext import ..@debug, ..DEBUG_LEVEL - """ request(ConnectionPoolLayer, ::URI, ::Request, body) -> HTTP.Response @@ -20,7 +19,6 @@ Otherwise leave it open so that it can be reused. `IO` related exceptions from `Base` are wrapped in `HTTP.IOError`. See [`isioerror`](@ref). """ - abstract type ConnectionPoolLayer{Next <: Layer} <: Layer end export ConnectionPoolLayer @@ -39,9 +37,6 @@ function request(::Type{ConnectionPoolLayer{Next}}, url::URI, req, body; end end - -sockettype(url::URI, default) = url.scheme in ("wss", "https") ? SSLContext : - default - +sockettype(url::URI, default) = url.scheme in ("wss", "https") ? SSLContext : default end # module ConnectionRequest diff --git a/src/CookieRequest.jl b/src/CookieRequest.jl index e4f4c7bc2..af29aef9a 100644 --- a/src/CookieRequest.jl +++ b/src/CookieRequest.jl @@ -7,17 +7,14 @@ using ..Cookies using ..Pairs: getkv, setkv import ..@debug, ..DEBUG_LEVEL - const default_cookiejar = Dict{String, Set{Cookie}}() - """ request(CookieLayer, method, ::URI, headers, body) -> HTTP.Response Add locally stored Cookies to the request headers. Store new Cookies found in the response headers. """ - abstract type CookieLayer{Next <: Layer} <: Layer end export CookieLayer @@ -40,7 +37,6 @@ function request(::Type{CookieLayer{Next}}, return res end - function getcookies(cookies, url) tosend = Vector{Cookie}() @@ -64,7 +60,6 @@ function getcookies(cookies, url) return tosend end - function setcookies(cookies, host, headers) for (k,v) in filter(x->x[1]=="Set-Cookie", headers) @debug 1 "Set-Cookie: $v (from $host)" @@ -72,5 +67,4 @@ function setcookies(cookies, host, headers) end end - end # module CookieRequest diff --git a/src/ExceptionRequest.jl b/src/ExceptionRequest.jl index a826d5325..007f72226 100644 --- a/src/ExceptionRequest.jl +++ b/src/ExceptionRequest.jl @@ -6,13 +6,11 @@ import ..Layer, ..request import ..HTTP using ..Messages: iserror - """ request(ExceptionLayer, ::URI, ::Request, body) -> HTTP.Response Throw a `StatusError` if the request returns an error response status. """ - abstract type ExceptionLayer{Next <: Layer} <: Layer end export ExceptionLayer @@ -27,7 +25,6 @@ function request(::Type{ExceptionLayer{Next}}, a...; kw...) where Next return res end - """ The `Response` has a `4xx`, `5xx` or unrecognised status code. @@ -35,11 +32,9 @@ Fields: - `status::Int16`, the response status code. - `response` the [`HTTP.Response`](@ref) """ - struct StatusError <: Exception status::Int16 response::HTTP.Response end - end # module ExceptionRequest diff --git a/src/HTTP.jl b/src/HTTP.jl index f40264a1d..52b5eb65e 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -27,7 +27,6 @@ include("Messages.jl") ;using .Messages include("cookies.jl") ;using .Cookies include("Streams.jl") ;using .Streams - """ HTTP.request(method, url [, headers [, body]]; ]) -> HTTP.Response @@ -219,7 +218,7 @@ println(read("get_data.txt")) Stream body through buffer: ```julia -io = BufferStream() +io = Base.BufferStream() @async while !eof(io) bytes = readavailable(io)) println("GET data: \$bytes") @@ -287,7 +286,6 @@ HTTP.open("POST", "http://music.com/play") do io end ``` """ - request(method::String, url::URI, headers::Headers, body; kw...)::Response = request(HTTP.stack(;kw...), method, url, headers, body; kw...) @@ -303,7 +301,6 @@ function request(method, url, h=Header[], b=nobody; return request(string(method), uri, mkheaders(headers), body; kw...) end - """ HTTP.open(method, url, [,headers]) do io write(io, body) @@ -326,49 +323,38 @@ HTTP.open("GET", "https://tinyurl.com/bach-cello-suite-1-ogg") do http end ``` """ - open(f::Function, method::String, url, headers=Header[]; kw...)::Response = request(method, url, headers, nothing; iofunction=f, kw...) - """ HTTP.get(url [, headers]; ) -> HTTP.Response Shorthand for `HTTP.request("GET", ...)`. See [`HTTP.request`](@ref). """ - get(u, a...; kw...) = request("GET", u, a...; kw...) - """ HTTP.put(url, headers, body; ) -> HTTP.Response Shorthand for `HTTP.request("PUT", ...)`. See [`HTTP.request`](@ref). """ - put(u, h, b; kw...) = request("PUT", u, h, b; kw...) - """ HTTP.post(url, headers, body; ) -> HTTP.Response Shorthand for `HTTP.request("POST", ...)`. See [`HTTP.request`](@ref). """ - post(u, h, b; kw...) = request("POST", u, h, b; kw...) - """ HTTP.head(url; ) -> HTTP.Response Shorthand for `HTTP.request("HEAD", ...)`. See [`HTTP.request`](@ref). """ - head(u; kw...) = request("HEAD", u; kw...) - - """ ## Request Execution Stack @@ -423,7 +409,6 @@ CodeInfo(:(begin end)) ``` """ - abstract type Layer end include("RedirectRequest.jl"); using .RedirectRequest include("BasicAuthRequest.jl"); using .BasicAuthRequest @@ -549,7 +534,6 @@ relationship with [`HTTP.Response`](@ref), [`HTTP.Parsers`](@ref), ``` *See `docs/src/layers`[`.monopic`](http://monodraw.helftone.com).* """ - function stack(;redirect=true, basic_authorization=false, aws_authorization=false, @@ -578,7 +562,6 @@ function stack(;redirect=true, }}}}}}}}}}} end - include("client.jl") include("Handlers.jl") ;using .Handlers include("Servers.jl") ;using .Servers; using .Servers: listen @@ -588,11 +571,10 @@ include("WebSockets.jl") ;using .WebSockets include("precompile.jl") - import .ConnectionPool: Transaction, Connection function Base.parse(::Type{T}, str::AbstractString)::T where T <: Message - buffer = BufferStream() + buffer = Base.BufferStream() write(buffer, str) close(buffer) m = T() @@ -602,5 +584,4 @@ function Base.parse(::Type{T}, str::AbstractString)::T where T <: Message return m end - end # module diff --git a/src/Handlers.jl b/src/Handlers.jl index 7a9495fa6..35fe914e8 100644 --- a/src/Handlers.jl +++ b/src/Handlers.jl @@ -141,6 +141,6 @@ function handle(r::Router, req) return handle(handler, req) end -handle(hf::HandlerFunction, http::HTTP.Stream) = HTTP.handle_request(hf.func,http) +handle(hf::HandlerFunction, http::HTTP.Stream) = handle_request(hf.func,http) end # module diff --git a/src/IOExtras.jl b/src/IOExtras.jl index 7c6dad403..401b0154f 100644 --- a/src/IOExtras.jl +++ b/src/IOExtras.jl @@ -4,7 +4,6 @@ This module defines extensions to the `Base.IO` interface to support: - `startwrite`, `closewrite`, `startread` and `closeread` for streams with transactional semantics. """ - module IOExtras export bytes, ByteView, CodeUnits, IOError, isioerror, @@ -37,7 +36,6 @@ bytes(s::Vector{UInt8}) = s Is `exception` caused by a possibly recoverable IO error. """ - isioerror(e) = false isioerror(::Base.EOFError) = true isioerror(::Base.UVError) = true @@ -50,7 +48,6 @@ The request terminated with due to an IO-related error. Fields: - `e`, the error. """ - struct IOError <: Exception e message @@ -64,8 +61,6 @@ Base.show(io::IO, e::IOError) = print(io, "IOError(", e.e, " ", e.message, ")\n" Push bytes back into a connection (to be returned by the next read). """ - - function unread!(io::IOBuffer, bytes) l = length(bytes) if l == 0 @@ -105,12 +100,14 @@ start_close_read_write_doc = """ Signal start/end of write or read operations. """ - -@doc start_close_read_write_doc -> startwrite(io) = nothing -@doc start_close_read_write_doc -> closewrite(io) = nothing -@doc start_close_read_write_doc -> startread(io) = nothing -@doc start_close_read_write_doc -> closeread(io) = nothing - +start_close_read_write_doc +startwrite(io) = nothing +start_close_read_write_doc +closewrite(io) = nothing +start_close_read_write_doc +startread(io) = nothing +start_close_read_write_doc +closeread(io) = nothing using MbedTLS: SSLContext tcpsocket(io::SSLContext)::TCPSocket = io.bio @@ -142,7 +139,6 @@ const ByteView = typeof(view(UInt8[], 1:0)) Read from an `IO` stream until `find_delimiter(bytes)` returns non-zero. Return view of bytes up to the delimiter. """ - function Base.readuntil(io::IO, find_delimiter::Function #= Vector{UInt8} -> Int =# )::ByteView diff --git a/src/MessageRequest.jl b/src/MessageRequest.jl index 536a210b7..54dcaac62 100644 --- a/src/MessageRequest.jl +++ b/src/MessageRequest.jl @@ -8,16 +8,14 @@ using ..IOExtras using ..URIs using ..Messages import ..Messages: bodylength -using ..Headers -using ..Form - +import ..Headers +import ..Form """ request(MessageLayer, method, ::URI, headers, body) -> HTTP.Response Construct a [`Request`](@ref) object and set mandatory headers. """ - struct MessageLayer{Next <: Layer} <: Layer end export MessageLayer @@ -46,7 +44,6 @@ function request(::Type{MessageLayer{Next}}, return request(Next, url, req, body; iofunction=iofunction, kw...) end - bodylength(body) = unknown_length bodylength(body::AbstractVector{UInt8}) = length(body) bodylength(body::AbstractString) = sizeof(body) @@ -56,7 +53,6 @@ bodylength(body::Vector{T}) where T <: AbstractArray{UInt8,1} = sum(length, body bodylength(body::IOBuffer) = bytesavailable(body) bodylength(body::Vector{IOBuffer}) = sum(bytesavailable, body) - const body_is_a_stream = UInt8[] const body_was_streamed = bytes("[Message Body was streamed]") bodybytes(body) = body_is_a_stream @@ -67,5 +63,4 @@ bodybytes(body::AbstractString) = bytes(body) bodybytes(body::Vector) = length(body) == 1 ? bodybytes(body[1]) : body_is_a_stream - end # module MessageRequest diff --git a/src/Messages.jl b/src/Messages.jl index 03812506a..ab32474b4 100644 --- a/src/Messages.jl +++ b/src/Messages.jl @@ -55,8 +55,6 @@ The `HTTP.Message` structs represent the Message Body as `Vector{UInt8}`. Streaming of request and response bodies is handled by the [`HTTP.StreamLayer`](@ref) and the [`HTTP.Stream`](@ref) `<: IO` stream. """ - - module Messages export Message, Request, Response, HeaderSizeError, @@ -72,7 +70,7 @@ export Message, Request, Response, HeaderSizeError, import ..HTTP using ..Pairs -using ..@warn +import ..@warn using ..IOExtras using ..Parsers import ..@require, ..precondition_error @@ -80,14 +78,10 @@ import ..bytes, ..bytesavailable const unknown_length = typemax(Int) - abstract type Message end - - # HTTP Response - """ Response <: Message @@ -108,7 +102,6 @@ Represents a HTTP Response Message. - `request`, the `Request` that yielded this `Response`. """ - mutable struct Response <: Message version::VersionNumber status::Int16 @@ -147,25 +140,19 @@ function reset!(r::Response) end end - """ statustext(::Response) -> String `String` representation of a HTTP status code. e.g. `200 => "OK"`. """ - statustext(r::Response) = Base.get(STATUS_MESSAGES, r.status, "Unknown Code") - @deprecate status(r::Response) getfield(r, :status) @deprecate headers(r::Response) getfield(r, :headers) @deprecate body(r::Response) getfield(r, :body) - - # HTTP Request - """ Request <: Message @@ -192,7 +179,6 @@ Represents a HTTP Request Message. (e.g. in the case of a redirect). [RFC7230 6.4](https://tools.ietf.org/html/rfc7231#section-6.4) """ - mutable struct Request <: Message method::String target::String @@ -218,7 +204,6 @@ function Request(method::String, target, headers=[], body=UInt8[]; return r end - mkheaders(h::Headers) = h mkheaders(h)::Headers = Header[string(k) => string(v) for (k,v) in h] @@ -227,39 +212,30 @@ mkheaders(h)::Headers = Header[string(k) => string(v) for (k,v) in h] @deprecate headers(r::Request) getfield(r, :headers) @deprecate body(r::Request) getfield(r, :body) - - # HTTP Message state and type queries - """ issafe(::Request) https://tools.ietf.org/html/rfc7231#section-4.2.1 """ - issafe(r::Request) = r.method in ["GET", "HEAD", "OPTIONS", "TRACE"] - """ isidempotent(::Request) https://tools.ietf.org/html/rfc7231#section-4.2.2 """ - isidempotent(r::Request) = issafe(r) || r.method in ["PUT", "DELETE"] - """ iserror(::Response) Does this `Response` have an error status? """ - iserror(r::Response) = r.status != 0 && r.status != 100 && r.status != 101 && (r.status < 200 || r.status >= 300) && !isredirect(r) - """ isredirect(::Response) @@ -267,28 +243,23 @@ Does this `Response` have a redirect status? """ isredirect(r::Response) = r.status in (301, 302, 307, 308) - """ ischunked(::Message) Does the `Message` have a "Transfer-Encoding: chunked" header? """ - ischunked(m) = any(h->(lowercase(h[1]) == "transfer-encoding" && endswith(lowercase(h[2]), "chunked")), m.headers) - """ headerscomplete(::Message) Have the headers been read into this `Message`? """ - headerscomplete(r::Response) = r.status != 0 && r.status != 100 headerscomplete(r::Request) = r.method != "" - """ "The presence of a message body in a response depends on both the request method to which it is responding and the response status code. @@ -300,14 +271,12 @@ headerscomplete(r::Request) = r.method != "" include a message body, although the body might be of zero length." [RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) """ - bodylength(r::Response)::Int = r.request.method == "HEAD" ? 0 : r.status in [204, 304] ? 0 : (l = header(r, "Content-Length")) != "" ? parse(Int, l) : unknown_length - """ "The presence of a message body in a request is signaled by a Content-Length or Transfer-Encoding header field. Request message @@ -315,16 +284,12 @@ bodylength(r::Response)::Int = not define any use for a message body." [RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) """ - bodylength(r::Request)::Int = ischunked(r) ? unknown_length : parse(Int, header(r, "Content-Length", "0")) - - # HTTP header-fields - """ header(::Message, key [, default=""]) -> String @@ -335,7 +300,6 @@ header(h::Headers, k::AbstractString, d::AbstractString="") = getbyfirst(h, k, k => d, lceq)[2] lceq(a,b) = lowercase(a) == lowercase(b) - """ hasheader(::Message, key) -> Bool @@ -343,7 +307,6 @@ Does header value for `key` exist (case-insensitive)? """ hasheader(m, k::AbstractString) = header(m, k) != "" - """ hasheader(::Message, key, value) -> Bool @@ -352,7 +315,6 @@ Does header for `key` match `value` (both case-insensitive)? hasheader(m, k::AbstractString, v::AbstractString) = lowercase(header(m, k)) == lowercase(v) - """ setheader(::Message, key => value) @@ -363,13 +325,11 @@ setheader(h::Headers, v::Header) = setbyfirst(h, v, lceq) setheader(h::Headers, v::Pair) = setbyfirst(h, Header(SubString(v.first), SubString(v.second)), lceq) - """ defaultheader(::Message, key => value) Set header `value` for `key` if it is not already set. """ - function defaultheader(m, v::Pair) if header(m, first(v)) == "" setheader(m, v) @@ -377,7 +337,6 @@ function defaultheader(m, v::Pair) return end - """ appendheader(::Message, key => value) @@ -392,7 +351,6 @@ delimiter](https://stackoverflow.com/a/24502264) `Set-Cookie` headers are not comma-combined because [cookies often contain internal commas](https://tools.ietf.org/html/rfc6265#section-3). """ - function appendheader(m::Message, header::Header) c = m.headers k,v = header @@ -406,11 +364,8 @@ function appendheader(m::Message, header::Header) return end - - # HTTP payload body - #Like https://github.com/JuliaIO/FileIO.jl/blob/v0.6.1/src/FileIO.jl#L19 ? load(m::Message) = payload(m, String) @@ -431,26 +386,20 @@ function decode(m::Message, encoding::String)::Vector{UInt8} return m.body end - - # Writing HTTP Messages to IO streams - """ httpversion(::Message) e.g. `"HTTP/1.1"` """ - httpversion(m::Message) = "HTTP/$(m.version.major).$(m.version.minor)" - """ writestartline(::IO, ::Message) e.g. `"GET /path HTTP/1.1\\r\\n"` or `"HTTP/1.1 200 OK\\r\\n"` """ - function writestartline(io::IO, r::Request) write(io, "$(r.method) $(r.target) $(httpversion(r))\r\n") return @@ -461,14 +410,12 @@ function writestartline(io::IO, r::Response) return end - """ writeheaders(::IO, ::Message) Write `Message` start line and a line for each "name: value" pair and a trailing blank line. """ - function writeheaders(io::IO, m::Message) writestartline(io, m) for (name, value) in m.headers @@ -478,37 +425,31 @@ function writeheaders(io::IO, m::Message) return end - """ write(::IO, ::Message) Write start line, headers and body of HTTP Message. """ - function Base.write(io::IO, m::Message) writeheaders(io, m) write(io, m.body) return end - function Base.String(m::Message) io = IOBuffer() write(io, m) String(take!(io)) end - # Reading HTTP Messages from IO streams - """ readheaders(::IO, ::Message) Read headers (and startline) from an `IO` stream into a `Message` struct. Throw `EOFError` if input is incomplete. """ - function readheaders(io::IO, message::Message) bytes = String(readuntil(io, find_end_of_header)) bytes = parse_start_line!(bytes, message) @@ -530,12 +471,10 @@ function parse_header_fields!(bytes::SubString{String}, m::Message) return end - """ Read chunk-size from an `IO` stream. After the final zero size chunk, read trailers into a `Message` struct. """ - function readchunksize(io::IO, message::Message)::Int n = parse_chunk_size(readuntil(io, find_end_of_line)) if n == 0 @@ -547,21 +486,16 @@ function readchunksize(io::IO, message::Message)::Int return n end - - # Debug message printing - """ set_show_max(x) Set the maximum number of body bytes to be displayed by `show(::IO, ::Message)` """ - set_show_max(x) = global body_show_max = x body_show_max = 1000 - """ bodysummary(bytes) @@ -595,7 +529,6 @@ function Base.show(io::IO, m::Message) return end - const STATUS_MESSAGES = (()->begin v = fill("Unknown Code", 530) v[100] = "Continue" diff --git a/src/Pairs.jl b/src/Pairs.jl index a9b33096b..647ac6d15 100644 --- a/src/Pairs.jl +++ b/src/Pairs.jl @@ -11,7 +11,6 @@ Set `item` in a `collection`. If `first() of an exisiting item matches `first(item)` it is replaced. Otherwise the new `item` is inserted at the end of the `collection`. """ - function setbyfirst(c, item, eq = ==) k = first(item) if (i = compat_findfirst(x->eq(first(x), k), c)) > 0 @@ -28,7 +27,6 @@ end Get `item` from collection where `first(item)` matches `key`. """ - function getbyfirst(c, k, default=nothing, eq = ==) i = compat_findfirst(x->eq(first(x), k), c) return i > 0 ? c[i] : default @@ -41,7 +39,6 @@ end If `first(item)` does not match match `first()` of any existing items, insert the new `item` at the end of the `collection`. """ - function defaultbyfirst(c, item, eq = ==) k = first(item) if (i = compat_findfirst(x->eq(first(x), k), c)) == 0 @@ -56,7 +53,6 @@ end Set `value` for `key` in collection of key/value `Pairs`. """ - setkv(c, k, v) = setbyfirst(c, k => v) @@ -66,7 +62,6 @@ setkv(c, k, v) = setbyfirst(c, k => v) Get `value` for `key` in collection of key/value `Pairs`, where `first(item) == key` and `value = item[2]` """ - function getkv(c, k, default=nothing) i = compat_findfirst(x->first(x) == k, c) return i > 0 ? c[i][2] : default @@ -78,7 +73,6 @@ end Remove `key` from `collection` of key/value `Pairs`. """ - function rmkv(c, k, default=nothing) i = compat_findfirst(x->first(x) == k, c) if i > 0 diff --git a/src/Parsers.jl b/src/Parsers.jl index bb7de6a8f..e5ffe0691 100644 --- a/src/Parsers.jl +++ b/src/Parsers.jl @@ -15,8 +15,6 @@ defined in `Messages.jl`. However, the `Request` and `Response` structs must have field names compatible with those expected by the `parse_status_line!` and `parse_request_line!` functions. """ - - module Parsers export Header, Headers, @@ -32,7 +30,6 @@ const emptyheader = emptyss => emptyss const Header = Pair{SubString{String},SubString{String}} const Headers = Vector{Header} - """ Parser input was invalid. @@ -40,7 +37,6 @@ Fields: - `code`, error code - `bytes`, the offending input. """ - struct ParseError <: Exception code::Symbol bytes::SubString{String} @@ -49,16 +45,12 @@ end ParseError(code::Symbol, bytes="") = ParseError(code, first(split(String(bytes), '\n'))) - - # Regular expressions for parsing HTTP start-line and header-fields - """ https://tools.ietf.org/html/rfc7230#section-3.1.1 request-line = method SP request-target SP HTTP-version CRLF """ - const request_line_regex = r"""^ (?: \r? \n) ? # ignore leading blank line ([!#$%&'*+\-.^_`|~[:alnum:]]+) [ ]+ # 1. method = token (RFC7230 3.2.6) @@ -67,7 +59,6 @@ const request_line_regex = r"""^ \r? \n # CRLF """x - """ https://tools.ietf.org/html/rfc7230#section-3.1.2 status-line = HTTP-version SP status-code SP reason-phrase CRLF @@ -75,7 +66,6 @@ status-line = HTTP-version SP status-code SP reason-phrase CRLF See: [#190](https://github.com/JuliaWeb/HTTP.jl/issues/190#issuecomment-363314009) """ - const status_line_regex = r"""^ [ ]? # Issue #190 HTTP/(\d\.\d) [ ]+ # 1. version @@ -83,12 +73,10 @@ const status_line_regex = r"""^ \r? \n # CRLF """x - """ https://tools.ietf.org/html/rfc7230#section-3.2 header-field = field-name ":" OWS field-value OWS """ - const header_field_regex = r"""^ ([!#$%&'*+\-.^_`|~[:alnum:]]+) : # 1. field-name = token (RFC7230 3.2.6) [ \t]* # OWS @@ -98,12 +86,10 @@ const header_field_regex = r"""^ (?= [^ \t]) # no WS on next line """x - """ https://tools.ietf.org/html/rfc7230#section-3.2.4 obs-fold = CRLF 1*( SP / HTAB ) """ - const obs_fold_header_field_regex = r"""^ ([!#$%&'*+\-.^_`|~[:alnum:]]+) : # 1. field-name = token (RFC7230 3.2.6) [ \t]* # OWS @@ -115,23 +101,18 @@ const obs_fold_header_field_regex = r"""^ const empty_header_field_regex = r"^ \r? \n"x - - # HTTP start-line and header-field parsing - """ Arbitrary limit to protect against denial of service attacks. """ const header_size_limit = Int(0x10000) - """ find_end_of_header(bytes) -> length or 0 Find length of header delimited by `\\r\\n\\r\\n` or `\\n\\n`. """ - function find_end_of_header(bytes::AbstractVector{UInt8}) buf = 0xFFFFFFFF l = min(length(bytes), header_size_limit) @@ -155,13 +136,11 @@ function find_end_of_header(bytes::AbstractVector{UInt8}) return 0 end - """ Parse HTTP request-line `bytes` and set the `method`, `target` and `version` fields of `request`. Return a `SubString` containing the header-field lines. """ - function parse_request_line!(bytes::AbstractString, request)::SubString{String} re = request_line_regex if !exec(re, bytes) @@ -173,13 +152,11 @@ function parse_request_line!(bytes::AbstractString, request)::SubString{String} return nextbytes(re, bytes) end - """ Parse HTTP response-line `bytes` and set the `status` and `version` fields of `response`. Return a `SubString` containing the header-field lines. """ - function parse_status_line!(bytes::AbstractString, response)::SubString{String} re = status_line_regex if !exec(re, bytes) @@ -190,16 +167,12 @@ function parse_status_line!(bytes::AbstractString, response)::SubString{String} return nextbytes(re, bytes) end - """ Parse HTTP header-field. Return `Pair(field-name => field-value)` and a `SubString` containing the remaining header-field lines. """ - -function parse_header_field(bytes::SubString{String})::Tuple{Header, - SubString{String}} - +function parse_header_field(bytes::SubString{String})::Tuple{Header,SubString{String}} # First look for: field-name ":" field-value re = header_field_regex if exec(re, bytes) @@ -216,38 +189,35 @@ function parse_header_field(bytes::SubString{String})::Tuple{Header, # Finally look for obsolete line folding format: re = obs_fold_header_field_regex if exec(re, bytes) - unfold = SubString(strip(replace(group(2, re, bytes), r"\r?\n", ""))) + unfold = SubString(strip(replace(group(2, re, bytes), r"\r?\n"=>""))) return (group(1, re, bytes) => unfold), nextbytes(re, bytes) end throw(ParseError(:INVALID_HEADER_FIELD, bytes)) end - - # HTTP Chunked Transfer Coding - +@static if VERSION < v"0.7.0-DEV.2005" """ Find `\\n` in `bytes` """ - -@static if VERSION < v"0.7.0-DEV.2005" - find_end_of_line(bytes::AbstractVector{UInt8}) = - findfirst(x->x==UInt8('\n'), bytes) +find_end_of_line(bytes::AbstractVector{UInt8}) = + findfirst(x->x==UInt8('\n'), bytes) else - find_end_of_line(bytes::AbstractVector{UInt8}) = - (i = findfirst(equalto(UInt8('\n')), bytes)) == nothing ? 0 : i +""" +Find `\\n` in `bytes` +""" +find_end_of_line(bytes::AbstractVector{UInt8}) = + (i = findfirst(equalto(UInt8('\n')), bytes)) == nothing ? 0 : i end - """ find_end_of_trailer(bytes) -> length or 0 Find length of trailer delimited by `\\r\\n\\r\\n` (or starting with `\\r\\n`). [RFC7230 4.1](https://tools.ietf.org/html/rfc7230#section-4.1) """ - find_end_of_trailer(bytes::AbstractVector{UInt8}) = length(bytes) < 2 ? 0 : bytes[2] == UInt8('\n') ? 2 : @@ -259,7 +229,6 @@ Arbitrary limit to protect against denial of service attacks. """ const chunk_size_limit = typemax(Int32) - """ Parse HTTP chunk-size. Return number of bytes of chunk-data. @@ -267,7 +236,6 @@ Return number of bytes of chunk-data. chunk-size = 1*HEXDIG [RFC7230 4.1](https://tools.ietf.org/html/rfc7230#section-4.1) """ - function parse_chunk_size(bytes::AbstractVector{UInt8})::Int chunk_size = Int64(0) @@ -288,7 +256,6 @@ function parse_chunk_size(bytes::AbstractVector{UInt8})::Int throw(ParseError(:INVALID_CHUNK_SIZE, bytes)) end - const unhex = Int8[ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 @@ -300,7 +267,6 @@ const unhex = Int8[ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 ] - function __init__() Base.compile(status_line_regex) Base.compile(request_line_regex) @@ -309,5 +275,4 @@ function __init__() Base.compile(obs_fold_header_field_regex) end - end # module Parsers diff --git a/src/RedirectRequest.jl b/src/RedirectRequest.jl index 09325361c..18950039c 100644 --- a/src/RedirectRequest.jl +++ b/src/RedirectRequest.jl @@ -4,16 +4,14 @@ import ..Layer, ..request using ..URIs using ..Messages using ..Pairs: setkv -using ..Header +import ..Header import ..@debug, ..DEBUG_LEVEL - """ request(RedirectLayer, method, ::URI, headers, body) -> HTTP.Response Redirects the request in the case of 3xx response status. """ - abstract type RedirectLayer{Next <: Layer} <: Layer end export RedirectLayer diff --git a/src/RetryRequest.jl b/src/RetryRequest.jl index 59d3af365..aec644c55 100644 --- a/src/RetryRequest.jl +++ b/src/RetryRequest.jl @@ -7,7 +7,6 @@ using ..MessageRequest using ..Messages import ..@debug, ..DEBUG_LEVEL - """ request(RetryLayer, ::URI, ::Request, body) -> HTTP.Response @@ -21,7 +20,6 @@ Methods of `isrecoverable(e)` define which exception types lead to a retry. e.g. `HTTP.IOError`, `Base.DNSError`, `Base.EOFError` and `HTTP.StatusError` (if status is ``5xx`). """ - abstract type RetryLayer{Next <: Layer} <: Layer end export RetryLayer @@ -45,7 +43,6 @@ function request(::Type{RetryLayer{Next}}, url, req, body; retry_request(Next, url, req, body; kw...) end - isrecoverable(e) = false isrecoverable(e::IOError) = true isrecoverable(e::Base.DNSError) = true @@ -74,5 +71,4 @@ function no_retry_reason(ex, req) return String(take!(buf)) end - end # module RetryRequest diff --git a/src/Servers.jl b/src/Servers.jl index 2ca3ebaff..21885bec1 100644 --- a/src/Servers.jl +++ b/src/Servers.jl @@ -8,7 +8,6 @@ using ..ConnectionPool import ..@info, ..@warn, ..@error, ..@debug, ..@debugshow, ..DEBUG_LEVEL using MbedTLS: SSLConfig, SSLContext, setup!, associate!, hostname!, handshake! - if !isdefined(Base, :Nothing) const Nothing = Void const Cvoid = Void @@ -272,7 +271,6 @@ e.g. end ``` """ - listen(f, host, port; kw...) = listen(f, string(host), Int(port); kw...) function listen(f::Function, @@ -341,7 +339,6 @@ end Start a timeout monitor task to close the `Connection` if it is inactive. Create a `Transaction` object for each HTTP Request received. """ - function handle_connection(f::Function, c::Connection; reuse_limit::Int=nolimit, readtimeout::Int=0, kw...) @@ -383,7 +380,6 @@ Create a `HTTP.Stream` and parse the Request headers from a `HTTP.Transaction`. If there is a parse error, send an error Response. Otherwise, execute stream processing function `f`. """ - function handle_transaction(f::Function, t::Transaction; final_transaction::Bool=false, verbose::Bool=false, kw...) @@ -394,6 +390,8 @@ function handle_transaction(f::Function, t::Transaction; try startread(http) catch e + # @show typeof(e) + # @show fieldnames(e) if e isa EOFError && isempty(request.method) return # FIXME https://github.com/JuliaWeb/HTTP.jl/pull/178#pullrequestreview-92547066 @@ -434,7 +432,6 @@ function handle_transaction(f::Function, t::Transaction; return end - """ Execute stream processing function `f`. If there is an error and the stream is still open, @@ -442,7 +439,6 @@ send a 500 response with the error message. Close the `Stream` for read and write (in case `f` has not already done so). """ - function handle_stream(f::Function, http::Stream) try @@ -467,11 +463,9 @@ function handle_stream(f::Function, http::Stream) return end - """ Execute Request processing function `f(::HTTP.Request) -> HTTP.Response`. """ - function handle_request(f::Function, http::Stream) request::HTTP.Request = http.message request.body = read(http) @@ -481,5 +475,4 @@ function handle_request(f::Function, http::Stream) return end - end # module diff --git a/src/StreamRequest.jl b/src/StreamRequest.jl index 83a4122e2..0944f1454 100644 --- a/src/StreamRequest.jl +++ b/src/StreamRequest.jl @@ -8,7 +8,6 @@ import ..ConnectionPool using ..MessageRequest import ..@debug, ..DEBUG_LEVEL, ..printlncompact - """ request(StreamLayer, ::IO, ::Request, body) -> HTTP.Response @@ -20,7 +19,6 @@ immediately so that the transmission can be aborted if the `Response` status indicates that the server does not wish to receive the message body. [RFC7230 6.5](https://tools.ietf.org/html/rfc7230#section-6.5). """ - abstract type StreamLayer <: Layer end export StreamLayer @@ -75,7 +73,6 @@ function request(::Type{StreamLayer}, io::IO, request::Request, body; return response end - function writebody(http::Stream, req::Request, body) if req.body === body_is_a_stream @@ -111,7 +108,6 @@ end writechunk(http, req, body::IO) = writebodystream(http, req, body) writechunk(http, req, body) = write(http, body) - function readbody(http::Stream, res::Response, response_stream) if response_stream == nothing res.body = read(http) @@ -122,5 +118,4 @@ function readbody(http::Stream, res::Response, response_stream) end end - end # module StreamRequest diff --git a/src/Streams.jl b/src/Streams.jl index fc02873de..dfa5ede2c 100644 --- a/src/Streams.jl +++ b/src/Streams.jl @@ -17,7 +17,6 @@ import ..@require, ..precondition_error import ..@ensure, ..postcondition_error import ..@debug, ..DEBUG_LEVEL - mutable struct Stream{M <: Message, S <: IO} <: IO message::M stream::S @@ -26,7 +25,6 @@ mutable struct Stream{M <: Message, S <: IO} <: IO ntoread::Int end - """ Stream(::IO, ::Request) @@ -52,7 +50,6 @@ Creates a `HTTP.Stream` that wraps an existing `IO` stream. response to be read by another `Stream` that is waiting in `startread`. If a complete response has not been recieved, `closeread` throws `EOFError`. """ - Stream(r::M, io::S) where {M, S} = Stream{M,S}(r, io, false, false, 0) header(http::Stream, a...) = header(http.message, a...) @@ -62,15 +59,11 @@ getrawstream(http::Stream) = getrawstream(http.stream) IOExtras.isopen(http::Stream) = isopen(http.stream) - - # Writing HTTP Messages - messagetowrite(http::Stream{Response}) = http.message.request messagetowrite(http::Stream{Request}) = http.message.response - IOExtras.iswritable(http::Stream) = iswritable(http.stream) function IOExtras.startwrite(http::Stream) @@ -89,7 +82,6 @@ function IOExtras.startwrite(http::Stream) writeheaders(http.stream, m) end - function Base.unsafe_write(http::Stream, p::Ptr{UInt8}, n::UInt) if n == 0 return 0 @@ -105,13 +97,11 @@ function Base.unsafe_write(http::Stream, p::Ptr{UInt8}, n::UInt) write(http.stream, "\r\n") end - """ closebody(::Stream) Write the final `0` chunk if needed. """ - function closebody(http::Stream) if http.writechunked http.writechunked = false @@ -119,7 +109,6 @@ function closebody(http::Stream) end end - function IOExtras.closewrite(http::Stream{Response}) if !iswritable(http) return @@ -143,11 +132,8 @@ function IOExtras.closewrite(http::Stream{Request}) end end - - # Reading HTTP Messages - IOExtras.isreadable(http::Stream) = isreadable(http.stream) function IOExtras.startread(http::Stream) @@ -163,13 +149,11 @@ function IOExtras.startread(http::Stream) return http.message end - """ 100 Continue https://tools.ietf.org/html/rfc7230#section-5.6 https://tools.ietf.org/html/rfc7231#section-6.2.1 """ - function handle_continue(http::Stream{Response}) if http.message.status == 100 @debug 1 "āœ… Continue: $(http.stream)" @@ -188,7 +172,6 @@ function handle_continue(http::Stream{Request}) end end - function Base.eof(http::Stream) if !headerscomplete(http.message) startread(http) @@ -202,7 +185,6 @@ function Base.eof(http::Stream) return false end - function Base.readavailable(http::Stream)::ByteView @require headerscomplete(http.message) @@ -239,17 +221,14 @@ function Base.readavailable(http::Stream)::ByteView return bytes end - IOExtras.unread!(http::Stream, excess) = unread!(http.stream, excess) - function Base.read(http::Stream) buf = IOBuffer() write(buf, http) return take!(buf) end - """ isaborted(::Stream{Response}) @@ -260,7 +239,6 @@ Has the server signaled that it does not wish to receive the message body? immediately cease transmitting the body and close the connection." [RFC7230, 6.5](https://tools.ietf.org/html/rfc7230#section-6.5) """ - function isaborted(http::Stream{Response}) if iswritable(http.stream) && @@ -299,7 +277,6 @@ function IOExtras.closeread(http::Stream{Response}) return http.message end - function IOExtras.closeread(http::Stream{Request}) if incomplete(http) # Error if Message is not complete... @@ -311,5 +288,4 @@ function IOExtras.closeread(http::Stream{Request}) end end - end #module Streams diff --git a/src/Strings.jl b/src/Strings.jl index d2dc07a73..eb1105e55 100644 --- a/src/Strings.jl +++ b/src/Strings.jl @@ -9,14 +9,13 @@ escapeHTML(i::String) Returns a string with special HTML characters escaped: &, <, >, ", ' """ - function escapehtml(i::AbstractString) # Refer to http://stackoverflow.com/a/7382028/3822752 for spec. links - o = replace(i, "&", "&") - o = replace(o, "\"", """) - o = replace(o, "'", "'") - o = replace(o, "<", "<") - o = replace(o, ">", ">") + o = replace(i, "&" =>"&") + o = replace(o, "\""=>""") + o = replace(o, "'" =>"'") + o = replace(o, "<" =>"<") + o = replace(o, ">" =>">") return o end @@ -26,7 +25,6 @@ end Ensure the first character and characters that follow a '-' are uppercase. """ - function tocameldash!(s::String) toUpper = UInt8('A') - UInt8('a') bytes = Vector{UInt8}(s) @@ -55,7 +53,6 @@ tocameldash(s::AbstractString) = tocameldash!(String(s)) Convert from ISO8859_1 to UTF8. """ - function iso8859_1_to_utf8(bytes::Vector{UInt8}) io = IOBuffer() for b in bytes diff --git a/src/TimeoutRequest.jl b/src/TimeoutRequest.jl index 39a17a309..56527bd02 100644 --- a/src/TimeoutRequest.jl +++ b/src/TimeoutRequest.jl @@ -4,13 +4,11 @@ import ..Layer, ..request using ..ConnectionPool import ..@debug, ..DEBUG_LEVEL - """ request(TimeoutLayer, ::IO, ::Request, body) -> HTTP.Response Close `IO` if no data has been received for `timeout` seconds. """ - abstract type TimeoutLayer{Next <: Layer} <: Layer end export TimeoutLayer diff --git a/src/URIs.jl b/src/URIs.jl index 835bb8872..15c957dd0 100644 --- a/src/URIs.jl +++ b/src/URIs.jl @@ -47,7 +47,6 @@ e.g. `"\$path?\$query#\$fragment"`. The `HTTP.queryparams(::URI)` function returns a `Dict` containing the `query`. """ - struct URI uri::String scheme::SubString{String} @@ -112,7 +111,6 @@ const uri_reference_regex = """ https://tools.ietf.org/html/rfc3986#section-3 """ - function parse_uri(str::AbstractString; kw...) uri = parse_uri_reference(str; kw...) if isempty(uri.scheme) @@ -125,7 +123,6 @@ end """ https://tools.ietf.org/html/rfc3986#section-4.1 """ - function parse_uri_reference(str::AbstractString; strict = false) if !exec(uri_reference_regex, str) @@ -156,25 +153,25 @@ function ensurevalid(uri::URI) # https://tools.ietf.org/html/rfc3986#section-3.1 # ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) if !(uri.scheme === absent || - ismatch(r"^[[:alpha:]][[:alnum:]+-.]*$", uri.scheme)) + contains(uri.scheme, r"^[[:alpha:]][[:alnum:]+-.]*$")) throw(ParseError("Invalid URI scheme: $(uri.scheme)")) end # https://tools.ietf.org/html/rfc3986#section-3.2.2 # unreserved / pct-encoded / sub-delims if !(uri.host === absent || - ismatch(r"^[:[:alnum:]\-._~%!$&'()*+,;=]+$", uri.host)) + contains(uri.host, r"^[:[:alnum:]\-._~%!$&'()*+,;=]+$")) throw(ParseError("Invalid URI host: $(uri.host) $uri")) end # https://tools.ietf.org/html/rfc3986#section-3.2.3 # "port number in decimal" - if !(uri.port === absent || ismatch(r"^\d+$", uri.port)) + if !(uri.port === absent || contains(uri.port, r"^\d+$")) throw(ParseError("Invalid URI port: $(uri.port)")) end # https://tools.ietf.org/html/rfc3986#section-3.3 # unreserved / pct-encoded / sub-delims / ":" / "@" if !(uri.path === absent || - ismatch(r"^[/[:alnum:]\-._~%!$&'()*+,;=:@]*$", uri.path)) + contains(uri.path, r"^[/[:alnum:]\-._~%!$&'()*+,;=:@]*$")) throw(ParseError("Invalid URI path: $(uri.path)")) end @@ -192,7 +189,6 @@ end """ https://tools.ietf.org/html/rfc3986#section-4.3 """ - isabsolute(uri::URI) = !isempty(uri.scheme) && isempty(uri.fragment) && @@ -203,7 +199,6 @@ isabsolute(uri::URI) = https://tools.ietf.org/html/rfc7230#section-5.3.1 https://tools.ietf.org/html/rfc3986#section-3.3 """ - pathissabsolute(uri::URI) = startwith(uri.path, "/") @@ -219,7 +214,6 @@ pathissabsolute(uri::URI) = startwith(uri.path, "/") """ "request-target" per https://tools.ietf.org/html/rfc7230#section-5.3 """ - resource(uri::URI) = string( isempty(uri.path) ? "/" : uri.path, !isempty(uri.query) ? "?" : "", uri.query, !isempty(uri.fragment) ? "#" : "", uri.fragment) @@ -300,7 +294,6 @@ const uses_query = ["http", "wais", "imap", "https", "shttp", "mms", "gopher", " const uses_fragment = ["hdfs", "ftp", "hdl", "http", "gopher", "news", "nntp", "wais", "https", "shttp", "snews", "file", "prospero"] "checks if a `HTTP.URI` is valid" - function Base.isvalid(uri::URI) sch = uri.scheme isempty(sch) && throw(ArgumentError("can not validate relative URI")) diff --git a/src/client.jl b/src/client.jl index 42466c186..71c6e5ca7 100644 --- a/src/client.jl +++ b/src/client.jl @@ -109,7 +109,7 @@ function request(client::Client, method, url::URI; m = string(method) h = mkheaders(headers) if stream - setkv(newargs, :response_stream, BufferStream()) + setkv(newargs, :response_stream, Base.BufferStream()) end if isa(body, Dict) diff --git a/src/compat.jl b/src/compat.jl index 48e529aca..1ad744844 100644 --- a/src/compat.jl +++ b/src/compat.jl @@ -73,3 +73,15 @@ end if !isdefined(Base, :isnumeric) const isnumeric = isnumber end + +if !applicable(contains, "", r"") + contains(s::String, r::Regex) = ismatch(r, s) +end + +if !applicable(replace, "", ""=>"") + replace(s::String, p::Pair) = replace(s, p.first, p.second) +end + +if !isdefined(Base, :lastindex) + lastindex(x) = endof(x) +end \ No newline at end of file diff --git a/src/cookies.jl b/src/cookies.jl index f86c8b426..29a444778 100644 --- a/src/cookies.jl +++ b/src/cookies.jl @@ -32,10 +32,14 @@ module Cookies export Cookie, cookies -import ..Dates +@static if VERSION < v"0.7-" +using Base.Dates +else +using Dates +end import Base: == -using ..pairs +import ..pairs import ..compat_search using ..IOExtras: bytes using ..Parsers: Headers @@ -387,7 +391,7 @@ function isCookieDomainName(s::String) return ok end -sanitizeCookieName(n::String) = replace(replace(n, '\n', '-'), '\r', '-') +sanitizeCookieName(n::String) = replace(replace(n, '\n'=>'-'), '\r'=>'-') sanitizeCookieName(n) = sanitizeCookieName(String(n)) # http:#tools.ietf.org/html/rfc6265#section-4.1.1 diff --git a/src/parseutils.jl b/src/parseutils.jl index 93832c2ea..c28e18bf6 100644 --- a/src/parseutils.jl +++ b/src/parseutils.jl @@ -1,7 +1,6 @@ """ Execute a regular expression without the overhead of `Base.Regex` """ - exec(re::Regex, bytes, offset::Int=1) = Base.PCRE.exec(re.regex, bytes, offset-1, re.match_options, re.match_data) @@ -9,14 +8,12 @@ exec(re::Regex, bytes, offset::Int=1) = """ `SubString` containing the bytes following the matched regular expression. """ - nextbytes(re::Regex, bytes) = SubString(bytes, re.ovec[2]+1) """ `SubString` containing a regular expression match group. """ - group(i, re::Regex, bytes) = SubString(bytes, re.ovec[2i+1]+1, prevind(bytes, re.ovec[2i+2]+1)) diff --git a/test/async.jl b/test/async.jl index aee2c39ab..3dd9653f5 100644 --- a/test/async.jl +++ b/test/async.jl @@ -135,13 +135,13 @@ get_data_sums = Dict() for i = 1:count @async try url = "$s3url/http.jl.test/file$i" - buf = BufferStream() + buf = Base.BufferStream() r = nothing if mode == :open r = HTTP.open("GET", url, ["Content-Length" => 0]; aws_authorization=true, conf...) do http - buf = BufferStream() # in case of retry! + buf = Base.BufferStream() # in case of retry! while !eof(http) write(buf, readavailable(http)) sleep(rand(1:10)/1000) @@ -275,8 +275,8 @@ println("running async $count, 1:$num, $config, $http C") if rand(Bool) for attempt in 1:4 try - #println("GET $i $n BufferStream $attempt") - s = BufferStream() + #println("GET $i $n Base.BufferStream $attempt") + s = Base.BufferStream() r = HTTP.request( "GET", url; response_stream=s, config...) @assert r.status == 200 diff --git a/test/benchmark.jl b/test/benchmark.jl index 4bcdd8058..184c0ad5e 100644 --- a/test/benchmark.jl +++ b/test/benchmark.jl @@ -179,7 +179,7 @@ function go(count::Int) end t_init_start = time() - io = BufferStream() + io = Base.BufferStream() c = Connection("", "", pipeline_limit, io) t_init_done = time() @@ -217,7 +217,7 @@ function go(count::Int) # Ensure that the header and body are buffered in the Connection # object. Otherwise, the time spent in readheaders below is # dominated by readavailable() copying huge body data from the - # BufferStream. We want to measure the parsing performance. + # Base.BufferStream. We want to measure the parsing performance. unread!(http.stream, readavailable(http.stream)) t_setup = time() diff --git a/test/body.jl b/test/body.jl index 04fff864e..533e4cb2d 100644 --- a/test/body.jl +++ b/test/body.jl @@ -7,7 +7,7 @@ using HTTP.Messages @test String(take!(Body(Vector{UInt8}("Hello!")))) == "Hello!" @test String(take!(Body())) == "" - io = BufferStream() + io = Base.BufferStream() @async begin write(io, "Hello") sleep(0.1) @@ -22,7 +22,7 @@ using HTTP.Messages write(b, "!") @test String(take!(b)) == "Hello!" - io = BufferStream() + io = Base.BufferStream() b = Body(io) write(b, "Hello") write(b, "!") @@ -32,7 +32,7 @@ using HTTP.Messages buf = IOBuffer() show(buf, b) - @test String(take!(buf)) == "Hello!\nā‹®\nWaiting for BufferStream...\n" + @test String(take!(buf)) == "Hello!\nā‹®\nWaiting for Base.BufferStream...\n" write(b, "\nWorld!") close(io) diff --git a/test/client.jl b/test/client.jl index f51dafc52..c0d248744 100644 --- a/test/client.jl +++ b/test/client.jl @@ -58,7 +58,7 @@ for sch in ("http", "https") a = [JSON.parse(l) for l in split(chomp(String(bytes)), "\n")] totallen = length(bytes) # number of bytes to expect begin - io = BufferStream() + io = Base.BufferStream() r = HTTP.get("$sch://httpbin.org/stream/100"; response_stream=io) @test status(r) == 200 @@ -66,7 +66,7 @@ for sch in ("http", "https") @test a == b end - # body posting: Vector{UInt8}, String, IOStream, IOBuffer, BufferStream + # body posting: Vector{UInt8}, String, IOStream, IOBuffer, Base.BufferStream println("client body posting of various types") @test status(HTTP.post("$sch://httpbin.org/post"; body="hey")) == 200 @test status(HTTP.post("$sch://httpbin.org/post"; body=UInt8['h','e','y'])) == 200 @@ -77,7 +77,7 @@ for sch in ("http", "https") io = open(tmp) @test status(HTTP.post("$sch://httpbin.org/post"; body=io, enablechunked=false)) == 200 close(io); rm(tmp) - f = BufferStream() + f = Base.BufferStream() write(f, "hey") close(f) @test status(HTTP.post("$sch://httpbin.org/post"; body=f, enablechunked=false)) == 200 @@ -98,7 +98,7 @@ for sch in ("http", "https") io = open(tmp) @test_broken status(HTTP.post("$sch://httpbin.org/post"; body=io, #=chunksize=2=#)) == 200 close(io); rm(tmp) - f = BufferStream() + f = Base.BufferStream() write(f, "hey") close(f) @test_broken status(HTTP.post("$sch://httpbin.org/post"; body=f, #=chunksize=2=#)) == 200 @@ -151,7 +151,7 @@ for sch in ("http", "https") # asynchronous println("asynchronous client request body") begin - f = BufferStream() + f = Base.BufferStream() write(f, "hey") t = @async HTTP.post("$sch://httpbin.org/post"; body=f, enablechunked=false) #wait(f) # wait for the async call to write it's first data diff --git a/test/loopback.jl b/test/loopback.jl index 70bca18a8..d0214c1c8 100644 --- a/test/loopback.jl +++ b/test/loopback.jl @@ -5,7 +5,6 @@ using HTTP.Parsers using HTTP.Messages using HTTP.MessageRequest: bodylength - mutable struct FunctionIO <: IO f::Function buf::IOBuffer @@ -20,13 +19,12 @@ HTTP.bytesavailable(fio::FunctionIO) = (call(fio); HTTP.bytesavailable(fio.buf)) Base.readavailable(fio::FunctionIO) = (call(fio); readavailable(fio.buf)) Base.read(fio::FunctionIO, a...) = (call(fio); read(fio.buf, a...)) - mutable struct Loopback <: IO got_headers::Bool buf::IOBuffer - io::BufferStream + io::Base.BufferStream end -Loopback() = Loopback(false, IOBuffer(), BufferStream()) +Loopback() = Loopback(false, IOBuffer(), Base.BufferStream()) function reset(lb::Loopback) truncate(lb.buf, 0) @@ -41,20 +39,17 @@ Base.isopen(lb::Loopback) = isopen(lb.io) HTTP.ConnectionPool.tcpstatus(c::HTTP.ConnectionPool.Connection{Loopback}) = "šŸ¤– " - """ escapelines(string) Escape `string` and insert '\n' after escaped newline characters. """ - function escapelines(s) s = Base.escape_string(s) s = replace(s, "\\n", "\\n\n ") return string(" ", strip(s)) end - server_events = [] function on_headers(f, lb) @@ -100,7 +95,6 @@ function on_body(f, lb) end end - function Base.unsafe_write(lb::Loopback, p::Ptr{UInt8}, n::UInt) global server_events @@ -215,8 +209,6 @@ lbopen(f, req, headers) = end @test String(body) == "Hello World!" - - # "If [the response] indicates the server does not wish to receive the # message body and is closing the connection, the client SHOULD # immediately cease transmitting the body and close the connection." @@ -301,7 +293,6 @@ lbopen(f, req, headers) = return t2 - t1 end - server_events = [] t = async_test(;pipeline_limit=0) @show t @@ -388,7 +379,6 @@ lbopen(f, req, headers) = "Response: HTTP/1.1 200 OK <= (GET /delay5 HTTP/1.1)"] end - # "A user agent SHOULD NOT pipeline requests after a # non-idempotent method, until the final response # status code for that method has been received" diff --git a/test/messages.jl b/test/messages.jl index 6066a690d..7b5b7ca7b 100644 --- a/test/messages.jl +++ b/test/messages.jl @@ -107,7 +107,7 @@ using JSON #= @sync begin - io = BufferStream() + io = Base.BufferStream() @async begin for i = 1:100 sleep(0.1) @@ -129,7 +129,7 @@ using JSON @test r.status == 200 body = r.body - io = BufferStream() + io = Base.BufferStream() r = request(m, uri, response_stream=io) @test r.status == 200 @test read(io) == body @@ -140,7 +140,7 @@ using JSON for m in ["POST", "PUT", "DELETE", "PATCH"] uri = "$sch://httpbin.org/$(lowercase(m))" - io = BufferStream() + io = Base.BufferStream() r = request(m, uri, response_stream=io) @test r.status == 200 end diff --git a/test/parser.jl b/test/parser.jl index c47914f9a..429f7ba25 100644 --- a/test/parser.jl +++ b/test/parser.jl @@ -1640,7 +1640,7 @@ https://github.com/nodejs/http-parser/pull/64#issuecomment-2042429 respstr = "HTTP/1.1 200 OK\r\n" * "Fo@: Failure\r\n\r\n" @test_throws HTTP.ParseError parse(Response,respstr) - @test ismatch(r"INVALID_HEADER_FIELD", @errmsg(parse(Response,respstr))) + @test contains(@errmsg(parse(Response,respstr)), r"INVALID_HEADER_FIELD") respstr = "HTTP/1.1 200 OK\r\n" * "Foo\01\test: Bar\r\n\r\n" @test_throws HTTP.ParseError parse(Response,respstr) diff --git a/test/server.jl b/test/server.jl index a8a9ee546..0ced17c06 100644 --- a/test/server.jl +++ b/test/server.jl @@ -55,7 +55,7 @@ sleep(1.0) r = testget("http://127.0.0.1:$port/") -@test ismatch(r"HTTP/1.1 200 OK", r) +@test contains(r, r"HTTP/1.1 200 OK") rv = [] n = 5 @@ -69,7 +69,7 @@ m = 20 sleep(0.01) end for i = 1:n - @test length(filter(l->ismatch(r"HTTP/1.1 200 OK", l), + @test length(filter(l->contains(l, r"HTTP/1.1 200 OK"), split(rv[i], "\n"))) == n * m end @@ -84,13 +84,13 @@ x = "GET / HTTP/1.1\r\n$(repeat("Foo: Bar\r\n", 10000))\r\n" @show length(x) write(tcp, "GET / HTTP/1.1\r\n$(repeat("Foo: Bar\r\n", 10000))\r\n") sleep(0.1) -@test ismatch(r"HTTP/1.1 413 Request Entity Too Large", String(read(tcp))) +@test contains(String(read(tcp)), r"HTTP/1.1 413 Request Entity Too Large") # invalid HTTP tcp = connect(ip"127.0.0.1", port) sleep(0.1) write(tcp, "GET / HTP/1.1\r\n\r\n") -@test ismatch(r"HTTP/1.1 400 Bad Request", String(read(tcp))) +@test contains(String(read(tcp)), r"HTTP/1.1 400 Bad Request") # no URL @@ -98,7 +98,7 @@ tcp = connect(ip"127.0.0.1", port) write(tcp, "SOMEMETHOD HTTP/1.1\r\nContent-Length: 0\r\n\r\n") sleep(0.1) r = String(read(tcp)) -@test ismatch(r"HTTP/1.1 400 Bad Request", r) +@test contains(r, r"HTTP/1.1 400 Bad Request") # Expect: 100-continue diff --git a/test/urlparser.jl b/test/urlparser.jl index aeb377477..4dda0ac2f 100644 --- a/test/urlparser.jl +++ b/test/urlparser.jl @@ -194,7 +194,7 @@ function http_parser_parse_url(url::String) scheme = userinfo = host = port = path = query = fragment = empty mask = 0x00 - end_i = endof(url) + end_i = HTTP.lastindex(url) for i in eachindex(url) @inbounds p = url[i] olds = s