diff --git a/src/HTTP.jl b/src/HTTP.jl index 5f182b44c..1d2445764 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -1,9 +1,9 @@ -__precompile__() +__precompile__(true) module HTTP export Request, Response, FIFOBuffer -using MbedTLS, Compat +using MbedTLS const TLS = MbedTLS @@ -21,19 +21,27 @@ const CRLF = "\r\n" include("consts.jl") include("utils.jl") -include("fifobuffer.jl") -include("sniff.jl") include("uri.jl") +using .URIs +include("fifobuffer.jl") +using .FIFOBuffers include("cookies.jl") using .Cookies - include("multipart.jl") include("types.jl") + include("parser.jl") +include("sniff.jl") + include("client.jl") include("handlers.jl") using .Handlers include("server.jl") +using .Nitrogen + +function __init__() + global const DEFAULT_CLIENT = Client() +end end # module # @time HTTP.parse(HTTP.Response, "HTTP/1.1 200 OK\r\n\r\n") diff --git a/src/client.jl b/src/client.jl index 38f406a14..a10ac71ec 100644 --- a/src/client.jl +++ b/src/client.jl @@ -71,7 +71,6 @@ const DEFAULT_OPTIONS = :((DEFAULT_CHUNK_SIZE, true, 15.0, 15.0, nothing, 5, tru Client(logger::Option{IO}; args...) = Client(logger, RequestOptions($(DEFAULT_OPTIONS)...; args...)) Client(; args...) = Client(nothing, RequestOptions($(DEFAULT_OPTIONS)...; args...)) end -const DEFAULT_CLIENT = Client() function setclient!(client::Client) global const DEFAULT_CLIENT = client @@ -94,7 +93,7 @@ request(method, uri::String; verbose::Bool=false, query="", args...) = (@log(ver request(method, uri::URI; verbose::Bool=false, args...) = (@log(verbose, STDOUT, "using default client"); request(DEFAULT_CLIENT, convert(HTTP.Method, method), uri; verbose=verbose, args...)) function request(client::Client, method, uri::URI; headers::Headers=Headers(), - body=EMPTYBODY, + body=FIFOBuffers.EMPTYBODY, stream::Bool=false, verbose::Bool=false, args...) @@ -112,7 +111,6 @@ function request(client::Client, req::Request, opts::RequestOptions; history::Ve retryattempt = max(0, retryattempt) # ensure all Request options are set, using client.options if necessary # this works because req.options are null by default whereas client.options always have a default - update!(opts, client.options) not(opts.tlsconfig) && (opts.tlsconfig = TLS.SSLConfig(true)) @log(verbose, client.logger, "making $(method(req)) request for host: '$(host(uri(req)))' and resource: '$(resource(uri(req)))'") @@ -231,7 +229,7 @@ function request(client::Client, req::Request, opts::RequestOptions, conn::Conne write(conn.tcp, req, opts) end # create a Response to fill - response = Response(stream ? DEFAULT_CHUNK_SIZE : DEFAULT_MAX, req) + response = Response(stream ? DEFAULT_CHUNK_SIZE : FIFOBuffers.DEFAULT_MAX, req) # process the response reset!(client.parser) success = process!(client, conn, opts, host, method(req), response, Ref{Float64}(time()), retryattempt, stream, tsk, verbose) diff --git a/src/consts.jl b/src/consts.jl index 4e6b2904c..5d17679f5 100644 --- a/src/consts.jl +++ b/src/consts.jl @@ -150,7 +150,7 @@ const MethodMap = Dict( "LINK" => LINK, "UNLINK" => UNLINK, ) -Base.convert(::Type{HTTP.Method}, s::String) = MethodMap[s] +Base.convert(::Type{Method}, s::String) = MethodMap[s] # parsing codes @enum(ParsingErrorCode, @@ -447,33 +447,6 @@ const normal_url_char = Bool[ true, true, true, true, true, true, true, false, ] -@enum(http_parser_url_fields, - UF_SCHEME = 1 - , UF_HOSTNAME = 2 - , UF_PORT = 3 - , UF_PATH = 4 - , UF_QUERY = 5 - , UF_FRAGMENT = 6 - , UF_USERINFO = 7 - , UF_MAX = 8 -) -const UF_SCHEME_MASK = 0x01 -const UF_HOSTNAME_MASK = 0x02 -const UF_PORT_MASK = 0x04 -const UF_PATH_MASK = 0x08 -const UF_QUERY_MASK = 0x10 -const UF_FRAGMENT_MASK = 0x20 -const UF_USERINFO_MASK = 0x40 - -@inline function Base.getindex(A::Vector{T}, i::http_parser_url_fields) where {T} - @inbounds v = A[Int(i)] - return v -end -@inline function Base.setindex!(A::Vector{T}, v::T, i::http_parser_url_fields) where {T} - @inbounds v = setindex!(A, v, Int(i)) - return v -end - @enum(http_host_state, s_http_host_dead = 1, s_http_userinfo_start =2, diff --git a/src/fifobuffer.jl b/src/fifobuffer.jl index 1f9778189..e6866c373 100644 --- a/src/fifobuffer.jl +++ b/src/fifobuffer.jl @@ -1,3 +1,9 @@ +module FIFOBuffers + +import Base.== + +export FIFOBuffer + """ FIFOBuffer([max::Integer]) FIFOBuffer(string_or_bytes_vector) @@ -233,7 +239,6 @@ function Base.write(f::FIFOBuffer, bytes::AbstractVector{UInt8}) if current_task() == f.task return 0 else # async: block until there's room to write - @debug(DEBUG, @__LINE__, "FIFOBuffer write() waiting...") wait(f.cond) f.nb == f.len && return 0 end @@ -295,7 +300,6 @@ function Base.write(f::FIFOBuffer, bytes::Vector{UInt8}) if current_task() == f.task return 0 else # async: block until there's room to write - @debug(DEBUG, @__LINE__, "FIFOBuffer write() waiting...") wait(f.cond) f.nb == f.len && return 0 end @@ -347,3 +351,5 @@ function Base.write(f::FIFOBuffer, bytes::Vector{UInt8}) end Base.write(f::FIFOBuffer, str::String) = write(f, Vector{UInt8}(str)) + +end # module \ No newline at end of file diff --git a/src/parser.jl b/src/parser.jl index 5f0e43f29..5669507bb 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -61,7 +61,7 @@ function onurl(r, bytes, i, j) @debug(PARSING_DEBUG, @__LINE__, i - j + 1) @debug(PARSING_DEBUG, @__LINE__, "'$(String(bytes[i:j]))'") @debug(PARSING_DEBUG, @__LINE__, r.method) - uri = http_parser_parse_url(bytes, i, j - i + 1, r.method == CONNECT) + uri = URIs.http_parser_parse_url(bytes, i, j - i + 1, r.method == CONNECT) @debug(PARSING_DEBUG, @__LINE__, uri) setfield!(r, :uri, uri) return @@ -153,7 +153,7 @@ const DEFAULT_MAX_BODY = Int64(2)^32 # 4Gib const DEFAULT_PARSER = Parser() function parse!(r::Union{Request, Response}, parser, bytes, len=length(bytes); - lenient::Bool=true, host::String="", method::HTTP.Method=GET, + lenient::Bool=true, host::String="", method::Method=GET, maxuri::Int64=DEFAULT_MAX_URI, maxheader::Int64=DEFAULT_MAX_HEADER, maxbody::Int64=DEFAULT_MAX_BODY, maintask::Task=current_task())::Tuple{ParsingErrorCode, Bool, Bool, String} strict = !lenient @@ -373,7 +373,7 @@ function parse!(r::Union{Request, Response}, parser, bytes, len=length(bytes); parser.content_length = ULLONG_MAX @errorif(!isalpha(ch), HPE_INVALID_METHOD) - r.method = HTTP.Method(0) + r.method = Method(0) parser.index = 2 if ch == 'A' @@ -481,12 +481,12 @@ function parse!(r::Union{Request, Response}, parser, bytes, len=length(bytes); if r.method == CONNECT p_state = s_req_server_start end - p_state = parseurlchar(p_state, ch, strict) + p_state = URIs.parseurlchar(p_state, ch, strict) @errorif(p_state == s_dead, HPE_INVALID_URL) elseif @anyeq(p_state, s_req_schema, s_req_schema_slash, s_req_schema_slash_slash, s_req_server_start) @errorif(ch in (' ', CR, LF), HPE_INVALID_URL) - p_state = parseurlchar(p_state, ch, strict) + p_state = URIs.parseurlchar(p_state, ch, strict) @errorif(p_state == s_dead, HPE_INVALID_URL) elseif @anyeq(p_state, s_req_server, s_req_server_with_at, s_req_path, s_req_query_string_start, @@ -506,7 +506,7 @@ function parse!(r::Union{Request, Response}, parser, bytes, len=length(bytes); onurl(r, bytes, url_mark, p-1) url_mark = 0 else - p_state = parseurlchar(p_state, ch, strict) + p_state = URIs.parseurlchar(p_state, ch, strict) @errorif(p_state == s_dead, HPE_INVALID_URL) end diff --git a/src/server.jl b/src/server.jl index b7e31b293..abceef717 100644 --- a/src/server.jl +++ b/src/server.jl @@ -1,3 +1,8 @@ +module Nitrogen + +using ..HTTP, ..Handlers + +export Server, ServerOptions, serve #TODO: # add in "events" handling # dealing w/ cookies @@ -26,7 +31,7 @@ # buffer re-use for server/client wire-reading # easter egg (response 418) mutable struct ServerOptions - tlsconfig::TLS.SSLConfig + tlsconfig::HTTP.TLS.SSLConfig readtimeout::Float64 ratelimit::Rational{Int} maxuri::Int64 @@ -35,12 +40,12 @@ mutable struct ServerOptions support100continue::Bool end -ServerOptions(; tlsconfig::TLS.SSLConfig=TLS.SSLConfig(true), +ServerOptions(; tlsconfig::HTTP.TLS.SSLConfig=HTTP.TLS.SSLConfig(true), readtimeout::Float64=180.0, ratelimit::Rational{Int64}=Int64(5)//Int64(1), - maxuri::Int64=DEFAULT_MAX_URI, - maxheader::Int64=DEFAULT_MAX_HEADER, - maxbody::Int64=DEFAULT_MAX_BODY, + maxuri::Int64=HTTP.DEFAULT_MAX_URI, + maxheader::Int64=HTTP.DEFAULT_MAX_HEADER, + maxbody::Int64=HTTP.DEFAULT_MAX_BODY, support100continue::Bool=true) = ServerOptions(tlsconfig, readtimeout, ratelimit, maxbody, maxuri, maxheader, support100continue) @@ -55,9 +60,9 @@ kill the julia process, interrupt (ctrl/cmd+c) if main task, or send the kill si `put!(server.in, HTTP.KILL)`. Supported keyword arguments include: - * `cert`: if https, the cert file to use, as passed to `TLS.SSLConfig(cert, key)` - * `key`: if https, the key file to use, as passed to `TLS.SSLConfig(cert, key)` - * `tlsconfig`: pass in an already-constructed `TLS.SSLConfig` instance + * `cert`: if https, the cert file to use, as passed to `HTTP.TLS.SSLConfig(cert, key)` + * `key`: if https, the key file to use, as passed to `HTTP.TLS.SSLConfig(cert, key)` + * `tlsconfig`: pass in an already-constructed `HTTP.TLS.SSLConfig` instance * `readtimeout`: how long a client connection will be left open without receiving any bytes * `ratelimit`: a `Rational{Int}` of the form `5//1` indicating how many `messages//second` should be allowed per client IP address; requests exceeding the rate limit will be dropped * `maxuri`: the maximum size in bytes that a request uri can be; default 8000 @@ -65,7 +70,7 @@ Supported keyword arguments include: * `maxbody`: the maximum size in bytes that a request body can be; default 4gb * `support100continue`: a `Bool` indicating whether `Expect: 100-continue` headers should be supported for delayed request body sending; default = `true` """ -mutable struct Server{T <: Scheme, H <: Handler} +mutable struct Server{T <: HTTP.Scheme, H <: HTTP.Handler} handler::H logger::IO in::Channel{Any} @@ -80,40 +85,40 @@ function process!(server::Server{T, H}, parser, request, i, tcp, rl, starttime, startedprocessingrequest = error = alreadysent100continue = false rate = Float64(server.options.ratelimit.num) rl.allowance += 1.0 # because it was just decremented right before we got here - response = Response() - @log(verbose, logger, "processing on connection i=$i...") + response = HTTP.Response() + HTTP.@log(verbose, logger, "processing on connection i=$i...") try tsk = @async begin request.body.task = current_task() while isopen(tcp) update!(rl, server.options.ratelimit) if rl.allowance > rate - @log(verbose, server.logger, "throttling on connection i=$i") + HTTP.@log(verbose, server.logger, "throttling on connection i=$i") rl.allowance = rate end if rl.allowance < 1.0 - @log(verbose, server.logger, "sleeping on connection i=$i due to rate limiting") + HTTP.@log(verbose, server.logger, "sleeping on connection i=$i due to rate limiting") sleep(1.0) else rl.allowance -= 1.0 - @log(verbose, server.logger, "reading request bytes with readtimeout=$(options.readtimeout)") + HTTP.@log(verbose, server.logger, "reading request bytes with readtimeout=$(options.readtimeout)") buffer = readavailable(tcp) length(buffer) > 0 || break starttime[] = time() # reset the timeout while still receiving bytes errno, headerscomplete, messagecomplete, upgrade = HTTP.parse!(request, parser, buffer) startedprocessingrequest = true - if errno != HPE_OK + if errno != HTTP.HPE_OK # error in parsing the http request - @log(verbose, logger, "error parsing request on connection i=$i: $(ParsingErrorCodeMap[errno])") - if errno == HPE_INVALID_VERSION + HTTP.@log(verbose, logger, "error parsing request on connection i=$i: $(HTTP.ParsingErrorCodeMap[errno])") + if errno == HTTP.HPE_INVALID_VERSION response.status = 505 - elseif errno == HPE_HEADER_OVERFLOW + elseif errno == HTTP.HPE_HEADER_OVERFLOW response.status = 431 - elseif errno == HPE_URI_OVERFLOW + elseif errno == HTTP.HPE_URI_OVERFLOW response.status = 414 - elseif errno == HPE_BODY_OVERFLOW + elseif errno == HTTP.HPE_BODY_OVERFLOW response.status = 413 - elseif errno == HPE_INVALID_METHOD + elseif errno == HTTP.HPE_INVALID_METHOD response.status = 405 else response.status = 400 @@ -121,9 +126,9 @@ function process!(server::Server{T, H}, parser, request, i, tcp, rl, starttime, error = true elseif headerscomplete && Base.get(HTTP.headers(request), "Expect", "") == "100-continue" && !alreadysent100continue if options.support100continue - @log(verbose, logger, "sending 100 Continue response to get request body") - write(tcp, Response(100), options) - parser.state = s_body_identity + HTTP.@log(verbose, logger, "sending 100 Continue response to get request body") + write(tcp, HTTP.Response(100), options) + parser.state = HTTP.s_body_identity alreadysent100continue = true continue else @@ -131,34 +136,34 @@ function process!(server::Server{T, H}, parser, request, i, tcp, rl, starttime, error = true end elseif length(upgrade) > 0 - @log(verbose, logger, "received upgrade request on connection i=$i") + HTTP.@log(verbose, logger, "received upgrade request on connection i=$i") response.status = 501 - response.body = FIFOBuffer("upgrade requests are not currently supported") + response.body = HTTP.FIFOBuffer("upgrade requests are not currently supported") error = true elseif messagecomplete - @log(verbose, logger, "received request on connection i=$i") - verbose && (show(logger, request, RequestOptions()); println(logger)) + HTTP.@log(verbose, logger, "received request on connection i=$i") + verbose && (show(logger, request, HTTP.RequestOptions()); println(logger)) try - response = handle(handler, request, response) + response = Handlers.handle(handler, request, response) catch e response.status = 500 error = true - @log(verbose, logger, e) + HTTP.@log(verbose, logger, e) end - if http_should_keep_alive(parser, request) && !error + if HTTP.http_should_keep_alive(parser, request) && !error get!(HTTP.headers(response), "Connection", "keep-alive") - reset!(parser) - request = Request() + HTTP.reset!(parser) + request = HTTP.Request() else get!(HTTP.headers(response), "Connection", "close") close(tcp) end - @log(verbose, logger, "responding with response on connection i=$i") + HTTP.@log(verbose, logger, "responding with response on connection i=$i") verbose && (show(logger, response, options); println(logger)) try write(tcp, response, options) catch e - @log(verbose, logger, e) + HTTP.@log(verbose, logger, e) error = true end error && break @@ -172,23 +177,23 @@ function process!(server::Server{T, H}, parser, request, i, tcp, rl, starttime, sleep(0.001) end if !istaskdone(tsk) - @log(verbose, logger, "connection i=$i timed out waiting for request bytes") - startedprocessingrequest && write(tcp, Response(408), options) + HTTP.@log(verbose, logger, "connection i=$i timed out waiting for request bytes") + startedprocessingrequest && write(tcp, HTTP.Response(408), options) end finally close(tcp) end - @log(verbose, logger, "finished processing on connection i=$i") + HTTP.@log(verbose, logger, "finished processing on connection i=$i") return nothing end -initTLS!(::Type{http}, tcp, tlsconfig) = return tcp -function initTLS!(::Type{https}, tcp, tlsconfig) +initTLS!(::Type{HTTP.http}, tcp, tlsconfig) = return tcp +function initTLS!(::Type{HTTP.https}, tcp, tlsconfig) try - tls = TLS.SSLContext() - TLS.setup!(tls, tlsconfig) - TLS.associate!(tls, tcp) - TLS.handshake!(tls) + tls = HTTP.TLS.SSLContext() + HTTP.TLS.setup!(tls, tlsconfig) + HTTP.TLS.associate!(tls, tcp) + HTTP.TLS.handshake!(tls) return tls catch e close(tcp) @@ -212,7 +217,7 @@ end @enum Signals KILL function serve(server::Server{T, H}, host, port, verbose) where {T, H} - @log(verbose, server.logger, "starting server to listen on: $(host):$(port)") + HTTP.@log(verbose, server.logger, "starting server to listen on: $(host):$(port)") tcpserver = listen(host, port) ratelimits = Dict{IPAddr, RateLimit}() rate = Float64(server.options.ratelimit.num) @@ -224,8 +229,8 @@ function serve(server::Server{T, H}, host, port, verbose) where {T, H} end end while true - p = Parser() - request = Request() + p = HTTP.Parser() + request = HTTP.Request() try # accept blocks until a new connection is detected tcp = accept(tcpserver) @@ -233,34 +238,34 @@ function serve(server::Server{T, H}, host, port, verbose) where {T, H} rl = get!(ratelimits, ip, RateLimit(rate, now())) update!(rl, server.options.ratelimit) if rl.allowance > rate - @log(verbose, server.logger, "throttling $ip") + HTTP.@log(verbose, server.logger, "throttling $ip") rl.allowance = rate end if rl.allowance < 1.0 - @log(verbose, server.logger, "discarding connection from $ip due to rate limiting") + HTTP.@log(verbose, server.logger, "discarding connection from $ip due to rate limiting") close(tcp) else rl.allowance -= 1.0 - @log(verbose, server.logger, "new tcp connection accepted, reading request...") + HTTP.@log(verbose, server.logger, "new tcp connection accepted, reading request...") let server=server, p=p, request=request, i=i, tcp=tcp, rl=rl - @async process!(server, p, request, i, initTLS!(T, tcp, server.options.tlsconfig::TLS.SSLConfig), rl, Ref{Float64}(time()), verbose) + @async process!(server, p, request, i, initTLS!(T, tcp, server.options.tlsconfig::HTTP.TLS.SSLConfig), rl, Ref{Float64}(time()), verbose) end i += 1 end catch e if typeof(e) <: InterruptException - @log(verbose, server.logger, "interrupt detected, shutting down...") + HTTP.@log(verbose, server.logger, "interrupt detected, shutting down...") interrupt() break else if !isopen(tcpserver) - @log(verbose, server.logger, "server TCPServer is closed, shutting down...") + HTTP.@log(verbose, server.logger, "server TCPServer is closed, shutting down...") # Server was closed while waiting to accept client. Exit gracefully. interrupt() break end - @log(verbose, server.logger, "error encountered: $e") - @log(verbose, server.logger, "resuming serving...") + HTTP.@log(verbose, server.logger, "error encountered: $e") + HTTP.@log(verbose, server.logger, "resuming serving...") end end end @@ -268,16 +273,16 @@ function serve(server::Server{T, H}, host, port, verbose) where {T, H} return end -Server(h::Function, l::IO; cert::String="", key::String="", args...) = Server(HandlerFunction(h), l; cert=cert, key=key, args...) -function Server(handler::H=HandlerFunction((req, rep) -> Response("Hello World!")), +Server(h::Function, l::IO; cert::String="", key::String="", args...) = Server(HTTP.HandlerFunction(h), l; cert=cert, key=key, args...) +function Server(handler::H=HTTP.HandlerFunction((req, rep) -> HTTP.Response("Hello World!")), logger::IO=STDOUT; cert::String="", key::String="", - args...) where {H <: Handler} + args...) where {H <: HTTP.Handler} if cert != "" && key != "" - server = Server{https, H}(handler, logger, Channel(1), Channel(1), ServerOptions(; tlsconfig=TLS.SSLConfig(cert, key), args...)) + server = Server{HTTP.https, H}(handler, logger, Channel(1), Channel(1), ServerOptions(; tlsconfig=HTTP.TLS.SSLConfig(cert, key), args...)) else - server = Server{http, H}(handler, logger, Channel(1), Channel(1), ServerOptions(; args...)) + server = Server{HTTP.http, H}(handler, logger, Channel(1), Channel(1), ServerOptions(; args...)) end return server end @@ -296,7 +301,7 @@ function serve end serve(server::Server, host=IPv4(127,0,0,1), port=8081; verbose::Bool=true) = serve(server, host, port, verbose) function serve(host::IPAddr, port::Int, - handler=(req, rep) -> Response("Hello World!"), + handler=(req, rep) -> HTTP.Response("Hello World!"), logger::I=STDOUT; cert::String="", key::String="", @@ -307,10 +312,12 @@ function serve(host::IPAddr, port::Int, end serve(; host::IPAddr=IPv4(127,0,0,1), port::Int=8081, - handler=(req, rep) -> Response("Hello World!"), + handler=(req, rep) -> HTTP.Response("Hello World!"), logger::IO=STDOUT, cert::String="", key::String="", verbose::Bool=true, args...) = serve(host, port, handler, logger; cert=cert, key=key, verbose=verbose, args...) + +end # module \ No newline at end of file diff --git a/src/uri.jl b/src/uri.jl index 5334ceae7..5629b2983 100644 --- a/src/uri.jl +++ b/src/uri.jl @@ -1,14 +1,22 @@ +module URIs + +import Base.== + +include("urlparser.jl") + +export URI, URL, + hasscheme, scheme, + hashostname, hostname, + haspath, path, + hasquery, query, + hasfragment, fragment, + hasuserinfo, userinfo, + hasport, port, + resource, host, + escape, unescape, + splitpath + # URI -struct Offset - off::UInt16 - len::UInt16 -end -Offset() = Offset(0, 0) -Base.getindex(A::Vector{UInt8}, o::Offset) = A[o.off:(o.off + o.len - 1)] -Base.isempty(o::Offset) = o.off == 0x0000 && o.len == 0x0000 -==(a::Offset, b::Offset) = a.off == b.off && a.len == b.len -const EMPTYOFFSET = Offset() - """ HTTP.URI(host; userinfo="", path="", query="", fragment="", isconnect=false) HTTP.URI(; scheme="", hostname="", port="", ...) @@ -84,7 +92,7 @@ Base.parse(::Type{URI}, str::String; isconnect::Bool=false) = http_parser_parse_ ((!hasport(a) || !hasport(b)) || (port(a) == port(b))) # accessors -for uf in instances(HTTP.http_parser_url_fields) +for uf in instances(http_parser_url_fields) uf == UF_MAX && break nm = lowercase(string(uf)[4:end]) has = Symbol(string("has", nm)) @@ -144,7 +152,6 @@ function Base.isvalid(uri::URI) return true end -lower(c::UInt8) = c | 0x20 # RFC3986 Unreserved Characters (and '~' Unsafe per RFC1738). @inline shouldencode(c) = !((c >= UInt8('A') && c <= UInt8('Z')) || (c >= UInt8('a') && c <= UInt8('z')) @@ -232,211 +239,4 @@ function splitpath(p::String, starting=2) return elems end -# url parsing -function parseurlchar(s, ch::Char, strict::Bool) - @anyeq(ch, ' ', '\r', '\n') && return s_dead - strict && (ch == '\t' || ch == '\f') && return s_dead - - if s == s_req_spaces_before_url - (ch == '/' || ch == '*') && return s_req_path - isalpha(ch) && return s_req_schema - elseif s == s_req_schema - isalphanum(ch) && return s - ch == ':' && return s_req_schema_slash - elseif s == s_req_schema_slash - ch == '/' && return s_req_schema_slash_slash - isurlchar(ch) && return s_req_path - elseif s == s_req_schema_slash_slash - ch == '/' && return s_req_server_start - isurlchar(ch) && return s_req_path - elseif s == s_req_server_with_at - ch == '@' && return s_dead - ch == '/' && return s_req_path - ch == '?' && return s_req_query_string_start - (isuserinfochar(ch) || ch == '[' || ch == ']') && return s_req_server - elseif s == s_req_server_start || s == s_req_server - ch == '/' && return s_req_path - ch == '?' && return s_req_query_string_start - ch == '@' && return s_req_server_with_at - (isuserinfochar(ch) || ch == '[' || ch == ']') && return s_req_server - elseif s == s_req_path - (isurlchar(ch) || ch == '@') && return s - ch == '?' && return s_req_query_string_start - ch == '#' && return s_req_fragment_start - elseif s == s_req_query_string_start || s == s_req_query_string - isurlchar(ch) && return s_req_query_string - ch == '?' && return s_req_query_string - ch == '#' && return s_req_fragment_start - elseif s == s_req_fragment_start - isurlchar(ch) && return s_req_fragment - ch == '?' && return s_req_fragment - ch == '#' && return s - elseif s == s_req_fragment - isurlchar(ch) && return s - (ch == '?' || ch == '#') && return s - end - #= We should never fall out of the switch above unless there's an error =# - return s_dead; -end - -function http_parse_host_char(s::http_host_state, ch) - if s == s_http_userinfo || s == s_http_userinfo_start - ch == '@' && return s_http_host_start - isuserinfochar(ch) && return s_http_userinfo - elseif s == s_http_host_start - ch == '[' && return s_http_host_v6_start - ishostchar(ch) && return s_http_host - elseif s == s_http_host - ishostchar(ch) && return s_http_host - ch == ':' && return s_http_host_port_start - elseif s == s_http_host_v6_end - ch == ':' && return s_http_host_port_start - elseif s == s_http_host_v6 - ch == ']' && return s_http_host_v6_end - (ishex(ch) || ch == ':' || ch == '.') && return s_http_host_v6 - s == s_http_host_v6 && ch == '%' && return s_http_host_v6_zone_start - elseif s == s_http_host_v6_start - (ishex(ch) || ch == ':' || ch == '.') && return s_http_host_v6 - s == s_http_host_v6 && ch == '%' && return s_http_host_v6_zone_start - elseif s == s_http_host_v6_zone - ch == ']' && return s_http_host_v6_end - (isalphanum(ch) || @anyeq(ch, '%', '.', '-', '_', '~')) && return s_http_host_v6_zone - elseif s == s_http_host_v6_zone_start - (isalphanum(ch) || @anyeq(ch, '%', '.', '-', '_', '~')) && return s_http_host_v6_zone - elseif s == s_http_host_port || s == s_http_host_port_start - isnum(ch) && return s_http_host_port - end - return s_http_host_dead -end - -function http_parse_host(buf, host::Offset, foundat) - portoff = portlen = uioff = uilen = UInt16(0) - off = len = UInt16(0) - s = ifelse(foundat, s_http_userinfo_start, s_http_host_start) - - for i = host.off:(host.off + host.len - 0x0001) - p = Char(buf[i]) - new_s = http_parse_host_char(s, p) - new_s == s_http_host_dead && throw(ParsingError("encountered invalid host character: \n$(String(buf))\n$(lpad("", i-1, "-"))^")) - if new_s == s_http_host - if s != s_http_host - off = i - end - len += 0x0001 - - elseif new_s == s_http_host_v6 - if s != s_http_host_v6 - off = i - end - len += 0x0001 - - elseif new_s == s_http_host_v6_zone_start || new_s == s_http_host_v6_zone - len += 0x0001 - - elseif new_s == s_http_host_port - if s != s_http_host_port - portoff = i - portlen = 0x0000 - end - portlen += 0x0001 - - elseif new_s == s_http_userinfo - if s != s_http_userinfo - uioff = i - uilen = 0x0000 - end - uilen += 0x0001 - end - s = new_s - end - if @anyeq(s, s_http_host_start, s_http_host_v6_start, s_http_host_v6, s_http_host_v6_zone_start, - s_http_host_v6_zone, s_http_host_port_start, s_http_userinfo, s_http_userinfo_start) - throw(ParsingError("ended in unexpected parsing state: $s")) - end - # (host, port, userinfo) - return Offset(off, len), Offset(portoff, portlen), Offset(uioff, uilen) -end - -function http_parser_parse_url(buf, startind=1, buflen=length(buf), isconnect::Bool=false) - s = ifelse(isconnect, s_req_server_start, s_req_spaces_before_url) - old_uf = UF_MAX - off = len = 0 - foundat = false - offsets = Offset[Offset(), Offset(), Offset(), Offset(), Offset(), Offset(), Offset()] - mask = 0x00 - for i = startind:(startind + buflen - 1) - @inbounds p = Char(buf[i]) - olds = s - s = parseurlchar(s, p, false) - if s == s_dead - throw(ParsingError("encountered invalid url character for parsing state = $(ParsingStateCode(olds)): \n$(String(buf))\n$(lpad("", i-1, "-"))^")) - elseif @anyeq(s, s_req_schema_slash, s_req_schema_slash_slash, s_req_server_start, s_req_query_string_start, s_req_fragment_start) - continue - elseif s == s_req_schema - uf = UF_SCHEME - mask |= UF_SCHEME_MASK - elseif s == s_req_server_with_at - foundat = true - uf = UF_HOSTNAME - mask |= UF_HOSTNAME_MASK - elseif s == s_req_server - uf = UF_HOSTNAME - mask |= UF_HOSTNAME_MASK - elseif s == s_req_path - uf = UF_PATH - mask |= UF_PATH_MASK - elseif s == s_req_query_string - uf = UF_QUERY - mask |= UF_QUERY_MASK - elseif s == s_req_fragment - uf = UF_FRAGMENT - mask |= UF_FRAGMENT_MASK - else - throw(ParsingError("ended in unexpected parsing state: $s")) - end - if uf == old_uf - len += 1 - continue - end - if old_uf != UF_MAX - offsets[old_uf] = Offset(off, len) - end - off = i - len = 1 - old_uf = uf - end - if old_uf != UF_MAX - offsets[old_uf] = Offset(off, len) - end - check = ~(UF_HOSTNAME_MASK | UF_PATH_MASK) - if (mask & UF_SCHEME_MASK > 0) && (mask | check == check) - throw(ParsingError("URI must include host or path with scheme")) - end - if mask & UF_HOSTNAME_MASK > 0 - host, port, userinfo = http_parse_host(buf, offsets[UF_HOSTNAME], foundat) - if !isempty(host) - offsets[UF_HOSTNAME] = host - mask |= UF_HOSTNAME_MASK - end - if !isempty(port) - offsets[UF_PORT] = port - mask |= UF_PORT_MASK - end - if !isempty(userinfo) - offsets[UF_USERINFO] = userinfo - mask |= UF_USERINFO_MASK - end - end - # CONNECT requests can only contain "hostname:port" - if isconnect - chk = HTTP.UF_HOSTNAME_MASK | HTTP.UF_PORT_MASK - ((mask | chk) > chk) && throw(ParsingError("connect requests must contain and can only contain both hostname and port")) - end - return URI(buf, (offsets[UF_SCHEME], - offsets[UF_HOSTNAME], - offsets[UF_PORT], - offsets[UF_PATH], - offsets[UF_QUERY], - offsets[UF_FRAGMENT], - offsets[UF_USERINFO])) -end +end # module \ No newline at end of file diff --git a/src/urlparser.jl b/src/urlparser.jl new file mode 100644 index 000000000..76c3fb98c --- /dev/null +++ b/src/urlparser.jl @@ -0,0 +1,253 @@ +include("consts.jl") +include("utils.jl") + +struct URLParsingError <: Exception + msg::String +end +Base.show(io::IO, p::URLParsingError) = println("HTTP.URLParsingError: ", p.msg) + +struct Offset + off::UInt16 + len::UInt16 +end +Offset() = Offset(0, 0) +Base.getindex(A::Vector{UInt8}, o::Offset) = A[o.off:(o.off + o.len - 1)] +Base.isempty(o::Offset) = o.off == 0x0000 && o.len == 0x0000 +==(a::Offset, b::Offset) = a.off == b.off && a.len == b.len +const EMPTYOFFSET = Offset() + +@enum(http_parser_url_fields, + UF_SCHEME = 1 + , UF_HOSTNAME = 2 + , UF_PORT = 3 + , UF_PATH = 4 + , UF_QUERY = 5 + , UF_FRAGMENT = 6 + , UF_USERINFO = 7 + , UF_MAX = 8 +) +const UF_SCHEME_MASK = 0x01 +const UF_HOSTNAME_MASK = 0x02 +const UF_PORT_MASK = 0x04 +const UF_PATH_MASK = 0x08 +const UF_QUERY_MASK = 0x10 +const UF_FRAGMENT_MASK = 0x20 +const UF_USERINFO_MASK = 0x40 + +@inline function Base.getindex(A::Vector{T}, i::http_parser_url_fields) where {T} + @inbounds v = A[Int(i)] + return v +end +@inline function Base.setindex!(A::Vector{T}, v::T, i::http_parser_url_fields) where {T} + @inbounds v = setindex!(A, v, Int(i)) + return v +end + +# url parsing +function parseurlchar(s, ch::Char, strict::Bool) + @anyeq(ch, ' ', '\r', '\n') && return s_dead + strict && (ch == '\t' || ch == '\f') && return s_dead + + if s == s_req_spaces_before_url + (ch == '/' || ch == '*') && return s_req_path + isalpha(ch) && return s_req_schema + elseif s == s_req_schema + isalphanum(ch) && return s + ch == ':' && return s_req_schema_slash + elseif s == s_req_schema_slash + ch == '/' && return s_req_schema_slash_slash + isurlchar(ch) && return s_req_path + elseif s == s_req_schema_slash_slash + ch == '/' && return s_req_server_start + isurlchar(ch) && return s_req_path + elseif s == s_req_server_with_at + ch == '@' && return s_dead + ch == '/' && return s_req_path + ch == '?' && return s_req_query_string_start + (isuserinfochar(ch) || ch == '[' || ch == ']') && return s_req_server + elseif s == s_req_server_start || s == s_req_server + ch == '/' && return s_req_path + ch == '?' && return s_req_query_string_start + ch == '@' && return s_req_server_with_at + (isuserinfochar(ch) || ch == '[' || ch == ']') && return s_req_server + elseif s == s_req_path + (isurlchar(ch) || ch == '@') && return s + ch == '?' && return s_req_query_string_start + ch == '#' && return s_req_fragment_start + elseif s == s_req_query_string_start || s == s_req_query_string + isurlchar(ch) && return s_req_query_string + ch == '?' && return s_req_query_string + ch == '#' && return s_req_fragment_start + elseif s == s_req_fragment_start + isurlchar(ch) && return s_req_fragment + ch == '?' && return s_req_fragment + ch == '#' && return s + elseif s == s_req_fragment + isurlchar(ch) && return s + (ch == '?' || ch == '#') && return s + end + #= We should never fall out of the switch above unless there's an error =# + return s_dead; +end + +function http_parse_host_char(s::http_host_state, ch) + if s == s_http_userinfo || s == s_http_userinfo_start + ch == '@' && return s_http_host_start + isuserinfochar(ch) && return s_http_userinfo + elseif s == s_http_host_start + ch == '[' && return s_http_host_v6_start + ishostchar(ch) && return s_http_host + elseif s == s_http_host + ishostchar(ch) && return s_http_host + ch == ':' && return s_http_host_port_start + elseif s == s_http_host_v6_end + ch == ':' && return s_http_host_port_start + elseif s == s_http_host_v6 + ch == ']' && return s_http_host_v6_end + (ishex(ch) || ch == ':' || ch == '.') && return s_http_host_v6 + s == s_http_host_v6 && ch == '%' && return s_http_host_v6_zone_start + elseif s == s_http_host_v6_start + (ishex(ch) || ch == ':' || ch == '.') && return s_http_host_v6 + s == s_http_host_v6 && ch == '%' && return s_http_host_v6_zone_start + elseif s == s_http_host_v6_zone + ch == ']' && return s_http_host_v6_end + (isalphanum(ch) || @anyeq(ch, '%', '.', '-', '_', '~')) && return s_http_host_v6_zone + elseif s == s_http_host_v6_zone_start + (isalphanum(ch) || @anyeq(ch, '%', '.', '-', '_', '~')) && return s_http_host_v6_zone + elseif s == s_http_host_port || s == s_http_host_port_start + isnum(ch) && return s_http_host_port + end + return s_http_host_dead +end + +function http_parse_host(buf, host::Offset, foundat) + portoff = portlen = uioff = uilen = UInt16(0) + off = len = UInt16(0) + s = ifelse(foundat, s_http_userinfo_start, s_http_host_start) + + for i = host.off:(host.off + host.len - 0x0001) + p = Char(buf[i]) + new_s = http_parse_host_char(s, p) + new_s == s_http_host_dead && throw(URLParsingError("encountered invalid host character: \n$(String(buf))\n$(lpad("", i-1, "-"))^")) + if new_s == s_http_host + if s != s_http_host + off = i + end + len += 0x0001 + + elseif new_s == s_http_host_v6 + if s != s_http_host_v6 + off = i + end + len += 0x0001 + + elseif new_s == s_http_host_v6_zone_start || new_s == s_http_host_v6_zone + len += 0x0001 + + elseif new_s == s_http_host_port + if s != s_http_host_port + portoff = i + portlen = 0x0000 + end + portlen += 0x0001 + + elseif new_s == s_http_userinfo + if s != s_http_userinfo + uioff = i + uilen = 0x0000 + end + uilen += 0x0001 + end + s = new_s + end + if @anyeq(s, s_http_host_start, s_http_host_v6_start, s_http_host_v6, s_http_host_v6_zone_start, + s_http_host_v6_zone, s_http_host_port_start, s_http_userinfo, s_http_userinfo_start) + throw(URLParsingError("ended in unexpected parsing state: $s")) + end + # (host, port, userinfo) + return Offset(off, len), Offset(portoff, portlen), Offset(uioff, uilen) +end + +function http_parser_parse_url(buf, startind=1, buflen=length(buf), isconnect::Bool=false) + s = ifelse(isconnect, s_req_server_start, s_req_spaces_before_url) + old_uf = UF_MAX + off = len = 0 + foundat = false + offsets = Offset[Offset(), Offset(), Offset(), Offset(), Offset(), Offset(), Offset()] + mask = 0x00 + for i = startind:(startind + buflen - 1) + @inbounds p = Char(buf[i]) + olds = s + s = parseurlchar(s, p, false) + if s == s_dead + throw(URLParsingError("encountered invalid url character for parsing state = $(ParsingStateCode(olds)): \n$(String(buf))\n$(lpad("", i-1, "-"))^")) + elseif @anyeq(s, s_req_schema_slash, s_req_schema_slash_slash, s_req_server_start, s_req_query_string_start, s_req_fragment_start) + continue + elseif s == s_req_schema + uf = UF_SCHEME + mask |= UF_SCHEME_MASK + elseif s == s_req_server_with_at + foundat = true + uf = UF_HOSTNAME + mask |= UF_HOSTNAME_MASK + elseif s == s_req_server + uf = UF_HOSTNAME + mask |= UF_HOSTNAME_MASK + elseif s == s_req_path + uf = UF_PATH + mask |= UF_PATH_MASK + elseif s == s_req_query_string + uf = UF_QUERY + mask |= UF_QUERY_MASK + elseif s == s_req_fragment + uf = UF_FRAGMENT + mask |= UF_FRAGMENT_MASK + else + throw(URLParsingError("ended in unexpected parsing state: $s")) + end + if uf == old_uf + len += 1 + continue + end + if old_uf != UF_MAX + offsets[old_uf] = Offset(off, len) + end + off = i + len = 1 + old_uf = uf + end + if old_uf != UF_MAX + offsets[old_uf] = Offset(off, len) + end + check = ~(UF_HOSTNAME_MASK | UF_PATH_MASK) + if (mask & UF_SCHEME_MASK > 0) && (mask | check == check) + throw(URLParsingError("URI must include host or path with scheme")) + end + if mask & UF_HOSTNAME_MASK > 0 + host, port, userinfo = http_parse_host(buf, offsets[UF_HOSTNAME], foundat) + if !isempty(host) + offsets[UF_HOSTNAME] = host + mask |= UF_HOSTNAME_MASK + end + if !isempty(port) + offsets[UF_PORT] = port + mask |= UF_PORT_MASK + end + if !isempty(userinfo) + offsets[UF_USERINFO] = userinfo + mask |= UF_USERINFO_MASK + end + end + # CONNECT requests can only contain "hostname:port" + if isconnect + chk = UF_HOSTNAME_MASK | UF_PORT_MASK + ((mask | chk) > chk) && throw(URLParsingError("connect requests must contain and can only contain both hostname and port")) + end + return URI(buf, (offsets[UF_SCHEME], + offsets[UF_HOSTNAME], + offsets[UF_PORT], + offsets[UF_PATH], + offsets[UF_QUERY], + offsets[UF_FRAGMENT], + offsets[UF_USERINFO])) +end \ No newline at end of file diff --git a/src/utils.jl b/src/utils.jl index 2d5ac2e83..1d6a78f65 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -71,6 +71,7 @@ macro anyeq(var, vals...) return esc(ret) end +@inline lower(c::UInt8) = c | 0x20 @inline lower(c) = Char(UInt32(c) | 0x20) @inline isurlchar(c) = c > '\u80' ? true : normal_url_char[Int(c) + 1] @inline ismark(c) = @anyeq(c, '-', '_', '.', '!', '~', '*', '\'', '(', ')') diff --git a/test/cookies.jl b/test/cookies.jl index 06e3b8eb6..82d72dccd 100644 --- a/test/cookies.jl +++ b/test/cookies.jl @@ -7,7 +7,7 @@ @test HTTP.Cookies.pathmatch(c, "/any/path") @test !HTTP.Cookies.pathmatch(c, "/nottherightpath") -const writesetcookietests = [ + writesetcookietests = [ (HTTP.Cookie("cookie-1", "v\$1"), "cookie-1=v\$1"), (HTTP.Cookie("cookie-2", "two", maxage=3600), "cookie-2=two; Max-Age=3600"), (HTTP.Cookie("cookie-3", "three", domain=".example.com"), "cookie-3=three; Domain=example.com"), diff --git a/test/server.jl b/test/server.jl index 106ea6f80..a778a4a86 100644 --- a/test/server.jl +++ b/test/server.jl @@ -4,7 +4,7 @@ server = HTTP.Server() tsk = @async HTTP.serve(server) sleep(1.0) -put!(server.in, HTTP.KILL) +put!(server.in, HTTP.Nitrogen.KILL) sleep(0.1) @test istaskdone(tsk) @@ -20,14 +20,14 @@ r = HTTP.get("http://127.0.0.1:8081/"; readtimeout=30) @test HTTP.status(r) == 200 @test String(take!(r)) == "" -print(readstring(serverlog)) +print(String(read(serverlog))) # invalid HTTP sleep(2.0) tcp = connect(ip"127.0.0.1", 8081) write(tcp, "GET / HTP/1.1\r\n\r\n") sleep(2.0) -log = readstring(serverlog) +log = String(read(serverlog)) print(log) @test contains(log, "invalid HTTP version") @@ -38,7 +38,7 @@ tcp = connect(ip"127.0.0.1", 8081) write(tcp, "BADMETHOD / HTTP/1.1\r\n\r\n") sleep(2.0) -log = readstring(serverlog) +log = String(read(serverlog)) print(log) @test contains(log, "invalid HTTP method") @@ -49,7 +49,7 @@ tcp = connect(ip"127.0.0.1", 8081) write(tcp, "POST / HTTP/1.1\r\nContent-Length: 15\r\nExpect: 100-continue\r\n\r\n") sleep(2.0) -log = readstring(serverlog) +log = String(read(serverlog)) @test contains(log, "sending 100 Continue response to get request body") client = String(readavailable(tcp)) @@ -57,7 +57,7 @@ client = String(readavailable(tcp)) write(tcp, "Body of Request") sleep(2.0) -log = readstring(serverlog) +log = String(read(serverlog)) client = String(readavailable(tcp)) print(client) @@ -66,7 +66,7 @@ print(client) @test contains(client, "Content-Length: 15\r\n") @test contains(client, "\r\n\r\nBody of Request") -put!(server.in, HTTP.KILL) +put!(server.in, HTTP.Nitrogen.KILL) # test readtimeout, before sending anything and then mid-request diff --git a/test/uri.jl b/test/uri.jl index 8249e0b5f..fd026f761 100644 --- a/test/uri.jl +++ b/test/uri.jl @@ -2,11 +2,11 @@ mutable struct URLTest name::String url::String isconnect::Bool - offsets::NTuple{7, HTTP.Offset} + offsets::NTuple{7, HTTP.URIs.Offset} shouldthrow::Bool end -URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(nm, url, isconnect, ntuple(x->HTTP.Offset(), 7), shouldthrow) +URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(nm, url, isconnect, ntuple(x->HTTP.URIs.Offset(), 7), shouldthrow) @testset "HTTP.URI" begin # constructor @@ -44,9 +44,9 @@ URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(n @test parse(HTTP.URI, "hdfs://user:password@hdfshost:9000/root/folder/file.csv") == HTTP.URI(hostname="hdfshost", path="/root/folder/file.csv", scheme="hdfs", port=9000, userinfo="user:password") @test parse(HTTP.URI, "http://google.com:80/some/path") == HTTP.URI(hostname="google.com", path="/some/path") - @test isempty(HTTP.Offset()) + @test isempty(HTTP.URIs.Offset()) @test HTTP.lower(UInt8('A')) == UInt8('a') - @test HTTP.hexstring(1) == "%01" + @test HTTP.URIs.hexstring(1) == "%01" @test HTTP.escape(Dict("key1"=>"value1", "key2"=>["value2", "value3"])) == "key2=value2&key2=value3&key1=value1" @@ -67,172 +67,172 @@ URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(n # Error paths # Non-ASCII characters - @test_throws HTTP.ParsingError parse(HTTP.URI, "http://🍕.com") + @test_throws HTTP.URIs.URLParsingError parse(HTTP.URI, "http://🍕.com") # Unexpected start of URL - @test_throws HTTP.ParsingError parse(HTTP.URI, ".google.com") + @test_throws HTTP.URIs.URLParsingError parse(HTTP.URI, ".google.com") # Unexpected character after scheme - @test_throws HTTP.ParsingError parse(HTTP.URI, "ht!tp://google.com") + @test_throws HTTP.URIs.URLParsingError parse(HTTP.URI, "ht!tp://google.com") # Issue #27 @test HTTP.escape("t est\n") == "t%20est%0A" @testset "HTTP.parse(HTTP.URI, str)" begin - const urltests = URLTest[ + urltests = URLTest[ URLTest("proxy request" ,"http://hostname/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(8, 8) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(16, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(8, 8) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(16, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("proxy request with port" ,"http://hostname:444/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(8, 8) # UF_HOST - ,HTTP.Offset(17, 3) # UF_PORT - ,HTTP.Offset(20, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(8, 8) # UF_HOST + ,HTTP.URIs.Offset(17, 3) # UF_PORT + ,HTTP.URIs.Offset(20, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("CONNECT request" ,"hostname:443" ,true - ,(HTTP.Offset(0, 0) # UF_SCHEMA - ,HTTP.Offset(1, 8) # UF_HOST - ,HTTP.Offset(10, 3) # UF_PORT - ,HTTP.Offset(0, 0) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(0, 0) # UF_SCHEMA + ,HTTP.URIs.Offset(1, 8) # UF_HOST + ,HTTP.URIs.Offset(10, 3) # UF_PORT + ,HTTP.URIs.Offset(0, 0) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("proxy ipv6 request" ,"http://[1:2::3:4]/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(9, 8) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(18, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(9, 8) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(18, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("proxy ipv6 request with port" ,"http://[1:2::3:4]:67/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(9, 8) # UF_HOST - ,HTTP.Offset(19, 2) # UF_PORT - ,HTTP.Offset(21, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(9, 8) # UF_HOST + ,HTTP.URIs.Offset(19, 2) # UF_PORT + ,HTTP.URIs.Offset(21, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("CONNECT ipv6 address" ,"[1:2::3:4]:443" ,true - ,(HTTP.Offset(0, 0) # UF_SCHEMA - ,HTTP.Offset(2, 8) # UF_HOST - ,HTTP.Offset(12, 3) # UF_PORT - ,HTTP.Offset(0, 0) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(0, 0) # UF_SCHEMA + ,HTTP.URIs.Offset(2, 8) # UF_HOST + ,HTTP.URIs.Offset(12, 3) # UF_PORT + ,HTTP.URIs.Offset(0, 0) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("ipv4 in ipv6 address" ,"http://[2001:0000:0000:0000:0000:0000:1.9.1.1]/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(9,37) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(47, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(9,37) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(47, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("extra ? in query string" ,"http://a.tbcdn.cn/p/fp/2010c/??fp-header-min.css,fp-base-min.css,fp-channel-min.css,fp-product-min.css,fp-mall-min.css,fp-category-min.css,fp-sub-min.css,fp-gdp4p-min.css,fp-css3-min.css,fp-misc-min.css?t=20101022.css" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(8,10) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(18,12) # UF_PATH - ,HTTP.Offset(31,187) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(8,10) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(18,12) # UF_PATH + ,HTTP.URIs.Offset(31,187) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("space URL encoded" ,"/toto.html?toto=a%20b" ,false - ,(HTTP.Offset(0, 0) # UF_SCHEMA - ,HTTP.Offset(0, 0) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(1,10) # UF_PATH - ,HTTP.Offset(12,10) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(0, 0) # UF_SCHEMA + ,HTTP.URIs.Offset(0, 0) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(1,10) # UF_PATH + ,HTTP.URIs.Offset(12,10) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("URL fragment" ,"/toto.html#titi" ,false - ,(HTTP.Offset(0, 0) # UF_SCHEMA - ,HTTP.Offset(0, 0) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(1,10) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(12, 4) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(0, 0) # UF_SCHEMA + ,HTTP.URIs.Offset(0, 0) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(1,10) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(12, 4) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("complex URL fragment" ,"http://www.webmasterworld.com/r.cgi?f=21&d=8405&url=http://www.example.com/index.html?foo=bar&hello=world#midpage" ,false - ,(HTTP.Offset( 1, 4) # UF_SCHEMA - ,HTTP.Offset( 8, 22) # UF_HOST - ,HTTP.Offset( 0, 0) # UF_PORT - ,HTTP.Offset( 30, 6) # UF_PATH - ,HTTP.Offset( 37, 69) # UF_QUERY - ,HTTP.Offset(107, 7) # UF_FRAGMENT - ,HTTP.Offset( 0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset( 1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset( 8, 22) # UF_HOST + ,HTTP.URIs.Offset( 0, 0) # UF_PORT + ,HTTP.URIs.Offset( 30, 6) # UF_PATH + ,HTTP.URIs.Offset( 37, 69) # UF_QUERY + ,HTTP.URIs.Offset(107, 7) # UF_FRAGMENT + ,HTTP.URIs.Offset( 0, 0) # UF_USERINFO ) ,false ), URLTest("complex URL from node js url parser doc" ,"http://host.com:8080/p/a/t/h?query=string#hash" ,false - ,( HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(8, 8) # UF_HOST - ,HTTP.Offset(17, 4) # UF_PORT - ,HTTP.Offset(21, 8) # UF_PATH - ,HTTP.Offset(30,12) # UF_QUERY - ,HTTP.Offset(43, 4) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,( HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(8, 8) # UF_HOST + ,HTTP.URIs.Offset(17, 4) # UF_PORT + ,HTTP.URIs.Offset(21, 8) # UF_PATH + ,HTTP.URIs.Offset(30,12) # UF_QUERY + ,HTTP.URIs.Offset(43, 4) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("complex URL with basic auth from node js url parser doc" ,"http://a:b@host.com:8080/p/a/t/h?query=string#hash" ,false - ,( HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(12, 8) # UF_HOST - ,HTTP.Offset(21, 4) # UF_PORT - ,HTTP.Offset(25, 8) # UF_PATH - ,HTTP.Offset(34,12) # UF_QUERY - ,HTTP.Offset(47, 4) # UF_FRAGMENT - ,HTTP.Offset(8, 3) # UF_USERINFO + ,( HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(12, 8) # UF_HOST + ,HTTP.URIs.Offset(21, 4) # UF_PORT + ,HTTP.URIs.Offset(25, 8) # UF_PATH + ,HTTP.URIs.Offset(34,12) # UF_QUERY + ,HTTP.URIs.Offset(47, 4) # UF_FRAGMENT + ,HTTP.URIs.Offset(8, 3) # UF_USERINFO ) ,false ), URLTest("double @" @@ -270,13 +270,13 @@ URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(n ), URLTest("proxy basic auth with space url encoded" ,"http://a%20:b@host.com/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(15, 8) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(23, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(8, 6) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(15, 8) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(23, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(8, 6) # UF_USERINFO ) ,false ), URLTest("carriage return in URL" @@ -290,13 +290,13 @@ URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(n ), URLTest("proxy basic auth with double :" ,"http://a::b@host.com/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(13, 8) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(21, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(8, 4) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(13, 8) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(21, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(8, 4) # UF_USERINFO ) ,false ), URLTest("line feed in URL" @@ -306,13 +306,13 @@ URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(n ), URLTest("proxy empty basic auth" ,"http://@hostname/fo" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(9, 8) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(17, 3) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(9, 8) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(17, 3) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("proxy line feed in hostname" @@ -330,13 +330,13 @@ URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(n ), URLTest("proxy basic auth with unreservedchars" ,"http://a!;-_!=+\$@host.com/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(18, 8) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(26, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(8, 9) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(18, 8) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(26, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(8, 9) # UF_USERINFO ) ,false ), URLTest("proxy only empty basic auth" @@ -354,25 +354,25 @@ URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(n ), URLTest("ipv6 address with Zone ID" ,"http://[fe80::a%25eth0]/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(9,14) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(24, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(9,14) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(24, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("ipv6 address with Zone ID, but '%' is not percent-encoded" ,"http://[fe80::a%eth0]/" ,false - ,(HTTP.Offset(1, 4) # UF_SCHEMA - ,HTTP.Offset(9,12) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(22, 1) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(1, 4) # UF_SCHEMA + ,HTTP.URIs.Offset(9,12) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(22, 1) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("ipv6 address ending with '%'" @@ -390,25 +390,25 @@ URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(n ), URLTest("tab in URL" ,"/foo\tbar/" ,false - ,(HTTP.Offset(0, 0) # UF_SCHEMA - ,HTTP.Offset(0, 0) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(1, 9) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(0, 0) # UF_SCHEMA + ,HTTP.URIs.Offset(0, 0) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(1, 9) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ), URLTest("form feed in URL" ,"/foo\fbar/" ,false - ,(HTTP.Offset(0, 0) # UF_SCHEMA - ,HTTP.Offset(0, 0) # UF_HOST - ,HTTP.Offset(0, 0) # UF_PORT - ,HTTP.Offset(1, 9) # UF_PATH - ,HTTP.Offset(0, 0) # UF_QUERY - ,HTTP.Offset(0, 0) # UF_FRAGMENT - ,HTTP.Offset(0, 0) # UF_USERINFO + ,(HTTP.URIs.Offset(0, 0) # UF_SCHEMA + ,HTTP.URIs.Offset(0, 0) # UF_HOST + ,HTTP.URIs.Offset(0, 0) # UF_PORT + ,HTTP.URIs.Offset(1, 9) # UF_PATH + ,HTTP.URIs.Offset(0, 0) # UF_QUERY + ,HTTP.URIs.Offset(0, 0) # UF_FRAGMENT + ,HTTP.URIs.Offset(0, 0) # UF_USERINFO ) ,false ) @@ -417,7 +417,7 @@ URLTest(nm::String, url::String, isconnect::Bool, shouldthrow::Bool) = URLTest(n for u in urltests println("TEST - uri.jl: $(u.name)") if u.shouldthrow - @test_throws HTTP.ParsingError parse(HTTP.URI, u.url; isconnect=u.isconnect) + @test_throws HTTP.URIs.URLParsingError parse(HTTP.URI, u.url; isconnect=u.isconnect) else url = parse(HTTP.URI, u.url; isconnect=u.isconnect) @test u.offsets == url.offsets