Reorganize HTTP internals: introduce URIs, FIFOBuffers, and Nitrogen … (

#72)

* Reorganize HTTP internals: introduce URIs, FIFOBuffers, and Nitrogen (Server) modules

* Make new URIs module truly stand-alone from rest of HTTP

* additional reorg updates

src/HTTP.jl

@@ -1,9 +1,9 @@
-__precompile__()
+__precompile__(true)
 module HTTP
 
 export Request, Response, FIFOBuffer
 
-using MbedTLS, Compat
+using MbedTLS
 
 const TLS = MbedTLS
 
@@ -21,19 +21,27 @@ const CRLF = "\r\n"
 
 include("consts.jl")
 include("utils.jl")
-include("fifobuffer.jl")
-include("sniff.jl")
 include("uri.jl")
+using .URIs
+include("fifobuffer.jl")
+using .FIFOBuffers
 include("cookies.jl")
 using .Cookies
-
 include("multipart.jl")
 include("types.jl")
+
 include("parser.jl")
+include("sniff.jl")
+
 include("client.jl")
 include("handlers.jl")
 using .Handlers
 include("server.jl")
+using .Nitrogen
+
+function __init__()
+    global const DEFAULT_CLIENT = Client()
+end
 
 end # module
 # @time HTTP.parse(HTTP.Response, "HTTP/1.1 200 OK\r\n\r\n")

src/client.jl

@@ -71,7 +71,6 @@ const DEFAULT_OPTIONS = :((DEFAULT_CHUNK_SIZE, true, 15.0, 15.0, nothing, 5, tru
     Client(logger::Option{IO}; args...) = Client(logger, RequestOptions($(DEFAULT_OPTIONS)...; args...))
     Client(; args...) = Client(nothing, RequestOptions($(DEFAULT_OPTIONS)...; args...))
 end
-const DEFAULT_CLIENT = Client()
 
 function setclient!(client::Client)
     global const DEFAULT_CLIENT = client
@@ -94,7 +93,7 @@ request(method, uri::String; verbose::Bool=false, query="", args...) = (@log(ver
 request(method, uri::URI; verbose::Bool=false, args...) = (@log(verbose, STDOUT, "using default client"); request(DEFAULT_CLIENT, convert(HTTP.Method, method), uri; verbose=verbose, args...))
 function request(client::Client, method, uri::URI;
                     headers::Headers=Headers(),
-                    body=EMPTYBODY,
+                    body=FIFOBuffers.EMPTYBODY,
                     stream::Bool=false,
                     verbose::Bool=false,
                     args...)
@@ -112,7 +111,6 @@ function request(client::Client, req::Request, opts::RequestOptions; history::Ve
     retryattempt = max(0, retryattempt)
     # ensure all Request options are set, using client.options if necessary
     # this works because req.options are null by default whereas client.options always have a default
-
     update!(opts, client.options)
     not(opts.tlsconfig) && (opts.tlsconfig = TLS.SSLConfig(true))
     @log(verbose, client.logger, "making $(method(req)) request for host: '$(host(uri(req)))' and resource: '$(resource(uri(req)))'")
@@ -231,7 +229,7 @@ function request(client::Client, req::Request, opts::RequestOptions, conn::Conne
         write(conn.tcp, req, opts)
     end
     # create a Response to fill
-    response = Response(stream ? DEFAULT_CHUNK_SIZE : DEFAULT_MAX, req)
+    response = Response(stream ? DEFAULT_CHUNK_SIZE : FIFOBuffers.DEFAULT_MAX, req)
     # process the response
     reset!(client.parser)
     success = process!(client, conn, opts, host, method(req), response, Ref{Float64}(time()), retryattempt, stream, tsk, verbose)

src/consts.jl

@@ -150,7 +150,7 @@ const MethodMap = Dict(
     "LINK" => LINK,
     "UNLINK" => UNLINK,
 )
-Base.convert(::Type{HTTP.Method}, s::String) = MethodMap[s]
+Base.convert(::Type{Method}, s::String) = MethodMap[s]
 
 # parsing codes
 @enum(ParsingErrorCode,
@@ -447,33 +447,6 @@ const normal_url_char = Bool[
         true,   true,   true,   true,   true,   true,   true,   false,
 ]
 
-@enum(http_parser_url_fields,
-      UF_SCHEME   = 1
-    , UF_HOSTNAME = 2
-    , UF_PORT     = 3
-    , UF_PATH     = 4
-    , UF_QUERY    = 5
-    , UF_FRAGMENT = 6
-    , UF_USERINFO = 7
-    , UF_MAX      = 8
-)
-const UF_SCHEME_MASK = 0x01
-const UF_HOSTNAME_MASK = 0x02
-const UF_PORT_MASK = 0x04
-const UF_PATH_MASK = 0x08
-const UF_QUERY_MASK = 0x10
-const UF_FRAGMENT_MASK = 0x20
-const UF_USERINFO_MASK = 0x40
-
-@inline function Base.getindex(A::Vector{T}, i::http_parser_url_fields) where {T}
-    @inbounds v = A[Int(i)]
-    return v
-end
-@inline function Base.setindex!(A::Vector{T}, v::T, i::http_parser_url_fields) where {T}
-    @inbounds v = setindex!(A, v, Int(i))
-    return v
-end
-
 @enum(http_host_state,
     s_http_host_dead = 1,
     s_http_userinfo_start =2,

src/fifobuffer.jl

@@ -1,3 +1,9 @@
+module FIFOBuffers
+
+import Base.==
+
+export FIFOBuffer
+
 """
     FIFOBuffer([max::Integer])
     FIFOBuffer(string_or_bytes_vector)
@@ -233,7 +239,6 @@ function Base.write(f::FIFOBuffer, bytes::AbstractVector{UInt8})
             if current_task() == f.task
                 return 0
             else # async: block until there's room to write
-                @debug(DEBUG, @__LINE__, "FIFOBuffer write() waiting...")
                 wait(f.cond)
                 f.nb == f.len && return 0
             end
@@ -295,7 +300,6 @@ function Base.write(f::FIFOBuffer, bytes::Vector{UInt8})
             if current_task() == f.task
                 return 0
             else # async: block until there's room to write
-                @debug(DEBUG, @__LINE__, "FIFOBuffer write() waiting...")
                 wait(f.cond)
                 f.nb == f.len && return 0
             end
@@ -347,3 +351,5 @@ function Base.write(f::FIFOBuffer, bytes::Vector{UInt8})
 end
 
 Base.write(f::FIFOBuffer, str::String) = write(f, Vector{UInt8}(str))
+
+end # module

src/parser.jl

@@ -61,7 +61,7 @@ function onurl(r, bytes, i, j)
     @debug(PARSING_DEBUG, @__LINE__, i - j + 1)
     @debug(PARSING_DEBUG, @__LINE__, "'$(String(bytes[i:j]))'")
     @debug(PARSING_DEBUG, @__LINE__, r.method)
-    uri = http_parser_parse_url(bytes, i, j - i + 1, r.method == CONNECT)
+    uri = URIs.http_parser_parse_url(bytes, i, j - i + 1, r.method == CONNECT)
     @debug(PARSING_DEBUG, @__LINE__, uri)
     setfield!(r, :uri, uri)
     return
@@ -153,7 +153,7 @@ const DEFAULT_MAX_BODY = Int64(2)^32 # 4Gib
 const DEFAULT_PARSER = Parser()
 
 function parse!(r::Union{Request, Response}, parser, bytes, len=length(bytes);
-        lenient::Bool=true, host::String="", method::HTTP.Method=GET,
+        lenient::Bool=true, host::String="", method::Method=GET,
         maxuri::Int64=DEFAULT_MAX_URI, maxheader::Int64=DEFAULT_MAX_HEADER,
         maxbody::Int64=DEFAULT_MAX_BODY, maintask::Task=current_task())::Tuple{ParsingErrorCode, Bool, Bool, String}
     strict = !lenient
@@ -373,7 +373,7 @@ function parse!(r::Union{Request, Response}, parser, bytes, len=length(bytes);
             parser.content_length = ULLONG_MAX
             @errorif(!isalpha(ch), HPE_INVALID_METHOD)
 
-            r.method = HTTP.Method(0)
+            r.method = Method(0)
             parser.index = 2
 
             if ch == 'A'
@@ -481,12 +481,12 @@ function parse!(r::Union{Request, Response}, parser, bytes, len=length(bytes);
             if r.method == CONNECT
                 p_state = s_req_server_start
             end
-            p_state = parseurlchar(p_state, ch, strict)
+            p_state = URIs.parseurlchar(p_state, ch, strict)
             @errorif(p_state == s_dead, HPE_INVALID_URL)
 
         elseif @anyeq(p_state, s_req_schema, s_req_schema_slash, s_req_schema_slash_slash, s_req_server_start)
             @errorif(ch in (' ', CR, LF), HPE_INVALID_URL)
-            p_state = parseurlchar(p_state, ch, strict)
+            p_state = URIs.parseurlchar(p_state, ch, strict)
             @errorif(p_state == s_dead, HPE_INVALID_URL)
 
         elseif @anyeq(p_state, s_req_server, s_req_server_with_at, s_req_path, s_req_query_string_start,
@@ -506,7 +506,7 @@ function parse!(r::Union{Request, Response}, parser, bytes, len=length(bytes);
                 onurl(r, bytes, url_mark, p-1)
                 url_mark = 0
             else
-                p_state = parseurlchar(p_state, ch, strict)
+                p_state = URIs.parseurlchar(p_state, ch, strict)
                 @errorif(p_state == s_dead, HPE_INVALID_URL)
             end