Introduce AWS.Response

src/AWS.jl

@@ -32,6 +32,7 @@ include("AWSConfig.jl")
 include("AWSMetadata.jl")
 
 include(joinpath("utilities", "request.jl"))
+include(joinpath("utilities", "response.jl"))
 include(joinpath("utilities", "sign.jl"))
 
 
@@ -203,6 +204,9 @@ function (service::RestXMLService)(
     aws_config::AbstractAWSConfig=global_aws_config(),
 )
     return_headers = _pop!(args, "return_headers", false)
+    return_stream = _pop!(args, "return_stream", false)
+    return_raw = _pop!(args, "return_raw", false)
+    response_stream = _pop!(args, "response_stream", nothing)
 
     request = Request(;
         _extract_common_kw_args(service, args)...,
@@ -226,7 +230,8 @@ function (service::RestXMLService)(
 
     request.url = generate_service_url(aws_config, request.service, request.resource)
 
-    return submit_request(aws_config, request; return_headers=return_headers)
+    response = submit_request(aws_config, request)
+    return legacy(response; return_headers, return_stream, return_raw, response_stream)
 end
 
 
@@ -254,6 +259,9 @@ function (service::QueryService)(
 )
     POST_RESOURCE = "/"
     return_headers = _pop!(args, "return_headers", false)
+    return_stream = _pop!(args, "return_stream", false)
+    return_raw = _pop!(args, "return_raw", false)
+    response_stream = _pop!(args, "response_stream", nothing)
 
     request = Request(;
         _extract_common_kw_args(service, args)...,
@@ -268,7 +276,8 @@ function (service::QueryService)(
     args["Version"] = service.api_version
     request.content = HTTP.escapeuri(_flatten_query(service.name, args))
 
-    return submit_request(aws_config, request; return_headers=return_headers)
+    response = submit_request(aws_config, request)
+    return legacy(response; return_headers, return_stream, return_raw, response_stream)
 end
 
 """
@@ -295,6 +304,9 @@ function (service::JSONService)(
 )
     POST_RESOURCE = "/"
     return_headers = _pop!(args, "return_headers", false)
+    return_stream = _pop!(args, "return_stream", false)
+    return_raw = _pop!(args, "return_raw", false)
+    response_stream = _pop!(args, "response_stream", nothing)
 
     request = Request(;
         _extract_common_kw_args(service,args)...,
@@ -307,7 +319,8 @@ function (service::JSONService)(
     request.headers["Content-Type"] = "application/x-amz-json-$(service.json_version)"
     request.headers["X-Amz-Target"] = "$(service.target).$(operation)"
 
-    return submit_request(aws_config, request; return_headers=return_headers)
+    response = submit_request(aws_config, request)
+    return legacy(response; return_headers, return_stream, return_raw, response_stream)
 end
 
 """
@@ -334,6 +347,9 @@ function (service::RestJSONService)(
     aws_config::AbstractAWSConfig=global_aws_config(),
 )
     return_headers = _pop!(args, "return_headers", false)
+    return_stream = _pop!(args, "return_stream", false)
+    return_raw = _pop!(args, "return_raw", false)
+    response_stream = _pop!(args, "response_stream", nothing)
 
     request = Request(;
         _extract_common_kw_args(service, args)...,
@@ -350,7 +366,8 @@ function (service::RestJSONService)(
     request.headers["Content-Type"] = "application/json"
     request.content = json(args)
 
-    return submit_request(aws_config, request; return_headers=return_headers)
+    response = submit_request(aws_config, request)
+    return legacy(response; return_headers, return_stream, return_raw, response_stream)
 end
 
 end  # module AWS

src/AWSMetadata.jl

@@ -1,3 +1,4 @@
+
 module AWSMetadata
 
 using Base64

src/utilities/request.jl

@@ -8,30 +8,24 @@ Base.@kwdef mutable struct Request
     resource::String=""
     url::String=""
 
-    return_stream::Bool=false
-    response_stream::Union{<:IO, Nothing}=nothing
     http_options::AbstractDict{Symbol,<:Any}=LittleDict{Symbol,String}()
-    return_raw::Bool=false
     response_dict_type::Type{<:AbstractDict}=LittleDict
 end
 
 
 """
-    submit_request(aws::AbstractAWSConfig, request::Request; return_headers::Bool=false)
+    submit_request(aws::AbstractAWSConfig, request::Request)
 
 Submit the request to AWS.
 
 # Arguments
 - `aws::AbstractAWSConfig`: AWSConfig containing credentials and other information for fulfilling the request, default value is the global configuration
 - `request::Request`: All the information about making a request to AWS
 
-# Keywords
-- `return_headers::Bool=false`: True if you want the headers from the response returned back
-
 # Returns
-- `Tuple or Dict`: Tuple if returning_headers, otherwise just return a Dict of the response body
+- `Response`: A struct containing the response
 """
-function submit_request(aws::AbstractAWSConfig, request::Request; return_headers::Bool=false)
+function submit_request(aws::AbstractAWSConfig, request::Request)
     response = nothing
     TOO_MANY_REQUESTS = 429
     EXPIRED_ERROR_CODES = ["ExpiredToken", "ExpiredTokenException", "RequestExpired"]
@@ -91,69 +85,28 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
         end
     end
 
-    response_dict_type = request.response_dict_type
-
-    # For HEAD request, return headers...
-    if request.request_method == "HEAD"
-        return response_dict_type(response.headers)
-    end
-
-    # Return response stream if requested...
-    if request.return_stream
-        return request.response_stream
-    end
-
-    # Return raw data if requested...
-    if request.return_raw
-        return (return_headers ? (response.body, response.headers) : response.body)
-    end
-
-    # Parse response data according to mimetype...
-    mime = HTTP.header(response, "Content-Type", "")
-
-    if isempty(mime)
-        if length(response.body) > 5 && response.body[1:5] == b"<?xml"
-            mime = "text/xml"
-        end
-    end
-
-    body = String(copy(response.body))
-
-    if occursin(r"/xml", mime)
-        xml_dict_type = response_dict_type{Union{Symbol, String}, Any}
-        body = parse_xml(body)
-        root = XMLDict.root(body.x)
-
-        return (return_headers ? (xml_dict(root, xml_dict_type), response_dict_type(response.headers)) : xml_dict(root, xml_dict_type))
-    elseif occursin(r"/x-amz-json-1.[01]$", mime) || endswith(mime, "json")
-        info = isempty(response.body) ? nothing : JSON.parse(body, dicttype=response_dict_type)
-        return (return_headers ? (info, response_dict_type(response.headers)) : info)
-    elseif startswith(mime, "text/")
-        return (return_headers ? (body, response_dict_type(response.headers)) : body)
-    else
-        return (return_headers ? (response.body, response.headers) : response.body)
-    end
+    return response
 end
 
 
 function _http_request(request::Request)
     @repeat 4 try
         http_stack = HTTP.stack(redirect=false, retry=false, aws_authorization=false)
 
-        if request.return_stream && request.response_stream === nothing
-            request.response_stream = Base.BufferStream()
-        end
+        response_stream = Base.BufferStream()
 
-        return HTTP.request(
+        r = HTTP.request(
             http_stack,
             request.request_method,
             HTTP.URI(request.url),
             HTTP.mkheaders(request.headers),
             request.content;
             require_ssl_verification=false,
-            response_stream=request.response_stream,
+            response_stream=response_stream,
             request.http_options...
         )
+
+        return Response(r, response_stream)
     catch e
         # Base.IOError is needed because HTTP.jl can often have errors that aren't
         # caught and wrapped in an HTTP.IOError

src/utilities/response.jl

@@ -0,0 +1,75 @@
+# TODO: Include only the fields we care about
+struct Response
+    response::HTTP.Response
+    body::BufferStream
+end
+
+function Base.getproperty(r::Response, f::Symbol)
+    if f === :headers
+        r.response.headers
+    else
+        getfield(r, f)
+    end
+end
+
+function mime_type(r::Response)
+    # Parse response data according to mimetype...
+    mime = HTTP.header(r.response, "Content-Type", "")
+
+    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
+
+    # Reading consumes the buffer so this is unsafe currently
+    # if isempty(mime) && read(r.body, 5) == b"<?xml"
+    #     "xml"
+
+    # e.g. "application/xml" or "text/xml"
+    if occursin(r"/xml", mime)
+        MIME(Symbol("application/xml"))
+    elseif occursin(r"/x-amz-json-1.[01]$", mime) || endswith(mime, "json")
+        MIME(Symbol("application/json"))
+    elseif startswith(mime, "text/")
+        MIME(Symbol("text/plain"))
+    else
+        MIME(Symbol("text/plain"))
+    end
+end
+
+
+# TODO: Interface isn't perfect. For example "text/plain" completely ignores `T`
+
+Base.parse(::Type{T}, r::Response) where T = parse(T, r, mime_type(r))
+
+function Base.parse(::Type{T}, r::Response, ::MIME"application/xml") where T <: AbstractDict
+    xml = parse_xml(String(r.body))
+    root = XMLDict.root(xml.x)  # TODO: Why x?
+    return xml_dict(root, T))
+end
+
+function Base.parse(::Type{T}, r::Response, ::MIME"application/json") where T <: AbstractDict
+    return JSON.parse(r.body, dicttype=T)
+end
+
+function Base.parse(::Type{T}, r::Response, ::MIME"text/plain") where T <: AbstractDict
+    return String(r.body)
+end
+
+function legacy(r::Response; response_dict_type::Type, return_headers=false, return_stream=false, return_raw=false, response_stream=nothing)
+    # For HEAD request, return headers...
+    if r.response.request.method == "HEAD"
+        return response_dict_type(r.response.headers)
+    end
+
+    # Return response stream if requested...
+    if return_stream
+        return write(response_stream, read(r.body))
+    end
+
+    # Return raw data if requested...
+    if return_raw
+        content = read(r.body)
+        return return_headers ? (content, r.response.headers) : content
+    end
+
+    content = parse(response_dict_type, r)
+    return return_headers ? (content, r.response.headers) : content
+end

src/utilities/utilities.jl

@@ -61,9 +61,6 @@ function _extract_common_kw_args(service, args)
     return (
         service=service.name,
         api_version=service.api_version,
-        return_stream=_pop!(args, "return_stream", false),
-        return_raw=_pop!(args, "return_raw", false),
-        response_stream=_pop!(args, "response_stream", nothing),
         headers=LittleDict{String, String}(_pop!(args, "headers", [])),
         http_options=_pop!(args, "http_options", LittleDict{Symbol, String}()),
         response_dict_type=_pop!(args, "response_dict_type", LittleDict),