diff --git a/src/AWS.jl b/src/AWS.jl index fe4e8c3e18..4bbc7649ed 100644 --- a/src/AWS.jl +++ b/src/AWS.jl @@ -1,7 +1,8 @@ module AWS -using Compat: Compat, @something +using Base: BufferStream using Base64 +using Compat: Compat, @something using Dates using Downloads: Downloads, Downloader, Curl using HTTP @@ -32,6 +33,7 @@ include("AWSConfig.jl") include("AWSMetadata.jl") include(joinpath("utilities", "request.jl")) +include(joinpath("utilities", "response.jl")) include(joinpath("utilities", "sign.jl")) include(joinpath("utilities", "downloads_backend.jl")) @@ -200,7 +202,7 @@ Perform a RestXML request to AWS. - `aws::AbstractAWSConfig`: AWSConfig containing credentials and other information for fulfilling the request, default value is the global configuration # Returns -- `Tuple or Dict`: If `return_headers` is passed in through `args` a Tuple containing the Headers and Response will be returned, otherwise just a Dict +- `AWS.Response`: A struct containing the response details """ function (service::RestXMLService)( request_method::String, @@ -208,7 +210,7 @@ function (service::RestXMLService)( args::AbstractDict{String,<:Any}=Dict{String,Any}(); aws_config::AbstractAWSConfig=global_aws_config(), ) - return_headers = _pop!(args, "return_headers", false) + _delete_outdated_kw_args!(args) request = Request(; _extract_common_kw_args(service, args)..., @@ -234,7 +236,7 @@ function (service::RestXMLService)( aws_config, service.endpoint_prefix, request.resource ) - return submit_request(aws_config, request; return_headers=return_headers) + return submit_request(aws_config, request) end """ @@ -253,7 +255,7 @@ Perform a Query request to AWS. - `aws::AbstractAWSConfig`: AWSConfig containing credentials and other information for fulfilling the request, default value is the global configuration # Returns -- `Tuple or Dict`: If `return_headers` is passed in through `args` a Tuple containing the Headers and Response will be returned, otherwise just a Dict +- `AWS.Response`: A struct containing the response details """ function (service::QueryService)( operation::String, @@ -261,7 +263,7 @@ function (service::QueryService)( aws_config::AbstractAWSConfig=global_aws_config(), ) POST_RESOURCE = "/" - return_headers = _pop!(args, "return_headers", false) + _delete_outdated_kw_args!(args) request = Request(; _extract_common_kw_args(service, args)..., @@ -276,7 +278,7 @@ function (service::QueryService)( args["Version"] = service.api_version request.content = HTTP.escapeuri(_flatten_query(service.signing_name, args)) - return submit_request(aws_config, request; return_headers=return_headers) + return submit_request(aws_config, request) end """ @@ -295,7 +297,7 @@ Perform a JSON request to AWS. - `aws::AbstractAWSConfig`: AWSConfig containing credentials and other information for fulfilling the request, default value is the global configuration # Returns -- `Tuple or Dict`: If `return_headers` is passed in through `args` a Tuple containing the Headers and Response will be returned, otherwise just a Dict +- `AWS.Response`: A struct containing the response details """ function (service::JSONService)( operation::String, @@ -303,7 +305,7 @@ function (service::JSONService)( aws_config::AbstractAWSConfig=global_aws_config(), ) POST_RESOURCE = "/" - return_headers = _pop!(args, "return_headers", false) + _delete_outdated_kw_args!(args) request = Request(; _extract_common_kw_args(service, args)..., @@ -316,7 +318,7 @@ function (service::JSONService)( request.headers["Content-Type"] = "application/x-amz-json-$(service.json_version)" request.headers["X-Amz-Target"] = "$(service.target).$(operation)" - return submit_request(aws_config, request; return_headers=return_headers) + return submit_request(aws_config, request) end """ @@ -336,7 +338,7 @@ Perform a RestJSON request to AWS. - `aws::AbstractAWSConfig`: AWSConfig containing credentials and other information for fulfilling the request, default value is the global configuration # Returns -- `Tuple or Dict`: If `return_headers` is passed in through `args` a Tuple containing the Headers and Response will be returned, otherwise just a Dict +- `AWS.Response`: A struct containing the response details """ function (service::RestJSONService)( request_method::String, @@ -344,7 +346,7 @@ function (service::RestJSONService)( args::AbstractDict{String,<:Any}=Dict{String,String}(); aws_config::AbstractAWSConfig=global_aws_config(), ) - return_headers = _pop!(args, "return_headers", false) + _delete_outdated_kw_args!(args) request = Request(; _extract_common_kw_args(service, args)..., @@ -363,7 +365,7 @@ function (service::RestJSONService)( request.headers["Content-Type"] = "application/json" request.content = json(args) - return submit_request(aws_config, request; return_headers=return_headers) + return submit_request(aws_config, request) end function __init__() diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index e3eb6b20b6..be39003b70 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -243,7 +243,7 @@ function ec2_instance_credentials(profile::AbstractString) path = dot_aws_config_file() ini = Inifile() if isfile(path) - ini = read(ini, path) + ini = Base.read(ini, path) end # Any profile except default must specify the credential_source as Ec2InstanceMetadata. @@ -373,7 +373,7 @@ function dot_aws_credentials(profile=nothing) credential_file = @mock dot_aws_credentials_file() if isfile(credential_file) - ini = read(Inifile(), credential_file) + ini = Base.read(Inifile(), credential_file) p = @something profile _aws_get_profile() access_key, secret_key, token = _aws_get_credential_details(p, ini) @@ -404,7 +404,7 @@ function dot_aws_config(profile=nothing) config_file = @mock dot_aws_config_file() if isfile(config_file) - ini = read(Inifile(), config_file) + ini = Base.read(Inifile(), config_file) p = @something profile _aws_get_profile() access_key, secret_key, token = _aws_get_credential_details(p, ini) diff --git a/src/AWSExceptions.jl b/src/AWSExceptions.jl index 9566f3bce2..db76e0c710 100644 --- a/src/AWSExceptions.jl +++ b/src/AWSExceptions.jl @@ -5,6 +5,8 @@ using JSON using XMLDict using XMLDict: XMLDictElement +const BODY_STREAMED_PLACEHOLDER = b"[Message Body was streamed]" + export AWSException, ProtocolNotDefined, InvalidFileName, NoCredentials struct ProtocolNotDefined <: Exception @@ -27,18 +29,38 @@ struct AWSException <: Exception message::String info::Union{XMLDictElement,Dict,String,Nothing} cause::HTTP.StatusError + streamed_body::Union{String,Nothing} end + function Base.show(io::IO, e::AWSException) - message = isempty(e.message) ? "" : (" -- " * e.message) - return println(io, "AWSException: ", string(e.code, message, "\n", e.cause)) + print(io, AWSException, ": ", e.code) + !isempty(e.message) && print(io, " -- ", e.message) + print(io, "\n\n", e.cause) + + if e.streamed_body !== nothing + print(io, "\n\n") + if isempty(e.streamed_body) + printstyled(io, "(empty body)"; bold=true) + else + print(io, e.streamed_body) + end + end + + println(io) + return nothing end -http_message(e::HTTP.StatusError) = String(copy(e.response.body)) http_status(e::HTTP.StatusError) = e.status content_type(e::HTTP.StatusError) = HTTP.header(e.response, "Content-Type") is_valid_xml_string(str) = startswith(str, '<') -function AWSException(e::HTTP.StatusError) +AWSException(e::HTTP.StatusError) = AWSException(e, String(copy(e.response.body))) + +function AWSException(e::HTTP.StatusError, body::IO) + return AWSException(e, String(take!(body))) +end + +function AWSException(e::HTTP.StatusError, body::AbstractString) code = string(http_status(e)) message = "AWSException" info = Dict{String,Dict}() @@ -50,7 +72,7 @@ function AWSException(e::HTTP.StatusError) # Extract API error code from JSON error message... if occursin(r"^application/x-amz-json-1\.[01]$", content_type(e)) - info = JSON.parse(http_message(e)) + info = JSON.parse(body) if haskey(info, "__type") code = rsplit(info["__type"], '#'; limit=2)[end] end @@ -58,8 +80,8 @@ function AWSException(e::HTTP.StatusError) # Extract API error code from XML error message... error_content_types = ["", "application/xml", "text/xml"] - if content_type(e) in error_content_types && is_valid_xml_string(http_message(e)) - info = parse_xml(http_message(e)) + if content_type(e) in error_content_types && is_valid_xml_string(body) + info = parse_xml(body) end # There are times when Errors or Error are returned back @@ -72,7 +94,9 @@ function AWSException(e::HTTP.StatusError) message = get(info, "Message", message) message = get(info, "message", message) - return AWSException(code, message, info, e) + streamed_body = e.response.body == BODY_STREAMED_PLACEHOLDER ? body : nothing + + return AWSException(code, message, info, e, streamed_body) end end diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index f5a469c328..46a597d584 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -5,9 +5,9 @@ function _can_read_file(file_name::String) false end end -_begins_with_ec2(file_name::String) = return uppercase(String(read(file_name, 3))) == "EC2" +_begins_with_ec2(file_name::String) = uppercase(String(Base.read(file_name, 3))) == "EC2" function _ends_with_ec2(file_name::String) - return endswith(strip(uppercase(read(file_name, String))), "EC2") + return endswith(strip(uppercase(Base.read(file_name, String))), "EC2") end """ @@ -49,7 +49,7 @@ end function _aws_profile_config(config_file::AbstractString, profile) isfile(config_file) || return Dict() - return _aws_profile_config(read(Inifile(), config_file), profile) + return _aws_profile_config(Base.read(Inifile(), config_file), profile) end function _aws_profile_config(config_file::Nothing, profile) diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl index 150d185815..f2497b2300 100644 --- a/src/utilities/downloads_backend.jl +++ b/src/utilities/downloads_backend.jl @@ -35,10 +35,10 @@ body_length(x::AbstractString) = sizeof(x) read_body(x::IOBuffer) = take!(x) function read_body(x::IO) close(x) - return read(x) + return Base.read(x) end -function _http_request(backend::DownloadsBackend, request) +function _http_request(backend::DownloadsBackend, request::Request, response_stream::IO) # If we pass `output`, older versions of Downloads.jl will # expect a message body in the response. Specifically, it sets # @@ -55,23 +55,16 @@ function _http_request(backend::DownloadsBackend, request) # Therefore, we do not pass an `output` when the `request_method` is `HEAD`. # (Note: this is fixed on the latest Downloads.jl, but we include this workaround # for compatability). - if request.response_stream === nothing - request.response_stream = IOBuffer() - end output_arg = if request.request_method == "HEAD" NamedTuple() else - (; output=request.response_stream) + (; output=response_stream) end # If we're going to return the stream, we don't want to read the body into an # HTTP.Response we're never going to use. If we do that, the returned stream # will have no data available (and reading from it could hang forever). - body_arg = if request.request_method == "HEAD" || request.return_stream - () -> NamedTuple() - else - () -> (; body=read_body(request.response_stream)) - end + body_arg = () -> NamedTuple() # HTTP.jl sets this header automatically. request.headers["Content-Length"] = string(body_length(request.content)) @@ -122,7 +115,9 @@ function _http_request(backend::DownloadsBackend, request) ), ) end - return http_response + + seekstart(response_stream) + return AWS.Response(http_response, response_stream) catch e @delay_retry if ( (isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500) || diff --git a/src/utilities/request.jl b/src/utilities/request.jl index b75bee0f9e..3d558f4355 100644 --- a/src/utilities/request.jl +++ b/src/utilities/request.jl @@ -19,7 +19,6 @@ function HTTPBackend(; kwargs...) HTTPBackend(LittleDict(kwargs)) end end - # populated in `__init__` const DEFAULT_BACKEND = Ref{AbstractBackend}() @@ -33,16 +32,12 @@ Base.@kwdef mutable struct Request resource::String = "" url::String = "" - return_stream::Bool = false - response_stream::Union{<:IO,Nothing} = nothing http_options::AbstractDict{Symbol,<:Any} = LittleDict{Symbol,String}() - return_raw::Bool = false - response_dict_type::Type{<:AbstractDict} = LittleDict backend::AbstractBackend = DEFAULT_BACKEND[] end """ - submit_request(aws::AbstractAWSConfig, request::Request; return_headers::Bool=false) + submit_request(aws::AbstractAWSConfig, request::Request) Submit the request to AWS. @@ -50,15 +45,10 @@ Submit the request to AWS. - `aws::AbstractAWSConfig`: AWSConfig containing credentials and other information for fulfilling the request, default value is the global configuration - `request::Request`: All the information about making a request to AWS -# Keywords -- `return_headers::Bool=false`: True if you want the headers from the response returned back - # Returns -- `Tuple or Dict`: Tuple if returning_headers, otherwise just return a Dict of the response body +- `AWS.Response`: A struct containing the response details """ -function submit_request( - aws::AbstractAWSConfig, request::Request; return_headers::Bool=false -) +function submit_request(aws::AbstractAWSConfig, request::Request) response = nothing TOO_MANY_REQUESTS = 429 EXPIRED_ERROR_CODES = ["ExpiredToken", "ExpiredTokenException", "RequestExpired"] @@ -78,10 +68,13 @@ function submit_request( request.headers["User-Agent"] = user_agent[] request.headers["Host"] = HTTP.URI(request.url).host + # TODO: Why did we originally use Base.BufferStream for streaming data? + stream = IOBuffer() + @repeat 3 try credentials(aws) === nothing || sign!(aws, request) - response = @mock _http_request(request.backend, request) + response = @mock _http_request(request.backend, request, stream) if response.status in REDIRECT_ERROR_CODES if HTTP.header(response, "Location") != "" @@ -93,7 +86,7 @@ function submit_request( end catch e if e isa HTTP.StatusError - e = AWSException(e) + e = AWSException(e, stream) end @retry if :message in fieldnames(typeof(e)) && @@ -132,77 +125,28 @@ function submit_request( end end - response_dict_type = request.response_dict_type - - # For HEAD request, return headers... - if request.request_method == "HEAD" - return response_dict_type(response.headers) - end - - # Return response stream if requested... - if request.return_stream - return request.response_stream - end - - # Return raw data if requested... - if request.return_raw - return (return_headers ? (response.body, response.headers) : response.body) - end - - # Parse response data according to mimetype... - mime = HTTP.header(response, "Content-Type", "") - - if isempty(mime) - if length(response.body) > 5 && response.body[1:5] == b" ""] + @testset "invalid content type" begin + headers = Pair["Content-Type" => ""] + body = "" - response = Patches._response(; headers=expected_headers, body=expected_body) - apply(Patches._aws_http_request_patch(response)) do - @testset "body" begin - result = AWS.submit_request(aws, request) - - @test String(result) == expected_body - end - - @testset "body and headers" begin - body, headers = AWS.submit_request( - aws, request; return_headers=true - ) - - @test String(body) == expected_body - @test headers == expected_headers - end - end + r = Patches._response(; headers=headers, body=body) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) end - @testset "text/xml" begin - expected_body_type = LittleDict{Union{Symbol,String},Any} - expected_body = _expected_body(Patches.body, expected_body_type) - expected_header_type = LittleDict{SubString{String},SubString{String}} - expected_headers = Pair["Content-Type" => "text/xml"] - - response = Patches._response(; headers=expected_headers) - apply(Patches._aws_http_request_patch(response)) do - @testset "body" begin - result = AWS.submit_request(aws, request) - - @test result isa expected_body_type - @test result == expected_body - end - - @testset "body and headers" begin - body, headers = AWS.submit_request( - aws, request; return_headers=true - ) - - @test body == expected_body - @test body isa expected_body_type - - @test headers == LittleDict(expected_headers) - @test headers isa expected_header_type - end - end - end + @test_throws MethodError AWS.read(response) end - @testset "xml" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - ) + @testset "text/xml" begin + headers = Pair["Content-Type" => "text/xml"] + expected_body_type = OrderedDict{Union{String,Symbol},Any} + expected_body = _expected_xml(Patches.body, expected_body_type) - expected_body_type = LittleDict{Union{Symbol,String},Any} - expected_body = _expected_body(Patches.body, expected_body_type) - expected_headers = Pair["Content-Type" => "application/xml"] - - response = Patches._response(; headers=expected_headers) - apply(Patches._aws_http_request_patch(response)) do - @testset "body" begin - result = AWS.submit_request(aws, request) - - @test result == expected_body - @test result isa expected_body_type - end + r = Patches._response(; headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) + end - @testset "body and headers" begin - body, headers = AWS.submit_request(aws, request; return_headers=true) + content = AWS.read(response) + @test content isa expected_body_type + @test content == expected_body + end - @test body == expected_body - @test body isa expected_body_type + @testset "application/xml" begin + headers = Pair["Content-Type" => "application/xml"] + expected_body_type = OrderedDict{Union{String,Symbol},Any} + expected_body = _expected_xml(Patches.body, expected_body_type) - @test headers == LittleDict(expected_headers) - @test headers isa LittleDict{SubString{String},SubString{String}} - end + r = Patches._response(; headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) end + + content = AWS.read(response) + @test content isa expected_body_type + @test content == expected_body end - @testset "JSON" begin - json_headers = ["Content-Type" => "application/json"] - body = Dict{String,Any}( - "Marker" => nothing, - "VaultList" => Any[Dict{String,Any}( - "VaultName" => "test", - "SizeInBytes" => 0, - "NumberOfArchives" => 0, - "CreationDate" => "2020-06-22T03:14:41.754Z", - "VaultARN" => "arn:aws:glacier:us-east-1:000:vaults/test", - "LastInventoryDate" => nothing, - )], + @testset "application/json" begin + headers = ["Content-Type" => "application/json"] + body = JSON.json( + Dict{String,Any}( + "Marker" => nothing, + "VaultList" => Any[Dict{String,Any}( + "VaultName" => "test", + "SizeInBytes" => 0, + "NumberOfArchives" => 0, + "CreationDate" => "2020-06-22T03:14:41.754Z", + "VaultARN" => "arn:aws:glacier:us-east-1:000:vaults/test", + "LastInventoryDate" => nothing, + )], + ), ) - json_body = JSON.json(body) - - expected_body_type = LittleDict{String,Any} - expected_body = JSON.parse(json_body; dicttype=LittleDict) - response = Patches._response(; body=json_body, headers=json_headers) - apply(Patches._aws_http_request_patch(response)) do - @testset "body" begin - result = AWS.submit_request(aws, request) + expected_body_type = OrderedDict{String,Any} + expected_body = JSON.parse(body; dicttype=expected_body_type) - @test result isa expected_body_type - @test result == expected_body - end - - @testset "body and headers" begin - body, headers = AWS.submit_request(aws, request; return_headers=true) - - @test body == expected_body - @test body isa expected_body_type - - @test headers == LittleDict(json_headers) - @test headers isa LittleDict{SubString{String},SubString{String}} - end + r = Patches._response(; body=body, headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) end - end - @testset "Text" begin - request = Request(; - service="s3", - api_version="api_version", - request_method="GET", - url="https://s3.us-east-1.amazonaws.com/sample-bucket", - ) + content = AWS.read(response) + @test content isa expected_body_type + @test content == expected_body + end - expected_headers = ["Content-Type" => "text/html"] + @testset "text/html" begin + headers = ["Content-Type" => "text/html"] expected_body = Patches.body - expected_header_type = LittleDict{SubString{String},SubString{String}} - response = Patches._response(; headers=expected_headers) - apply(Patches._aws_http_request_patch(response)) do - @testset "body" begin - result = AWS.submit_request(aws, request) - - @test result isa String - @test result == expected_body - end - - @testset "body and headers" begin - body, headers = AWS.submit_request(aws, request; return_headers=true) - - @test body == expected_body - @test body isa String - - @test headers == LittleDict(expected_headers) - @test headers isa expected_header_type - end + r = Patches._response(; headers=headers) + response = apply(Patches._aws_http_request_patch(r)) do + AWS.submit_request(aws, request) end + + content = AWS.read(response) + @test content isa String + @test content == expected_body end end end @@ -368,7 +278,7 @@ struct TestBackend <: AWS.AbstractBackend param::Int end -function AWS._http_request(backend::TestBackend, request) +function AWS._http_request(backend::TestBackend, ::AWS.Request, ::IO) return backend.param end @@ -380,28 +290,30 @@ end url="https://s3.us-east-1.amazonaws.com/sample-bucket", backend=AWS.HTTPBackend(), ) - apply(Patches._http_options_patch) do + io = IOBuffer() + + apply(Patches._http_options_patches) do # No default options - @test isempty(AWS._http_request(request.backend, request)) + @test isempty(AWS._http_request(request.backend, request, io)) # We can pass HTTP options via the backend custom_backend = AWS.HTTPBackend(Dict(:connection_limit => 5)) @test custom_backend isa AWS.AbstractBackend - @test AWS._http_request(custom_backend, request) == Dict(:connection_limit => 5) + @test AWS._http_request(custom_backend, request, io) == Dict(:connection_limit => 5) # We can pass options per-request request.http_options = Dict(:pipeline_limit => 20) - @test AWS._http_request(request.backend, request) == Dict(:pipeline_limit => 20) - @test AWS._http_request(custom_backend, request) == + @test AWS._http_request(request.backend, request, io) == Dict(:pipeline_limit => 20) + @test AWS._http_request(custom_backend, request, io) == Dict(:pipeline_limit => 20, :connection_limit => 5) # per-request options override backend options: custom_backend = AWS.HTTPBackend(Dict(:pipeline_limit => 5)) - @test AWS._http_request(custom_backend, request) == Dict(:pipeline_limit => 20) + @test AWS._http_request(custom_backend, request, io) == Dict(:pipeline_limit => 20) end request.backend = TestBackend(2) - @test AWS._http_request(request.backend, request) == 2 + @test AWS._http_request(request.backend, request, io) == 2 request = Request(; service="s3", @@ -410,7 +322,7 @@ end url="https://s3.us-east-1.amazonaws.com/sample-bucket", backend=TestBackend(4), ) - @test AWS._http_request(request.backend, request) == 4 + @test AWS._http_request(request.backend, request, io) == 4 # Let's test setting the default backend prev_backend = AWS.DEFAULT_BACKEND[] @@ -422,7 +334,7 @@ end request_method="GET", url="https://s3.us-east-1.amazonaws.com/sample-bucket", ) - @test AWS._http_request(request.backend, request) == 3 + @test AWS._http_request(request.backend, request, io) == 3 finally AWS.DEFAULT_BACKEND[] = prev_backend end @@ -541,6 +453,27 @@ end end end +@testset "STS" begin + @testset "high-level" begin + @service STS + + response = STS.get_caller_identity() + d = r["GetCallerIdentityResult"] + + @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) + @test startswith(d["Arn"], "arn:aws:sts:") + @test all(isdigit, d["Account"]) + end + + @testset "low-level" begin + response = AWSServices.sts("GetCallerIdentity") + + @test Set(keys(d)) == Set(["Arn", "UserId", "Account"]) + @test startswith(d["Arn"], "arn:aws:sts:") + @test all(isdigit, d["Account"]) + end +end + @testset "json" begin @testset "high-level secrets manager" begin @service Secrets_Manager @@ -839,10 +772,8 @@ end @test length([result["Contents"]]) == max_keys # GET with an IO target - mktemp() do f, io - S3.get_object(bucket_name, file_name, Dict("response_stream" => io)) - @test read(f, String) == body - end + response = S3.get_object(bucket_name, file_name) + @test read(response.body, String) == body finally # DELETE with parameters operation S3.delete_object(bucket_name, file_name) diff --git a/test/issues.jl b/test/issues.jl index 469e66be4c..002e00f4f9 100644 --- a/test/issues.jl +++ b/test/issues.jl @@ -54,6 +54,7 @@ end end end +# TODO: Evaluate usefulness of this test @testset "issue 324" begin body = "Hello World!" file_name = "streaming.bin" @@ -61,20 +62,13 @@ end try S3.create_bucket(bucket_name) S3.put_object(bucket_name, file_name, Dict("body" => body)) - resp = S3.get_object(bucket_name, file_name) - @test String(resp) == body + response = S3.get_object(bucket_name, file_name) + @test String(response) == body # ERROR: MethodError: no method matching iterate(::Base.BufferStream) # => BUG: header `response_stream` is pushed into the query... - io = Base.BufferStream() - S3.get_object( - bucket_name, file_name, Dict("response_stream" => io, "return_stream" => true) - ) - if bytesavailable(io) > 0 - @test String(readavailable(io)) == body - else - @test "no body data was available" == body - end + response = S3.get_object(bucket_name, file_name) + @test String(take!(response.body)) == body finally S3.delete_object(bucket_name, file_name) diff --git a/test/minio.jl b/test/minio.jl index 4194284ee5..12bcfb9a88 100644 --- a/test/minio.jl +++ b/test/minio.jl @@ -37,10 +37,8 @@ S3.put_object("anewbucket", "foo/baz", Dict("body" => "a secondnested object")) @test String(S3.get_object("anewbucket", "myobject")) == "Hi from Minio" # Test retrieving an object into a stream target -mktemp() do f, io - S3.get_object("anewbucket", "myobject", Dict("response_stream" => io)) - @test read(f, String) == "Hi from Minio" -end +response = S3.get_object("anewbucket", "myobject") +@test read(response.body, String) == "Hi from Minio" # Test listing objs = S3.list_objects_v2("anewbucket") diff --git a/test/patch.jl b/test/patch.jl index aca9d19c16..1af87a6069 100644 --- a/test/patch.jl +++ b/test/patch.jl @@ -53,13 +53,16 @@ function _response(; response.version = version response.status = status response.headers = headers - response.body = Vector{UInt8}(body) + response.body = b"[Message Body was streamed]" - return response + b = IOBuffer(body) + + return AWS.Response(response, b) end -function _aws_http_request_patch(response::HTTP.Messages.Response=_response()) - return @patch AWS._http_request(::AWS.AbstractBackend, request::Request) = response +function _aws_http_request_patch(response::AWS.Response=_response()) + p = @patch AWS._http_request(::AWS.AbstractBackend, request::Request, ::IO) = response + return p end _cred_file_patch = @patch function dot_aws_credentials_file() @@ -115,11 +118,14 @@ end # except `require_ssl_verification` and `response_stream`. This is used to # test which other options are being passed to `HTTP.Request` inside of # `_http_request`. -_http_options_patch = @patch function HTTP.request(args...; kwargs...) - options = Dict(kwargs) - delete!(options, :require_ssl_verification) - delete!(options, :response_stream) - return options -end +_http_options_patches = [ + @patch function HTTP.request(args...; kwargs...) + options = Dict(kwargs) + delete!(options, :require_ssl_verification) + delete!(options, :response_stream) + return options + end + @patch AWS.Response(options, args...) = options +] end