Skip to content

Commit

Permalink
Merge pull request #308 from mmd-osm/patch/compressedupload
Browse files Browse the repository at this point in the history 
Added testcase for compressed changeset uploads
  • Loading branch information
@mmd-osm
mmd-osm authored Nov 20, 2023
2 parents 1613cb0 + b14a3d2 commit 341effa
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 14 deletions.
56 changes: 51 additions & 5 deletions test/test_apidb_backend_changeset_uploads.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "cgimap/routes.hpp"
#include "cgimap/process_request.hpp"
#include "cgimap/output_buffer.hpp"
#include "cgimap/zlib.hpp"

#include "cgimap/api06/changeset_upload/osmchange_handler.hpp"
#include "cgimap/api06/changeset_upload/osmchange_input_format.hpp"
Expand All @@ -24,6 +25,29 @@

namespace {

std::string get_compressed_payload()
{
std::stringstream body;
std::stringstream output;

std::string payload = R"(<?xml version="1.0" encoding="UTF-8"?>
<osmChange version="0.6" generator="iD">
<create>
<node id="-5" lon="11" lat="46" version="0" changeset="1">
<tag k="highway" v="bus_stop" />
</node>
</create>
</osmChange>)";

// gzip compress payload
test_output_buffer tob(output, body);
zlib_output_buffer zlib_ob(tob, zlib_output_buffer::gzip);
zlib_ob.write(payload.data(), payload.size());
zlib_ob.close();

return body.str();
}

template <typename T>
void assert_equal(const T& a, const T&b, const std::string &message) {
if (a != b) {
Expand Down Expand Up @@ -2369,7 +2393,7 @@ namespace {
process_request(req, limiter, generator, route, *sel_factory, upd_factory.get(), nullptr);

if (req.response_status() != 409)
std::runtime_error("Expected HTTP 409 Conflict: Cannot add more elements to changeset");
throw std::runtime_error("Expected HTTP 409 Conflict: Cannot add more elements to changeset");
}

// Try to add a node to a changeset that is already closed
Expand All @@ -2390,7 +2414,7 @@ namespace {
process_request(req, limiter, generator, route, *sel_factory, upd_factory.get(), nullptr);

if (req.response_status() != 409)
std::runtime_error("Expected HTTP 409 Conflict: Changeset already closed");
throw std::runtime_error("Expected HTTP 409 Conflict: Changeset already closed");
}

// Try to add a nodes, ways, relations to a changeset
Expand Down Expand Up @@ -2453,7 +2477,7 @@ namespace {
// std::cout << "Response was:\n----------------------\n" << req.buffer().str() << "\n";

if (req.response_status() != 200)
std::runtime_error("Expected HTTP 200 OK: Create new node");
throw std::runtime_error("Expected HTTP 200 OK: Create new node");
}

// Try to add, modify and delete nodes, ways, relations in changeset
Expand Down Expand Up @@ -2509,7 +2533,7 @@ namespace {
// std::cout << "Response was:\n----------------------\n" << req.buffer().str() << "\n";

if (req.response_status() != 200)
std::runtime_error("Expected HTTP 200 OK: add, modify and delete nodes, ways, relations in changeset");
throw std::runtime_error("Expected HTTP 200 OK: add, modify and delete nodes, ways, relations in changeset");
}

// Multiple operations on the same node id -1
Expand Down Expand Up @@ -2545,8 +2569,30 @@ namespace {
// std::cout << "Response was:\n----------------------\n" << req.buffer().str() << "\n";

if (req.response_status() != 200)
std::runtime_error("Expected HTTP 200 OK: Multiple operations on the same node id -1");
throw std::runtime_error("Expected HTTP 200 OK: Multiple operations on the same node id -1");

}

// Compressed upload
{
// set up request headers from test case
test_request req;
req.set_header("REQUEST_METHOD", "POST");
req.set_header("REQUEST_URI", "/api/0.6/changeset/1/upload");
req.set_header("HTTP_AUTHORIZATION", baseauth);
req.set_header("REMOTE_ADDR", "127.0.0.1");
req.set_header("HTTP_CONTENT_ENCODING", "gzip");

req.set_payload(get_compressed_payload());

// execute the request
process_request(req, limiter, generator, route, *sel_factory, upd_factory.get(), nullptr);

// std::cerr << "Response status: " << req.response_status() << "\n";
// std::cerr << "Response was:\n----------------------\n" << req.buffer().str() << "\n";

if (req.response_status() != 200)
throw std::runtime_error("Expected HTTP 200 OK: Compressed upload");
}

}
Expand Down
16 changes: 8 additions & 8 deletions test/test_parse_changeset_input.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ void test_changeset_xml() {
"test_changeset_xml::001: Expected exception");
} catch (http::exception &e) {
if (e.code() != 400) {
std::runtime_error("test_changeset_xml::001: Expected HTTP 400");
throw std::runtime_error("test_changeset_xml::001: Expected HTTP 400");
}
}

Expand All @@ -42,7 +42,7 @@ void test_changeset_xml() {
"test_changeset_xml::invalid:001: Expected exception");
} catch (http::exception &e) {
if (e.code() != 400) {
std::runtime_error("test_changeset_xml::invalid:001: Expected HTTP 400");
throw std::runtime_error("test_changeset_xml::invalid:001: Expected HTTP 400");
}
}

Expand All @@ -53,7 +53,7 @@ void test_changeset_xml() {
"test_changeset_xml::invalid:002: Expected exception");
} catch (http::exception &e) {
if (e.code() != 400) {
std::runtime_error("test_changeset_xml::invalid:002: Expected HTTP 400");
throw std::runtime_error("test_changeset_xml::invalid:002: Expected HTTP 400");
}
}

Expand All @@ -64,7 +64,7 @@ void test_changeset_xml() {
"test_changeset_xml::invalid:003 Expected exception");
} catch (http::exception &e) {
if (e.code() != 400) {
std::runtime_error("test_changeset_xml::invalid:003: Expected HTTP 400");
throw std::runtime_error("test_changeset_xml::invalid:003: Expected HTTP 400");
}
}

Expand All @@ -75,7 +75,7 @@ void test_changeset_xml() {
"test_changeset_xml::invalid:004 Expected exception");
} catch (http::exception &e) {
if (e.code() != 400) {
std::runtime_error("test_changeset_xml::invalid:004 Expected HTTP 400");
throw std::runtime_error("test_changeset_xml::invalid:004 Expected HTTP 400");
}
}

Expand All @@ -86,7 +86,7 @@ void test_changeset_xml() {
"test_changeset_xml::invalid:005 Expected exception");
} catch (http::exception &e) {
if (e.code() != 400) {
std::runtime_error("test_changeset_xml::invalid:005: Expected HTTP 400");
throw std::runtime_error("test_changeset_xml::invalid:005: Expected HTTP 400");
}
}

Expand All @@ -97,7 +97,7 @@ void test_changeset_xml() {
"test_changeset_xml::invalid:006 Expected exception");
} catch (http::exception &e) {
if (e.code() != 400) {
std::runtime_error("test_changeset_xml::invalid:006: Expected HTTP 400");
throw std::runtime_error("test_changeset_xml::invalid:006: Expected HTTP 400");
}
}

Expand All @@ -108,7 +108,7 @@ void test_changeset_xml() {
"test_changeset_xml::invalid:007: Expected exception");
} catch (http::exception &e) {
if (e.code() != 400) {
std::runtime_error("test_changeset_xml::invalid:007: Expected HTTP 400");
throw std::runtime_error("test_changeset_xml::invalid:007: Expected HTTP 400");
}
}

Expand Down
47 changes: 46 additions & 1 deletion test/test_request.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
#include "test_request.hpp"
#include "cgimap/options.hpp"
#include "cgimap/request_helpers.hpp"

#include <cassert>
#include <fmt/core.h>

test_output_buffer::test_output_buffer(std::ostream &out, std::ostream &body)
: m_out(out), m_body(body), m_written(0) {
Expand Down Expand Up @@ -41,7 +43,50 @@ const char *test_request::get_param(const char *key) const {
}

const std::string test_request::get_payload() {
return m_payload;


// TODO: still a bit too much duplication from fcgi_request.cpp::get_payload

const unsigned int BUFFER_LEN = 512000;

// fetch and parse the content length
const char *content_length_str = m_params.find("CONTENT_LENGTH") != m_params.end() ? m_params["CONTENT_LENGTH"].c_str() : nullptr;
const char *content_encoding = m_params.find("HTTP_CONTENT_ENCODING") != m_params.end() ? m_params["HTTP_CONTENT_ENCODING"].c_str() : nullptr;

auto content_encoding_handler = http::get_content_encoding_handler(
std::string(content_encoding == nullptr ? "" : content_encoding));

unsigned long content_length = 0;
unsigned long curr_content_length = 0;
unsigned long result_length = 0;

if (content_length_str)
content_length = http::parse_content_length(content_length_str);

std::array<char, BUFFER_LEN> content_buffer{};

std::string result;

std::string content(m_payload);
result_length += content.length();

// Decompression according to Content-Encoding header (null op, if header is not set)
try {
std::string content_decompressed = content_encoding_handler->decompress(content);
result += content_decompressed;
} catch (std::bad_alloc& e) {
throw http::server_error("Decompression failed due to memory issue");
} catch (std::runtime_error& e) {
throw http::bad_request("Payload cannot be decompressed according to Content-Encoding");
}

if (result.length() > global_settings::get_payload_max_size())
throw http::payload_too_large(fmt::format("Payload exceeds limit of {:d} bytes", global_settings::get_payload_max_size()));

if (content_length > 0 && result_length != content_length)
throw http::server_error("HTTP Header field 'Content-Length' differs from actual payload length");

return result;
}

void test_request::set_payload(const std::string& payload) {
Expand Down

0 comments on commit 341effa

Please sign in to comment.