Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[breakchange][feat] async_upload support offset #621

Merged
merged 1 commit into from
Aug 8, 2024
+111 −19
Merged

[breakchange][feat] async_upload support offset #621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 24 additions & 8 deletions include/cinatra/coro_http_client.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
: coro_http_client(executor->get_asio_executor()) {}

bool init_config(const config &conf) {
config_ = conf;
if (conf.conn_timeout_duration.has_value()) {
set_conn_timeout(*conf.conn_timeout_duration);
}
Expand Down Expand Up @@ -207,6 +208,8 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {

coro_io::ExecutorWrapper<> &get_executor() { return executor_wrapper_; }

const config &get_config() { return config_; }

#ifdef CINATRA_ENABLE_SSL
bool init_ssl(int verify_mode, const std::string &base_path,
const std::string &cert_file, const std::string &sni_hostname) {
Expand Down Expand Up @@ -875,14 +878,16 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
}

async_simple::coro::Lazy<void> send_file_no_chunked_with_copy(
std::string_view source, std::error_code &ec, std::size_t length) {
std::string_view source, std::error_code &ec, std::size_t length,
std::size_t offset) {
if (length <= 0) {
co_return;
}
std::string file_data;
detail::resize(file_data, std::min(max_single_part_size_, length));
coro_io::coro_file file{};
file.open(source, std::ios::in);
file.seek(offset, std::ios::cur);
if (!file.is_open()) {
ec = std::make_error_code(std::errc::bad_file_descriptor);
co_return;
Expand Down Expand Up @@ -920,15 +925,15 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
};
async_simple::coro::Lazy<void> send_file_without_copy(
const std::filesystem::path &source, std::error_code &ec,
std::size_t length) {
std::size_t length, std::size_t offset) {
fd_guard guard(source.c_str());
if (guard.fd < 0) [[unlikely]] {
ec = std::make_error_code(std::errc::bad_file_descriptor);
co_return;
}
std::size_t actual_len = 0;
std::tie(ec, actual_len) =
co_await coro_io::async_sendfile(socket_->impl_, guard.fd, 0, length);
std::tie(ec, actual_len) = co_await coro_io::async_sendfile(
socket_->impl_, guard.fd, offset, length);
if (ec) [[unlikely]] {
co_return;
}
Expand Down Expand Up @@ -1006,7 +1011,8 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
template <typename S, typename Source>
async_simple::coro::Lazy<resp_data> async_upload(
S uri, http_method method, Source source /* file */,
int64_t content_length = -1,
uint64_t offset = 0 /*file offset*/,
int64_t content_length = -1 /*upload size*/,
req_content_type content_type = req_content_type::text,
std::unordered_map<std::string, std::string> headers = {}) {
std::error_code ec{};
Expand Down Expand Up @@ -1050,6 +1056,12 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
co_return resp_data{std::make_error_code(std::errc::invalid_argument),
404};
}
content_length -= offset;
if (content_length < 0) {
CINATRA_LOG_ERROR << "the offset is larger than the end of file";
co_return resp_data{std::make_error_code(std::errc::invalid_argument),
404};
}
}

assert(content_length >= 0);
Expand Down Expand Up @@ -1088,6 +1100,7 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
}

if constexpr (is_stream_file) {
source->seekg(offset, std::ios::cur);
std::string file_data;
detail::resize(file_data, std::min<std::size_t>(max_single_part_size_,
content_length));
Expand Down Expand Up @@ -1116,15 +1129,17 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
if (!has_init_ssl_) {
#endif
co_await send_file_without_copy(std::filesystem::path{source}, ec,
content_length);
content_length, offset);
#ifdef CINATRA_ENABLE_SSL
}
else {
co_await send_file_no_chunked_with_copy(source, ec, content_length);
co_await send_file_no_chunked_with_copy(source, ec, content_length,
offset);
}
#endif
#else
co_await send_file_no_chunked_with_copy(source, ec, content_length);
co_await send_file_no_chunked_with_copy(source, ec, content_length,
offset);
#endif
}
else {
Expand Down Expand Up @@ -2432,6 +2447,7 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
std::string resp_chunk_str_;
std::span<char> out_buf_;
bool should_reset_ = false;
config config_;

#ifdef CINATRA_ENABLE_GZIP
bool enable_ws_deflate_ = false;
Expand Down
98 changes: 87 additions & 11 deletions tests/test_cinatra.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,10 @@ bool create_file(std::string_view filename, size_t file_size = 1024) {
return false;
}

std::string str(file_size, 'A');
std::string str;
for (int i = 0; i < file_size; ++i) {
str.push_back(rand() % 26 + 'A');
}
out.write(str.data(), str.size());
return true;
}
Expand Down Expand Up @@ -1146,40 +1149,45 @@ TEST_CASE("test coro_http_client multipart upload") {

TEST_CASE("test coro_http_client upload") {
auto test_upload_by_file_path = [](std::string filename,
std::size_t offset = 0,
std::size_t r_size = SIZE_MAX,
bool should_failed = false) {
coro_http_client client{};
client.add_header("filename", filename);
client.add_header("offset", std::to_string(offset));
if (r_size != SIZE_MAX)
client.add_header("filesize", std::to_string(r_size));
std::string uri = "http://127.0.0.1:8090/upload";
cinatra::resp_data result;
if (r_size != SIZE_MAX) {
auto lazy = client.async_upload(uri, http_method::PUT, filename, r_size);
auto lazy =
client.async_upload(uri, http_method::PUT, filename, offset, r_size);
result = async_simple::coro::syncAwait(lazy);
}
else {
auto lazy = client.async_upload(uri, http_method::PUT, filename);
auto lazy = client.async_upload(uri, http_method::PUT, filename, offset);
result = async_simple::coro::syncAwait(lazy);
}
CHECK(((result.status == 200) ^ should_failed));
};
auto test_upload_by_stream = [](std::string filename,
auto test_upload_by_stream = [](std::string filename, std::size_t offset = 0,
std::size_t r_size = SIZE_MAX,
bool should_failed = false) {
coro_http_client client{};
client.add_header("filename", filename);
client.add_header("offset", std::to_string(offset));
if (r_size != SIZE_MAX)
client.add_header("filesize", std::to_string(r_size));
std::string uri = "http://127.0.0.1:8090/upload";
std::ifstream ifs(filename, std::ios::binary);
cinatra::resp_data result;
if (r_size != SIZE_MAX) {
auto lazy = client.async_upload(uri, http_method::PUT, filename, r_size);
auto lazy =
client.async_upload(uri, http_method::PUT, filename, offset, r_size);
result = async_simple::coro::syncAwait(lazy);
}
else {
auto lazy = client.async_upload(uri, http_method::PUT, filename);
auto lazy = client.async_upload(uri, http_method::PUT, filename, offset);
result = async_simple::coro::syncAwait(lazy);
}
CHECK(((result.status == 200) ^ should_failed));
Expand All @@ -1189,6 +1197,7 @@ TEST_CASE("test coro_http_client upload") {
bool should_failed = false) {
coro_http_client client{};
client.add_header("filename", filename);
client.add_header("offset", "0");
if (r_size != SIZE_MAX)
client.add_header("filesize", std::to_string(r_size));
std::string uri = "http://127.0.0.1:8090/upload";
Expand All @@ -1210,7 +1219,7 @@ TEST_CASE("test coro_http_client upload") {
}
else {
auto lazy =
client.async_upload(uri, http_method::PUT, async_read, r_size);
client.async_upload(uri, http_method::PUT, async_read, 0, r_size);
result = async_simple::coro::syncAwait(lazy);
CHECK(((result.status == 200) ^ should_failed));
}
Expand All @@ -1231,15 +1240,31 @@ TEST_CASE("test coro_http_client upload") {
file.write(req.get_body().data(), req.get_body().size());
file.flush();
file.close();

size_t offset = 0;
std::string offset_s = std::string{req.get_header_value("offset")};
if (!offset_s.empty()) {
offset = stoull(offset_s);
}

std::string filesize = std::string{req.get_header_value("filesize")};

if (!filesize.empty()) {
sz = stoull(filesize);
}
else {
sz = std::filesystem::file_size(oldpath);
sz -= offset;
}

CHECK(!filename.empty());
CHECK(sz == std::filesystem::file_size(newpath));
std::ifstream ifs(oldpath);
ifs.seekg(offset, std::ios::cur);
std::string str;
str.resize(sz);
ifs.read(str.data(), sz);
CHECK(str == req.get_body());
resp.set_status_and_content(status_type::ok, std::string(filename));
co_return;
});
Expand Down Expand Up @@ -1274,8 +1299,8 @@ TEST_CASE("test coro_http_client upload") {
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, r_size);
test_upload_by_stream(filename, r_size);
test_upload_by_file_path(filename, 0, r_size);
test_upload_by_stream(filename, 0, r_size);
test_upload_by_coro(filename, r_size);
}
}
Expand All @@ -1292,11 +1317,62 @@ TEST_CASE("test coro_http_client upload") {
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, r_size, true);
test_upload_by_stream(filename, r_size, true);
test_upload_by_file_path(filename, 0, r_size, true);
test_upload_by_stream(filename, 0, r_size, true);
test_upload_by_coro(filename, r_size, true);
}
}
// upload with offset
{
auto sizes = {std::pair{1024 * 1024, 1'000'000},
std::pair{2'000'000, 1'999'999}, std::pair{200, 1},
std::pair{100, 0}, std::pair{0, 0}};
for (auto [size, offset] : sizes) {
std::error_code ec{};
fs::remove(filename, ec);
if (ec) {
std::cout << ec << "\n";
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, offset);
test_upload_by_stream(filename, offset);
}
}
// upload with size & offset
{
auto sizes = {std::tuple{1024 * 1024, 500'000, 500'000},
std::tuple{2'000'000, 1'999'999, 1}, std::tuple{200, 1, 199},
std::tuple{100, 100, 0}};
for (auto [size, offset, r_size] : sizes) {
std::error_code ec{};
fs::remove(filename, ec);
if (ec) {
std::cout << ec << "\n";
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, offset, r_size);
test_upload_by_stream(filename, offset, r_size);
}
}
// upload with too large size & offset
{
auto sizes = {std::tuple{1024 * 1024, 1'000'000, 50'000},
std::tuple{2'000'000, 1'999'999, 2}, std::tuple{200, 1, 200},
std::tuple{100, 100, 1}};
for (auto [size, offset, r_size] : sizes) {
std::error_code ec{};
fs::remove(filename, ec);
if (ec) {
std::cout << ec << "\n";
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, offset, r_size, true);
test_upload_by_stream(filename, offset, r_size, true);
}
}
}

TEST_CASE("test coro_http_client chunked upload and download") {
Expand Down
Loading