Skip to content

Commit

Permalink
feat:add br support (#600)
Browse files Browse the repository at this point in the history
  • Loading branch information
helintongh authored Jun 24, 2024
1 parent 1e6e704 commit 929936e
Show file tree
Hide file tree
Showing 11 changed files with 249 additions and 22 deletions.
21 changes: 21 additions & 0 deletions cmake/FindBrotli.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
include(FindPackageHandleStandardArgs)

find_path(BROTLI_INCLUDE_DIR "brotli/decode.h")

find_library(BROTLICOMMON_LIBRARY NAMES brotlicommon)
find_library(BROTLIDEC_LIBRARY NAMES brotlidec)
find_library(BROTLIENC_LIBRARY NAMES brotlienc)

find_package_handle_standard_args(Brotli
FOUND_VAR
BROTLI_FOUND
REQUIRED_VARS
BROTLIDEC_LIBRARY
BROTLICOMMON_LIBRARY
BROTLI_INCLUDE_DIR
FAIL_MESSAGE
"Could NOT find Brotli"
)

set(BROTLI_INCLUDE_DIRS ${BROTLI_INCLUDE_DIR})
set(BROTLI_LIBRARIES ${BROTLIDEC_LIBRARY} ${BROTLIENC_LIBRARY} ${BROTLICOMMON_LIBRARY})
12 changes: 12 additions & 0 deletions cmake/develop.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,12 @@ if(ENABLE_METRIC_JSON)
message(STATUS "Enable serialize metric to json")
endif()

set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}")

SET(ENABLE_GZIP OFF)
SET(ENABLE_SSL OFF)
SET(ENABLE_CLIENT_SSL OFF)
SET(ENABLE_BROTLI OFF)

if (ENABLE_SSL)
add_definitions(-DCINATRA_ENABLE_SSL)
Expand Down Expand Up @@ -106,4 +109,13 @@ if (ENABLE_GZIP)
find_package(ZLIB REQUIRED)
endif()

if (ENABLE_BROTLI)
find_package(Brotli REQUIRED)
if (Brotli_FOUND)
message(STATUS "Brotli found")
add_definitions(-DCINATRA_ENABLE_BROTLI)
endif (Brotli_FOUND)
endif(ENABLE_BROTLI)


add_definitions(-DCORO_HTTP_PRINT_REQ_HEAD)
6 changes: 6 additions & 0 deletions example/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ if (ENABLE_GZIP)
target_link_libraries(benchmark PRIVATE ${ZLIB_LIBRARIES})
endif()

if (ENABLE_BROTLI)
include_directories(${BROTLI_INCLUDE_DIRS})
target_link_libraries(${project_name} ${BROTLI_LIBRARIES})
target_link_libraries(benchmark PRIVATE ${BROTLI_LIBRARIES})
endif()

if (ENABLE_SIMD STREQUAL "AARCH64")
if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
add_library(neon INTERFACE IMPORTED)
Expand Down
79 changes: 79 additions & 0 deletions include/cinatra/brzip.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
#pragma once
#include <brotli/decode.h>
#include <brotli/encode.h>

#include <array>
#include <sstream>
#include <string>
#include <string_view>

namespace cinatra::br_codec {

#define BROTLI_BUFFER_SIZE 1024

inline bool brotli_compress(std::string_view input, std::string &output) {
auto instance = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
std::array<uint8_t, BROTLI_BUFFER_SIZE> buffer;
std::stringstream result;

size_t available_in = input.size(), available_out = buffer.size();
const uint8_t *next_in = reinterpret_cast<const uint8_t *>(input.data());
uint8_t *next_out = buffer.data();

do {
int ret = BrotliEncoderCompressStream(instance, BROTLI_OPERATION_FINISH,
&available_in, &next_in,
&available_out, &next_out, nullptr);
if (!ret)
return false;
result.write(reinterpret_cast<const char *>(buffer.data()),
buffer.size() - available_out);
available_out = buffer.size();
next_out = buffer.data();
} while (!(available_in == 0 && BrotliEncoderIsFinished(instance)));

BrotliEncoderDestroyInstance(instance);
output = result.str();
return true;
}

inline bool brotli_decompress(std::string_view input,
std::string &decompressed) {
if (input.size() == 0)
return false;

size_t available_in = input.size();
auto next_in = (const uint8_t *)(input.data());
decompressed = std::string(available_in * 3, 0);
size_t available_out = decompressed.size();
auto next_out = (uint8_t *)(decompressed.data());
size_t total_out{0};
bool done = false;
auto s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
while (!done) {
auto result = BrotliDecoderDecompressStream(
s, &available_in, &next_in, &available_out, &next_out, &total_out);
if (result == BROTLI_DECODER_RESULT_SUCCESS) {
decompressed.resize(total_out);
BrotliDecoderDestroyInstance(s);
return true;
}
else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
if (total_out != decompressed.size()) {
return false;
}
decompressed.resize(total_out * 2);
next_out = (uint8_t *)(decompressed.data() + total_out);
available_out = total_out;
}
else {
decompressed.resize(0);
BrotliDecoderDestroyInstance(s);
return true;
}
}
return true;
}
} // namespace cinatra::br_codec

// namespace cinatra::br_codec
46 changes: 36 additions & 10 deletions include/cinatra/coro_http_client.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
#ifdef CINATRA_ENABLE_GZIP
#include "gzip.hpp"
#endif
#ifdef CINATRA_ENABLE_BROTLI
#include "brzip.hpp"
#endif
#include "cinatra_log_wrapper.hpp"
#include "http_parser.hpp"
#include "multipart.hpp"
Expand Down Expand Up @@ -1436,19 +1439,20 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
if (is_redirect)
redirect_uri_ = parser_.get_header_value("Location");

#ifdef CINATRA_ENABLE_GZIP
if (!parser_.get_header_value("Content-Encoding").empty()) {
if (parser_.get_header_value("Content-Encoding").find("gzip") !=
std::string_view::npos)
encoding_type_ = content_encoding::gzip;
else if (parser_.get_header_value("Content-Encoding").find("deflate") !=
std::string_view::npos)
encoding_type_ = content_encoding::deflate;
else if (parser_.get_header_value("Content-Encoding").find("br") !=
std::string_view::npos)
encoding_type_ = content_encoding::br;
}
else {
encoding_type_ = content_encoding::none;
}
#endif

size_t content_len = (size_t)parser_.body_len();
#ifdef BENCHMARK_TEST
Expand Down Expand Up @@ -1557,24 +1561,42 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
std::string_view reply(data_ptr, content_len);
#ifdef CINATRA_ENABLE_GZIP
if (encoding_type_ == content_encoding::gzip) {
std::string unziped_str;
bool r = gzip_codec::uncompress(reply, unziped_str);
uncompressed_str_.clear();
bool r = gzip_codec::uncompress(reply, uncompressed_str_);
if (r)
data.resp_body = unziped_str;
data.resp_body = uncompressed_str_;
else
data.resp_body = reply;
}
else if (encoding_type_ == content_encoding::deflate) {
std::string inflate_str;
bool r = gzip_codec::inflate(reply, inflate_str);
uncompressed_str_.clear();
bool r = gzip_codec::inflate(reply, uncompressed_str_);
if (r)
data.resp_body = inflate_str;
data.resp_body = uncompressed_str_;
else
data.resp_body = reply;
}
else
#endif
data.resp_body = reply;
#if defined(CINATRA_ENABLE_BROTLI) && defined(CINATRA_ENABLE_GZIP)
else if (encoding_type_ == content_encoding::br)
#endif
#if defined(CINATRA_ENABLE_BROTLI) && !defined(CINATRA_ENABLE_GZIP)
if (encoding_type_ == content_encoding::br)
#endif
#ifdef CINATRA_ENABLE_BROTLI
{
uncompressed_str_.clear();
bool r = br_codec::brotli_decompress(reply, uncompressed_str_);
if (r)
data.resp_body = uncompressed_str_;
else
data.resp_body = reply;
}
#endif
#if defined(CINATRA_ENABLE_BROTLI) || defined(CINATRA_ENABLE_GZIP)
else
#endif
data.resp_body = reply;

head_buf_.consume(content_len);
}
Expand Down Expand Up @@ -2101,7 +2123,11 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
bool enable_ws_deflate_ = false;
bool is_server_support_ws_deflate_ = false;
std::string inflate_str_;
#endif
content_encoding encoding_type_ = content_encoding::none;

#if defined(CINATRA_ENABLE_BROTLI) || defined(CINATRA_ENABLE_GZIP)
std::string uncompressed_str_;
#endif

#ifdef BENCHMARK_TEST
Expand Down
2 changes: 2 additions & 0 deletions include/cinatra/coro_http_request.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,8 @@ class coro_http_request {
return content_encoding::gzip;
else if (encoding_type.find("deflate") != std::string_view::npos)
return content_encoding::deflate;
else if (encoding_type.find("br") != std::string_view::npos)
return content_encoding::br;
else
return content_encoding::none;
}
Expand Down
48 changes: 41 additions & 7 deletions include/cinatra/coro_http_response.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
#ifdef CINATRA_ENABLE_GZIP
#include "gzip.hpp"
#endif
#ifdef CINATRA_ENABLE_BROTLI
#include "brzip.hpp"
#endif
#include "picohttpparser.h"
#include "response_cv.hpp"
#include "time_util.hpp"
Expand Down Expand Up @@ -69,7 +72,7 @@ class coro_http_response {
if (client_encoding_type.empty() ||
client_encoding_type.find("gzip") != std::string_view::npos) {
std::string encode_str;
bool r = gzip_codec::compress(content, encode_str, true);
bool r = gzip_codec::compress(content, encode_str);
if (!r) {
set_status_and_content(status_type::internal_server_error,
"gzip compress error");
Expand All @@ -87,8 +90,11 @@ class coro_http_response {
content_ = std::move(content);
}
}
has_set_content_ = true;
return;
}
else if (encoding == content_encoding::deflate) {

if (encoding == content_encoding::deflate) {
if (client_encoding_type.empty() ||
client_encoding_type.find("deflate") != std::string_view::npos) {
std::string deflate_str;
Expand All @@ -110,16 +116,44 @@ class coro_http_response {
content_ = std::move(content);
}
}
has_set_content_ = true;
return;
}
else
#endif
{
if (is_view) {
content_view_ = content;

#ifdef CINATRA_ENABLE_BROTLI
if (encoding == content_encoding::br) {
if (client_encoding_type.empty() ||
client_encoding_type.find("br") != std::string_view::npos) {
std::string br_str;
bool r = br_codec::brotli_compress(content, br_str);
if (!r) {
set_status_and_content(status_type::internal_server_error,
"br compress error");
}
else {
add_header("Content-Encoding", "br");
set_content(std::move(br_str));
}
}
else {
content_ = std::move(content);
if (is_view) {
content_view_ = content;
}
else {
content_ = std::move(content);
}
}
has_set_content_ = true;
return;
}
#endif

if (is_view) {
content_view_ = content;
}
else {
content_ = std::move(content);
}
has_set_content_ = true;
}
Expand Down
2 changes: 1 addition & 1 deletion include/cinatra/define.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ enum class http_method {
OPTIONS,
DEL,
};
enum class content_encoding { gzip, deflate, none };
enum class content_encoding { gzip, deflate, br, none };
constexpr inline auto GET = http_method::GET;
constexpr inline auto POST = http_method::POST;
constexpr inline auto DEL = http_method::DEL;
Expand Down
10 changes: 10 additions & 0 deletions press_tool/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ if (ENABLE_GZIP)
target_link_libraries(${project_name} ${ZLIB_LIBRARIES})
endif()

if (ENABLE_BROTLI)
include_directories(${BROTLI_INCLUDE_DIRS})
target_link_libraries(${project_name} ${BROTLI_LIBRARIES})
endif()

if (ENABLE_SIMD STREQUAL "AARCH64")
if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
add_library(neon INTERFACE IMPORTED)
Expand Down Expand Up @@ -78,4 +83,9 @@ if (ENABLE_PRESS_TOOL_TESTS)
target_link_libraries(${unittest_press_tool} ${ZLIB_LIBRARIES})
endif()

if (ENABLE_BROTLI)
include_directories(${BROTLI_INCLUDE_DIRS})
target_link_libraries(${unittest_press_tool} ${BROTLI_LIBRARIES})
endif()

endif()
5 changes: 5 additions & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ if (ENABLE_GZIP)
target_link_libraries(${project_name} ${ZLIB_LIBRARIES})
endif()

if (ENABLE_BROTLI)
include_directories(${BROTLI_INCLUDE_DIRS})
target_link_libraries(${project_name} ${BROTLI_LIBRARIES})
endif()

# test_coro_file
option(ENABLE_FILE_IO_URING "enable io_uring" OFF)
if(ENABLE_FILE_IO_URING)
Expand Down
Loading

0 comments on commit 929936e

Please sign in to comment.