-
Notifications
You must be signed in to change notification settings - Fork 592
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #21450 from BenPope/backport-pr-20827-v24.1.x
[CORE-5540] [v24.1.x] Pandaproxy: Avoid large allocations whilst serializing JSON
- Loading branch information
Showing
36 changed files
with
625 additions
and
396 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| // Copyright 2024 Redpanda Data, Inc. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.md | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0 | ||
|
|
||
| #pragma once | ||
|
|
||
| #include "bytes/iobuf.h" | ||
| #include "json/encodings.h" | ||
|
|
||
| namespace json { | ||
|
|
||
| namespace impl { | ||
|
|
||
| /** | ||
| * \brief An in-memory output stream with non-contiguous memory allocation. | ||
| */ | ||
| template<typename Encoding> | ||
| struct generic_chunked_buffer { | ||
| using Ch = Encoding::Ch; | ||
|
|
||
| /** | ||
| * \defgroup Implement rapidjson::Stream | ||
| */ | ||
| /**@{*/ | ||
|
|
||
| void Put(Ch c) { _impl.append(&c, sizeof(Ch)); } | ||
| void Flush() {} | ||
|
|
||
| //! Get the size of string in bytes in the string buffer. | ||
| size_t GetSize() const { return _impl.size_bytes(); } | ||
|
|
||
| //! Get the length of string in Ch in the string buffer. | ||
| size_t GetLength() const { return _impl.size_bytes() / sizeof(Ch); } | ||
|
|
||
| void Reserve(size_t s) { _impl.reserve(s); } | ||
|
|
||
| void Clear() { _impl.clear(); } | ||
|
|
||
| /**@}*/ | ||
|
|
||
| iobuf as_iobuf() && { return std::move(_impl); } | ||
|
|
||
| private: | ||
| iobuf _impl; | ||
| }; | ||
|
|
||
| } // namespace impl | ||
|
|
||
| template<typename Encoding> | ||
| using generic_chunked_buffer = impl::generic_chunked_buffer<Encoding>; | ||
|
|
||
| using chunked_buffer = generic_chunked_buffer<UTF8<>>; | ||
|
|
||
| } // namespace json |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| // Copyright 2024 Redpanda Data, Inc. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.md | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0 | ||
|
|
||
| #pragma once | ||
|
|
||
| #include "bytes/streambuf.h" | ||
| #include "json/encodings.h" | ||
| #include "json/istreamwrapper.h" | ||
|
|
||
| namespace json { | ||
|
|
||
| namespace impl { | ||
|
|
||
| /** | ||
| * \brief An in-memory input stream with non-contiguous memory allocation. | ||
| */ | ||
| template<typename Encoding = ::json::UTF8<>> | ||
| class chunked_input_stream { | ||
| public: | ||
| using Ch = Encoding::Ch; | ||
|
|
||
| explicit chunked_input_stream(iobuf&& buf) | ||
| : _buf(std::move(buf)) | ||
| , _is(_buf) | ||
| , _sis{&_is} | ||
| , _isw(_sis) {} | ||
|
|
||
| /** | ||
| * \defgroup Implement rapidjson::Stream | ||
| */ | ||
| /**@{*/ | ||
|
|
||
| Ch Peek() const { return _isw.Peek(); } | ||
| Ch Peek4() const { return _isw.Peek4(); } | ||
| Ch Take() { return _isw.Take(); } | ||
| size_t Tell() const { return _isw.Tell(); } | ||
| void Put(Ch ch) { return _isw.Put(ch); } | ||
| Ch* PutBegin() { return _isw.PutBegin(); } | ||
| size_t PutEnd(Ch* ch) { return _isw.PutEnd(ch); } | ||
| void Flush() { return _isw.Flush(); } | ||
|
|
||
| /**@}*/ | ||
|
|
||
| private: | ||
| iobuf _buf; | ||
| iobuf_istreambuf _is; | ||
| std::istream _sis; | ||
| ::json::IStreamWrapper _isw; | ||
| }; | ||
|
|
||
| } // namespace impl | ||
|
|
||
| template<typename Encoding> | ||
| using generic_chunked_input_stream = impl::chunked_input_stream<Encoding>; | ||
|
|
||
| using chunked_input_stream = generic_chunked_input_stream<UTF8<>>; | ||
|
|
||
| } // namespace json |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.