From d36729378fb368ad8d147593c565866a5b9ed1d8 Mon Sep 17 00:00:00 2001 From: Marcin Kowalczyk Date: Thu, 1 Aug 2024 10:58:41 +0200 Subject: [PATCH] Let types explicitly indicate support for being passed to `ExternalRef` constructors: `RiegeliSupportsExternalRef()` and `RiegeliSupportsExternalRefWhole()`. They also indicate whether their substrings are stable when the object is moved, and whether passing a copy of an object is more efficient than copying the data. `ExternalRef::From()` are like `ExternalRef` constructors, but `RiegeliSupportsExternalRef()` or `RiegeliSupportsExternalRefWhole()` is not needed. The caller is responsible for using an appropriate type of the external object. This mechanism replaces `ToExternalRef()` member functions in several classes, and opts certain types into it. A uniform mechanism allows certain functions to wrap objects in `ExternalRef` automatically. Also, callers are not forced to remember whether specifying `substr` is valid for the moved object (this property differs e.g. between `std::string` and `std::vector`). Allow types to provide a predicate for objects whose data must be copied, e.g. because it is stored inline: `RiegeliExternalCopy()`. Remove `ExternalRef` constructor which always makes a copy. This is no longer needed given `RiegeliExternalCopy()`. Call `RiegeliTo{ChainBlock,Cord,ExternalData}(T*, substr)` if defined with `substr` also when `ExternalRef` constructor was called without `substr`. Allow types to provide a function which indicates that a subobject should be wrapped instead: `RiegeliExternalDelegate()`. Express appending `ExternalRef` to `Chain` as `Chain::Append(ExternalRef)` instead of `ExternalRef::AppendTo(Chain&)`. Same for prepending. This is inconsistent with `Cord` but is more natural in isolation, and allows for automatic wrapping of objects in `ExternalRef` (see below). Convert data to `ExternalRef` automatically when this is indicated as supported in `{,Backward}Writer::Write()` and in conversions to `Chain`. Add `Chain::Reset(ExternalRef)` and optimize the behavior of `riegeli::Reset(Cord&, ExternalRef)`. Do not bother including the overhead of wrapper objects when checking whether wrapping an object would be wasteful, for simplicity. This was inaccurate anyway when `RiegeliTo{ChainBlock,Cord,ExternalData}()` is used. Add assertions that the specified substring of an external object is indeed a substring if the object supports `riegeli::ToStringView()`. Change `Chain::BlockIterator` to yield a new type `Chain::BlockRef` instead of `absl::string_view`. This allows conversion to `ExternalRef` to be defined on `Chain::BlockRef` rather than `Chain::BlockIterator`, which works also when iterators are implicit in a `for` loop. PiperOrigin-RevId: 658318102 --- riegeli/base/BUILD | 27 +- riegeli/base/buffer.h | 26 +- riegeli/base/chain.cc | 87 +- riegeli/base/chain.h | 7 +- riegeli/base/chain_base.h | 122 +- riegeli/base/chain_details.h | 399 +++-- riegeli/base/compact_string.h | 64 +- riegeli/base/cord_utils.cc | 9 + riegeli/base/cord_utils.h | 12 +- riegeli/base/dependency.h | 3 +- riegeli/base/external_data.h | 16 + riegeli/base/external_ref.h | 7 +- riegeli/base/external_ref_base.h | 1747 +++++++++++++++------ riegeli/base/external_ref_support.h | 181 +++ riegeli/base/intrusive_shared_ptr.h | 12 + riegeli/base/shared_buffer.h | 53 +- riegeli/base/shared_ptr.h | 16 + riegeli/base/sized_shared_buffer.h | 76 +- riegeli/bytes/BUILD | 4 + riegeli/bytes/backward_writer.cc | 11 +- riegeli/bytes/backward_writer.h | 67 +- riegeli/bytes/buffered_reader.cc | 22 +- riegeli/bytes/chain_backward_writer.cc | 2 +- riegeli/bytes/chain_reader.cc | 17 +- riegeli/bytes/chain_writer.cc | 17 +- riegeli/bytes/cord_backward_writer.cc | 2 +- riegeli/bytes/cord_writer.cc | 2 +- riegeli/bytes/fd_mmap_reader.cc | 5 +- riegeli/bytes/pullable_reader.cc | 34 +- riegeli/bytes/pushable_backward_writer.cc | 6 +- riegeli/bytes/pushable_writer.cc | 4 +- riegeli/bytes/reader_factory.cc | 28 +- riegeli/bytes/resizable_writer.cc | 2 +- riegeli/bytes/restricted_chain_writer.cc | 2 +- riegeli/bytes/string_writer.cc | 2 +- riegeli/bytes/writer.cc | 9 - riegeli/bytes/writer.h | 87 +- riegeli/chunk_encoding/BUILD | 2 + riegeli/chunk_encoding/chunk_encoder.h | 34 +- riegeli/records/BUILD | 2 + riegeli/records/record_writer.cc | 13 - riegeli/records/record_writer.h | 32 +- riegeli/snappy/snappy_streams.cc | 2 +- riegeli/snappy/snappy_writer.cc | 2 +- riegeli/tensorflow/io/BUILD | 1 + riegeli/tensorflow/io/file_reader.cc | 23 +- riegeli/tensorflow/io/file_writer.cc | 2 +- 47 files changed, 2150 insertions(+), 1150 deletions(-) create mode 100644 riegeli/base/external_ref_support.h diff --git a/riegeli/base/BUILD b/riegeli/base/BUILD index b470a07f..166e954c 100644 --- a/riegeli/base/BUILD +++ b/riegeli/base/BUILD @@ -361,6 +361,17 @@ cc_library( deps = ["@com_google_absl//absl/strings"], ) +cc_library( + name = "external_ref_support", + hdrs = ["external_ref_support.h"], + deps = [ + ":external_data", + ":to_string_view", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/meta:type_traits", + ], +) + cc_library( name = "shared_ptr", hdrs = [ @@ -375,6 +386,7 @@ cc_library( ":compare", ":constexpr", ":external_data", + ":external_ref_support", ":initializer", ":new_aligned", ":reset", @@ -394,7 +406,6 @@ cc_library( ":buffering", ":estimated_allocated_size", ":external_data", - ":external_ref", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/strings", ], @@ -409,7 +420,6 @@ cc_library( ":assert", ":buffer", ":external_data", - ":external_ref", ":initializer", ":shared_ptr", "@com_google_absl//absl/base:core_headers", @@ -425,7 +435,6 @@ cc_library( ":arithmetic", ":assert", ":buffering", - ":external_ref", ":shared_buffer", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/strings", @@ -449,6 +458,7 @@ cc_library( ":compare", ":cord_utils", ":external_data", + ":external_ref_support", ":global", ":initializer", ":memory_estimator", @@ -471,13 +481,19 @@ cc_library( cc_library( name = "chain", hdrs = ["chain.h"], - deps = [":chain_and_external_ref"], + deps = [ + ":chain_and_external_ref", + ":external_ref_support", + ], ) cc_library( name = "external_ref", hdrs = ["external_ref.h"], - deps = [":chain_and_external_ref"], + deps = [ + ":chain_and_external_ref", + ":external_ref_support", + ], ) cc_library( @@ -490,7 +506,6 @@ cc_library( ":compare", ":estimated_allocated_size", ":external_data", - ":external_ref", ":new_aligned", "@com_google_absl//absl/base:config", "@com_google_absl//absl/base:core_headers", diff --git a/riegeli/base/buffer.h b/riegeli/base/buffer.h index 42f0c5af..7865a346 100644 --- a/riegeli/base/buffer.h +++ b/riegeli/base/buffer.h @@ -17,7 +17,6 @@ #include -#include #include #include @@ -27,7 +26,6 @@ #include "riegeli/base/buffering.h" #include "riegeli/base/estimated_allocated_size.h" #include "riegeli/base/external_data.h" -#include "riegeli/base/external_ref.h" namespace riegeli { @@ -61,28 +59,8 @@ class // Returns the usable data size. It can be greater than the requested size. size_t capacity() const { return capacity_; } - // Converts a substring of `*this` to `ExternalRef`. - // - // `storage` must outlive usages of the returned `ExternalRef`. - // - // Precondition: - // if `!substr.empty()` then `substr` is a substring of - // [`data()`..`data() + capacity()`). - ExternalRef ToExternalRef(absl::string_view substr, - ExternalRef::StorageSubstr&& storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) && { - if (!substr.empty()) { - RIEGELI_ASSERT(std::greater_equal<>()(substr.data(), data())) - << "Failed precondition of Buffer::ToExternalRef(): " - "substring not contained in the buffer"; - RIEGELI_ASSERT(std::less_equal<>()(substr.data() + substr.size(), - data() + capacity())) - << "Failed precondition of Buffer::ToExternalRef(): " - "substring not contained in the buffer"; - } - return ExternalRef(std::move(*this), substr, std::move(storage)); - } + // Indicate support for `ExternalRef(Buffer&&, substr)`. + friend void RiegeliSupportsExternalRef(Buffer*) {} // Support `ExternalRef`. friend size_t RiegeliExternalMemory(const Buffer* self) { diff --git a/riegeli/base/chain.cc b/riegeli/base/chain.cc index fb9d291d..cd80ea59 100644 --- a/riegeli/base/chain.cc +++ b/riegeli/base/chain.cc @@ -21,7 +21,6 @@ #include #include #include -#include #include #include "absl/base/attributes.h" @@ -422,6 +421,17 @@ absl::Cord Chain::Block::ToCord(absl::string_view substr) && { return absl::MakeCordFromExternal(substr, [block = std::move(block_)] {}); } +absl::Cord Chain::Block::ToCord(absl::string_view substr) const& { + if (const FlatCordBlock* const cord_ptr = + block_->checked_external_object()) { + if (substr.size() == cord_ptr->src().size()) return cord_ptr->src(); + return cord_ptr->src().Subcord( + PtrDistance(absl::string_view(*cord_ptr).data(), substr.data()), + substr.size()); + } + return absl::MakeCordFromExternal(substr, [block = block_] {}); +} + void Chain::Block::DumpStructure(absl::string_view substr, std::ostream& out) const { out << "[block] { offset: " @@ -464,24 +474,6 @@ void Chain::Reset(absl::string_view src) { Initialize(src); } -template ::value, int>> -void Chain::Reset(Src&& src) { - size_ = 0; - if (begin_ != end_ && ClearSlow()) { - const size_t size = src.size(); - // `std::move(src)` is correct and `std::forward(src)` is not - // necessary: `Src` is always `std::string`, never an lvalue reference. - Append(std::move(src), Options().set_size_hint(size)); - return; - } - // `std::move(src)` is correct and `std::forward(src)` is not necessary: - // `Src` is always `std::string`, never an lvalue reference. - Initialize(std::move(src)); -} - -template void Chain::Reset(std::string&& src); - void Chain::Reset(Block src) { size_ = 0; UnrefBlocks(); @@ -523,26 +515,6 @@ void Chain::InitializeSlow(absl::string_view src) { Append(src, options); } -template ::value, int>> -void Chain::InitializeSlow(Src&& src) { - RIEGELI_ASSERT_GT(src.size(), kMaxShortDataSize) - << "Failed precondition of Chain::InitializeSlow(string&&): " - "string too short, use Initialize() instead"; - if (Wasteful( - RawBlock::kExternalAllocatedSize() + src.capacity() + 1, - src.size())) { - // Not `std::move(src)`: forward to `InitializeSlow(absl::string_view)`. - InitializeSlow(src); - return; - } - // `std::move(src)` is correct and `std::forward(src)` is not necessary: - // `Src` is always `std::string`, never an lvalue reference. - Initialize(Block(std::move(src))); -} - -template void Chain::InitializeSlow(std::string&& src); - inline void Chain::Initialize(const absl::Cord& src) { RIEGELI_ASSERT_EQ(size_, 0u) << "Failed precondition of Chain::Initialize(const Cord&): " @@ -571,8 +543,8 @@ inline void Chain::InitializeFromCord(CordRef&& src) { return; } } - const size_t size = src.size(); - AppendCordSlow(std::forward(src), Options().set_size_hint(size)); + AppendCordSlow(std::forward(src), + Options().set_size_hint(src.size())); } inline void Chain::Initialize(const Chain& src) { @@ -1423,16 +1395,6 @@ void Chain::Append(absl::string_view src, Options options) { } } -template ::value, int>> -void Chain::Append(Src&& src, Options options) { - // `std::move(src)` is correct and `std::forward(src)` is not necessary: - // `Src` is always `std::string`, never an lvalue reference. - ExternalRef(std::move(src)).AppendTo(*this, options); -} - -template void Chain::Append(std::string&& src, Options options); - void Chain::Append(const Chain& src, Options options) { AppendChain(src, options); } @@ -1714,16 +1676,6 @@ void Chain::Prepend(absl::string_view src, Options options) { } } -template ::value, int>> -void Chain::Prepend(Src&& src, Options options) { - // `std::move(src)` is correct and `std::forward(src)` is not necessary: - // `Src` is always `std::string`, never an lvalue reference. - ExternalRef(std::move(src)).PrependTo(*this, options); -} - -template void Chain::Prepend(std::string&& src, Options options); - void Chain::Prepend(const Chain& src, Options options) { PrependChain(src, options); } @@ -2035,11 +1987,8 @@ void Chain::RemoveSuffix(size_t length, Options options) { data.remove_suffix(length); // Compensate for increasing `size_` by `Append()`. size_ -= data.size(); - if (data.size() <= kMaxBytesToCopy) { - Append(data, options); - return; - } - Append(Block(riegeli::Invoker(MakeBlock(), std::move(last)), data), options); + Append(ExternalRef(riegeli::Invoker(MakeBlock(), std::move(last)), data), + options); } void Chain::RemovePrefix(size_t length, Options options) { @@ -2085,11 +2034,7 @@ void Chain::RemovePrefix(size_t length, Options options) { data.remove_prefix(length); // Compensate for increasing `size_` by `Prepend()`. size_ -= data.size(); - if (data.size() <= kMaxBytesToCopy) { - Prepend(data, options); - return; - } - Prepend(Block(riegeli::Invoker(MakeBlock(), std::move(first)), data), + Prepend(ExternalRef(riegeli::Invoker(MakeBlock(), std::move(first)), data), options); } diff --git a/riegeli/base/chain.h b/riegeli/base/chain.h index a4fb0a06..57bee272 100644 --- a/riegeli/base/chain.h +++ b/riegeli/base/chain.h @@ -15,8 +15,9 @@ #ifndef RIEGELI_BASE_CHAIN_H_ #define RIEGELI_BASE_CHAIN_H_ -#include "riegeli/base/chain_base.h" // IWYU pragma: export -#include "riegeli/base/chain_details.h" // IWYU pragma: export -#include "riegeli/base/external_ref_base.h" // IWYU pragma: keep +#include "riegeli/base/chain_base.h" // IWYU pragma: export +#include "riegeli/base/chain_details.h" // IWYU pragma: export +#include "riegeli/base/external_ref_base.h" // IWYU pragma: keep +#include "riegeli/base/external_ref_support.h" // IWYU pragma: keep #endif // RIEGELI_BASE_CHAIN_H_ diff --git a/riegeli/base/chain_base.h b/riegeli/base/chain_base.h index a8b14ffc..00ac72aa 100644 --- a/riegeli/base/chain_base.h +++ b/riegeli/base/chain_base.h @@ -28,6 +28,7 @@ #include #include "absl/base/attributes.h" +#include "absl/meta/type_traits.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -37,6 +38,7 @@ #include "riegeli/base/buffering.h" #include "riegeli/base/compare.h" #include "riegeli/base/external_data.h" +#include "riegeli/base/external_ref_support.h" #include "riegeli/base/initializer.h" #include "riegeli/base/intrusive_shared_ptr.h" #include "riegeli/base/memory_estimator.h" @@ -46,6 +48,8 @@ namespace riegeli { +class ExternalRef; + // A `Chain` represents a sequence of bytes. It supports efficient appending and // prepending, and sharing memory with other `Chain`s and other types. It does // not support efficient random access. @@ -150,7 +154,7 @@ class Chain : public WithCompare { }; class Block; - struct MakeBlock; + class BlockRef; class BlockIterator; class Blocks; struct BlockAndChar; @@ -178,17 +182,21 @@ class Chain : public WithCompare { constexpr Chain() = default; // Converts from a string-like type. - // - // `std::string&&` is accepted with a template to avoid implicit conversions - // to `std::string` which can be ambiguous against `absl::string_view` - // (e.g. `const char*`). explicit Chain(absl::string_view src); template ::value, int> = 0> + std::enable_if_t< + absl::conjunction< + SupportsToStringView, + absl::negation>>::value, + int> = 0> explicit Chain(Src&& src); explicit Chain(Block src); explicit Chain(const absl::Cord& src); explicit Chain(absl::Cord&& src); + explicit Chain(ExternalRef src); + template ::value, int> = 0> + explicit Chain(Src&& src); Chain(const Chain& that); Chain& operator=(const Chain& that); @@ -207,11 +215,19 @@ class Chain : public WithCompare { ABSL_ATTRIBUTE_REINITIALIZES void Reset(); ABSL_ATTRIBUTE_REINITIALIZES void Reset(absl::string_view src); template ::value, int> = 0> + std::enable_if_t< + absl::conjunction< + SupportsToStringView, + absl::negation>>::value, + int> = 0> ABSL_ATTRIBUTE_REINITIALIZES void Reset(Src&& src); ABSL_ATTRIBUTE_REINITIALIZES void Reset(Block src); ABSL_ATTRIBUTE_REINITIALIZES void Reset(const absl::Cord& src); ABSL_ATTRIBUTE_REINITIALIZES void Reset(absl::Cord&& src); + ABSL_ATTRIBUTE_REINITIALIZES void Reset(ExternalRef src); + template ::value, int> = 0> + ABSL_ATTRIBUTE_REINITIALIZES void Reset(Src&& src); // Removes all data. ABSL_ATTRIBUTE_REINITIALIZES void Clear(); @@ -291,13 +307,13 @@ class Chain : public WithCompare { Options options = Options()); // Appends/prepends a string-like type. - // - // `std::string&&` is accepted with a template to avoid implicit conversions - // to `std::string` which can be ambiguous against `absl::string_view` - // (e.g. `const char*`). void Append(absl::string_view src, Options options = Options()); template ::value, int> = 0> + std::enable_if_t< + absl::conjunction< + SupportsToStringView, + absl::negation>>::value, + int> = 0> void Append(Src&& src, Options options = Options()); void Append(const Chain& src, Options options = Options()); void Append(Chain&& src, Options options = Options()); @@ -305,9 +321,21 @@ class Chain : public WithCompare { void Append(Block&& src, Options options = Options()); void Append(const absl::Cord& src, Options options = Options()); void Append(absl::Cord&& src, Options options = Options()); + void Append(ExternalRef src); + void Append(ExternalRef src, Options options); + template ::value, int> = 0> + void Append(Src&& src); + template ::value, int> = 0> + void Append(Src&& src, Options options); void Prepend(absl::string_view src, Options options = Options()); template ::value, int> = 0> + std::enable_if_t< + absl::conjunction< + SupportsToStringView, + absl::negation>>::value, + int> = 0> void Prepend(Src&& src, Options options = Options()); void Prepend(const Chain& src, Options options = Options()); void Prepend(Chain&& src, Options options = Options()); @@ -315,6 +343,14 @@ class Chain : public WithCompare { void Prepend(Block&& src, Options options = Options()); void Prepend(const absl::Cord& src, Options options = Options()); void Prepend(absl::Cord&& src, Options options = Options()); + void Prepend(ExternalRef src); + void Prepend(ExternalRef src, Options options); + template ::value, int> = 0> + void Prepend(Src&& src); + template ::value, int> = 0> + void Prepend(Src&& src, Options options); // `AppendFrom(iter, length)` is equivalent to // `Append(absl::Cord::AdvanceAndRead(&iter, length))` but more efficient. @@ -370,6 +406,7 @@ class Chain : public WithCompare { private: class BlockPtrPtr; + struct MakeBlock; struct ExternalMethods; template struct ExternalMethodsFor; @@ -457,12 +494,6 @@ class Chain : public WithCompare { void Initialize(absl::string_view src); void InitializeSlow(absl::string_view src); - template ::value, int> = 0> - void Initialize(Src&& src); - template ::value, int> = 0> - void InitializeSlow(Src&& src); void Initialize(Block src); void Initialize(const absl::Cord& src); void Initialize(absl::Cord&& src); @@ -595,9 +626,8 @@ class Chain::RawBlock { explicit RawBlock(const size_t* raw_capacity); // Constructs an external block containing an external object of type `T`, - // and sets block data to `RiegeliToStringView(&new_object)`, - // `absl::string_view(new_object)`, or `absl::Span(new_object)`. - // This constructor is public for `NewAligned()`. + // and sets block data to `riegeli::ToStringView(new_object)`. This + // constructor is public for `NewAligned()`. template explicit RawBlock(Initializer object); @@ -750,17 +780,11 @@ class Chain::Block { // The `object` parameter supports `riegeli::Maker(args...)` to construct // `T` in-place. // - // If the `substr` parameter is given, `substr` must be valid for the new - // object. + // If the `substr` parameter is given, `substr` must be owned by the object + // after it gets created or moved. // - // If the `substr` parameter is not given, `T` must support any of: - // ``` - // // Returns contents of the object. Called when it is moved to its final - // // location. - // friend absl::string_view RiegeliToStringView(const T* self); - // explicit operator absl::string_view() const; - // explicit operator absl::Span() const; - // ``` + // If the `substr` parameter is not given, `T` must support + // `riegeli::ToStringView()`. // // `T` may also support the following member functions, either with or without // the `substr` parameter, with the following definitions assumed by default: @@ -788,8 +812,11 @@ class Chain::Block { // `substr` parameter passed to `FromExternal()`. Having `substr` available in // these functions might avoid storing `substr` in the external object. template >::value, - int> = 0> + std::enable_if_t< + absl::conjunction< + absl::negation, Block>>, + SupportsToStringView>>::value, + int> = 0> explicit Block(T&& object); template explicit Block(T&& object, absl::string_view substr); @@ -800,6 +827,28 @@ class Chain::Block { Block(Block&& that) = default; Block& operator=(Block&& that) = default; + explicit operator absl::string_view() const { + if (block_ == nullptr) return absl::string_view(); + return absl::string_view(*block_); + } + + bool empty() const { return block_ == nullptr || block_->empty(); } + size_t size() const { + if (block_ == nullptr) return 0; + return block_->size(); + } + const char* data() const { + if (block_ == nullptr) return nullptr; + return block_->data_begin(); + } + + // Indicate support for: + // * `ExternalRef(const Block&)` + // * `ExternalRef(Block&&)` + // * `ExternalRef(const Block&, substr)` + // * `ExternalRef(Block&&, substr)` + friend void RiegeliSupportsExternalRef(const Block*) {} + // Support `ExternalRef`. friend size_t RiegeliExternalMemory(const Block* self) { return self->ExternalMemory(); @@ -814,6 +863,9 @@ class Chain::Block { friend absl::Cord RiegeliToCord(Block* self, absl::string_view substr) { return std::move(*self).ToCord(substr); } + friend absl::Cord RiegeliToCord(const Block* self, absl::string_view substr) { + return self->ToCord(substr); + } // Support `ExternalRef`. friend ExternalStorage RiegeliToExternalStorage(Block* self) { @@ -837,6 +889,7 @@ class Chain::Block { friend class Chain; // For `Block()` and `raw_block()`. explicit Block(RawBlock* block); + explicit Block(RawBlock* block, absl::string_view substr); explicit Block(IntrusiveSharedPtr block); const IntrusiveSharedPtr& raw_block() const& { return block_; } @@ -845,6 +898,7 @@ class Chain::Block { size_t ExternalMemory() const; Block ToChainBlock(absl::string_view substr) &&; absl::Cord ToCord(absl::string_view substr) &&; + absl::Cord ToCord(absl::string_view substr) const&; ExternalStorage ToExternalStorage() &&; void DumpStructure(absl::string_view substr, std::ostream& out) const; diff --git a/riegeli/base/chain_details.h b/riegeli/base/chain_details.h index ff2a9391..68977764 100644 --- a/riegeli/base/chain_details.h +++ b/riegeli/base/chain_details.h @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -43,13 +42,12 @@ #include "riegeli/base/compare.h" #include "riegeli/base/external_data.h" #include "riegeli/base/external_ref_base.h" +#include "riegeli/base/external_ref_support.h" #include "riegeli/base/initializer.h" #include "riegeli/base/intrusive_shared_ptr.h" -#include "riegeli/base/invoker.h" #include "riegeli/base/memory_estimator.h" #include "riegeli/base/new_aligned.h" #include "riegeli/base/ownership.h" -#include "riegeli/base/temporary_storage.h" #include "riegeli/base/to_string_view.h" namespace riegeli { @@ -117,6 +115,64 @@ struct Chain::MakeBlock { Block operator()(RawBlock* block) const { return Block(block); } }; +class Chain::BlockRef { + public: + BlockRef(const BlockRef& that) = default; + BlockRef& operator=(const BlockRef& that) = default; + + /*implicit*/ operator absl::string_view() const; + + bool empty() const; + const char* data() const; + size_t size() const; + + // Indicate support for: + // * `ExternalRef(BlockRef)` + // * `ExternalRef(BlockRef, substr)` + friend void RiegeliSupportsExternalRef(const BlockRef*) {} + + // Support `ExternalRef`. + friend bool RiegeliExternalCopy(const BlockRef* self) { + return self->ExternalCopy(); + } + + // Support `ExternalRef`. + friend Chain::Block RiegeliToChainBlock(const BlockRef* self, + absl::string_view substr) { + return self->ToChainBlock(substr); + } + + // Support `ExternalRef`. + template + friend void RiegeliExternalDelegate(const BlockRef* self, + absl::string_view substr, + Callback&& delegate_to) { + self->ExternalDelegate(substr, std::forward(delegate_to)); + } + + // Returns a pointer to the external object if this is an external block + // holding an object of type `T`, otherwise returns `nullptr`. + template + const T* external_object() const; + + private: + friend class Chain; // For `BlockRef()`. + + explicit BlockRef(const Chain* chain, BlockPtrPtr ptr) + : chain_(chain), ptr_(ptr) {} + + bool ExternalCopy() const; + Chain::Block ToChainBlock(absl::string_view substr) const; + template + void ExternalDelegate(absl::string_view substr, Callback&& delegate_to) const; + + const Chain* chain_; + // If `*chain_` has short data, `kBeginShortData`. + // If `*chain_` has block pointers, a pointer to an element of the block + // pointer array. + BlockPtrPtr ptr_; +}; + class Chain::BlockIterator : public WithCompare { public: using iterator_concept = std::random_access_iterator_tag; @@ -124,7 +180,7 @@ class Chain::BlockIterator : public WithCompare { // `LegacyForwardIterator` requirement and above require `reference` to be // a true reference type. using iterator_category = std::input_iterator_tag; - using value_type = absl::string_view; + using value_type = BlockRef; using reference = value_type; using difference_type = ptrdiff_t; @@ -138,8 +194,6 @@ class Chain::BlockIterator : public WithCompare { reference ref_; }; - using BlockMaker = InvokerType; - BlockIterator() = default; explicit BlockIterator(const Chain* chain, size_t block_index); @@ -190,49 +244,13 @@ class Chain::BlockIterator : public WithCompare { return a + n; } - // Converts the block pointed to `*this` to `ExternalRef`. - // - // `temporary_storage` and `external_storage` must outlive usages of the - // returned `ExternalRef`. - // - // Precondition: `*this` is not past the end iterator - ExternalRef ToExternalRef( - TemporaryStorage&& temporary_storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = TemporaryStorage(), - ExternalRef::StorageSubstr&& external_storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) const; - - // Converts a substring of the block pointed to `*this` to `ExternalRef`. - // - // `temporary_storage` and `external_storage` must outlive usages of the - // returned `ExternalRef`. - // - // Preconditions: - // `*this` is not past the end iterator - // if `!substr.empty()` then `substr` is a substring of `**this` - ExternalRef ToExternalRef( - absl::string_view substr, - TemporaryStorage&& temporary_storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = TemporaryStorage(), - ExternalRef::StorageSubstr&& external_storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) const; - - // Returns a pointer to the external object if this points to an external - // block holding an object of type `T`, otherwise returns `nullptr`. - // - // Precondition: `*this` is not past the end iterator - template - const T* external_object() const; - private: friend class Chain; static constexpr BlockPtrPtr kBeginShortData{0}; static constexpr BlockPtrPtr kEndShortData{sizeof(BlockPtr)}; - explicit BlockIterator(const Chain* chain, BlockPtrPtr ptr) noexcept; + explicit BlockIterator(const Chain* chain, BlockPtrPtr ptr); size_t CharIndexInChainInternal() const; @@ -240,13 +258,14 @@ class Chain::BlockIterator : public WithCompare { // If `chain_ == nullptr`, `kBeginShortData`. // If `*chain_` has no block pointers and no short data, `kEndShortData`. // If `*chain_` has short data, `kBeginShortData` or `kEndShortData`. - // If `*chain_` has block pointers, a pointer to the block pointer array. + // If `*chain_` has block pointers, a pointer to an element of the block + // pointer array. BlockPtrPtr ptr_ = kBeginShortData; }; class Chain::Blocks { public: - using value_type = absl::string_view; + using value_type = BlockRef; using reference = value_type; using const_reference = reference; using iterator = BlockIterator; @@ -440,8 +459,7 @@ void RiegeliDumpStructure(const std::string* self, std::ostream& out); template struct Chain::ExternalMethodsFor { // Creates an external block containing an external object constructed from - // `object`, and sets block data to `RiegeliToStringView(&new_object)`, - // `absl::string_view(new_object)`, or `absl::Span(new_object)`. + // `object`, and sets block data to `riegeli::ToStringView(new_object)`. static IntrusiveSharedPtr NewBlock(Initializer object); // Creates an external block containing an external object constructed from @@ -694,6 +712,66 @@ inline Chain::BlockPtrPtr Chain::BlockPtrPtr::operator-(ptrdiff_t n) const { return BlockPtrPtr::from_ptr(as_ptr() - n); } +inline Chain::BlockRef::operator absl::string_view() const { + if (ptr_ == BlockIterator::kBeginShortData) { + return chain_->short_data(); + } else { + return absl::string_view(*ptr_.as_ptr()->block_ptr); + } +} + +inline bool Chain::BlockRef::empty() const { + return ptr_ != BlockIterator::kBeginShortData && + ptr_.as_ptr()->block_ptr->empty(); +} + +inline const char* Chain::BlockRef::data() const { + if (ptr_ == BlockIterator::kBeginShortData) { + return chain_->short_data_begin(); + } else { + return ptr_.as_ptr()->block_ptr->data_begin(); + } +} + +inline size_t Chain::BlockRef::size() const { + if (ptr_ == BlockIterator::kBeginShortData) { + return chain_->size_; + } else { + return ptr_.as_ptr()->block_ptr->size(); + } +} + +inline bool Chain::BlockRef::ExternalCopy() const { + return ptr_ == BlockIterator::kBeginShortData; +} + +inline Chain::Block Chain::BlockRef::ToChainBlock( + absl::string_view substr) const { + RIEGELI_ASSERT(ptr_ != BlockIterator::kBeginShortData) + << "Failed precondition of RiegeliToChainBlock(const Chain::BlockRef*): " + "case excluded by RiegeliExternalCopy()"; + return Block(ptr_.as_ptr()->block_ptr, substr); +} + +template +inline void Chain::BlockRef::ExternalDelegate(absl::string_view substr, + Callback&& delegate_to) const { + RIEGELI_ASSERT(ptr_ != BlockIterator::kBeginShortData) + << "Failed precondition of " + "RiegeliExternalDelegate(const Chain::BlockRef*): " + "case excluded by RiegeliExternalCopy()"; + std::forward(delegate_to)(Block(ptr_.as_ptr()->block_ptr), substr); +} + +template +inline const T* Chain::BlockRef::external_object() const { + if (ptr_ == BlockIterator::kBeginShortData) { + return nullptr; + } else { + return ptr_.as_ptr()->block_ptr->checked_external_object(); + } +} + inline Chain::BlockIterator::BlockIterator(const Chain* chain, size_t block_index) : chain_(chain), @@ -703,8 +781,7 @@ inline Chain::BlockIterator::BlockIterator(const Chain* chain, : BlockPtrPtr::from_ptr(chain_->begin_)) + IntCast(block_index)) {} -inline Chain::BlockIterator::BlockIterator(const Chain* chain, - BlockPtrPtr ptr) noexcept +inline Chain::BlockIterator::BlockIterator(const Chain* chain, BlockPtrPtr ptr) : chain_(chain), ptr_(ptr) {} inline size_t Chain::BlockIterator::block_index() const { @@ -726,11 +803,7 @@ inline Chain::BlockIterator::reference Chain::BlockIterator::operator*() const { RIEGELI_ASSERT(ptr_ != kEndShortData) << "Failed precondition of Chain::BlockIterator::operator*: " "iterator is end()"; - if (ptr_ == kBeginShortData) { - return chain_->short_data(); - } else { - return absl::string_view(*ptr_.as_ptr()->block_ptr); - } + return BlockRef(chain_, ptr_); } inline Chain::BlockIterator::pointer Chain::BlockIterator::operator->() const { @@ -786,61 +859,13 @@ inline Chain::BlockIterator::reference Chain::BlockIterator::operator[]( return *(*this + n); } -inline ExternalRef Chain::BlockIterator::ToExternalRef( - TemporaryStorage&& temporary_storage, - ExternalRef::StorageSubstr&& external_storage) const { - RIEGELI_ASSERT(ptr_ != kEndShortData) - << "Failed precondition of Chain::BlockIterator::ToExternalRef(): " - "iterator is end()"; - if (ptr_ == kBeginShortData) { - return ExternalRef(chain_->short_data(), std::move(external_storage)); - } else { - return ExternalRef(std::move(temporary_storage) - .emplace(MakeBlock(), ptr_.as_ptr()->block_ptr), - absl::string_view(*ptr_.as_ptr()->block_ptr), - std::move(external_storage)); - } -} - -inline ExternalRef Chain::BlockIterator::ToExternalRef( - absl::string_view substr, TemporaryStorage&& temporary_storage, - ExternalRef::StorageSubstr&& external_storage) const { - RIEGELI_ASSERT(ptr_ != kEndShortData) - << "Failed precondition of Chain::BlockIterator::ToExternalRef(): " - "iterator is end()"; - if (!substr.empty()) { - RIEGELI_ASSERT(std::greater_equal<>()(substr.data(), (*this)->data())) - << "Failed precondition of SizedSharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - RIEGELI_ASSERT(std::less_equal<>()(substr.data() + substr.size(), - (*this)->data() + (*this)->size())) - << "Failed precondition of SizedSharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - } - if (ptr_ == kBeginShortData) { - return ExternalRef(substr, std::move(external_storage)); - } else { - return ExternalRef(std::move(temporary_storage) - .emplace(MakeBlock(), ptr_.as_ptr()->block_ptr), - substr, std::move(external_storage)); - } -} - -template -inline const T* Chain::BlockIterator::external_object() const { - RIEGELI_ASSERT(ptr_ != kEndShortData) - << "Failed precondition of Chain::BlockIterator::external_object(): " - "iterator is end()"; - if (ptr_ == kBeginShortData) { - return nullptr; - } else { - return ptr_.as_ptr()->block_ptr->checked_external_object(); - } -} - template < typename T, - std::enable_if_t>::value, int>> + std::enable_if_t< + absl::conjunction< + absl::negation, Chain::Block>>, + SupportsToStringView>>::value, + int>> inline Chain::Block::Block(T&& object) : block_(ExternalMethodsFor>::NewBlock( std::forward(object))) {} @@ -850,6 +875,19 @@ inline Chain::Block::Block(T&& object, absl::string_view substr) : block_(ExternalMethodsFor>::NewBlock( std::forward(object), substr)) {} +inline Chain::Block::Block(RawBlock* block, absl::string_view substr) { + if (block->size() == substr.size()) { + block_.Reset(block, kShareOwnership); + return; + } + if (const Block* const block_ptr = block->checked_external_object()) { + // `block` is already a `Block`. Refer to its target instead. + block = block_ptr->block_.get(); + } + block_.Reset(block, kShareOwnership); + block_ = ExternalMethodsFor::NewBlock(std::move(*this), substr); +} + inline Chain::Block::Block(RawBlock* block) { if (const Block* const block_ptr = block->checked_external_object()) { // `block` is already a `Block`. Refer to its target instead. @@ -906,41 +944,33 @@ inline Chain::Blocks::reference Chain::Blocks::operator[](size_type n) const { RIEGELI_ASSERT_LT(n, size()) << "Failed precondition of Chain::Blocks::operator[]: " "block index out of range"; - if (ABSL_PREDICT_FALSE(chain_->begin_ == chain_->end_)) { - return chain_->short_data(); - } else { - return absl::string_view(*chain_->begin_[n].block_ptr); - } + return BlockRef(chain_, chain_->begin_ == chain_->end_ + ? BlockIterator::kBeginShortData + : BlockPtrPtr::from_ptr(chain_->begin_ + n)); } inline Chain::Blocks::reference Chain::Blocks::at(size_type n) const { RIEGELI_CHECK_LT(n, size()) << "Failed precondition of Chain::Blocks::at(): " "block index out of range"; - if (ABSL_PREDICT_FALSE(chain_->begin_ == chain_->end_)) { - return chain_->short_data(); - } else { - return absl::string_view(*chain_->begin_[n].block_ptr); - } + return BlockRef(chain_, chain_->begin_ == chain_->end_ + ? BlockIterator::kBeginShortData + : BlockPtrPtr::from_ptr(chain_->begin_ + n)); } inline Chain::Blocks::reference Chain::Blocks::front() const { RIEGELI_ASSERT(!empty()) << "Failed precondition of Chain::Blocks::front(): no blocks"; - if (ABSL_PREDICT_FALSE(chain_->begin_ == chain_->end_)) { - return chain_->short_data(); - } else { - return absl::string_view(*chain_->begin_[0].block_ptr); - } + return BlockRef(chain_, chain_->begin_ == chain_->end_ + ? BlockIterator::kBeginShortData + : BlockPtrPtr::from_ptr(chain_->begin_)); } inline Chain::Blocks::reference Chain::Blocks::back() const { RIEGELI_ASSERT(!empty()) << "Failed precondition of Chain::Blocks::back(): no blocks"; - if (ABSL_PREDICT_FALSE(chain_->begin_ == chain_->end_)) { - return chain_->short_data(); - } else { - return absl::string_view(*chain_->end_[-1].block_ptr); - } + return BlockRef(chain_, chain_->begin_ == chain_->end_ + ? BlockIterator::kBeginShortData + : BlockPtrPtr::from_ptr(chain_->end_ - 1)); } template @@ -950,18 +980,28 @@ constexpr size_t Chain::kExternalAllocatedSize() { inline Chain::Chain(absl::string_view src) { Initialize(src); } -template ::value, int>> +template < + typename Src, + std::enable_if_t< + absl::conjunction, + absl::negation>>::value, + int>> inline Chain::Chain(Src&& src) { - // `std::move(src)` is correct and `std::forward(src)` is not necessary: - // `Src` is always `std::string`, never an lvalue reference. - Initialize(std::move(src)); + Initialize(riegeli::ToStringView(src)); } inline Chain::Chain(Block src) { if (src.raw_block() != nullptr) Initialize(std::move(src)); } +inline Chain::Chain(ExternalRef src) { std::move(src).InitializeTo(*this); } + +template ::value, int>> +inline Chain::Chain(Src&& src) { + ExternalRef(std::forward(src)).InitializeTo(*this); +} + inline Chain::Chain(Chain&& that) noexcept : size_(std::exchange(that.size_, 0)) { // Use `std::memcpy()` instead of copy constructor to silence @@ -1015,7 +1055,23 @@ inline Chain::~Chain() { inline void Chain::Reset() { Clear(); } -extern template void Chain::Reset(std::string&& src); +template < + typename Src, + std::enable_if_t< + absl::conjunction, + absl::negation>>::value, + int>> +inline void Chain::Reset(Src&& src) { + Reset(riegeli::ToStringView(src)); +} + +inline void Chain::Reset(ExternalRef src) { std::move(src).AssignTo(*this); } + +template ::value, int>> +inline void Chain::Reset(Src&& src) { + ExternalRef(std::forward(src)).AssignTo(*this); +} inline void Chain::Clear() { size_ = 0; @@ -1036,26 +1092,6 @@ inline void Chain::Initialize(absl::string_view src) { InitializeSlow(src); } -template ::value, int>> -inline void Chain::Initialize(Src&& src) { - RIEGELI_ASSERT_EQ(size_, 0u) - << "Failed precondition of Chain::Initialize(string&&): " - "size not reset"; - if (src.size() <= kMaxShortDataSize) { - if (src.empty()) return; - EnsureHasHere(); - size_ = src.size(); - std::memcpy(short_data_begin(), src.data(), src.size()); - return; - } - // `std::move(src)` is correct and `std::forward(src)` is not necessary: - // `Src` is always `std::string`, never an lvalue reference. - InitializeSlow(std::move(src)); -} - -extern template void Chain::InitializeSlow(std::string&& src); - inline void Chain::Initialize(Block src) { size_ = src.raw_block()->size(); (end_++)->block_ptr = std::move(src).raw_block().Release(); @@ -1142,8 +1178,61 @@ inline absl::Span Chain::PrependFixedBuffer(size_t length, return PrependBuffer(length, length, length, options); } -extern template void Chain::Append(std::string&& src, Options options); -extern template void Chain::Prepend(std::string&& src, Options options); +template < + typename Src, + std::enable_if_t< + absl::conjunction, + absl::negation>>::value, + int>> +inline void Chain::Append(Src&& src, Options options) { + Append(riegeli::ToStringView(src), options); +} + +template < + typename Src, + std::enable_if_t< + absl::conjunction, + absl::negation>>::value, + int>> +inline void Chain::Prepend(Src&& src, Options options) { + Prepend(riegeli::ToStringView(src), options); +} + +inline void Chain::Append(ExternalRef src) { std::move(src).AppendTo(*this); } + +inline void Chain::Append(ExternalRef src, Options options) { + std::move(src).AppendTo(*this, options); +} + +inline void Chain::Prepend(ExternalRef src) { std::move(src).PrependTo(*this); } + +inline void Chain::Prepend(ExternalRef src, Options options) { + std::move(src).PrependTo(*this, options); +} + +template ::value, int>> +inline void Chain::Append(Src&& src) { + ExternalRef(std::forward(src)).AppendTo(*this); +} + +template ::value, int>> +inline void Chain::Append(Src&& src, Options options) { + ExternalRef(std::forward(src)).AppendTo(*this, options); +} + +template ::value, int>> +inline void Chain::Prepend(Src&& src) { + ExternalRef(std::forward(src)).PrependTo(*this); +} + +template ::value, int>> +inline void Chain::Prepend(Src&& src, Options options) { + ExternalRef(std::forward(src)).PrependTo(*this, options); +} template HashState Chain::HashValue(HashState hash_state) const { diff --git a/riegeli/base/compact_string.h b/riegeli/base/compact_string.h index 52a9001c..3c79c214 100644 --- a/riegeli/base/compact_string.h +++ b/riegeli/base/compact_string.h @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -33,7 +32,6 @@ #include "riegeli/base/assert.h" #include "riegeli/base/compare.h" #include "riegeli/base/external_data.h" -#include "riegeli/base/external_ref.h" #include "riegeli/base/new_aligned.h" namespace riegeli { @@ -227,50 +225,25 @@ class dest->append(src); } - // Converts `*this` to `ExternalRef`. - // - // `storage` must outlive usages of the returned `ExternalRef`. - ExternalRef ToExternalRef( - ExternalRef::StorageSubstr&& storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) && { - const uintptr_t tag = repr_ & 7; - if (tag == 1) { - return ExternalRef(absl::string_view(inline_data(), inline_size()), - std::move(storage)); - } - const absl::string_view data(allocated_data(), allocated_size_for_tag(tag)); - return ExternalRef(std::move(*this), data, std::move(storage)); - } + // Indicate support for: + // * `ExternalRef(CompactString&&)` + // * `ExternalRef(CompactString&&, substr)` + friend void RiegeliSupportsExternalRef(CompactString*) {} - // Converts a substring of `*this` to `ExternalRef`. - // - // `storage` must outlive usages of the returned `ExternalRef`. - // - // Precondition: if `!substr.empty()` then `substr` is a substring of `**this` - ExternalRef ToExternalRef( - absl::string_view substr, - ExternalRef::StorageSubstr&& storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) && { - if (!substr.empty()) { - RIEGELI_ASSERT(std::greater_equal<>()(substr.data(), data())) - << "Failed precondition of CompactString::ToExternalRef(): " - "substring not contained in the string"; - RIEGELI_ASSERT( - std::less_equal<>()(substr.data() + substr.size(), data() + size())) - << "Failed precondition of CompactString::ToExternalRef(): " - "substring not contained in the string"; - } - const uintptr_t tag = repr_ & 7; - if (tag == 1) return ExternalRef(substr, std::move(storage)); - return ExternalRef(std::move(*this), substr, std::move(storage)); + // Support `ExternalRef`. + friend bool RiegeliExternalCopy(const CompactString* self) { + return (self->repr_ & 7) == 1; } // Support `ExternalRef`. friend size_t RiegeliExternalMemory(const CompactString* self) { const uintptr_t tag = self->repr_ & 7; - if (ABSL_PREDICT_FALSE(tag == 1)) return 0; + if (tag == 1) { + RIEGELI_ASSERT_UNREACHABLE() + << "Failed precondition of " + "RiegeliExternalMemory(const CompactString*): " + "case excluded by RiegeliExternalCopy()"; + } const size_t offset = tag == 0 ? 2 * sizeof(size_t) : IntCast(tag); return offset + self->allocated_capacity_for_tag(tag); } @@ -279,7 +252,16 @@ class friend ExternalStorage RiegeliToExternalStorage(CompactString* self) { return ExternalStorage( reinterpret_cast(std::exchange(self->repr_, kDefaultRepr)), - [](void* ptr) { DeleteRepr(reinterpret_cast(ptr)); }); + [](void* ptr) { + const uintptr_t repr = reinterpret_cast(ptr); + if ((repr & 7) == 1) { + RIEGELI_ASSERT_UNREACHABLE() + << "Failed precondition of " + "RiegeliExternalStorage(CompactString*): " + "case excluded by RiegeliExternalCopy()"; + } + DeleteRepr(repr); + }); } // Support `ExternalRef` and `Chain::Block`. diff --git a/riegeli/base/cord_utils.cc b/riegeli/base/cord_utils.cc index b1f597d2..8149d502 100644 --- a/riegeli/base/cord_utils.cc +++ b/riegeli/base/cord_utils.cc @@ -48,6 +48,15 @@ absl::Cord MakeBlockyCord(absl::string_view src) { return dest; } +void AssignToBlockyCord(absl::string_view src, absl::Cord& dest) { + if (src.size() <= absl::CordBuffer::kDefaultLimit) { + dest = src; + return; + } + dest.Clear(); + AppendToBlockyCord(src, dest); +} + void AppendToBlockyCord(absl::string_view src, absl::Cord& dest) { if (src.empty()) return; { diff --git a/riegeli/base/cord_utils.h b/riegeli/base/cord_utils.h index d6579b1c..8e1d97d8 100644 --- a/riegeli/base/cord_utils.h +++ b/riegeli/base/cord_utils.h @@ -35,10 +35,6 @@ namespace cord_internal { RIEGELI_INLINE_CONSTEXPR(size_t, kFlatOverhead, sizeof(size_t) + sizeof(uint32_t) + sizeof(uint8_t)); -// `sizeof(absl::cord_internal::CordRepExternal)`. Does not have to be -// accurate. -RIEGELI_INLINE_CONSTEXPR(size_t, kSizeOfCordRepExternal, 4 * sizeof(void*)); - // The `block_size` parameter for `absl::CordBuffer::CreateWithCustomLimit()`. RIEGELI_INLINE_CONSTEXPR(size_t, kCordBufferBlockSize, UnsignedMin(kDefaultMaxBlockSize, @@ -82,14 +78,14 @@ void AppendCordToString(const absl::Cord& src, std::string& dest); // Variants of `absl::Cord` operations with different block sizing tradeoffs: // * `MakeBlockyCord(src)` is like `absl::Cord(src)`. +// * `AssignToBlockyCord(src, dest)` is like `dest = src`. // * `AppendToBlockyCord(src, dest)` is like `dest.Append(src)`. // * `PrependToBlockyCord(src, dest)` is like `dest.Prepend(src)`. // -// They assume that the `absl::Cord` is constructed from fragments of reasonable -// sizes, with adjacent sizes being not too small. -// -// They avoid splitting `src` into 4083-byte fragments and avoid overallocation. +// They avoid splitting `src` into 4083-byte fragments and avoid overallocation, +// without guarantees. absl::Cord MakeBlockyCord(absl::string_view src); +void AssignToBlockyCord(absl::string_view src, absl::Cord& dest); void AppendToBlockyCord(absl::string_view src, absl::Cord& dest); void PrependToBlockyCord(absl::string_view src, absl::Cord& dest); diff --git a/riegeli/base/dependency.h b/riegeli/base/dependency.h index 8d8c1d3d..7236d23e 100644 --- a/riegeli/base/dependency.h +++ b/riegeli/base/dependency.h @@ -329,8 +329,7 @@ class DependencyImpl< }; // Specialization of `DependencyImpl` when -// `DependencyManagerRef` supports `RiegeliToStringView(&object)` or is -// explicitly convertible to `absl::string_view` or `absl::Span`. +// `DependencyManagerRef` supports `riegeli::ToStringView(object)`. template class DependencyImpl< absl::string_view, Manager, diff --git a/riegeli/base/external_data.h b/riegeli/base/external_data.h index 0b6b0df1..b8c06bb0 100644 --- a/riegeli/base/external_data.h +++ b/riegeli/base/external_data.h @@ -16,6 +16,7 @@ #define RIEGELI_BASE_EXTERNAL_DATA_H_ #include +#include #include "absl/strings/string_view.h" @@ -27,14 +28,29 @@ namespace riegeli { // and `ExternalStorage::get_deleter() -> void (*)(void*)`. using ExternalStorage = std::unique_ptr; +// Support `ExternalRef`. +inline ExternalStorage RiegeliToExternalStorage(ExternalStorage* self) { + return std::move(*self); +} + // Type-erased external object with its deleter and a substring of a byte array // it ows. struct ExternalData { + // Indicate support for: + // * `ExternalRef(ExternalData&&)` + // * `ExternalRef(ExternalData&&, substr)` + friend void RiegeliSupportsExternalRef(ExternalData*) {} + // Support `ExternalRef`. friend absl::string_view RiegeliToStringView(const ExternalData* self) { return self->substr; } + // Support `ExternalRef`. + friend ExternalStorage RiegeliToExternalStorage(ExternalData* self) { + return std::move(self->storage); + } + ExternalStorage storage; // Must outlive usages of `substr`. absl::string_view substr; }; diff --git a/riegeli/base/external_ref.h b/riegeli/base/external_ref.h index 35ee8213..f6242bbb 100644 --- a/riegeli/base/external_ref.h +++ b/riegeli/base/external_ref.h @@ -15,8 +15,9 @@ #ifndef RIEGELI_BASE_EXTERNAL_REF_H_ #define RIEGELI_BASE_EXTERNAL_REF_H_ -#include "riegeli/base/chain_base.h" // IWYU pragma: keep -#include "riegeli/base/chain_details.h" // IWYU pragma: keep -#include "riegeli/base/external_ref_base.h" // IWYU pragma: export +#include "riegeli/base/chain_base.h" // IWYU pragma: keep +#include "riegeli/base/chain_details.h" // IWYU pragma: keep +#include "riegeli/base/external_ref_base.h" // IWYU pragma: export +#include "riegeli/base/external_ref_support.h" // IWYU pragma: export #endif // RIEGELI_BASE_EXTERNAL_REF_H_ diff --git a/riegeli/base/external_ref_base.h b/riegeli/base/external_ref_base.h index 1c4d5962..ec0d0997 100644 --- a/riegeli/base/external_ref_base.h +++ b/riegeli/base/external_ref_base.h @@ -19,10 +19,10 @@ #include +#include #include #include #include -#include #include #include @@ -35,6 +35,7 @@ #include "riegeli/base/chain_base.h" #include "riegeli/base/cord_utils.h" #include "riegeli/base/external_data.h" +#include "riegeli/base/external_ref_support.h" #include "riegeli/base/initializer.h" #include "riegeli/base/temporary_storage.h" #include "riegeli/base/to_string_view.h" @@ -42,37 +43,6 @@ namespace riegeli { -// Support `ExternalRef`. -inline size_t RiegeliExternalMemory(ABSL_ATTRIBUTE_UNUSED const void* self) { - return 0; -} - -// Support `ExternalRef`. -inline size_t RiegeliExternalMemory(const std::string* self) { - // Do not bother checking for short string optimization. Such strings will - // likely not be considered wasteful anyway. - return self->capacity() + 1; -} - -// Support `ExternalRef`. -template -inline ExternalStorage RiegeliToExternalStorage(std::unique_ptr* self) { - return ExternalStorage(const_cast*>(self->release()), - [](void* ptr) { delete static_cast(ptr); }); -} - -// Support `ExternalRef`. -template -inline ExternalStorage RiegeliToExternalStorage(std::unique_ptr* self) { - return ExternalStorage(const_cast*>(self->release()), - [](void* ptr) { delete[] static_cast(ptr); }); -} - -// Support `ExternalRef`. -inline ExternalStorage RiegeliToExternalStorage(ExternalStorage* self) { - return std::move(*self); -} - // `ExternalRef` specifies a byte array in a way which allows sharing it with // other objects without copying if that is considered more efficient than // copying. It mediates between the producer and the consumer of the data during @@ -86,8 +56,9 @@ inline ExternalStorage RiegeliToExternalStorage(ExternalStorage* self) { // constructed object is not needed otherwise. // // `ExternalRef` can be converted to `absl::string_view`, `Chain`, `absl::Cord`, -// or `ExternalData`, or appended or prepended to a `Chain` or `absl::Cord`. -// It can be consumed at most once. +// or `ExternalData`, or assigned, appended, or prepended to a `Chain` or +// `absl::Cord`. Apart from conversion to `absl::string_view` it can be consumed +// at most once. // // In contrast to `Chain::Block` and `absl::MakeCordFromExternal()`, // `ExternalRef` chooses between sharing the object and copying the data, @@ -102,18 +73,44 @@ inline ExternalStorage RiegeliToExternalStorage(ExternalStorage* self) { // The expected interface of the object which owns the data is a superset of the // interfaces expected by `Chain::Block` and `absl::MakeCordFromExternal()`. // -// If the `substr` parameter to `ExternalRef` constructor is given, `substr` -// must be valid for the new object if it gets created. -// -// If the `substr` parameter is not given, `T` must support any of: +// `ExternalRef` constructors require the external object type to indicate +// that it supports `ExternalRef` by providing one of the following functions +// (only their presence is checked, they are never called): // ``` -// // Returns contents of the object. Called when it is moved to its final -// // location. -// friend absl::string_view RiegeliToStringView(const T* self); -// explicit operator absl::string_view() const; -// explicit operator absl::Span() const; +// // Indicate support for `ExternalRef(T&&, substr)`. +// // +// // `substr` must be owned by the object if it gets created or moved, unless +// // `RiegeliExternalCopy()` (see below) recognizes cases when it is not. +// // +// // If `T` supports `riegeli::ToStringView()`, then also indicates support +// // for `ExternalRef(T&&)`. +// // +// // The parameter can also have type `const T*`. This also indicates +// // support for `ExternalRef(const T&, substr)` and possibly +// // `ExternalRef(const T&)`, i.e. that `T` is copyable and copying it is +// // more efficient than copying the data. +// // +// // If the `ExternalRef` is later converted to `absl::Cord` and +// // `absl::MakeCordFromExternal()` gets used, then this avoids an allocation +// // by taking advantage of the promise that `substr` will be owned also by +// // the moved object (`absl::MakeCordFromExternal()` requires knowing the +// // data before specifying the object to be moved). +// friend void RiegeliSupportsExternalRef(T*) {} +// +// // Indicate support for `ExternalRef(T&&)` as long as `T` supports +// // `riegeli::ToStringView()`. +// // +// // The parameter can also have type `const T*`. This also indicates support +// // for `ExternalRef(const T&)`, i.e. that `T` is copyable and copying it is +// // more efficient than copying the data. +// friend void RiegeliSupportsExternalRefWhole(T*) {} // ``` // +// `ExternalRef::From()` are like `ExternalRef` constructors, but +// `RiegeliSupportsExternalRef()` or `RiegeliSupportsExternalRefWhole()` is not +// needed. The caller is responsible for using an appropriate type of the +// external object. +// // `T` may also support the following member functions, either with or without // the `substr` parameter, with the following definitions assumed by default: // ``` @@ -128,6 +125,15 @@ inline ExternalStorage RiegeliToExternalStorage(ExternalStorage* self) { // // mutable copy of the object. // void operator()(absl::string_view substr) && {} // +// // If this returns `true`, the data will be copied instead of wrapping the +// // object. The data does not need to be stable while the object is moved. +// // `RiegeliExternalMemory()`, `RiegeliToChainBlock()`, `RiegeliToCord()`, +// // `RiegeliToExternalData()`, `RiegeliToExternalStorage()`, nor +// // `RiegeliExternalDelegate()` will not be called. +// // +// // Typically this indicates an object with short data stored inline. +// friend bool RiegeliExternalCopy(const T* self) { return false; } +// // // Returns an approximate amount of memory allocated by the object, // // excluding data stored inside the object itself. // // @@ -150,8 +156,8 @@ inline ExternalStorage RiegeliToExternalStorage(ExternalStorage* self) { // // and the object is passed by const or lvalue reference, this will be // // called on a mutable copy of the object. // // -// // The `substr` parameter is given here when the `substr` parameter to -// // `ExternalRef` constructor was given. +// // If the `substr` parameter was given to `ExternalRef` constructor, the +// // `substr` parameter is required here, otherwise it is optional. // friend Chain::Block RiegeliToChainBlock(T* self, absl::string_view substr); // // // Converts `*self` or its `substr` to `absl::Cord`, if this can be done @@ -162,8 +168,8 @@ inline ExternalStorage RiegeliToExternalStorage(ExternalStorage* self) { // // and the object is passed by const or lvalue reference, this will be // // called on a mutable copy of the object. // // -// // The `substr` parameter is given here when the `substr` parameter to -// // `ExternalRef` constructor was given. +// // If the `substr` parameter was given to `ExternalRef` constructor, the +// // `substr` parameter is required here, otherwise it is optional. // friend absl::Cord RiegeliToCord(T* self, absl::string_view substr); // // // Converts `*self` to `ExternalData`, if this can be done more efficiently @@ -174,23 +180,45 @@ inline ExternalStorage RiegeliToExternalStorage(ExternalStorage* self) { // // and the object is passed by const or lvalue reference, this will be // // called on a mutable copy of the object. // // -// // Used for conversion to `ExternalData` when the `substr` parameter to -// // `ExternalRef` constructor was not given. -// friend ExternalData RiegeliToExternalData(T* self); +// // If the `substr` parameter was given to `ExternalRef` constructor, the +// // `substr` parameter is required here, otherwise it is optional. +// friend ExternalData RiegeliToExternalData(T* self, +// absl::string_view substr); +// +// // This can be defined instead of `RiegeliToExternalData()` with `substr`, +// // which would return `ExternalData` with the same `substr`. +// friend ExternalStorage RiegeliToExternalStorage(T* self); // -// // Convert `*self` to `ExternalStorage`, if this can be done more -// // efficiently than allocating the object on the heap, e.g. if the object -// // fits in a pointer. Can modify `*self`. `operator()` will no longer be -// // called. +// // If defined, indicates a subobject to wrap instead of the whole object. +// // It must call `std::forward(delegate_to)(subobject)` or +// // `std::forward(delegate_to)(subobject, substr)`, preferably +// // with `std::move(subobject)`. `delegate_to` must be called exactly once. +// // +// // Typically this indicates a smaller object which is sufficient to keep +// // the data alive, or the active variant if the object stores one of +// // multiple subobjects. +// // +// // `RiegeliToChainBlock()`, `RiegeliToCord()`, `RiegeliToExternalData()`, +// // and `RiegeliToExternalStorage()`, if defined, are used in preference to +// // this. +// // +// // The subobject will be processed like by `ExternalRef::From()`, including +// // the possibility of further delegation, except that `Initializer` is not +// // supported. The subobject must not be `*self`. // // // // The `self` parameter can also have type `const T*`. If it has type `T*` // // and the object is passed by const or lvalue reference, this will be // // called on a mutable copy of the object. // // -// // Used for conversion to `ExternalData` when the `substr` parameter to -// // `ExternalRef` constructor was given. The same `substr` will be used in -// // the returned `ExternalData`. -// friend ExternalStorage RiegeliToExternalStorage(T* self); +// // The `substr` parameter is optional here. If absent here while the +// // `substr` parameter was given to `ExternalRef` constructor, then it is +// // propagated. If absent here while the `substr` parameter was not given +// // to `ExternalRef` constructor, then `riegeli::ToStringView(subobject)` +// // must be supported. If present here, it can be used to specify the data +// // if `riegeli::ToStringView(subobject)` is not supported. +// template +// friend void RiegeliExternalDelegate(T* self, absl::string_view substr, +// Callback&& delegate_to); // // // Shows internal structure in a human-readable way, for debugging. // // @@ -220,15 +248,16 @@ class ExternalRef { using UseStringViewFunction = void (*)(void* context, absl::string_view data); using UseChainBlockFunction = void (*)(void* context, Chain::Block data); using UseCordFunction = void (*)(void* context, absl::Cord data); + using UseExternalDataFunction = void (*)(void* context, ExternalData data); template - static bool Wasteful(const T& object, size_t extra_allocated, size_t used) { + static bool Wasteful(const T& object, size_t used) { size_t allocated = RiegeliExternalMemory(&object); - if (extra_allocated > std::numeric_limits::max() - allocated) { + if (sizeof(T) > std::numeric_limits::max() - allocated) { RIEGELI_ASSERT_UNREACHABLE() << "Result of RiegeliExternalMemory() " "suspiciously close to size_t range"; } - allocated += extra_allocated; + allocated += sizeof(T); return allocated >= used && riegeli::Wasteful(allocated, used); } @@ -254,7 +283,8 @@ class ExternalRef { template ::value, int> = 0> static void CallOperatorWhole(T&& object) { - std::forward(object)(riegeli::ToStringView(object)); + const absl::string_view data = riegeli::ToStringView(object); + std::forward(object)(data); } template = 0> static void CallOperatorWhole(T&& object) { absl::remove_cvref_t copy(object); - std::move(copy)(riegeli::ToStringView(copy)); + const absl::string_view data = riegeli::ToStringView(copy); + std::move(copy)(data); } template < typename T, @@ -312,7 +343,8 @@ class ExternalRef { int> = 0> static void CallOperatorSubstr(T&& object, absl::string_view substr) { absl::remove_cvref_t copy(object); - std::move(copy)(riegeli::ToStringView(copy)); + const absl::string_view data = riegeli::ToStringView(copy); + std::move(copy)(data); } template < typename T, @@ -333,95 +365,415 @@ class ExternalRef { ABSL_ATTRIBUTE_UNUSED absl::string_view substr) {} template - struct PointerType { - using type = T*; - }; - template - struct PointerType { - using type = const T*; - }; - template - struct PointerType { - using type = T*; - }; - - template - using PointerTypeT = typename PointerType::type; + static external_ref_internal::PointerTypeT Pointer(T&& object) { + return &object; + } +#if RIEGELI_DEBUG + template ::value, int> = 0> + static void AssertSubstr(const T& object, absl::string_view substr) { + if (!substr.empty()) { + const absl::string_view whole = riegeli::ToStringView(object); + RIEGELI_ASSERT(std::greater_equal<>()(substr.data(), whole.data())) + << "Failed precondition of ExternalRef: " + "substring not contained in whole data"; + RIEGELI_ASSERT(std::less_equal<>()(substr.data() + substr.size(), + whole.data() + whole.size())) + << "Failed precondition of ExternalRef: " + "substring not contained in whole data"; + } + } + template ::value, int> = 0> +#else template - static PointerTypeT Pointer(T&& object) { - return &object; +#endif + static void AssertSubstr(ABSL_ATTRIBUTE_UNUSED const T& object, + ABSL_ATTRIBUTE_UNUSED absl::string_view substr) { } - template - struct HasRiegeliToChainBlockWhole : std::false_type {}; + template + struct HasRiegeliExternalDelegateWhole : std::false_type {}; + + template + struct HasRiegeliExternalDelegateWhole< + T, Callback, + absl::void_t>(), + std::declval()))>> : std::true_type {}; + + template + struct HasRiegeliExternalDelegateSubstr : std::false_type {}; + + template + struct HasRiegeliExternalDelegateSubstr< + T, Callback, + absl::void_t>(), + std::declval(), std::declval()))>> + : std::true_type {}; - template - struct HasRiegeliToChainBlockWhole< - T, std::enable_if_t>())), - Chain::Block>::value>> : std::true_type {}; + template + struct HasExternalDelegateWhole + : absl::disjunction, + HasRiegeliExternalDelegateSubstr> {}; - template ::value, int> = 0> - static Chain::Block ToChainBlockExternal(T&& object) { - return RiegeliToChainBlock(ExternalRef::Pointer(std::forward(object))); + template ::value, int> = 0> + static void ExternalDelegateWhole(T&& object, Callback&& delegate_to) { + RiegeliExternalDelegate(ExternalRef::Pointer(std::forward(object)), + std::forward(delegate_to)); } template < - typename T, + typename T, typename Callback, std::enable_if_t< absl::conjunction< - absl::negation>, - HasRiegeliToChainBlockWhole>>::value, + absl::negation>, + HasRiegeliExternalDelegateSubstr>::value, int> = 0> - static Chain::Block ToChainBlockExternal(T&& object) { - return RiegeliToChainBlock( - ExternalRef::Pointer(absl::remove_cvref_t(object))); + static void ExternalDelegateWhole(T&& object, Callback&& delegate_to) { + const absl::string_view data = riegeli::ToStringView(object); + RiegeliExternalDelegate(ExternalRef::Pointer(std::forward(object)), data, + std::forward(delegate_to)); } - template >::value, - int> = 0> - static Chain::Block ToChainBlockExternal(T&& object) { - return Chain::Block(std::forward(object)); + + template ::value, int> = 0> + static void ExternalDelegateWhole( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data, + Callback&& delegate_to) { + RiegeliExternalDelegate(ExternalRef::Pointer(std::forward(object)), + std::forward(delegate_to)); + } + template < + typename T, typename Callback, + std::enable_if_t< + absl::conjunction< + absl::negation>, + HasRiegeliExternalDelegateSubstr>::value, + int> = 0> + static void ExternalDelegateWhole(T&& object, absl::string_view data, + Callback&& delegate_to) { + RiegeliExternalDelegate(ExternalRef::Pointer(std::forward(object)), data, + std::forward(delegate_to)); } + template + struct HasExternalDelegateSubstr + : absl::disjunction, + HasRiegeliExternalDelegateWhole> {}; + + template ::value, int> = 0> + static void ExternalDelegateSubstr(T&& object, absl::string_view substr, + Callback&& delegate_to) { + RiegeliExternalDelegate(ExternalRef::Pointer(std::forward(object)), + substr, std::forward(delegate_to)); + } + template < + typename T, typename Callback, + std::enable_if_t< + absl::conjunction< + absl::negation>, + HasRiegeliExternalDelegateWhole>::value, + int> = 0> + static void ExternalDelegateSubstr( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view substr, + Callback&& delegate_to) { + RiegeliExternalDelegate(ExternalRef::Pointer(std::forward(object)), + std::forward(delegate_to)); + } + + template + struct HasRiegeliToChainBlockWhole : std::false_type {}; + + template + struct HasRiegeliToChainBlockWhole< + T, std::enable_if_t>())), + Chain::Block>::value>> : std::true_type {}; + template struct HasRiegeliToChainBlockSubstr : std::false_type {}; template struct HasRiegeliToChainBlockSubstr< T, std::enable_if_t>(), - std::declval())), + decltype(RiegeliToChainBlock( + std::declval>(), + std::declval())), Chain::Block>::value>> : std::true_type {}; + template + struct HasToChainBlockWhole + : absl::disjunction, + HasRiegeliToChainBlockSubstr> {}; + template ::value, int> = 0> - static Chain::Block ToChainBlockExternal(T&& object, - absl::string_view substr) { + std::enable_if_t::value, int> = 0> + static Chain::Block ToChainBlockWhole(T&& object) { + return RiegeliToChainBlock(ExternalRef::Pointer(std::forward(object))); + } + template >, + HasRiegeliToChainBlockSubstr>::value, + int> = 0> + static Chain::Block ToChainBlockWhole(T&& object) { + const absl::string_view data = riegeli::ToStringView(object); return RiegeliToChainBlock(ExternalRef::Pointer(std::forward(object)), - substr); + data); } - template < - typename T, - std::enable_if_t< - absl::conjunction< - absl::negation>, - HasRiegeliToChainBlockSubstr>>::value, - int> = 0> - static Chain::Block ToChainBlockExternal(T&& object, - absl::string_view substr) { - return RiegeliToChainBlock( - ExternalRef::Pointer(absl::remove_cvref_t(object)), substr); + + template ::value, int> = 0> + static Chain::Block ToChainBlockWhole( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) { + return RiegeliToChainBlock(ExternalRef::Pointer(std::forward(object))); } - template >::value, - int> = 0> - static Chain::Block ToChainBlockExternal(T&& object, - absl::string_view substr) { - return Chain::Block(std::forward(object), substr); + template >, + HasRiegeliToChainBlockSubstr>::value, + int> = 0> + static Chain::Block ToChainBlockWhole(T&& object, absl::string_view data) { + return RiegeliToChainBlock(ExternalRef::Pointer(std::forward(object)), + data); } + template + using HasToChainBlockSubstr = HasRiegeliToChainBlockSubstr; + + template ::value, int> = 0> + static Chain::Block ToChainBlockSubstr(T&& object, absl::string_view substr) { + return RiegeliToChainBlock(ExternalRef::Pointer(std::forward(object)), + substr); + } + + template + class ConverterToChainBlockWhole { + public: + ConverterToChainBlockWhole(const ConverterToChainBlockWhole&) = delete; + ConverterToChainBlockWhole& operator=(const ConverterToChainBlockWhole&) = + delete; + + template ::value, int> = 0> + void operator()(SubT&& subobject) && { + // The constructor processes the subobject. + const absl::string_view data = riegeli::ToStringView(subobject); + ConverterToChainBlockWhole converter( + std::forward(subobject), data, context_, use_string_view_, + use_chain_block_); + } + + template + void operator()(SubT&& subobject, absl::string_view substr) && { + // The constructor processes the subobject. + ConverterToChainBlockSubstr converter( + std::forward(subobject), substr, context_, use_string_view_, + use_chain_block_); + } + + private: + friend class ExternalRef; + + ABSL_ATTRIBUTE_ALWAYS_INLINE + explicit ConverterToChainBlockWhole(T&& object, absl::string_view data, + void* context, + UseStringViewFunction use_string_view, + UseChainBlockFunction use_chain_block) + : context_(context), + use_string_view_(use_string_view), + use_chain_block_(use_chain_block) { + if (RiegeliExternalCopy(&object) || Wasteful(object, data.size())) { + use_string_view_(context_, data); + ExternalRef::CallOperatorWhole(std::forward(object)); + return; + } + std::move(*this).Callback(std::forward(object), data); + } + + template < + typename DependentT = T, + std::enable_if_t::value, int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object, + absl::string_view data) && { + use_chain_block_(context_, ExternalRef::ToChainBlockWhole( + std::forward(object), data)); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction< + absl::negation>, + HasToChainBlockWhole>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) && { + use_chain_block_(context_, ExternalRef::ToChainBlockWhole( + absl::remove_cvref_t(object))); + } + template >>, + HasExternalDelegateWhole< + DependentT, ConverterToChainBlockWhole>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object, + absl::string_view data) && { + ExternalRef::ExternalDelegateWhole(std::forward(object), data, + std::move(*this)); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction< + absl::negation< + HasToChainBlockWhole>>, + absl::negation>, + HasExternalDelegateWhole, + ConverterToChainBlockWhole>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) && { + ExternalRef::ExternalDelegateWhole(absl::remove_cvref_t(object), + std::move(*this)); + } + template >>, + absl::negation, + ConverterToChainBlockWhole>>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) && { + use_chain_block_(context_, Chain::Block(std::forward(object))); + } + + void* context_; + UseStringViewFunction use_string_view_; + UseChainBlockFunction use_chain_block_; + }; + + template + class ConverterToChainBlockSubstr { + public: + ConverterToChainBlockSubstr(const ConverterToChainBlockSubstr&) = delete; + ConverterToChainBlockSubstr& operator=(const ConverterToChainBlockSubstr&) = + delete; + + template + void operator()(SubT&& subobject) && { + std::move (*this)(std::forward(subobject), substr_); + } + + template + void operator()(SubT&& subobject, absl::string_view substr) && { + RIEGELI_ASSERT_EQ(substr_.size(), substr.size()) + << "ExternalRef: size mismatch"; + // The constructor processes the subobject. + ConverterToChainBlockSubstr converter( + std::forward(subobject), substr, context_, use_string_view_, + use_chain_block_); + } + + private: + friend class ExternalRef; + + ABSL_ATTRIBUTE_ALWAYS_INLINE + explicit ConverterToChainBlockSubstr(T&& object, absl::string_view substr, + void* context, + UseStringViewFunction use_string_view, + UseChainBlockFunction use_chain_block) + : substr_(substr), + context_(context), + use_string_view_(use_string_view), + use_chain_block_(use_chain_block) { + AssertSubstr(object, substr_); + if (RiegeliExternalCopy(&object) || Wasteful(object, substr_.size())) { + use_string_view_(context_, substr_); + ExternalRef::CallOperatorSubstr(std::forward(object), substr_); + return; + } + std::move(*this).Callback(std::forward(object)); + } + + template < + typename DependentT = T, + std::enable_if_t::value, int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + use_chain_block_(context_, ExternalRef::ToChainBlockSubstr( + std::forward(object), substr_)); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction< + absl::negation>, + HasToChainBlockSubstr>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + use_chain_block_(context_, ExternalRef::ToChainBlockSubstr( + absl::remove_cvref_t(object), substr_)); + } + template >>, + HasExternalDelegateSubstr< + DependentT, ConverterToChainBlockSubstr>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + ExternalRef::ExternalDelegateSubstr(std::forward(object), substr_, + std::move(*this)); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction< + absl::negation< + HasToChainBlockSubstr>>, + absl::negation>, + HasExternalDelegateSubstr, + ConverterToChainBlockSubstr>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + ExternalRef::ExternalDelegateSubstr(absl::remove_cvref_t(object), + substr_, std::move(*this)); + } + template >>, + absl::negation, + ConverterToChainBlockSubstr>>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + use_chain_block_(context_, + Chain::Block(std::forward(object), substr_)); + } + + absl::string_view substr_; + void* context_; + UseStringViewFunction use_string_view_; + UseChainBlockFunction use_chain_block_; + }; + template class ObjectForCordWhole { public: @@ -445,93 +797,328 @@ class ExternalRef { std::unique_ptr ptr_; }; + template + class ObjectForCordSubstr { + public: + explicit ObjectForCordSubstr(Initializer object) + : object_(std::move(object)) {} + + ObjectForCordSubstr(ObjectForCordSubstr&& that) = default; + ObjectForCordSubstr& operator=(ObjectForCordSubstr&& that) = default; + + void operator()(absl::string_view substr) && { + ExternalRef::CallOperatorSubstr(std::move(object_), substr); + } + + T& operator*() { return object_; } + const T& operator*() const { return object_; } + + private: + ABSL_ATTRIBUTE_NO_UNIQUE_ADDRESS T object_; + }; + template struct HasRiegeliToCordWhole : std::false_type {}; template struct HasRiegeliToCordWhole< T, std::enable_if_t>())), + decltype(RiegeliToCord( + std::declval>())), absl::Cord>::value>> : std::true_type {}; + template + struct HasRiegeliToCordSubstr : std::false_type {}; + + template + struct HasRiegeliToCordSubstr< + T, std::enable_if_t>(), + std::declval())), + absl::Cord>::value>> : std::true_type {}; + + template + struct HasToCordWhole + : absl::disjunction, HasRiegeliToCordSubstr> { + }; + template ::value, int> = 0> - static absl::Cord ToCordExternal(T&& object) { + static absl::Cord ToCordWhole(T&& object) { return RiegeliToCord(ExternalRef::Pointer(std::forward(object))); } template >, - HasRiegeliToCordWhole>>::value, + absl::conjunction>, + HasRiegeliToCordSubstr>::value, int> = 0> - static absl::Cord ToCordExternal(T&& object) { - return RiegeliToCord(ExternalRef::Pointer(absl::remove_cvref_t(object))); - } - template < - typename T, - std::enable_if_t>::value, - int> = 0> - static absl::Cord ToCordExternal(T&& object) { - ObjectForCordWhole> object_for_cord( - std::forward(object)); - return absl::MakeCordFromExternal(riegeli::ToStringView(*object_for_cord), - std::move(object_for_cord)); + static absl::Cord ToCordWhole(T&& object) { + const absl::string_view data = riegeli::ToStringView(object); + return RiegeliToCord(ExternalRef::Pointer(std::forward(object)), data); } - template - struct HasRiegeliToCordSubstr : std::false_type {}; + template ::value, int> = 0> + static absl::Cord ToCordWhole(T&& object, + ABSL_ATTRIBUTE_UNUSED absl::string_view data) { + return RiegeliToCord(ExternalRef::Pointer(std::forward(object))); + } + template >, + HasRiegeliToCordSubstr>::value, + int> = 0> + static absl::Cord ToCordWhole(T&& object, absl::string_view data) { + return RiegeliToCord(ExternalRef::Pointer(std::forward(object)), data); + } template - struct HasRiegeliToCordSubstr< - T, std::enable_if_t>(), - std::declval())), - absl::Cord>::value>> : std::true_type {}; + using HasToCordSubstr = HasRiegeliToCordSubstr; + + template ::value, int> = 0> + static absl::Cord ToCordSubstr(T&& object, absl::string_view substr) { + return RiegeliToCord(ExternalRef::Pointer(std::forward(object)), substr); + } template - class ObjectForCordSubstr { + class ConverterToCordWhole { public: - explicit ObjectForCordSubstr(Initializer object) - : object_(std::move(object)) {} + ConverterToCordWhole(const ConverterToCordWhole&) = delete; + ConverterToCordWhole& operator=(const ConverterToCordWhole&) = delete; + + template ::value, int> = 0> + void operator()(SubT&& subobject) && { + // The constructor processes the subobject. + const absl::string_view data = riegeli::ToStringView(subobject); + ConverterToCordWhole converter(std::forward(subobject), data, + context_, use_string_view_, + use_cord_); + } - ObjectForCordSubstr(ObjectForCordSubstr&& that) = default; - ObjectForCordSubstr& operator=(ObjectForCordSubstr&& that) = default; + template + void operator()(SubT&& subobject, absl::string_view substr) && { + // The constructor processes the subobject. + ConverterToCordSubstr converter(std::forward(subobject), + substr, context_, use_string_view_, + use_cord_); + } - void operator()(absl::string_view substr) && { - ExternalRef::CallOperatorSubstr(std::move(object_), substr); + private: + friend class ExternalRef; + + ABSL_ATTRIBUTE_ALWAYS_INLINE + explicit ConverterToCordWhole(T&& object, absl::string_view data, + void* context, + UseStringViewFunction use_string_view, + UseCordFunction use_cord) + : context_(context), + use_string_view_(use_string_view), + use_cord_(use_cord) { + if (RiegeliExternalCopy(&object) || Wasteful(object, data.size())) { + use_string_view_(context_, data); + ExternalRef::CallOperatorWhole(std::forward(object)); + return; + } + std::move(*this).Callback(std::forward(object), data); } - T& operator*() { return object_; } - const T& operator*() const { return object_; } + template ::value, int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object, + absl::string_view data) && { + use_cord_(context_, + ExternalRef::ToCordWhole(std::forward(object), data)); + } + template >, + HasToCordWhole>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) && { + use_cord_(context_, ExternalRef::ToCordWhole(absl::remove_cvref_t( + std::forward(object)))); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction>>, + HasExternalDelegateWhole< + DependentT, ConverterToCordWhole>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object, + absl::string_view data) && { + ExternalRef::ExternalDelegateWhole(std::forward(object), data, + std::move(*this)); + } + template >>, + absl::negation>, + HasExternalDelegateWhole, + ConverterToCordWhole>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) && { + ExternalRef::ExternalDelegateWhole(absl::remove_cvref_t(object), + std::move(*this)); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction< + absl::negation< + HasToCordWhole>>, + absl::negation, ConverterToCordWhole>>, + SupportsExternalRefSubstr>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object, + absl::string_view data) && { + // If the type indicates that substrings are stable, then + // `ObjectForCordSubstr` can be used instead of `ObjectForCordWhole`. + use_cord_(context_, absl::MakeCordFromExternal( + data, ObjectForCordSubstr>( + std::forward(object)))); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction< + absl::negation< + HasToCordWhole>>, + absl::negation, ConverterToCordWhole>>, + absl::negation>>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) && { + ObjectForCordWhole> object_for_cord( + std::forward(object)); + const absl::string_view moved_data = + riegeli::ToStringView(*object_for_cord); + use_cord_(context_, absl::MakeCordFromExternal( + moved_data, std::move(object_for_cord))); + } - private: - ABSL_ATTRIBUTE_NO_UNIQUE_ADDRESS T object_; + void* context_; + UseStringViewFunction use_string_view_; + UseCordFunction use_cord_; }; - template ::value, int> = 0> - static absl::Cord ToCordExternal(T&& object, absl::string_view substr) { - return RiegeliToCord(ExternalRef::Pointer(std::forward(object)), substr); - } - template >, - HasRiegeliToCordSubstr>>::value, - int> = 0> - static absl::Cord ToCordExternal(T&& object, absl::string_view substr) { - return RiegeliToCord(ExternalRef::Pointer(absl::remove_cvref_t(object)), - substr); - } - template < - typename T, - std::enable_if_t>::value, - int> = 0> - static absl::Cord ToCordExternal(T&& object, absl::string_view substr) { - return absl::MakeCordFromExternal( - substr, ObjectForCordSubstr>(std::forward(object))); - } + template + class ConverterToCordSubstr { + public: + ConverterToCordSubstr(const ConverterToCordSubstr&) = delete; + ConverterToCordSubstr& operator=(const ConverterToCordSubstr&) = delete; + + template + void operator()(SubT&& subobject) && { + std::move (*this)(std::forward(subobject), substr_); + } + + template + void operator()(SubT&& subobject, absl::string_view substr) && { + RIEGELI_ASSERT_EQ(substr_.size(), substr.size()) + << "ExternalRef: size mismatch"; + // The constructor processes the subobject. + ConverterToCordSubstr converter(std::forward(subobject), + substr, context_, use_string_view_, + use_cord_); + } + + private: + friend class ExternalRef; + + ABSL_ATTRIBUTE_ALWAYS_INLINE + explicit ConverterToCordSubstr(T&& object, absl::string_view substr, + void* context, + UseStringViewFunction use_string_view, + UseCordFunction use_cord) + : substr_(substr), + context_(context), + use_string_view_(use_string_view), + use_cord_(use_cord) { + AssertSubstr(object, substr_); + if (RiegeliExternalCopy(&object) || Wasteful(object, substr_.size())) { + use_string_view_(context_, substr_); + ExternalRef::CallOperatorSubstr(std::forward(object), substr_); + return; + } + std::move(*this).Callback(std::forward(object)); + } + + template ::value, int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + use_cord_(context_, + ExternalRef::ToCordSubstr(std::forward(object), substr_)); + } + template >, + HasToCordSubstr>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + use_cord_(context_, ExternalRef::ToCordSubstr( + absl::remove_cvref_t(object), substr_)); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction>>, + HasExternalDelegateSubstr< + DependentT, ConverterToCordSubstr>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + ExternalRef::ExternalDelegateSubstr(std::forward(object), substr_, + std::move(*this)); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction< + absl::negation< + HasToCordSubstr>>, + absl::negation>, + HasExternalDelegateSubstr, + ConverterToCordSubstr>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + ExternalRef::ExternalDelegateSubstr(absl::remove_cvref_t(object), + substr_, std::move(*this)); + } + template >>, + absl::negation, + ConverterToCordSubstr>>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + use_cord_(context_, absl::MakeCordFromExternal( + substr_, ObjectForCordSubstr>( + std::forward(object)))); + } + + absl::string_view substr_; + void* context_; + UseStringViewFunction use_string_view_; + UseCordFunction use_cord_; + }; template class ExternalObjectWhole { @@ -540,7 +1127,7 @@ class ExternalRef { : object_(std::move(object)) {} ~ExternalObjectWhole() { - ExternalRef::CallOperatorWhole(std::forward(object_)); + ExternalRef::CallOperatorWhole(std::move(object_)); } T& operator*() { return object_; } @@ -561,7 +1148,7 @@ class ExternalRef { absl::string_view substr) : object_(std::move(object)), substr_(substr) {} - ~ExternalObjectSubstr() { std::forward(object_)(substr_); } + ~ExternalObjectSubstr() { std::move(object_)(substr_); } T& operator*() { return object_; } const T& operator*() const { return object_; } @@ -582,121 +1169,340 @@ class ExternalRef { }; template - struct HasRiegeliToExternalData : std::false_type {}; + struct HasRiegeliToExternalDataWhole : std::false_type {}; + + template + struct HasRiegeliToExternalDataWhole< + T, std::enable_if_t>())), + ExternalData>::value>> : std::true_type {}; + + template + struct HasRiegeliToExternalDataSubstr : std::false_type {}; + + template + struct HasRiegeliToExternalDataSubstr< + T, std::enable_if_t>(), + std::declval())), + ExternalData>::value>> : std::true_type {}; + + template + struct HasRiegeliToExternalStorage : std::false_type {}; template - struct HasRiegeliToExternalData< + struct HasRiegeliToExternalStorage< T, std::enable_if_t>())), - ExternalData>::value>> : std::true_type {}; + decltype(RiegeliToExternalStorage( + std::declval>())), + ExternalStorage>::value>> : std::true_type {}; + + template + struct HasToExternalDataSubstr + : absl::disjunction, + HasRiegeliToExternalStorage> {}; template ::value, int> = 0> - static ExternalData ToExternalDataExternal(T&& object) { - return RiegeliToExternalData(ExternalRef::Pointer(std::forward(object))); - } - template >, - HasRiegeliToExternalData>>::value, - int> = 0> - static ExternalData ToExternalDataExternal(T&& object) { - return RiegeliToExternalData( - ExternalRef::Pointer(absl::remove_cvref_t(object))); + std::enable_if_t::value, int> = 0> + static ExternalData ToExternalDataSubstr(T&& object, + absl::string_view substr) { + return RiegeliToExternalData(ExternalRef::Pointer(std::forward(object)), + substr); } + template < typename T, std::enable_if_t< - !HasRiegeliToExternalData>::value, int> = 0> - static ExternalData ToExternalDataExternal(T&& object) { - auto* const storage = - new ExternalObjectWhole>(std::forward(object)); + absl::disjunction>, + HasRiegeliToExternalStorage>::value, + int> = 0> + static ExternalData ToExternalDataSubstr(T&& object, + absl::string_view substr) { return ExternalData{ - ExternalStorage( - storage, - [](void* ptr) { - delete static_cast>*>(ptr); - }), - riegeli::ToStringView(**storage)}; + RiegeliToExternalStorage(ExternalRef::Pointer(std::forward(object))), + substr}; } - template - struct HasRiegeliToExternalStorage : std::false_type {}; - template - struct HasRiegeliToExternalStorage< - T, - std::enable_if_t>())), - ExternalStorage>::value>> : std::true_type {}; + struct HasToExternalDataWhole + : absl::disjunction, + HasToExternalDataSubstr> {}; template ::value, int> = 0> - static ExternalData ToExternalDataExternal(T&& object, - absl::string_view substr) { - return ExternalData{ - RiegeliToExternalStorage(ExternalRef::Pointer(std::forward(object))), - substr}; + std::enable_if_t::value, int> = 0> + static ExternalData ToExternalDataWhole(T&& object) { + return RiegeliToExternalData(ExternalRef::Pointer(std::forward(object))); } template < typename T, std::enable_if_t< - absl::conjunction< - absl::negation>, - HasRiegeliToExternalStorage>>::value, + absl::conjunction>, + HasToExternalDataSubstr>::value, int> = 0> - static ExternalData ToExternalDataExternal(T&& object, - absl::string_view substr) { - return ExternalData{RiegeliToExternalStorage(ExternalRef::Pointer( - absl::remove_cvref_t(object))), - substr}; + static ExternalData ToExternalDataWhole(T&& object) { + const absl::string_view data = riegeli::ToStringView(object); + return ExternalRef::ToExternalDataSubstr(std::forward(object), data); } - template >::value, - int> = 0> - static ExternalData ToExternalDataExternal(T&& object, - absl::string_view substr) { - return ExternalData{ - ExternalStorage( - new ExternalObjectSubstr>(std::forward(object), - substr), - [](void* ptr) { - delete static_cast>*>(ptr); - }), - substr}; + + template ::value, int> = 0> + static ExternalData ToExternalDataWhole( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) { + return RiegeliToExternalData(ExternalRef::Pointer(std::forward(object))); + } + template < + typename T, + std::enable_if_t< + absl::conjunction>, + HasToExternalDataSubstr>::value, + int> = 0> + static ExternalData ToExternalDataWhole(T&& object, absl::string_view data) { + return ExternalRef::ToExternalDataSubstr(std::forward(object), data); } - class StorageBase { + template + class ConverterToExternalDataWhole { public: - StorageBase() = default; - - StorageBase(const StorageBase&) = delete; - StorageBase& operator=(const StorageBase&) = delete; + ConverterToExternalDataWhole(const ConverterToExternalDataWhole&) = delete; + ConverterToExternalDataWhole& operator=( + const ConverterToExternalDataWhole&) = delete; + + template ::value, int> = 0> + void operator()(SubT&& subobject) && { + // The constructor processes the subobject. + const absl::string_view data = riegeli::ToStringView(subobject); + ConverterToExternalDataWhole converter( + std::forward(subobject), data, context_, use_external_data_); + } - protected: - void Initialize(absl::string_view substr) { substr_ = substr; } + template + void operator()(SubT&& subobject, absl::string_view substr) && { + // The constructor processes the subobject. + ConverterToExternalDataSubstr converter( + std::forward(subobject), substr, context_, use_external_data_); + } private: friend class ExternalRef; - static void ToChainBlock( - StorageBase* storage, ABSL_ATTRIBUTE_UNUSED size_t max_bytes_to_copy, - void* context, UseStringViewFunction use_string_view, - ABSL_ATTRIBUTE_UNUSED UseChainBlockFunction use_chain_block) { - use_string_view(context, storage->substr()); + ABSL_ATTRIBUTE_ALWAYS_INLINE + explicit ConverterToExternalDataWhole( + T&& object, absl::string_view data, void* context, + UseExternalDataFunction use_external_data) + : context_(context), use_external_data_(use_external_data) { + if (RiegeliExternalCopy(&object) || Wasteful(object, data.size())) { + use_external_data_(context_, ExternalDataCopy(data)); + ExternalRef::CallOperatorWhole(std::forward(object)); + return; + } + std::move(*this).Callback(std::forward(object), data); + } + + template < + typename DependentT = T, + std::enable_if_t::value, int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object, + absl::string_view data) && { + use_external_data_(context_, ExternalRef::ToExternalDataWhole( + std::forward(object), data)); + } + template < + typename DependentT = T, + std::enable_if_t>, + HasToExternalDataWhole< + absl::remove_cvref_t>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) && { + use_external_data_(context_, ExternalRef::ToExternalDataWhole( + absl::remove_cvref_t(object))); + } + template >>, + HasExternalDelegateWhole< + DependentT, ConverterToExternalDataWhole>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object, + absl::string_view data) && { + ExternalRef::ExternalDelegateWhole(std::forward(object), data, + std::move(*this)); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction< + absl::negation< + HasToExternalDataWhole>>, + absl::negation>, + HasExternalDelegateWhole, + ConverterToExternalDataWhole>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback( + T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) && { + ExternalRef::ExternalDelegateWhole(absl::remove_cvref_t(object), + std::move(*this)); + } + template >>, + absl::negation, + ConverterToExternalDataWhole>>>::value, + int> = 0> + void Callback(T&& object, ABSL_ATTRIBUTE_UNUSED absl::string_view data) { + auto* const storage = + new ExternalObjectWhole>(std::forward(object)); + const absl::string_view moved_data = riegeli::ToStringView(**storage); + use_external_data_( + context_, + ExternalData{ + ExternalStorage( + storage, + [](void* ptr) { + delete static_cast>*>( + ptr); + }), + moved_data}); } - static void ToCord(StorageBase* storage, - ABSL_ATTRIBUTE_UNUSED size_t max_bytes_to_copy, - void* context, UseStringViewFunction use_string_view, - ABSL_ATTRIBUTE_UNUSED UseCordFunction use_cord) { - use_string_view(context, storage->substr()); + void* context_; + UseExternalDataFunction use_external_data_; + }; + + template + class ConverterToExternalDataSubstr { + public: + ConverterToExternalDataSubstr(const ConverterToExternalDataSubstr&) = + delete; + ConverterToExternalDataSubstr& operator=( + const ConverterToExternalDataSubstr&) = delete; + + template + void operator()(SubT&& subobject) && { + std::move (*this)(std::forward(subobject), substr_); + } + + template + void operator()(SubT&& subobject, absl::string_view substr) && { + RIEGELI_ASSERT_EQ(substr_.size(), substr.size()) + << "ExternalRef: size mismatch"; + // The constructor processes the subobject. + ConverterToExternalDataSubstr converter( + std::forward(subobject), substr, context_, use_external_data_); } - static ExternalData ToExternalData(StorageBase* storage) { - return ExternalDataCopy(storage->substr()); + private: + friend class ExternalRef; + + ABSL_ATTRIBUTE_ALWAYS_INLINE + explicit ConverterToExternalDataSubstr( + T&& object, absl::string_view substr, void* context, + UseExternalDataFunction use_external_data) + : substr_(substr), + context_(context), + use_external_data_(use_external_data) { + AssertSubstr(object, substr_); + if (RiegeliExternalCopy(&object) || Wasteful(object, substr_.size())) { + use_external_data_(context_, ExternalDataCopy(substr_)); + ExternalRef::CallOperatorSubstr(std::forward(object), substr_); + return; + } + std::move(*this).Callback(std::forward(object)); + } + + template < + typename DependentT = T, + std::enable_if_t::value, int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + use_external_data_(context_, ExternalRef::ToExternalDataSubstr( + std::forward(object), substr_)); + } + template >, + HasToExternalDataSubstr< + absl::remove_cvref_t>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + use_external_data_(context_, + ExternalRef::ToExternalDataSubstr( + absl::remove_cvref_t(object), substr_)); } + template >>, + HasExternalDelegateSubstr< + DependentT, ConverterToExternalDataSubstr>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + ExternalRef::ExternalDelegateSubstr(std::forward(object), substr_, + std::move(*this)); + } + template < + typename DependentT = T, + std::enable_if_t< + absl::conjunction>>, + absl::negation>, + HasExternalDelegateSubstr< + absl::remove_cvref_t, + ConverterToExternalDataSubstr>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + ExternalRef::ExternalDelegateSubstr(absl::remove_cvref_t(object), + substr_, std::move(*this)); + } + template >>, + absl::negation, + ConverterToExternalDataSubstr>>>::value, + int> = 0> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Callback(T&& object) && { + use_external_data_( + context_, + ExternalData{ + ExternalStorage( + new ExternalObjectSubstr>( + std::forward(object), substr_), + [](void* ptr) { + delete static_cast>*>( + ptr); + }), + substr_}); + } + + absl::string_view substr_; + void* context_; + UseExternalDataFunction use_external_data_; + }; + + class StorageBase { + protected: + StorageBase() = default; + + StorageBase(const StorageBase&) = delete; + StorageBase& operator=(const StorageBase&) = delete; + + void Initialize(absl::string_view substr) { substr_ = substr; } + + private: + friend class ExternalRef; bool empty() const { return substr_.empty(); } const char* data() const { return substr_.data(); } @@ -729,39 +1535,33 @@ class ExternalRef { void* context, UseStringViewFunction use_string_view, UseChainBlockFunction use_chain_block) { - T&& object = StorageWholeWithoutCallOperator::object(storage); - if (storage->size() <= max_bytes_to_copy || - Wasteful(object, Chain::kExternalAllocatedSize(), - storage->size())) { + if (storage->size() <= max_bytes_to_copy) { use_string_view(context, storage->substr()); return; } - use_chain_block( - context, ExternalRef::ToChainBlockExternal(std::forward(object))); + // The constructor processes the object. + ConverterToChainBlockWhole converter(object(storage), + storage->substr(), context, + use_string_view, use_chain_block); } static void ToCord(StorageBase* storage, size_t max_bytes_to_copy, void* context, UseStringViewFunction use_string_view, UseCordFunction use_cord) { - T&& object = StorageWholeWithoutCallOperator::object(storage); - if (storage->size() <= max_bytes_to_copy || - Wasteful(object, - cord_internal::kSizeOfCordRepExternal + - sizeof(ObjectForCordWhole>) + sizeof(T), - storage->size())) { + if (storage->size() <= max_bytes_to_copy) { use_string_view(context, storage->substr()); return; } - use_cord(context, ExternalRef::ToCordExternal(std::forward(object))); + // The constructor processes the object. + ConverterToCordWhole converter(object(storage), storage->substr(), + context, use_string_view, use_cord); } - static ExternalData ToExternalData(StorageBase* storage) { - T&& object = StorageWholeWithoutCallOperator::object(storage); - if (Wasteful(object, sizeof(ExternalObjectWhole>), - storage->size())) { - return ExternalDataCopy(storage->substr()); - } - return ExternalRef::ToExternalDataExternal(std::forward(object)); + static void ToExternalData(StorageBase* storage, void* context, + UseExternalDataFunction use_external_data) { + // The constructor processes the object. + ConverterToExternalDataWhole converter( + object(storage), storage->substr(), context, use_external_data); } static T&& object(StorageBase* storage) { @@ -803,14 +1603,14 @@ class ExternalRef { void* context, UseStringViewFunction use_string_view, UseChainBlockFunction use_chain_block) { - if (storage->size() <= max_bytes_to_copy || - Wasteful(object(storage), Chain::kExternalAllocatedSize(), - storage->size())) { + if (storage->size() <= max_bytes_to_copy) { use_string_view(context, storage->substr()); return; } - use_chain_block( - context, ExternalRef::ToChainBlockExternal(ExtractObject(storage))); + // The constructor processes the object. + ConverterToChainBlockWhole converter(ExtractObject(storage), + storage->substr(), context, + use_string_view, use_chain_block); } static void ToCord(StorageBase* storage, size_t max_bytes_to_copy, @@ -820,34 +1620,18 @@ class ExternalRef { use_string_view(context, storage->substr()); return; } - T&& object = ExtractObject(storage); - if (Wasteful(object, - cord_internal::kSizeOfCordRepExternal + - sizeof(ObjectForCordWhole>) + sizeof(T), - storage->size())) { - use_string_view(context, storage->substr()); - ExternalRef::CallOperatorSubstr(std::forward(object), - storage->substr()); - return; - } - use_cord(context, ExternalRef::ToCordExternal(std::forward(object))); - } - - static ExternalData ToExternalData(StorageBase* storage) { - T&& object = ExtractObject(storage); - if (Wasteful(object, sizeof(ExternalObjectWhole>), - storage->size())) { - ExternalData result = ExternalDataCopy(storage->substr()); - ExternalRef::CallOperatorSubstr(std::forward(object), - storage->substr()); - return result; - } - return ExternalRef::ToExternalDataExternal(std::forward(object)); + // The constructor processes the object. + ConverterToCordWhole converter(ExtractObject(storage), + storage->substr(), context, + use_string_view, use_cord); } - static const T& object(const StorageBase* storage) { - return *static_cast(storage) - ->object_; + static void ToExternalData(StorageBase* storage, void* context, + UseExternalDataFunction use_external_data) { + // The constructor processes the object. + ConverterToExternalDataWhole converter(ExtractObject(storage), + storage->substr(), context, + use_external_data); } static T&& ExtractObject(StorageBase* storage) { @@ -880,15 +1664,10 @@ class ExternalRef { use_string_view(context, storage->substr()); return; } - TemporaryStorage temporary_storage; - T&& object = initializer(storage).Reference(std::move(temporary_storage)); - if (Wasteful(object, Chain::kExternalAllocatedSize(), - storage->size())) { - use_string_view(context, storage->substr()); - return; - } - use_chain_block(context, ExternalRef::ToChainBlockExternal( - std::forward(object), storage->substr())); + // The constructor processes the object. + ConverterToChainBlockSubstr converter( + initializer(storage).Reference(), storage->substr(), context, + use_string_view, use_chain_block); } static void ToCord(StorageBase* storage, size_t max_bytes_to_copy, @@ -898,28 +1677,18 @@ class ExternalRef { use_string_view(context, storage->substr()); return; } - TemporaryStorage temporary_storage; - T&& object = initializer(storage).Reference(std::move(temporary_storage)); - if (Wasteful(object, - cord_internal::kSizeOfCordRepExternal + - sizeof(ObjectForCordSubstr>), - storage->size())) { - use_string_view(context, storage->substr()); - return; - } - use_cord(context, ExternalRef::ToCordExternal(std::forward(object), - storage->substr())); + // The constructor processes the object. + ConverterToCordSubstr converter(initializer(storage).Reference(), + storage->substr(), context, + use_string_view, use_cord); } - static ExternalData ToExternalData(StorageBase* storage) { - TemporaryStorage temporary_storage; - T&& object = initializer(storage).Reference(std::move(temporary_storage)); - if (Wasteful(object, sizeof(ExternalObjectWhole>), - storage->size())) { - return ExternalDataCopy(storage->substr()); - } - return ExternalRef::ToExternalDataExternal(std::forward(object), - storage->substr()); + static void ToExternalData(StorageBase* storage, void* context, + UseExternalDataFunction use_external_data) { + // The constructor processes the object. + ConverterToExternalDataSubstr converter( + initializer(storage).Reference(), storage->substr(), context, + use_external_data); } static Initializer initializer(StorageBase* storage) { @@ -961,47 +1730,35 @@ class ExternalRef { void* context, UseStringViewFunction use_string_view, UseChainBlockFunction use_chain_block) { - if (storage->size() <= max_bytes_to_copy || - Wasteful(object(storage), Chain::kExternalAllocatedSize(), - storage->size())) { + if (storage->size() <= max_bytes_to_copy) { use_string_view(context, storage->substr()); return; } - use_chain_block(context, ExternalRef::ToChainBlockExternal( - ExtractObject(storage), storage->substr())); + // The constructor processes the object. + ConverterToChainBlockSubstr converter( + ExtractObject(storage), storage->substr(), context, use_string_view, + use_chain_block); } static void ToCord(StorageBase* storage, size_t max_bytes_to_copy, void* context, UseStringViewFunction use_string_view, UseCordFunction use_cord) { - if (storage->size() <= max_bytes_to_copy || - Wasteful(object(storage), - cord_internal::kSizeOfCordRepExternal + - sizeof(ObjectForCordSubstr>), - storage->size())) { + if (storage->size() <= max_bytes_to_copy) { use_string_view(context, storage->substr()); return; } - use_cord(context, ExternalRef::ToCordExternal(ExtractObject(storage), - storage->substr())); - } - - static ExternalData ToExternalData(StorageBase* storage) { - T&& object = ExtractObject(storage); - if (Wasteful(object, sizeof(ExternalObjectSubstr>), - storage->size())) { - ExternalData result = ExternalDataCopy(storage->substr()); - ExternalRef::CallOperatorSubstr(std::forward(object), - storage->substr()); - return result; - } - return ExternalRef::ToExternalDataExternal(std::forward(object), - storage->substr()); + // The constructor processes the object. + ConverterToCordSubstr converter(ExtractObject(storage), + storage->substr(), context, + use_string_view, use_cord); } - static const T& object(const StorageBase* storage) { - return *static_cast(storage) - ->object_; + static void ToExternalData(StorageBase* storage, void* context, + UseExternalDataFunction use_external_data) { + // The constructor processes the object. + ConverterToExternalDataSubstr converter(ExtractObject(storage), + storage->substr(), context, + use_external_data); } static T&& ExtractObject(StorageBase* storage) { @@ -1049,51 +1806,27 @@ class ExternalRef { }; public: - // The type of the `storage` parameter for the constructor which copies the - // data. - using StorageCopy = StorageBase; - - // The type of the `storage` parameter for the constructor which takes - // an external object supporting `RiegeliToStringView(&object)`, - // `absl::string_view(object)`, or `absl::Span(object)`. + // The type of the `storage` parameter for the constructor and + // `ExternalRef::From()` which take an external object supporting + // `riegeli::ToStringView()`. template using StorageWhole = typename StorageWholeImpl::type; - // The type of the `storage` parameter for the constructor which takes an - // external object and its substring. + // The type of the `storage` parameter for the constructor and + // `ExternalRef::From()` which take an external object and its substring. template using StorageSubstr = typename StorageSubstrImpl::type; - // Constructs an `ExternalRef` from `data` which will be copied. No external - // object will be created. - // - // `storage` must outlive usages of the returned `ExternalRef`. - // - // `StorageSubstr` derives from `StorageCopy` and can also be used for - // this constructor. This is useful in a function which conditionally uses - // either of these constructors. - explicit ExternalRef( - absl::string_view data = absl::string_view(), - StorageCopy&& storage ABSL_ATTRIBUTE_LIFETIME_BOUND = StorageCopy()) - : methods_(&kMethods), storage_(&storage) { - storage.Initialize(data); - } - // Constructs an `ExternalRef` from an external object or its `Initializer`. // See class comments for expectations on the external object. // - // The object must be explicitly convertible to `absl::string_view`. - // - // If the result of conversion to `absl::string_view` is already known and it - // remains valid when the object gets moved, it is better to use the overload - // below which passes `substr`, because this may avoid creating the external - // object altogether, and because conversion to `absl::Cord` avoids an - // allocation which makes the object stable. + // The object must support `riegeli::ToStringView()`. // // `storage` must outlive usages of the returned `ExternalRef`. - template >::value, int> = 0> + template < + typename Arg, + std::enable_if_t>::value, + int> = 0> explicit ExternalRef(Arg&& arg ABSL_ATTRIBUTE_LIFETIME_BOUND, StorageWhole&& storage ABSL_ATTRIBUTE_LIFETIME_BOUND = @@ -1105,10 +1838,13 @@ class ExternalRef { // Constructs an `ExternalRef` from an external object or its `Initializer`. // See class comments for expectations on the external object. // - // `substr` must be valid for the new object if it gets created. + // `substr` must be owned by the object if it gets created or moved. // // `storage` must outlive usages of the returned `ExternalRef`. - template + template < + typename Arg, + std::enable_if_t< + SupportsExternalRefSubstr>::value, int> = 0> explicit ExternalRef( Arg&& arg ABSL_ATTRIBUTE_LIFETIME_BOUND, absl::string_view substr, StorageSubstr&& storage ABSL_ATTRIBUTE_LIFETIME_BOUND = @@ -1120,6 +1856,33 @@ class ExternalRef { ExternalRef(ExternalRef&& that) = default; ExternalRef& operator=(ExternalRef&& that) = default; + // Like `ExternalRef` constructor, but `RiegeliSupportsExternalRef()` or + // `RiegeliSupportsExternalRefWhole()` is not needed. The caller is + // responsible for using an appropriate type of the external object. + template < + typename Arg, + std::enable_if_t< + SupportsToStringView>::value, int> = 0> + static ExternalRef From(Arg&& arg ABSL_ATTRIBUTE_LIFETIME_BOUND, + StorageWhole&& storage + ABSL_ATTRIBUTE_LIFETIME_BOUND = + StorageWhole()) { + storage.Initialize(std::forward(arg)); + return ExternalRef(&kMethods>, &storage); + } + + // Like `ExternalRef` constructor, but `RiegeliSupportsExternalRef()` is not + // needed. The caller is responsible for using an appropriate type of the + // external object. + template + static ExternalRef From( + Arg&& arg ABSL_ATTRIBUTE_LIFETIME_BOUND, absl::string_view substr, + StorageSubstr&& storage ABSL_ATTRIBUTE_LIFETIME_BOUND = + StorageSubstr()) { + storage.Initialize(std::forward(arg), substr); + return ExternalRef(&kMethods>, &storage); + } + // Returns `true` if the data size is 0. bool empty() const { return storage_->empty(); } @@ -1136,22 +1899,11 @@ class ExternalRef { return storage_->substr(); } - // Converts the data to `Chain`. - explicit operator Chain() && { - Chain result; - // Destruction of a just default-constructed `Chain` can be optimized out. - // Construction in place is more efficient than assignment. - result.~Chain(); - methods_->to_chain_block( - storage_, Chain::kMaxBytesToCopyToEmpty, &result, - [](void* context, absl::string_view data) { - new (context) Chain(data); - }, - [](void* context, Chain::Block data) { - new (context) Chain(std::move(data)); - }); - return result; - } + // The data can be converted to `Chain` using: + // * `Chain::Chain(ExternalRef)` + // * `Chain::Reset(ExternalRef)` or `riegeli::Reset(Chain&, ExternalRef)` + // * `Chain::Append(ExternalRef)` + // * `Chain::Prepend(ExternalRef)` // Converts the data to `absl::Cord`. explicit operator absl::Cord() && { @@ -1170,6 +1922,107 @@ class ExternalRef { return result; } + // Support `riegeli::Reset(absl::Cord&, ExternalRef)`. + friend void RiegeliReset(absl::Cord& dest, ExternalRef src) { + src.methods_->to_cord( + src.storage_, cord_internal::kMaxBytesToCopyToEmptyCord, &dest, + [](void* context, absl::string_view data) { + cord_internal::AssignToBlockyCord(data, + *static_cast(context)); + }, + [](void* context, absl::Cord data) { + *static_cast(context) = std::move(data); + }); + } + + // Appends the data to `dest`. + void AppendTo(absl::Cord& dest) && { + methods_->to_cord( + storage_, cord_internal::MaxBytesToCopyToCord(dest), &dest, + [](void* context, absl::string_view data) { + cord_internal::AppendToBlockyCord(data, + *static_cast(context)); + }, + [](void* context, absl::Cord data) { + static_cast(context)->Append(std::move(data)); + }); + } + + // Prepends the data to `dest`. + void PrependTo(absl::Cord& dest) && { + methods_->to_cord( + storage_, cord_internal::MaxBytesToCopyToCord(dest), &dest, + [](void* context, absl::string_view data) { + cord_internal::PrependToBlockyCord( + data, *static_cast(context)); + }, + [](void* context, absl::Cord data) { + static_cast(context)->Prepend(std::move(data)); + }); + } + + // Returns a type-erased external object with its deleter and data. + explicit operator ExternalData() && { + ExternalData result{ExternalStorage(nullptr, nullptr), absl::string_view()}; + // Destruction of just constructed `ExternalData` can be optimized out. + // Construction in place is more efficient than assignment. + result.~ExternalData(); + methods_->to_external_data(storage_, &result, + [](void* context, ExternalData data) { + new (context) ExternalData(std::move(data)); + }); + return result; + } + + private: + // For `InitializeTo()`, `AssignTo()`, `AppendTo()`, and `PrependTo()`. + friend class Chain; + + struct Methods { + // Calls once either `use_string_view` or `use_chain_block`. + void (*to_chain_block)(StorageBase* storage, size_t max_bytes_to_copy, + void* context, UseStringViewFunction use_string_view, + UseChainBlockFunction use_chain_block); + // Calls once either `use_string_view` or `use_cord`. + void (*to_cord)(StorageBase* storage, size_t max_bytes_to_copy, + void* context, UseStringViewFunction use_string_view, + UseCordFunction use_cord); + // Calls once `use_external_data`. + void (*to_external_data)(StorageBase* storage, void* context, + UseExternalDataFunction use_external_data); + }; + + explicit ExternalRef(const Methods* methods, StorageBase* storage) + : methods_(methods), storage_(storage) {} + + // Assigns the data to `dest` which is expected to be just + // default-constructed. + void InitializeTo(Chain& dest) && { + // Destruction of a just default-constructed `Chain` can be optimized out. + // Construction in place is more efficient than assignment. + dest.~Chain(); + methods_->to_chain_block( + storage_, Chain::kMaxBytesToCopyToEmpty, &dest, + [](void* context, absl::string_view data) { + new (context) Chain(data); + }, + [](void* context, Chain::Block data) { + new (context) Chain(std::move(data)); + }); + } + + // Assigns the data to `dest`. + void AssignTo(Chain& dest) && { + methods_->to_chain_block( + storage_, Chain::kMaxBytesToCopyToEmpty, &dest, + [](void* context, absl::string_view data) { + static_cast(context)->Reset(data); + }, + [](void* context, Chain::Block data) { + static_cast(context)->Reset(std::move(data)); + }); + } + // Appends the data to `dest`. void AppendTo(Chain& dest) && { methods_->to_chain_block( @@ -1196,19 +2049,6 @@ class ExternalRef { }); } - // Appends the data to `dest`. - void AppendTo(absl::Cord& dest) && { - methods_->to_cord( - storage_, cord_internal::MaxBytesToCopyToCord(dest), &dest, - [](void* context, absl::string_view data) { - cord_internal::AppendToBlockyCord(data, - *static_cast(context)); - }, - [](void* context, absl::Cord data) { - static_cast(context)->Append(std::move(data)); - }); - } - // Prepends the data to `dest`. void PrependTo(Chain& dest) && { methods_->to_chain_block( @@ -1235,35 +2075,6 @@ class ExternalRef { }); } - // Prepends the data to `dest`. - void PrependTo(absl::Cord& dest) && { - methods_->to_cord( - storage_, cord_internal::MaxBytesToCopyToCord(dest), &dest, - [](void* context, absl::string_view data) { - cord_internal::PrependToBlockyCord( - data, *static_cast(context)); - }, - [](void* context, absl::Cord data) { - static_cast(context)->Prepend(std::move(data)); - }); - } - - // Returns a type-erased external object with its deleter and data. - operator ExternalData() && { return methods_->to_external_data(storage_); } - - private: - struct Methods { - // Calls once either `use_string_view` or `use_chain_block`. - void (*to_chain_block)(StorageBase* storage, size_t max_bytes_to_copy, - void* context, UseStringViewFunction use_string_view, - UseChainBlockFunction use_chain_block); - // Calls once either `use_string_view` or `use_cord`. - void (*to_cord)(StorageBase* storage, size_t max_bytes_to_copy, - void* context, UseStringViewFunction use_string_view, - UseCordFunction use_cord); - ExternalData (*to_external_data)(StorageBase* storage); - }; - template static constexpr Methods kMethods = {Impl::ToChainBlock, Impl::ToCord, Impl::ToExternalData}; diff --git a/riegeli/base/external_ref_support.h b/riegeli/base/external_ref_support.h new file mode 100644 index 00000000..06c1934c --- /dev/null +++ b/riegeli/base/external_ref_support.h @@ -0,0 +1,181 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RIEGELI_BASE_EXTERNAL_REF_SUPPORT_H_ +#define RIEGELI_BASE_EXTERNAL_REF_SUPPORT_H_ + +#include + +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/meta/type_traits.h" +#include "riegeli/base/external_data.h" +#include "riegeli/base/to_string_view.h" + +namespace riegeli { + +// Default implementation for `ExternalRef` support. +inline bool RiegeliExternalCopy(ABSL_ATTRIBUTE_UNUSED const void* self) { + return false; +} + +// Default implementation for `ExternalRef` support. +inline size_t RiegeliExternalMemory(ABSL_ATTRIBUTE_UNUSED const void* self) { + return 0; +} + +// Indicate support for `ExternalRef(std::string&&)`. +void RiegeliSupportsExternalRefWhole(std::string*); + +inline size_t RiegeliExternalMemory(const std::string* self) { + // Do not bother checking for short string optimization. Such strings will + // likely not be considered wasteful anyway. + return self->capacity() + 1; +} + +// Indicate support for: +// * `ExternalRef(std::vector&&)` +// * `ExternalRef(std::vector&&, substr)` +template +void RiegeliSupportsExternalRef(std::vector*); + +template +inline size_t RiegeliExternalMemory(const std::vector* self) { + return self->capacity() * sizeof(T); +} + +// Indicate support for `ExternalRef(std::unique_ptr&&, substr)`. +template +void RiegeliSupportsExternalRef(std::unique_ptr*); + +template < + typename T, typename Deleter, + std::enable_if_t>, + absl::negation>>::value, + int> = 0> +inline size_t RiegeliExternalMemory(const std::unique_ptr* self) { + size_t memory = RiegeliExternalMemory(&self->get_deleter()); + if (*self != nullptr) { + memory += sizeof(T) + RiegeliExternalMemory(self->get()); + } + return memory; +} + +template +inline ExternalStorage RiegeliToExternalStorage(std::unique_ptr* self) { + return ExternalStorage(const_cast*>(self->release()), + [](void* ptr) { delete static_cast(ptr); }); +} + +template +inline ExternalStorage RiegeliToExternalStorage(std::unique_ptr* self) { + return ExternalStorage(const_cast*>(self->release()), + [](void* ptr) { delete[] static_cast(ptr); }); +} + +// Indicate support for: +// * `ExternalRef(const std::shared_ptr&, substr)` +// * `ExternalRef(std::shared_ptr&&, substr)` +template +void RiegeliSupportsExternalRef(const std::shared_ptr*); + +namespace external_ref_internal { + +// Reflects the layout of a control block of `std::shared_ptr` from libc++. +struct SharedPtrControlBlock { + virtual ~SharedPtrControlBlock() = default; + long shared_count; + long weak_count; +}; + +} // namespace external_ref_internal + +template < + typename T, typename Deleter, + std::enable_if_t>, + absl::negation>>::value, + int> = 0> +inline size_t RiegeliExternalMemory(const std::shared_ptr* self) { + if (*self == nullptr) return 0; + return sizeof(external_ref_internal::SharedPtrControlBlock) + sizeof(T) + + RiegeliExternalMemory(self->get()); +} + +template +inline size_t RiegeliExternalMemory(const std::shared_ptr* self) { + if (*self == nullptr) return 0; + size_t memory = + sizeof(external_ref_internal::SharedPtrControlBlock) + sizeof(T[size]); + for (size_t i = 0; i < size; ++i) { + memory += RiegeliExternalMemory(&(*self)[i]); + } + return memory; +} + +namespace external_ref_internal { + +template +struct PointerType { + using type = T*; +}; +template +struct PointerType { + using type = const T*; +}; +template +struct PointerType { + using type = T*; +}; + +template +using PointerTypeT = typename PointerType::type; + +template +struct HasRiegeliSupportsExternalRefWhole : std::false_type {}; + +template +struct HasRiegeliSupportsExternalRefWhole< + T, absl::void_t>()))>> : std::true_type {}; + +template +struct HasRiegeliSupportsExternalRef : std::false_type {}; + +template +struct HasRiegeliSupportsExternalRef< + T, absl::void_t>()))>> : std::true_type {}; + +} // namespace external_ref_internal + +template +struct SupportsExternalRefWhole + : absl::conjunction< + absl::disjunction< + external_ref_internal::HasRiegeliSupportsExternalRefWhole, + external_ref_internal::HasRiegeliSupportsExternalRef>, + SupportsToStringView> {}; + +template +struct SupportsExternalRefSubstr + : external_ref_internal::HasRiegeliSupportsExternalRef {}; + +} // namespace riegeli + +#endif // RIEGELI_BASE_EXTERNAL_REF_SUPPORT_H_ diff --git a/riegeli/base/intrusive_shared_ptr.h b/riegeli/base/intrusive_shared_ptr.h index 943bbb77..3523d17f 100644 --- a/riegeli/base/intrusive_shared_ptr.h +++ b/riegeli/base/intrusive_shared_ptr.h @@ -27,6 +27,7 @@ #include "riegeli/base/assert.h" #include "riegeli/base/compare.h" #include "riegeli/base/external_data.h" +#include "riegeli/base/external_ref_support.h" // IWYU pragma: keep #include "riegeli/base/initializer.h" #include "riegeli/base/ownership.h" #include "riegeli/base/reset.h" @@ -264,6 +265,17 @@ class // Allow Nullability annotations on `IntrusiveSharedPtr`. using absl_nullability_compatible = void; + // Indicate support for: + // * `ExternalRef(const IntrusiveSharedPtr&, substr)` + // * `ExternalRef(IntrusiveSharedPtr&&, substr)` + friend void RiegeliSupportsExternalRef(const IntrusiveSharedPtr*) {} + + // Support `ExternalRef`. + friend size_t RiegeliExternalMemory(const IntrusiveSharedPtr* self) { + if (*self == nullptr) return 0; + return sizeof(T) + RiegeliExternalMemory(self->get()); + } + // Support `ExternalRef`. friend ExternalStorage RiegeliToExternalStorage(IntrusiveSharedPtr* self) { return ExternalStorage(const_cast*>(self->Release()), diff --git a/riegeli/base/shared_buffer.h b/riegeli/base/shared_buffer.h index 34bb5c52..58f832b2 100644 --- a/riegeli/base/shared_buffer.h +++ b/riegeli/base/shared_buffer.h @@ -17,16 +17,13 @@ #include -#include #include -#include #include "absl/base/attributes.h" #include "absl/strings/string_view.h" #include "riegeli/base/assert.h" #include "riegeli/base/buffer.h" #include "riegeli/base/external_data.h" -#include "riegeli/base/external_ref.h" #include "riegeli/base/maker.h" #include "riegeli/base/shared_ptr.h" @@ -72,57 +69,19 @@ class SharedBuffer { // Returns the usable data size. It can be greater than the requested size. size_t capacity() const; - // Converts a substring of `*this` to `ExternalRef`. - // - // `storage` must outlive usages of the returned `ExternalRef`. - // - // Precondition: - // if `!substr.empty()` then `substr` is a substring of - // [`data()`..`data() + capacity()`). - ExternalRef ToExternalRef( - absl::string_view substr, - ExternalRef::StorageSubstr&& storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) const& { - if (!substr.empty()) { - RIEGELI_ASSERT(std::greater_equal<>()(substr.data(), data())) - << "Failed precondition of SharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - RIEGELI_ASSERT(std::less_equal<>()(substr.data() + substr.size(), - data() + capacity())) - << "Failed precondition of SharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - } - return ExternalRef(*this, substr, std::move(storage)); - } - ExternalRef ToExternalRef( - absl::string_view substr, - ExternalRef::StorageSubstr&& storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) && { - if (!substr.empty()) { - RIEGELI_ASSERT(std::greater_equal<>()(substr.data(), data())) - << "Failed precondition of SharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - RIEGELI_ASSERT(std::less_equal<>()(substr.data() + substr.size(), - data() + capacity())) - << "Failed precondition of SharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - } - return ExternalRef(std::move(*this), substr, std::move(storage)); - } + // Indicate support for: + // * `ExternalRef(const SharedBuffer&, substr)` + // * `ExternalRef(SharedBuffer&&, substr)` + friend void RiegeliSupportsExternalRef(const SharedBuffer*) {} // Support `ExternalRef`. friend size_t RiegeliExternalMemory(const SharedBuffer* self) { - if (self->buffer_ == nullptr) return 0; - return sizeof(size_t) + sizeof(Buffer) + self->buffer_->capacity(); + return RiegeliExternalMemory(&self->buffer_); } // Support `ExternalRef`. friend ExternalStorage RiegeliToExternalStorage(SharedBuffer* self) { - return ExternalStorage(self->buffer_.Release(), [](void* ptr) { - SharedPtr::DeleteReleased(static_cast(ptr)); - }); + return RiegeliToExternalStorage(&self->buffer_); } // Support `ExternalRef` and `Chain::Block`. diff --git a/riegeli/base/shared_ptr.h b/riegeli/base/shared_ptr.h index 65a1a083..9c7269fc 100644 --- a/riegeli/base/shared_ptr.h +++ b/riegeli/base/shared_ptr.h @@ -29,6 +29,7 @@ #include "riegeli/base/assert.h" #include "riegeli/base/compare.h" #include "riegeli/base/external_data.h" +#include "riegeli/base/external_ref_support.h" // IWYU pragma: keep #include "riegeli/base/initializer.h" #include "riegeli/base/new_aligned.h" #include "riegeli/base/ref_count.h" @@ -211,6 +212,21 @@ class // Allow Nullability annotations on `IntrusiveSharedPtr`. using absl_nullability_compatible = void; + // Indicate support for: + // * `ExternalRef(const SharedPtr&, substr)` + // * `ExternalRef(SharedPtr&&, substr)` + friend void RiegeliSupportsExternalRef(const SharedPtr*) {} + + // Support `ExternalRef`. + friend size_t RiegeliExternalMemory(const SharedPtr* self) { + static constexpr size_t kOffset = + !std::has_virtual_destructor::value + ? RoundUp(sizeof(RefCount)) + : RoundUp(sizeof(Control)); + if (*self == nullptr) return 0; + return kOffset + sizeof(T) + RiegeliExternalMemory(self->get()); + } + // Support `ExternalRef`. friend ExternalStorage RiegeliToExternalStorage(SharedPtr* self) { return ExternalStorage( diff --git a/riegeli/base/sized_shared_buffer.h b/riegeli/base/sized_shared_buffer.h index 693857aa..86fc2f7f 100644 --- a/riegeli/base/sized_shared_buffer.h +++ b/riegeli/base/sized_shared_buffer.h @@ -17,7 +17,6 @@ #include -#include #include #include @@ -27,7 +26,6 @@ #include "riegeli/base/arithmetic.h" #include "riegeli/base/assert.h" #include "riegeli/base/buffering.h" -#include "riegeli/base/external_ref.h" #include "riegeli/base/shared_buffer.h" namespace riegeli { @@ -125,61 +123,27 @@ class void RemoveSuffix(size_t length); void RemovePrefix(size_t length); - // Converts `*this` to `ExternalRef`. - // - // `storage` must outlive usages of the returned `ExternalRef`. - ExternalRef ToExternalRef( - ExternalRef::StorageSubstr&& storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) const& { - return ToExternalRef(absl::string_view(*this), std::move(storage)); - } - ExternalRef ToExternalRef( - ExternalRef::StorageSubstr&& storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) && { - return std::move(*this).ToExternalRef(absl::string_view(*this), - std::move(storage)); + // Indicate support for: + // * `ExternalRef(const SizedSharedBuffer&)` + // * `ExternalRef(SizedSharedBuffer&&)` + // * `ExternalRef(const SizedSharedBuffer&, substr)` + // * `ExternalRef(SizedSharedBuffer&&, substr)` + friend void RiegeliSupportsExternalRef(const SizedSharedBuffer*) {} + + // Support `ExternalRef`. + template + friend void RiegeliExternalDelegate(SizedSharedBuffer* self, + absl::string_view substr, + Callback&& delegate_to) { + self->data_ = nullptr; + self->size_ = 0; + std::forward(delegate_to)(std::move(self->buffer_), substr); } - - // Converts a substring of `*this` to `ExternalRef`. - // - // `storage` must outlive usages of the returned `ExternalRef`. - // - // Precondition: if `!substr.empty()` then `substr` is a substring of `**this` - ExternalRef ToExternalRef( - absl::string_view substr, - ExternalRef::StorageSubstr&& storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) const& { - if (!substr.empty()) { - RIEGELI_ASSERT(std::greater_equal<>()(substr.data(), data_)) - << "Failed precondition of SizedSharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - RIEGELI_ASSERT( - std::less_equal<>()(substr.data() + substr.size(), data_ + size_)) - << "Failed precondition of SizedSharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - } - return buffer_.ToExternalRef(substr, std::move(storage)); - } - ExternalRef ToExternalRef( - absl::string_view substr, - ExternalRef::StorageSubstr&& storage - ABSL_ATTRIBUTE_LIFETIME_BOUND = - ExternalRef::StorageSubstr()) && { - if (!substr.empty()) { - RIEGELI_ASSERT(std::greater_equal<>()(substr.data(), data_)) - << "Failed precondition of SizedSharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - RIEGELI_ASSERT( - std::less_equal<>()(substr.data() + substr.size(), data_ + size_)) - << "Failed precondition of SizedSharedBuffer::ToExternalRef(): " - "substring not contained in the buffer"; - } - data_ = nullptr; - size_ = 0; - return std::move(buffer_).ToExternalRef(substr, std::move(storage)); + template + friend void RiegeliExternalDelegate(const SizedSharedBuffer* self, + absl::string_view substr, + Callback&& delegate_to) { + std::forward(delegate_to)(self->buffer_, substr); } // Support `MemoryEstimator`. diff --git a/riegeli/bytes/BUILD b/riegeli/bytes/BUILD index 379e5a56..6f987429 100644 --- a/riegeli/bytes/BUILD +++ b/riegeli/bytes/BUILD @@ -204,6 +204,7 @@ cc_library( "//riegeli/base:buffering", "//riegeli/base:chain", "//riegeli/base:cord_utils", + "//riegeli/base:external_ref", "//riegeli/base:object", "//riegeli/base:sized_shared_buffer", "//riegeli/base:types", @@ -293,6 +294,7 @@ cc_library( "//riegeli/base:assert", "//riegeli/base:buffering", "//riegeli/base:chain", + "//riegeli/base:external_ref", "//riegeli/base:object", "//riegeli/base:sized_shared_buffer", "//riegeli/base:types", @@ -691,6 +693,7 @@ cc_library( "//riegeli/base:buffering", "//riegeli/base:chain", "//riegeli/base:dependency", + "//riegeli/base:external_ref", "//riegeli/base:initializer", "//riegeli/base:object", "//riegeli/base:stable_dependency", @@ -854,6 +857,7 @@ cc_library( "//riegeli/base:buffering", "//riegeli/base:chain", "//riegeli/base:dependency", + "//riegeli/base:external_ref", "//riegeli/base:initializer", "//riegeli/base:moving_dependency", "//riegeli/base:object", diff --git a/riegeli/bytes/backward_writer.cc b/riegeli/bytes/backward_writer.cc index b56011ba..d0dc4412 100644 --- a/riegeli/bytes/backward_writer.cc +++ b/riegeli/bytes/backward_writer.cc @@ -19,8 +19,6 @@ #include #include #include -#include -#include #include #include "absl/base/optimization.h" @@ -86,20 +84,13 @@ bool BackwardWriter::WriteSlow(absl::string_view src) { return true; } -bool BackwardWriter::WriteStringSlow(std::string&& src) { - RIEGELI_ASSERT_LT(UnsignedMin(available(), kMaxBytesToCopy), src.size()) - << "Failed precondition of BackwardWriter::WriteStringSlow(): " - "enough space available, use Write(string&&) instead"; - return WriteSlow(ExternalRef(std::move(src))); -} - bool BackwardWriter::WriteSlow(const Chain& src) { RIEGELI_ASSERT_LT(UnsignedMin(available(), kMaxBytesToCopy), src.size()) << "Failed precondition of BackwardWriter::WriteSlow(Chain): " "enough space available, use Write(Chain) instead"; for (Chain::Blocks::const_reverse_iterator iter = src.blocks().crbegin(); iter != src.blocks().crend(); ++iter) { - if (ABSL_PREDICT_FALSE(!Write(*iter))) return false; + if (ABSL_PREDICT_FALSE(!Write(absl::string_view(*iter)))) return false; } return true; } diff --git a/riegeli/bytes/backward_writer.h b/riegeli/bytes/backward_writer.h index a1bdb215..c2915051 100644 --- a/riegeli/bytes/backward_writer.h +++ b/riegeli/bytes/backward_writer.h @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -152,10 +151,6 @@ class BackwardWriter : public Object { // Writes a fixed number of bytes from `src` to the buffer and/or the // destination. The whole `src` is prepended, bytes are not reversed. // - // `std::string&&` is accepted with a template to avoid implicit conversions - // to `std::string` which can be ambiguous against `absl::string_view` - // (e.g. `const char*`). - // // Return values: // * `true` - success (`src.size()` bytes written) // * `false` - failure (a suffix of less than `src.size()` bytes written, @@ -166,19 +161,20 @@ class BackwardWriter : public Object { #endif bool Write(absl::string_view src); template , - absl::negation>>::value, - int> = 0> - bool Write(Src&& src); - template ::value, int> = 0> + std::enable_if_t< + absl::conjunction< + SupportsToStringView, + absl::negation>>::value, + int> = 0> bool Write(Src&& src); bool Write(const Chain& src); bool Write(Chain&& src); bool Write(const absl::Cord& src); bool Write(absl::Cord&& src); bool Write(ExternalRef src); + template ::value, int> = 0> + bool Write(Src&& src); // Writes a stringified value to the buffer and/or the destination. // @@ -206,8 +202,8 @@ class BackwardWriter : public Object { absl::conjunction< HasAbslStringify, absl::negation>, absl::negation>, - absl::negation>>:: - value, + absl::negation>, + absl::negation>>::value, int> = 0> bool Write(Src&& src); @@ -407,12 +403,11 @@ class BackwardWriter : public Object { // Precondition for `WriteSlow(absl::string_view)`: // `available() < src.size()` // - // Precondition for `WriteStringSlow()`, `WriteSlow(const Chain&)`, - // `WriteSlow(Chain&&)`, `WriteSlow(const absl::Cord&)`, - // `WriteSlow(ExternalRef)`, and `WriteSlow(absl::Cord&&): + // Precondition for `WriteSlow(const Chain&)`, `WriteSlow(Chain&&)`, + // `WriteSlow(const absl::Cord&)`, `WriteSlow(absl::Cord&&), and + // `WriteSlow(ExternalRef)`: // `UnsignedMin(available(), kMaxBytesToCopy) < src.size()` virtual bool WriteSlow(absl::string_view src); - bool WriteStringSlow(std::string&& src); virtual bool WriteSlow(const Chain& src); virtual bool WriteSlow(Chain&& src); virtual bool WriteSlow(const absl::Cord& src); @@ -627,32 +622,14 @@ inline bool BackwardWriter::Write(absl::string_view src) { template < typename Src, - std::enable_if_t, - absl::negation>>::value, - int>> + std::enable_if_t< + absl::conjunction, + absl::negation>>::value, + int>> inline bool BackwardWriter::Write(Src&& src) { return Write(riegeli::ToStringView(src)); } -template ::value, int>> -inline bool BackwardWriter::Write(Src&& src) { - if (ABSL_PREDICT_TRUE(available() >= src.size() && - src.size() <= kMaxBytesToCopy)) { - // `std::memcpy(nullptr, _, 0)` is undefined. - if (ABSL_PREDICT_TRUE(!src.empty())) { - move_cursor(src.size()); - std::memcpy(cursor(), src.data(), src.size()); - } - return true; - } - AssertInitialized(cursor(), start_to_cursor()); - // `std::move(src)` is correct and `std::forward(src)` is not necessary: - // `Src` is always `std::string`, never an lvalue reference. - return WriteStringSlow(std::move(src)); -} - inline bool BackwardWriter::Write(const Chain& src) { #ifdef MEMORY_SANITIZER for (const absl::string_view fragment : src.blocks()) { @@ -732,6 +709,12 @@ inline bool BackwardWriter::Write(ExternalRef src) { return WriteSlow(std::move(src)); } +template ::value, int>> +inline bool BackwardWriter::Write(Src&& src) { + return Write(ExternalRef(src)); +} + inline bool BackwardWriter::Write(signed char src) { return write_int_internal::WriteSigned(src, *this); } @@ -786,8 +769,8 @@ template < absl::conjunction< HasAbslStringify, absl::negation>, absl::negation>, - absl::negation>>:: - value, + absl::negation>, + absl::negation>>::value, int>> inline bool BackwardWriter::Write(Src&& src) { RestrictedChainWriter chain_writer; diff --git a/riegeli/bytes/buffered_reader.cc b/riegeli/bytes/buffered_reader.cc index c597f4f0..f7287023 100644 --- a/riegeli/bytes/buffered_reader.cc +++ b/riegeli/bytes/buffered_reader.cc @@ -31,6 +31,7 @@ #include "riegeli/base/assert.h" #include "riegeli/base/buffering.h" #include "riegeli/base/chain.h" +#include "riegeli/base/external_ref.h" #include "riegeli/base/sized_shared_buffer.h" #include "riegeli/base/types.h" #include "riegeli/bytes/backward_writer.h" @@ -194,9 +195,8 @@ bool BufferedReader::ReadSlow(size_t length, Chain& dest) { if (flat_buffer.empty()) { // Not enough space in `buffer_`. Append available data to `dest` and make // a new buffer. - std::move(buffer_) - .ToExternalRef(absl::string_view(cursor(), available_length)) - .AppendTo(dest); + dest.Append(ExternalRef(std::move(buffer_), + absl::string_view(cursor(), available_length))); length -= available_length; buffer_.ClearAndShrink(buffer_length); if (ABSL_PREDICT_FALSE(buffer_length == 0)) { @@ -238,7 +238,7 @@ bool BufferedReader::ReadSlow(size_t length, Chain& dest) { break; } } - buffer_.ToExternalRef(absl::string_view(cursor(), length)).AppendTo(dest); + dest.Append(ExternalRef(buffer_, absl::string_view(cursor(), length))); move_cursor(length); return enough_read; } @@ -267,8 +267,8 @@ bool BufferedReader::ReadSlow(size_t length, absl::Cord& dest) { if (flat_buffer.empty()) { // Not enough space in `buffer_`. Append available data to `dest` and make // a new buffer. - std::move(buffer_) - .ToExternalRef(absl::string_view(cursor(), available_length)) + ExternalRef(std::move(buffer_), + absl::string_view(cursor(), available_length)) .AppendTo(dest); length -= available_length; buffer_.ClearAndShrink(buffer_length); @@ -311,7 +311,7 @@ bool BufferedReader::ReadSlow(size_t length, absl::Cord& dest) { break; } } - buffer_.ToExternalRef(absl::string_view(cursor(), length)).AppendTo(dest); + ExternalRef(buffer_, absl::string_view(cursor(), length)).AppendTo(dest); move_cursor(length); return enough_read; } @@ -354,8 +354,8 @@ bool BufferedReader::CopySlow(Position length, Writer& dest) { // Not enough space in `buffer_`. Append available data to `dest` and make // a new buffer. if (available_length > 0) { - const bool write_ok = dest.Write(std::move(buffer_).ToExternalRef( - absl::string_view(cursor(), available_length))); + const bool write_ok = dest.Write(ExternalRef( + std::move(buffer_), absl::string_view(cursor(), available_length))); if (ABSL_PREDICT_FALSE(!write_ok)) { buffer_.ClearAndShrink(buffer_length); set_buffer(); @@ -407,8 +407,8 @@ bool BufferedReader::CopySlow(Position length, Writer& dest) { break; } } - const bool write_ok = dest.Write(buffer_.ToExternalRef( - absl::string_view(cursor(), IntCast(length)))); + const bool write_ok = dest.Write(ExternalRef( + buffer_, absl::string_view(cursor(), IntCast(length)))); move_cursor(IntCast(length)); return write_ok && enough_read; } diff --git a/riegeli/bytes/chain_backward_writer.cc b/riegeli/bytes/chain_backward_writer.cc index 7b1abf12..25f77972 100644 --- a/riegeli/bytes/chain_backward_writer.cc +++ b/riegeli/bytes/chain_backward_writer.cc @@ -188,7 +188,7 @@ bool ChainBackwardWriterBase::WriteSlow(ExternalRef src) { return FailOverflow(); } move_start_pos(src.size()); - std::move(src).PrependTo(dest, options_); + dest.Prepend(std::move(src), options_); MakeBuffer(dest); return true; } diff --git a/riegeli/bytes/chain_reader.cc b/riegeli/bytes/chain_reader.cc index 9ba48def..138bbafc 100644 --- a/riegeli/bytes/chain_reader.cc +++ b/riegeli/bytes/chain_reader.cc @@ -28,6 +28,7 @@ #include "riegeli/base/assert.h" #include "riegeli/base/buffering.h" #include "riegeli/base/chain.h" +#include "riegeli/base/external_ref.h" #include "riegeli/base/types.h" #include "riegeli/bytes/backward_writer.h" #include "riegeli/bytes/pullable_reader.h" @@ -81,12 +82,12 @@ bool ChainReaderBase::ReadBehindScratch(size_t length, Chain& dest) { RIEGELI_ASSERT_LE(limit_pos(), src.size()) << "ChainReader source changed unexpectedly"; if (length <= available()) { - iter_.ToExternalRef(absl::string_view(cursor(), length)).AppendTo(dest); + dest.Append(ExternalRef(*iter_, absl::string_view(cursor(), length))); move_cursor(length); return true; } if (ABSL_PREDICT_FALSE(iter_ == src.blocks().cend())) return false; - iter_.ToExternalRef(absl::string_view(cursor(), available())).AppendTo(dest); + dest.Append(ExternalRef(*iter_, absl::string_view(cursor(), available()))); length -= available(); while (++iter_ != src.blocks().cend()) { RIEGELI_ASSERT_LE(iter_->size(), src.size() - limit_pos()) @@ -94,10 +95,10 @@ bool ChainReaderBase::ReadBehindScratch(size_t length, Chain& dest) { move_limit_pos(iter_->size()); if (length <= iter_->size()) { set_buffer(iter_->data(), iter_->size(), length); - iter_.ToExternalRef(absl::string_view(start(), length)).AppendTo(dest); + dest.Append(ExternalRef(*iter_, absl::string_view(start(), length))); return true; } - iter_.ToExternalRef().AppendTo(dest); + dest.Append(*iter_); length -= iter_->size(); } set_buffer(); @@ -119,12 +120,12 @@ bool ChainReaderBase::ReadBehindScratch(size_t length, absl::Cord& dest) { RIEGELI_ASSERT_LE(limit_pos(), src.size()) << "ChainReader source changed unexpectedly"; if (length <= available()) { - iter_.ToExternalRef(absl::string_view(cursor(), length)).AppendTo(dest); + ExternalRef(*iter_, absl::string_view(cursor(), length)).AppendTo(dest); move_cursor(length); return true; } if (ABSL_PREDICT_FALSE(iter_ == src.blocks().cend())) return false; - iter_.ToExternalRef(absl::string_view(cursor(), available())).AppendTo(dest); + ExternalRef(*iter_, absl::string_view(cursor(), available())).AppendTo(dest); length -= available(); while (++iter_ != src.blocks().cend()) { RIEGELI_ASSERT_LE(iter_->size(), src.size() - limit_pos()) @@ -132,10 +133,10 @@ bool ChainReaderBase::ReadBehindScratch(size_t length, absl::Cord& dest) { move_limit_pos(iter_->size()); if (length <= iter_->size()) { set_buffer(iter_->data(), iter_->size(), length); - iter_.ToExternalRef(absl::string_view(start(), length)).AppendTo(dest); + ExternalRef(*iter_, absl::string_view(start(), length)).AppendTo(dest); return true; } - iter_.ToExternalRef().AppendTo(dest); + ExternalRef(*iter_).AppendTo(dest); length -= iter_->size(); } set_buffer(); diff --git a/riegeli/bytes/chain_writer.cc b/riegeli/bytes/chain_writer.cc index ff465dd3..39b16639 100644 --- a/riegeli/bytes/chain_writer.cc +++ b/riegeli/bytes/chain_writer.cc @@ -76,12 +76,12 @@ inline void ChainWriterBase::MoveFromTail(size_t length, Chain& dest) { Chain::BlockIterator iter = tail_->blocks().cbegin(); size_t remaining = length; while (remaining > iter->size()) { - iter.ToExternalRef().AppendTo(dest, options_); + dest.Append(*iter, options_); remaining -= iter->size(); ++iter; } - iter.ToExternalRef(absl::string_view(iter->data(), remaining)) - .AppendTo(dest, options_); + dest.Append(ExternalRef(*iter, absl::string_view(iter->data(), remaining)), + options_); tail_->RemovePrefix(length, options_); } @@ -100,12 +100,13 @@ inline void ChainWriterBase::MoveToTail(size_t length, Chain& dest) { for (;;) { --iter; if (remaining <= iter->size()) break; - iter.ToExternalRef().PrependTo(*tail_, options_); + tail_->Prepend(*iter, options_); remaining -= iter->size(); } - iter.ToExternalRef( - absl::string_view(iter->data() + iter->size() - remaining, remaining)) - .PrependTo(*tail_, options_); + tail_->Prepend(ExternalRef(*iter, absl::string_view( + iter->data() + iter->size() - remaining, + remaining)), + options_); dest.RemoveSuffix(length, options_); } @@ -261,7 +262,7 @@ bool ChainWriterBase::WriteSlow(ExternalRef src) { } ShrinkTail(src.size()); move_start_pos(src.size()); - std::move(src).AppendTo(dest, options_); + dest.Append(std::move(src), options_); MakeBuffer(dest); return true; } diff --git a/riegeli/bytes/cord_backward_writer.cc b/riegeli/bytes/cord_backward_writer.cc index 973fbace..ab7f6bf2 100644 --- a/riegeli/bytes/cord_backward_writer.cc +++ b/riegeli/bytes/cord_backward_writer.cc @@ -73,7 +73,7 @@ inline void CordBackwardWriterBase::SyncBuffer(absl::Cord& dest) { dest.RemovePrefix(prefix_to_remove); } } else { - std::move(buffer_).ToExternalRef(data).PrependTo(dest); + ExternalRef(std::move(buffer_), data).PrependTo(dest); } set_buffer(); } diff --git a/riegeli/bytes/cord_writer.cc b/riegeli/bytes/cord_writer.cc index 7722db48..f94ffdd2 100644 --- a/riegeli/bytes/cord_writer.cc +++ b/riegeli/bytes/cord_writer.cc @@ -78,7 +78,7 @@ inline void CordWriterBase::SyncBuffer(absl::Cord& dest) { dest.Append(std::move(cord_buffer_)); } } else { - std::move(buffer_).ToExternalRef(data).AppendTo(dest); + ExternalRef(std::move(buffer_), data).AppendTo(dest); } set_buffer(); } diff --git a/riegeli/bytes/fd_mmap_reader.cc b/riegeli/bytes/fd_mmap_reader.cc index a446bfdb..2ebd2b67 100644 --- a/riegeli/bytes/fd_mmap_reader.cc +++ b/riegeli/bytes/fd_mmap_reader.cc @@ -169,7 +169,10 @@ class MMapBlock { MMapBlock(MMapBlock&& that) = default; MMapBlock& operator=(MMapBlock&& that) = default; - // Support `ExternalRef` and `Chain::Block`. + // Indicate support for `ExternalRef(MMapBlock&&, substr)`. + friend void RiegeliSupportsExternalRef(MMapBlock*) {} + + // Support `ExternalRef`, `Chain::Block`, and `absl::MakeCordFromExternal()`. void operator()(absl::string_view data) const; // Support `ExternalRef` and `Chain::Block`. diff --git a/riegeli/bytes/pullable_reader.cc b/riegeli/bytes/pullable_reader.cc index ec50c224..a3132305 100644 --- a/riegeli/bytes/pullable_reader.cc +++ b/riegeli/bytes/pullable_reader.cc @@ -33,6 +33,7 @@ #include "riegeli/base/buffering.h" #include "riegeli/base/chain.h" #include "riegeli/base/cord_utils.h" +#include "riegeli/base/external_ref.h" #include "riegeli/base/sized_shared_buffer.h" #include "riegeli/base/types.h" #include "riegeli/bytes/backward_writer.h" @@ -388,15 +389,14 @@ bool PullableReader::ReadSlow(size_t length, Chain& dest) { if (ABSL_PREDICT_FALSE(scratch_used())) { if (!ScratchEnds()) { if (available() >= length) { - scratch_->buffer.ToExternalRef(absl::string_view(cursor(), length)) - .AppendTo(dest); + dest.Append( + ExternalRef(scratch_->buffer, absl::string_view(cursor(), length))); move_cursor(length); return true; } length -= available(); - std::move(scratch_->buffer) - .ToExternalRef(absl::string_view(cursor(), available())) - .AppendTo(dest); + dest.Append(ExternalRef(std::move(scratch_->buffer), + absl::string_view(cursor(), available()))); ClearScratch(); } if (available() >= length && length <= kMaxBytesToCopy) { @@ -418,14 +418,14 @@ bool PullableReader::ReadSlow(size_t length, absl::Cord& dest) { if (ABSL_PREDICT_FALSE(scratch_used())) { if (!ScratchEnds()) { if (available() >= length) { - scratch_->buffer.ToExternalRef(absl::string_view(cursor(), length)) + ExternalRef(scratch_->buffer, absl::string_view(cursor(), length)) .AppendTo(dest); move_cursor(length); return true; } length -= available(); - std::move(scratch_->buffer) - .ToExternalRef(absl::string_view(cursor(), available())) + ExternalRef(std::move(scratch_->buffer), + absl::string_view(cursor(), available())) .AppendTo(dest); ClearScratch(); } @@ -445,15 +445,15 @@ bool PullableReader::CopySlow(Position length, Writer& dest) { if (ABSL_PREDICT_FALSE(scratch_used())) { if (!ScratchEnds()) { if (available() >= length) { - const bool write_ok = dest.Write(scratch_->buffer.ToExternalRef( - absl::string_view(cursor(), length))); + const bool write_ok = dest.Write( + ExternalRef(scratch_->buffer, absl::string_view(cursor(), length))); move_cursor(length); return write_ok; } length -= available(); - const bool write_ok = dest.Write( - std::move(scratch_->buffer) - .ToExternalRef(absl::string_view(cursor(), available()))); + const bool write_ok = + dest.Write(ExternalRef(std::move(scratch_->buffer), + absl::string_view(cursor(), available()))); ClearScratch(); if (ABSL_PREDICT_FALSE(!write_ok)) return false; } @@ -474,15 +474,15 @@ bool PullableReader::CopySlow(size_t length, BackwardWriter& dest) { Chain from_scratch; if (!ScratchEnds()) { if (available() >= length) { - const bool write_ok = dest.Write(scratch_->buffer.ToExternalRef( - absl::string_view(cursor(), length))); + const bool write_ok = dest.Write( + ExternalRef(scratch_->buffer, absl::string_view(cursor(), length))); move_cursor(length); return write_ok; } length -= available(); from_scratch = - Chain(std::move(scratch_->buffer) - .ToExternalRef(absl::string_view(cursor(), available()))); + Chain(ExternalRef(std::move(scratch_->buffer), + absl::string_view(cursor(), available()))); ClearScratch(); } if (available() >= length && length <= kMaxBytesToCopy) { diff --git a/riegeli/bytes/pushable_backward_writer.cc b/riegeli/bytes/pushable_backward_writer.cc index d94ffc44..4c656119 100644 --- a/riegeli/bytes/pushable_backward_writer.cc +++ b/riegeli/bytes/pushable_backward_writer.cc @@ -77,8 +77,8 @@ inline bool PushableBackwardWriter::SyncScratch() { RIEGELI_ASSERT(!scratch_used()) << "Moving should have left the source SizedSharedBuffer cleared"; const char* const data = buffer.data() + buffer.size() - length_to_write; - if (ABSL_PREDICT_FALSE(!Write(std::move(buffer).ToExternalRef( - absl::string_view(data, length_to_write))))) { + if (ABSL_PREDICT_FALSE(!Write(ExternalRef( + std::move(buffer), absl::string_view(data, length_to_write))))) { return false; } RIEGELI_ASSERT(!scratch_used()) @@ -192,7 +192,7 @@ bool PushableBackwardWriter::WriteBehindScratch(const Chain& src) { "scratch used"; for (Chain::Blocks::const_reverse_iterator iter = src.blocks().crbegin(); iter != src.blocks().crend(); ++iter) { - if (ABSL_PREDICT_FALSE(!Write(*iter))) return false; + if (ABSL_PREDICT_FALSE(!Write(absl::string_view(*iter)))) return false; } return true; } diff --git a/riegeli/bytes/pushable_writer.cc b/riegeli/bytes/pushable_writer.cc index a5da4847..49a62024 100644 --- a/riegeli/bytes/pushable_writer.cc +++ b/riegeli/bytes/pushable_writer.cc @@ -77,8 +77,8 @@ inline bool PushableWriter::SyncScratch() { RIEGELI_ASSERT(!scratch_used()) << "Moving should have left the source SizedSharedBuffer cleared"; const char* const data = buffer.data(); - if (ABSL_PREDICT_FALSE(!Write(std::move(buffer).ToExternalRef( - absl::string_view(data, length_to_write))))) { + if (ABSL_PREDICT_FALSE(!Write(ExternalRef( + std::move(buffer), absl::string_view(data, length_to_write))))) { return false; } RIEGELI_ASSERT(!scratch_used()) diff --git a/riegeli/bytes/reader_factory.cc b/riegeli/bytes/reader_factory.cc index 23827d79..52014b0b 100644 --- a/riegeli/bytes/reader_factory.cc +++ b/riegeli/bytes/reader_factory.cc @@ -34,6 +34,7 @@ #include "riegeli/base/assert.h" #include "riegeli/base/buffering.h" #include "riegeli/base/chain.h" +#include "riegeli/base/external_ref.h" #include "riegeli/base/types.h" #include "riegeli/bytes/buffer_options.h" #include "riegeli/bytes/pullable_reader.h" @@ -243,13 +244,12 @@ bool ReaderFactoryBase::ConcurrentReader::ReadBehindScratch(size_t length, << "Failed precondition of PullableReader::ReadBehindScratch(Chain&): " "scratch used"; if (length <= available()) { - iter_.ToExternalRef(absl::string_view(cursor(), length)).AppendTo(dest); + dest.Append(ExternalRef(*iter_, absl::string_view(cursor(), length))); move_cursor(length); return true; } if (iter_ != secondary_buffer_.blocks().cend()) { - iter_.ToExternalRef(absl::string_view(cursor(), available())) - .AppendTo(dest); + dest.Append(ExternalRef(*iter_, absl::string_view(cursor(), available()))); length -= available(); ++iter_; } @@ -259,11 +259,11 @@ bool ReaderFactoryBase::ConcurrentReader::ReadBehindScratch(size_t length, move_limit_pos(iter_->size()); if (length <= iter_->size()) { set_buffer(iter_->data(), iter_->size(), length); - iter_.ToExternalRef(absl::string_view(start(), start_to_cursor())) - .AppendTo(dest); + dest.Append( + ExternalRef(*iter_, absl::string_view(start(), start_to_cursor()))); return true; } - iter_.ToExternalRef().AppendTo(dest); + dest.Append(*iter_); length -= iter_->size(); ++iter_; } @@ -301,12 +301,12 @@ bool ReaderFactoryBase::ConcurrentReader::ReadBehindScratch(size_t length, << "Failed precondition of PullableReader::ReadBehindScratch(Cord&): " "scratch used"; if (length <= available()) { - iter_.ToExternalRef(absl::string_view(cursor(), length)).AppendTo(dest); + ExternalRef(*iter_, absl::string_view(cursor(), length)).AppendTo(dest); move_cursor(length); return true; } if (iter_ != secondary_buffer_.blocks().cend()) { - iter_.ToExternalRef(absl::string_view(cursor(), available())) + ExternalRef(*iter_, absl::string_view(cursor(), available())) .AppendTo(dest); length -= available(); ++iter_; @@ -317,11 +317,11 @@ bool ReaderFactoryBase::ConcurrentReader::ReadBehindScratch(size_t length, move_limit_pos(iter_->size()); if (length <= iter_->size()) { set_buffer(iter_->data(), iter_->size(), length); - iter_.ToExternalRef(absl::string_view(start(), start_to_cursor())) + ExternalRef(*iter_, absl::string_view(start(), start_to_cursor())) .AppendTo(dest); return true; } - iter_.ToExternalRef().AppendTo(dest); + ExternalRef(*iter_).AppendTo(dest); length -= iter_->size(); ++iter_; } @@ -357,7 +357,7 @@ bool ReaderFactoryBase::ConcurrentReader::CopyBehindScratch(Position length, "scratch used"; if (length <= available()) { if (ABSL_PREDICT_FALSE(!dest.Write( - iter_.ToExternalRef(absl::string_view(cursor(), length))))) { + ExternalRef(*iter_, absl::string_view(cursor(), length))))) { return false; } move_cursor(length); @@ -365,7 +365,7 @@ bool ReaderFactoryBase::ConcurrentReader::CopyBehindScratch(Position length, } if (iter_ != secondary_buffer_.blocks().cend()) { if (ABSL_PREDICT_FALSE(!dest.Write( - iter_.ToExternalRef(absl::string_view(cursor(), available()))))) { + ExternalRef(*iter_, absl::string_view(cursor(), available()))))) { return false; } length -= available(); @@ -378,9 +378,9 @@ bool ReaderFactoryBase::ConcurrentReader::CopyBehindScratch(Position length, if (length <= iter_->size()) { set_buffer(iter_->data(), iter_->size(), length); return dest.Write( - iter_.ToExternalRef(absl::string_view(start(), start_to_cursor()))); + ExternalRef(*iter_, absl::string_view(start(), start_to_cursor()))); } - if (ABSL_PREDICT_FALSE(!dest.Write(iter_.ToExternalRef()))) return false; + if (ABSL_PREDICT_FALSE(!dest.Write(*iter_))) return false; length -= iter_->size(); ++iter_; } diff --git a/riegeli/bytes/resizable_writer.cc b/riegeli/bytes/resizable_writer.cc index e5830d4a..0ec65c57 100644 --- a/riegeli/bytes/resizable_writer.cc +++ b/riegeli/bytes/resizable_writer.cc @@ -242,7 +242,7 @@ bool ResizableWriterBase::WriteSlow(ExternalRef src) { SyncSecondaryBuffer(); } move_start_pos(src.size()); - std::move(src).AppendTo(secondary_buffer_, options_); + secondary_buffer_.Append(std::move(src), options_); MakeSecondaryBuffer(); return true; } diff --git a/riegeli/bytes/restricted_chain_writer.cc b/riegeli/bytes/restricted_chain_writer.cc index 4e9a5eac..dcd85dbf 100644 --- a/riegeli/bytes/restricted_chain_writer.cc +++ b/riegeli/bytes/restricted_chain_writer.cc @@ -156,7 +156,7 @@ bool RestrictedChainWriter::WriteSlow(ExternalRef src) { return FailOverflow(); } move_start_pos(src.size()); - std::move(src).AppendTo(dest_); + dest_.Append(std::move(src)); MakeBuffer(); return true; } diff --git a/riegeli/bytes/string_writer.cc b/riegeli/bytes/string_writer.cc index 5cd2f2d5..d9c251f2 100644 --- a/riegeli/bytes/string_writer.cc +++ b/riegeli/bytes/string_writer.cc @@ -337,7 +337,7 @@ bool StringWriterBase::WriteSlow(ExternalRef src) { SyncSecondaryBuffer(); } move_start_pos(src.size()); - std::move(src).AppendTo(secondary_buffer_, options_); + secondary_buffer_.Append(std::move(src), options_); MakeSecondaryBuffer(); return true; } diff --git a/riegeli/bytes/writer.cc b/riegeli/bytes/writer.cc index 8bc90ead..4001d3b2 100644 --- a/riegeli/bytes/writer.cc +++ b/riegeli/bytes/writer.cc @@ -19,8 +19,6 @@ #include #include #include -#include -#include #include "absl/base/optimization.h" #include "absl/status/status.h" @@ -84,13 +82,6 @@ bool Writer::WriteSlow(absl::string_view src) { return true; } -bool Writer::WriteStringSlow(std::string&& src) { - RIEGELI_ASSERT_LT(UnsignedMin(available(), kMaxBytesToCopy), src.size()) - << "Failed precondition of Writer::WriteStringSlow(): " - "enough space available, use Write(string&&) instead"; - return WriteSlow(ExternalRef(std::move(src))); -} - bool Writer::WriteSlow(const Chain& src) { RIEGELI_ASSERT_LT(UnsignedMin(available(), kMaxBytesToCopy), src.size()) << "Failed precondition of Writer::WriteSlow(Chain): " diff --git a/riegeli/bytes/writer.h b/riegeli/bytes/writer.h index 3e4ee6c9..20d45950 100644 --- a/riegeli/bytes/writer.h +++ b/riegeli/bytes/writer.h @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -68,12 +67,9 @@ class WriterAbslStringifySink { Writer* dest() const { return dest_; } void Append(size_t length, char src); - void Append(absl::string_view src); - // `std::string&&` is accepted with a template to avoid implicit conversions - // to `std::string` which can be ambiguous against `absl::string_view` - // (e.g. `const char*`). template ::value, int> = 0> + std::enable_if_t::value, + int> = 0> void Append(Src&& src); friend void AbslFormatFlush(WriterAbslStringifySink* dest, absl::string_view src) { @@ -269,10 +265,6 @@ class Writer : public Object { // Writes a fixed number of bytes from `src` to the buffer and/or the // destination. // - // `std::string&&` is accepted with a template to avoid implicit conversions - // to `std::string` which can be ambiguous against `absl::string_view` - // (e.g. `const char*`). - // // Return values: // * `true` - success (`src.size()` bytes written) // * `false` - failure (less than `src.size()` bytes written, `!ok()`) @@ -282,19 +274,20 @@ class Writer : public Object { #endif bool Write(absl::string_view src); template , - absl::negation>>::value, - int> = 0> - bool Write(Src&& src); - template ::value, int> = 0> + std::enable_if_t< + absl::conjunction< + SupportsToStringView, + absl::negation>>::value, + int> = 0> bool Write(Src&& src); bool Write(const Chain& src); bool Write(Chain&& src); bool Write(const absl::Cord& src); bool Write(absl::Cord&& src); bool Write(ExternalRef src); + template ::value, int> = 0> + bool Write(Src&& src); // Writes a stringified value to the buffer and/or the destination. // @@ -322,8 +315,8 @@ class Writer : public Object { absl::conjunction< HasAbslStringify, absl::negation>, absl::negation>, - absl::negation>>:: - value, + absl::negation>, + absl::negation>>::value, int> = 0> bool Write(Src&& src); @@ -570,12 +563,11 @@ class Writer : public Object { // Precondition for `WriteSlow(absl::string_view)`: // `available() < src.size()` // - // Precondition for `WriteStringSlow()`, `WriteSlow(const Chain&)`, - // `WriteSlow(Chain&&)`, `WriteSlow(const absl::Cord&)`, - // `WriteSlow(absl::Cord&&), and `WriteSlow(ExternalRef)`: + // Precondition for `WriteSlow(const Chain&)`, `WriteSlow(Chain&&)`, + // `WriteSlow(const absl::Cord&)`, `WriteSlow(absl::Cord&&), and + // `WriteSlow(ExternalRef)`: // `UnsignedMin(available(), kMaxBytesToCopy) < src.size()` virtual bool WriteSlow(absl::string_view src); - bool WriteStringSlow(std::string&& src); virtual bool WriteSlow(const Chain& src); virtual bool WriteSlow(Chain&& src); virtual bool WriteSlow(const absl::Cord& src); @@ -733,16 +725,11 @@ inline void WriterAbslStringifySink::Append(size_t length, char src) { dest_->WriteChars(length, src); } -inline void WriterAbslStringifySink::Append(absl::string_view src) { - dest_->Write(src); -} - -template ::value, int>> +template < + typename Src, + std::enable_if_t::value, int>> inline void WriterAbslStringifySink::Append(Src&& src) { - // `std::move(src)` is correct and `std::forward(src)` is not necessary: - // `Src` is always `std::string`, never an lvalue reference. - dest_->Write(std::move(src)); + dest_->Write(std::forward(src)); } inline Writer::Writer(Writer&& that) noexcept @@ -851,32 +838,14 @@ inline bool Writer::Write(absl::string_view src) { template < typename Src, - std::enable_if_t, - absl::negation>>::value, - int>> + std::enable_if_t< + absl::conjunction, + absl::negation>>::value, + int>> inline bool Writer::Write(Src&& src) { return Write(riegeli::ToStringView(src)); } -template ::value, int>> -inline bool Writer::Write(Src&& src) { - if (ABSL_PREDICT_TRUE(available() >= src.size() && - src.size() <= kMaxBytesToCopy)) { - // `std::memcpy(nullptr, _, 0)` is undefined. - if (ABSL_PREDICT_TRUE(!src.empty())) { - std::memcpy(cursor(), src.data(), src.size()); - move_cursor(src.size()); - } - return true; - } - AssertInitialized(start(), start_to_cursor()); - // `std::move(src)` is correct and `std::forward(src)` is not necessary: - // `Src` is always `std::string`, never an lvalue reference. - return WriteStringSlow(std::move(src)); -} - inline bool Writer::Write(const Chain& src) { #ifdef MEMORY_SANITIZER for (const absl::string_view fragment : src.blocks()) { @@ -956,6 +925,12 @@ inline bool Writer::Write(ExternalRef src) { return WriteSlow(std::move(src)); } +template ::value, int>> +inline bool Writer::Write(Src&& src) { + return Write(ExternalRef(src)); +} + inline bool Writer::Write(signed char src) { return write_int_internal::WriteSigned(src, *this); } @@ -1010,8 +985,8 @@ template < absl::conjunction< HasAbslStringify, absl::negation>, absl::negation>, - absl::negation>>:: - value, + absl::negation>, + absl::negation>>::value, int>> inline bool Writer::Write(Src&& src) { WriterAbslStringifySink sink(this); diff --git a/riegeli/chunk_encoding/BUILD b/riegeli/chunk_encoding/BUILD index d92787c3..0eab7fc1 100644 --- a/riegeli/chunk_encoding/BUILD +++ b/riegeli/chunk_encoding/BUILD @@ -14,9 +14,11 @@ cc_library( "//riegeli/base:chain", "//riegeli/base:external_ref", "//riegeli/base:object", + "//riegeli/base:to_string_view", "//riegeli/bytes:writer", "//riegeli/messages:message_serialize", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/meta:type_traits", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", diff --git a/riegeli/chunk_encoding/chunk_encoder.h b/riegeli/chunk_encoding/chunk_encoder.h index 4bce192e..89c8b02b 100644 --- a/riegeli/chunk_encoding/chunk_encoder.h +++ b/riegeli/chunk_encoding/chunk_encoder.h @@ -18,17 +18,18 @@ #include #include -#include #include #include #include +#include "absl/meta/type_traits.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "google/protobuf/message_lite.h" #include "riegeli/base/chain.h" #include "riegeli/base/external_ref.h" #include "riegeli/base/object.h" +#include "riegeli/base/to_string_view.h" #include "riegeli/bytes/writer.h" #include "riegeli/chunk_encoding/constants.h" #include "riegeli/messages/message_serialize.h" @@ -51,10 +52,6 @@ class ChunkEncoder : public Object { // `AddRecord(google::protobuf::MessageLite)` serializes a proto message to // raw bytes beforehand. The remaining overloads accept raw bytes. // - // `std::string&&` is accepted with a template to avoid implicit conversions - // to `std::string` which can be ambiguous against `absl::string_view` - // (e.g. `const char*`). - // // Return values: // * `true` - success (`ok()`) // * `false` - failure (`!ok()`) @@ -63,13 +60,20 @@ class ChunkEncoder : public Object { SerializeOptions serialize_options); virtual bool AddRecord(absl::string_view record) = 0; template ::value, int> = 0> + std::enable_if_t< + absl::conjunction< + SupportsToStringView, + absl::negation>>::value, + int> = 0> bool AddRecord(Src&& record); virtual bool AddRecord(const Chain& record) = 0; virtual bool AddRecord(Chain&& record); virtual bool AddRecord(const absl::Cord& record) = 0; virtual bool AddRecord(absl::Cord&& record); virtual bool AddRecord(ExternalRef record) = 0; + template ::value, int> = 0> + bool AddRecord(Src&& src); // Add multiple records, expressed as concatenated record values and sorted // record end positions. @@ -120,12 +124,20 @@ inline bool ChunkEncoder::AddRecord( return AddRecord(record, SerializeOptions()); } -template ::value, int>> +template < + typename Src, + std::enable_if_t< + absl::conjunction, + absl::negation>>::value, + int>> inline bool ChunkEncoder::AddRecord(Src&& record) { - // `std::move(record)` is correct and `std::forward(record)` is not - // necessary: `Src` is always `std::string`, never an lvalue reference. - return AddRecord(ExternalRef(std::move(record))); + return AddRecord(riegeli::ToStringView(record)); +} + +template ::value, int>> +inline bool ChunkEncoder::AddRecord(Src&& src) { + return AddRecord(ExternalRef(std::forward(src))); } } // namespace riegeli diff --git a/riegeli/records/BUILD b/riegeli/records/BUILD index dc02eb60..cb1691bd 100644 --- a/riegeli/records/BUILD +++ b/riegeli/records/BUILD @@ -68,6 +68,7 @@ cc_library( "//riegeli/base:reset", "//riegeli/base:stable_dependency", "//riegeli/base:status", + "//riegeli/base:to_string_view", "//riegeli/base:types", "//riegeli/bytes:chain_writer", "//riegeli/bytes:writer", @@ -81,6 +82,7 @@ cc_library( "//riegeli/messages:message_serialize", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/meta:type_traits", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", diff --git a/riegeli/records/record_writer.cc b/riegeli/records/record_writer.cc index ad7f13e1..475cecd4 100644 --- a/riegeli/records/record_writer.cc +++ b/riegeli/records/record_writer.cc @@ -23,7 +23,6 @@ #include #include #include -#include #include #include @@ -930,18 +929,6 @@ bool RecordWriterBase::WriteRecord(absl::string_view record) { return WriteRecordImpl(record.size(), record); } -template ::value, int>> -bool RecordWriterBase::WriteRecord(Src&& record) { - if (ABSL_PREDICT_FALSE(!ok())) return false; - const size_t size = record.size(); - // `std::move(record)` is correct and `std::forward(record)` is not - // necessary: `Src` is always `std::string`, never an lvalue reference. - return WriteRecordImpl(size, std::move(record)); -} - -template bool RecordWriterBase::WriteRecord(std::string&& record); - bool RecordWriterBase::WriteRecord(const Chain& record) { if (ABSL_PREDICT_FALSE(!ok())) return false; return WriteRecordImpl(record.size(), record); diff --git a/riegeli/records/record_writer.h b/riegeli/records/record_writer.h index f2097c7a..e63c5066 100644 --- a/riegeli/records/record_writer.h +++ b/riegeli/records/record_writer.h @@ -20,12 +20,12 @@ #include #include -#include #include #include #include "absl/base/attributes.h" #include "absl/base/optimization.h" +#include "absl/meta/type_traits.h" #include "absl/status/status.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" @@ -42,6 +42,7 @@ #include "riegeli/base/recycling_pool.h" #include "riegeli/base/reset.h" #include "riegeli/base/stable_dependency.h" +#include "riegeli/base/to_string_view.h" #include "riegeli/base/types.h" #include "riegeli/bytes/writer.h" #include "riegeli/chunk_encoding/compressor_options.h" @@ -433,10 +434,6 @@ class RecordWriterBase : public Object { // `WriteRecord(google::protobuf::MessageLite)` serializes a proto message to // raw bytes beforehand. The remaining overloads accept raw bytes. // - // `std::string&&` is accepted with a template to avoid implicit conversions - // to `std::string` which can be ambiguous against `absl::string_view` - // (e.g. `const char*`). - // // Return values: // * `true` - success (`ok()`) // * `false` - failure (`!ok()`) @@ -444,13 +441,20 @@ class RecordWriterBase : public Object { SerializeOptions serialize_options = SerializeOptions()); bool WriteRecord(absl::string_view record); template ::value, int> = 0> + std::enable_if_t< + absl::conjunction< + SupportsToStringView, + absl::negation>>::value, + int> = 0> bool WriteRecord(Src&& record); bool WriteRecord(const Chain& record); bool WriteRecord(Chain&& record); bool WriteRecord(const absl::Cord& record); bool WriteRecord(absl::Cord&& record); bool WriteRecord(ExternalRef record); + template ::value, int> = 0> + bool WriteRecord(Src&& src); // Finalizes any open chunk and pushes buffered data to the destination. // If `Options::parallelism() > 0`, waits for any background writing to @@ -665,7 +669,21 @@ explicit RecordWriter(Dest&& dest, RecordWriterBase::Options options = // Implementation details follow. -extern template bool RecordWriterBase::WriteRecord(std::string&& record); +template < + typename Src, + std::enable_if_t< + absl::conjunction, + absl::negation>>::value, + int>> +inline bool RecordWriterBase::WriteRecord(Src&& record) { + return WriteRecord(riegeli::ToStringView(record)); +} + +template ::value, int>> +inline bool RecordWriterBase::WriteRecord(Src&& src) { + return WriteRecord(ExternalRef(std::forward(src))); +} template inline RecordWriter::RecordWriter(Initializer dest, Options options) diff --git a/riegeli/snappy/snappy_streams.cc b/riegeli/snappy/snappy_streams.cc index ff287b32..ab254200 100644 --- a/riegeli/snappy/snappy_streams.cc +++ b/riegeli/snappy/snappy_streams.cc @@ -59,7 +59,7 @@ char* WriterSnappySink::GetAppendBuffer(size_t length, char* scratch) { void WriterSnappySink::AppendAndTakeOwnership( char* src, size_t length, void (*deleter)(void*, const char*, size_t), void* deleter_arg) { - dest_->Write(ExternalRef( + dest_->Write(ExternalRef::From( [deleter, deleter_arg](absl::string_view data) { deleter(deleter_arg, data.data(), data.size()); }, diff --git a/riegeli/snappy/snappy_writer.cc b/riegeli/snappy/snappy_writer.cc index 78f5186f..cb69bd9a 100644 --- a/riegeli/snappy/snappy_writer.cc +++ b/riegeli/snappy/snappy_writer.cc @@ -258,7 +258,7 @@ bool SnappyWriterBase::WriteSlow(ExternalRef src) { return FailOverflow(); } move_start_pos(src.size()); - std::move(src).AppendTo(uncompressed_, options_); + uncompressed_.Append(std::move(src), options_); return true; } diff --git a/riegeli/tensorflow/io/BUILD b/riegeli/tensorflow/io/BUILD index 23bb377b..24b90e18 100644 --- a/riegeli/tensorflow/io/BUILD +++ b/riegeli/tensorflow/io/BUILD @@ -15,6 +15,7 @@ cc_library( "//riegeli/base:buffering", "//riegeli/base:chain", "//riegeli/base:dependency", + "//riegeli/base:external_ref", "//riegeli/base:initializer", "//riegeli/base:object", "//riegeli/base:reset", diff --git a/riegeli/tensorflow/io/file_reader.cc b/riegeli/tensorflow/io/file_reader.cc index d18bc0d3..fae255fd 100644 --- a/riegeli/tensorflow/io/file_reader.cc +++ b/riegeli/tensorflow/io/file_reader.cc @@ -35,6 +35,7 @@ #include "riegeli/base/assert.h" #include "riegeli/base/buffering.h" #include "riegeli/base/chain.h" +#include "riegeli/base/external_ref.h" #include "riegeli/base/initializer.h" #include "riegeli/base/object.h" #include "riegeli/base/reset.h" @@ -340,9 +341,8 @@ bool FileReaderBase::ReadSlow(size_t length, Chain& dest) { if (flat_buffer.empty()) { // Not enough space in `buffer_`. Append available data to `dest` and // make a new buffer. - std::move(buffer_) - .ToExternalRef(absl::string_view(cursor(), available_length)) - .AppendTo(dest); + dest.Append(ExternalRef(std::move(buffer_), + absl::string_view(cursor(), available_length))); length -= available_length; buffer_.ClearAndShrink(buffer_length); if (ABSL_PREDICT_FALSE(buffer_length == 0)) { @@ -364,7 +364,7 @@ bool FileReaderBase::ReadSlow(size_t length, Chain& dest) { if (buffer_.empty()) { dest.Append(absl::string_view(cursor(), length)); } else { - buffer_.ToExternalRef(absl::string_view(cursor(), length)).AppendTo(dest); + dest.Append(ExternalRef(buffer_, absl::string_view(cursor(), length))); } move_cursor(length); return enough_read; @@ -408,8 +408,8 @@ bool FileReaderBase::ReadSlow(size_t length, absl::Cord& dest) { if (flat_buffer.empty()) { // Not enough space in `buffer_`. Append available data to `dest` and // make a new buffer. - std::move(buffer_) - .ToExternalRef(absl::string_view(cursor(), available_length)) + ExternalRef(std::move(buffer_), + absl::string_view(cursor(), available_length)) .AppendTo(dest); length -= available_length; buffer_.ClearAndShrink(buffer_length); @@ -432,7 +432,7 @@ bool FileReaderBase::ReadSlow(size_t length, absl::Cord& dest) { if (buffer_.empty()) { dest.Append(absl::string_view(cursor(), length)); } else { - buffer_.ToExternalRef(absl::string_view(cursor(), length)).AppendTo(dest); + ExternalRef(buffer_, absl::string_view(cursor(), length)).AppendTo(dest); } move_cursor(length); return enough_read; @@ -494,8 +494,9 @@ bool FileReaderBase::CopySlow(Position length, Writer& dest) { // Not enough space in `buffer_`. Append available data to `dest` and // make a new buffer. if (available_length > 0) { - const bool write_ok = dest.Write(std::move(buffer_).ToExternalRef( - absl::string_view(cursor(), available_length))); + const bool write_ok = dest.Write( + ExternalRef(std::move(buffer_), + absl::string_view(cursor(), available_length))); if (ABSL_PREDICT_FALSE(!write_ok)) { buffer_.ClearAndShrink(buffer_length); set_buffer(); @@ -527,8 +528,8 @@ bool FileReaderBase::CopySlow(Position length, Writer& dest) { const bool write_ok = buffer_.empty() ? dest.Write(absl::string_view(cursor(), IntCast(length))) - : dest.Write(buffer_.ToExternalRef( - absl::string_view(cursor(), IntCast(length)))); + : dest.Write(ExternalRef( + buffer_, absl::string_view(cursor(), IntCast(length)))); move_cursor(IntCast(length)); return write_ok && enough_read; } diff --git a/riegeli/tensorflow/io/file_writer.cc b/riegeli/tensorflow/io/file_writer.cc index bcc7503c..c27abae7 100644 --- a/riegeli/tensorflow/io/file_writer.cc +++ b/riegeli/tensorflow/io/file_writer.cc @@ -144,7 +144,7 @@ inline bool FileWriterBase::SyncBuffer() { if (start_to_cursor() > kMaxBytesToCopy) { if (ABSL_PREDICT_FALSE(!ok())) return false; const absl::Cord data( - buffer_.ToExternalRef(absl::string_view(start(), start_to_cursor()))); + ExternalRef(buffer_, absl::string_view(start(), start_to_cursor()))); set_buffer(); return WriteInternal(data); }