From 0fe7304500421de25f57d4d0004a9601ec722d87 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Wed, 9 Feb 2022 15:16:21 +0800 Subject: [PATCH] fix format Signed-off-by: Yuan Zhou --- .../src/third_party/parallel_hashmap/btree.h | 7398 ++++++++--------- .../third_party/parallel_hashmap/meminfo.h | 281 +- .../src/third_party/parallel_hashmap/phmap.h | 7252 ++++++++-------- .../third_party/parallel_hashmap/phmap_base.h | 5363 ++++++------ .../third_party/parallel_hashmap/phmap_bits.h | 636 +- .../parallel_hashmap/phmap_config.h | 557 +- .../third_party/parallel_hashmap/phmap_dump.h | 324 +- .../parallel_hashmap/phmap_fwd_decl.h | 243 +- .../parallel_hashmap/phmap_utils.h | 409 +- 9 files changed, 10940 insertions(+), 11523 deletions(-) diff --git a/native-sql-engine/cpp/src/third_party/parallel_hashmap/btree.h b/native-sql-engine/cpp/src/third_party/parallel_hashmap/btree.h index cbfb8eeb3..b9b0d94da 100644 --- a/native-sql-engine/cpp/src/third_party/parallel_hashmap/btree.h +++ b/native-sql-engine/cpp/src/third_party/parallel_hashmap/btree.h @@ -35,3507 +35,3394 @@ #define PHMAP_BTREE_BTREE_CONTAINER_H_ #ifdef _MSC_VER - #pragma warning(push) - - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4324) // structure was padded due to alignment specifier - #pragma warning(disable : 4355) // 'this': used in base member initializer list - #pragma warning(disable : 4365) // conversion from 'int' to 'const unsigned __int64', signed/unsigned mismatch - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member - #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list - #pragma warning(disable : 5026) // move constructor was implicitly defined as deleted - #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted - #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified +#pragma warning(push) + +#pragma warning(disable : 4127) // conditional expression is constant +#pragma warning(disable : 4324) // structure was padded due to alignment specifier +#pragma warning(disable : 4355) // 'this': used in base member initializer list +#pragma warning(disable : 4365) // conversion from 'int' to 'const unsigned __int64', + // signed/unsigned mismatch +#pragma warning(disable : 4514) // unreferenced inline function has been removed +#pragma warning(disable : 4623) // default constructor was implicitly defined as deleted +#pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted +#pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted +#pragma warning(disable : 4710) // function not inlined +#pragma warning(disable : 4711) // selected for automatic inline expansion +#pragma warning(disable : 4820) // '6' bytes padding added after data member +#pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation + // order in braced initializer list +#pragma warning(disable : 5026) // move constructor was implicitly defined as deleted +#pragma warning( \ + disable : 5027) // move assignment operator was implicitly defined as deleted +#pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory + // load if /Qspectre switch specified #endif - #include #include #include #include #include -#include "phmap_fwd_decl.h" #include "phmap_base.h" +#include "phmap_fwd_decl.h" #if PHMAP_HAVE_STD_STRING_VIEW - #include +#include #endif // MSVC constructibility traits do not detect destructor properties and so our // implementations should not use them as a source-of-truth. #if defined(_MSC_VER) && !defined(__clang__) && !defined(__GNUC__) - #define PHMAP_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1 +#define PHMAP_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1 #endif namespace phmap { - // Defined and documented later on in this file. - template - struct is_trivially_destructible; +// Defined and documented later on in this file. +template +struct is_trivially_destructible; - // Defined and documented later on in this file. - template - struct is_trivially_move_assignable; +// Defined and documented later on in this file. +template +struct is_trivially_move_assignable; - namespace type_traits_internal { +namespace type_traits_internal { - // Silence MSVC warnings about the destructor being defined as deleted. +// Silence MSVC warnings about the destructor being defined as deleted. #if defined(_MSC_VER) && !defined(__GNUC__) - #pragma warning(push) - #pragma warning(disable : 4624) +#pragma warning(push) +#pragma warning(disable : 4624) #endif // defined(_MSC_VER) && !defined(__GNUC__) - template - union SingleMemberUnion { - T t; - }; +template +union SingleMemberUnion { + T t; +}; - // Restore the state of the destructor warning that was silenced above. +// Restore the state of the destructor warning that was silenced above. #if defined(_MSC_VER) && !defined(__GNUC__) - #pragma warning(pop) +#pragma warning(pop) #endif // defined(_MSC_VER) && !defined(__GNUC__) - template - struct IsTriviallyMoveConstructibleObject - : std::integral_constant< - bool, std::is_move_constructible< - type_traits_internal::SingleMemberUnion>::value && - phmap::is_trivially_destructible::value> {}; - - template - struct IsTriviallyCopyConstructibleObject - : std::integral_constant< - bool, std::is_copy_constructible< - type_traits_internal::SingleMemberUnion>::value && - phmap::is_trivially_destructible::value> {}; - - template - struct IsTriviallyMoveAssignableReference : std::false_type {}; - - template - struct IsTriviallyMoveAssignableReference - : phmap::is_trivially_move_assignable::type {}; - - template - struct IsTriviallyMoveAssignableReference - : phmap::is_trivially_move_assignable::type {}; - - } // namespace type_traits_internal - - - template - using void_t = typename type_traits_internal::VoidTImpl::type; - - - template - struct is_function - : std::integral_constant< - bool, !(std::is_reference::value || - std::is_const::type>::value)> {}; - - - namespace type_traits_internal { - - template - class is_trivially_copyable_impl { - using ExtentsRemoved = typename std::remove_all_extents::type; - static constexpr bool kIsCopyOrMoveConstructible = - std::is_copy_constructible::value || - std::is_move_constructible::value; - static constexpr bool kIsCopyOrMoveAssignable = - phmap::is_copy_assignable::value || - phmap::is_move_assignable::value; - - public: - static constexpr bool kValue = - (__has_trivial_copy(ExtentsRemoved) || !kIsCopyOrMoveConstructible) && - (__has_trivial_assign(ExtentsRemoved) || !kIsCopyOrMoveAssignable) && - (kIsCopyOrMoveConstructible || kIsCopyOrMoveAssignable) && - is_trivially_destructible::value && - // We need to check for this explicitly because otherwise we'll say - // references are trivial copyable when compiled by MSVC. - !std::is_reference::value; - }; - - template - struct is_trivially_copyable - : std::integral_constant< - bool, type_traits_internal::is_trivially_copyable_impl::kValue> {}; - } // namespace type_traits_internal - - namespace swap_internal { - - // Necessary for the traits. - using std::swap; - - // This declaration prevents global `swap` and `phmap::swap` overloads from being - // considered unless ADL picks them up. - void swap(); - - template - using IsSwappableImpl = decltype(swap(std::declval(), std::declval())); - - // NOTE: This dance with the default template parameter is for MSVC. - template (), std::declval()))>> - using IsNothrowSwappableImpl = typename std::enable_if::type; - - template - struct IsSwappable - : phmap::type_traits_internal::is_detected {}; - - template - struct IsNothrowSwappable - : phmap::type_traits_internal::is_detected {}; - - template ::value, int> = 0> - void Swap(T& lhs, T& rhs) noexcept(IsNothrowSwappable::value) { - swap(lhs, rhs); - } +template +struct IsTriviallyMoveConstructibleObject + : std::integral_constant< + bool, + std::is_move_constructible>::value && + phmap::is_trivially_destructible::value> {}; + +template +struct IsTriviallyCopyConstructibleObject + : std::integral_constant< + bool, + std::is_copy_constructible>::value && + phmap::is_trivially_destructible::value> {}; + +template +struct IsTriviallyMoveAssignableReference : std::false_type {}; + +template +struct IsTriviallyMoveAssignableReference + : phmap::is_trivially_move_assignable::type {}; + +template +struct IsTriviallyMoveAssignableReference + : phmap::is_trivially_move_assignable::type {}; + +} // namespace type_traits_internal + +template +using void_t = typename type_traits_internal::VoidTImpl::type; + +template +struct is_function + : std::integral_constant::value || + std::is_const::type>::value)> { +}; + +namespace type_traits_internal { + +template +class is_trivially_copyable_impl { + using ExtentsRemoved = typename std::remove_all_extents::type; + static constexpr bool kIsCopyOrMoveConstructible = + std::is_copy_constructible::value || + std::is_move_constructible::value; + static constexpr bool kIsCopyOrMoveAssignable = + phmap::is_copy_assignable::value || + phmap::is_move_assignable::value; + + public: + static constexpr bool kValue = + (__has_trivial_copy(ExtentsRemoved) || !kIsCopyOrMoveConstructible) && + (__has_trivial_assign(ExtentsRemoved) || !kIsCopyOrMoveAssignable) && + (kIsCopyOrMoveConstructible || kIsCopyOrMoveAssignable) && + is_trivially_destructible::value && + // We need to check for this explicitly because otherwise we'll say + // references are trivial copyable when compiled by MSVC. + !std::is_reference::value; +}; + +template +struct is_trivially_copyable + : std::integral_constant< + bool, type_traits_internal::is_trivially_copyable_impl::kValue> {}; +} // namespace type_traits_internal + +namespace swap_internal { + +// Necessary for the traits. +using std::swap; + +// This declaration prevents global `swap` and `phmap::swap` overloads from being +// considered unless ADL picks them up. +void swap(); + +template +using IsSwappableImpl = decltype(swap(std::declval(), std::declval())); + +// NOTE: This dance with the default template parameter is for MSVC. +template (), std::declval()))>> +using IsNothrowSwappableImpl = typename std::enable_if::type; + +template +struct IsSwappable : phmap::type_traits_internal::is_detected {}; + +template +struct IsNothrowSwappable + : phmap::type_traits_internal::is_detected {}; + +template ::value, int> = 0> +void Swap(T& lhs, T& rhs) noexcept(IsNothrowSwappable::value) { + swap(lhs, rhs); +} - using StdSwapIsUnconstrained = IsSwappable; +using StdSwapIsUnconstrained = IsSwappable; - } // namespace swap_internal +} // namespace swap_internal - namespace type_traits_internal { +namespace type_traits_internal { - // Make the swap-related traits/function accessible from this namespace. - using swap_internal::IsNothrowSwappable; - using swap_internal::IsSwappable; - using swap_internal::Swap; - using swap_internal::StdSwapIsUnconstrained; +// Make the swap-related traits/function accessible from this namespace. +using swap_internal::IsNothrowSwappable; +using swap_internal::IsSwappable; +using swap_internal::StdSwapIsUnconstrained; +using swap_internal::Swap; - } // namespace type_traits_internal +} // namespace type_traits_internal - namespace compare_internal { +namespace compare_internal { - using value_type = int8_t; +using value_type = int8_t; - template - struct Fail { - static_assert(sizeof(T) < 0, "Only literal `0` is allowed."); - }; +template +struct Fail { + static_assert(sizeof(T) < 0, "Only literal `0` is allowed."); +}; - template - struct OnlyLiteralZero { - constexpr OnlyLiteralZero(NullPtrT) noexcept {} // NOLINT +template +struct OnlyLiteralZero { + constexpr OnlyLiteralZero(NullPtrT) noexcept {} // NOLINT - template < - typename T, - typename = typename std::enable_if< - std::is_same::value || - (std::is_integral::value && !std::is_same::value)>::type, - typename = typename Fail::type> - OnlyLiteralZero(T); // NOLINT - }; + template ::value || + (std::is_integral::value && + !std::is_same::value)>::type, + typename = typename Fail::type> + OnlyLiteralZero(T); // NOLINT +}; - enum class eq : value_type { - equal = 0, - equivalent = equal, - nonequal = 1, - nonequivalent = nonequal, - }; +enum class eq : value_type { + equal = 0, + equivalent = equal, + nonequal = 1, + nonequivalent = nonequal, +}; - enum class ord : value_type { less = -1, greater = 1 }; +enum class ord : value_type { less = -1, greater = 1 }; - enum class ncmp : value_type { unordered = -127 }; +enum class ncmp : value_type { unordered = -127 }; #if defined(__cpp_inline_variables) && !defined(_MSC_VER) #define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) -#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) \ - static const type name; +#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) static const type name; -#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ - inline constexpr type type::name(init) +#define PHMAP_COMPARE_INLINE_INIT(type, name, init) inline constexpr type type::name(init) #else // __cpp_inline_variables -#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) \ - static const T name; +#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) static const T name; #define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) -#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ - template \ - const T compare_internal::type##_base::name(init) +#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ + template \ + const T compare_internal::type##_base::name(init) #endif // __cpp_inline_variables - // These template base classes allow for defining the values of the constants - // in the header file (for performance) without using inline variables (which - // aren't available in C++11). - template - struct weak_equality_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) - }; - - template - struct strong_equality_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) - }; - - template - struct partial_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(unordered) - }; - - template - struct weak_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - }; - - template - struct strong_ordering_base { - PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) - PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) - }; - - } // namespace compare_internal - - class weak_equality - : public compare_internal::weak_equality_base { - explicit constexpr weak_equality(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::weak_equality_base; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, nonequivalent) - - // Comparisons - friend constexpr bool operator==( - weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - weak_equality v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - weak_equality v) noexcept { - return 0 != v.value_; - } - - private: - compare_internal::value_type value_; - }; - PHMAP_COMPARE_INLINE_INIT(weak_equality, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(weak_equality, nonequivalent, - compare_internal::eq::nonequivalent); - - class strong_equality - : public compare_internal::strong_equality_base { - explicit constexpr strong_equality(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::strong_equality_base; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequivalent) - - // Conversion - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - // Comparisons - friend constexpr bool operator==( - strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - strong_equality v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - strong_equality v) noexcept { - return 0 != v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(strong_equality, equal, compare_internal::eq::equal); - PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequal, - compare_internal::eq::nonequal); - PHMAP_COMPARE_INLINE_INIT(strong_equality, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequivalent, - compare_internal::eq::nonequivalent); - - class partial_ordering - : public compare_internal::partial_ordering_base { - explicit constexpr partial_ordering(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - explicit constexpr partial_ordering(compare_internal::ord v) noexcept - : value_(static_cast(v)) {} - explicit constexpr partial_ordering(compare_internal::ncmp v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::partial_ordering_base; - - constexpr bool is_ordered() const noexcept { - return value_ != - compare_internal::value_type(compare_internal::ncmp::unordered); - } - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, greater) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, unordered) - - // Conversion - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - // Comparisons - friend constexpr bool operator==( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ == 0; - } - friend constexpr bool operator!=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return !v.is_ordered() || v.value_ != 0; - } - friend constexpr bool operator<( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ < 0; - } - friend constexpr bool operator<=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ <= 0; - } - friend constexpr bool operator>( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ > 0; - } - friend constexpr bool operator>=( - partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.is_ordered() && v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return !v.is_ordered() || 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - partial_ordering v) noexcept { - return v.is_ordered() && 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(partial_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, greater, - compare_internal::ord::greater); - PHMAP_COMPARE_INLINE_INIT(partial_ordering, unordered, - compare_internal::ncmp::unordered); - - class weak_ordering - : public compare_internal::weak_ordering_base { - explicit constexpr weak_ordering(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - explicit constexpr weak_ordering(compare_internal::ord v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::weak_ordering_base; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, greater) - - // Conversions - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - constexpr operator partial_ordering() const noexcept { // NOLINT - return value_ == 0 ? partial_ordering::equivalent - : (value_ < 0 ? partial_ordering::less - : partial_ordering::greater); - } - // Comparisons - friend constexpr bool operator==( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator<( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ < 0; - } - friend constexpr bool operator<=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ <= 0; - } - friend constexpr bool operator>( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ > 0; - } - friend constexpr bool operator>=( - weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - weak_ordering v) noexcept { - return 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - - PHMAP_COMPARE_INLINE_INIT(weak_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(weak_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(weak_ordering, greater, - compare_internal::ord::greater); - - class strong_ordering - : public compare_internal::strong_ordering_base { - explicit constexpr strong_ordering(compare_internal::eq v) noexcept - : value_(static_cast(v)) {} - explicit constexpr strong_ordering(compare_internal::ord v) noexcept - : value_(static_cast(v)) {} - friend struct compare_internal::strong_ordering_base; - - public: - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, less) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equal) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equivalent) - PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, greater) - - // Conversions - constexpr operator weak_equality() const noexcept { // NOLINT - return value_ == 0 ? weak_equality::equivalent - : weak_equality::nonequivalent; - } - constexpr operator strong_equality() const noexcept { // NOLINT - return value_ == 0 ? strong_equality::equal : strong_equality::nonequal; - } - constexpr operator partial_ordering() const noexcept { // NOLINT - return value_ == 0 ? partial_ordering::equivalent - : (value_ < 0 ? partial_ordering::less - : partial_ordering::greater); - } - constexpr operator weak_ordering() const noexcept { // NOLINT - return value_ == 0 - ? weak_ordering::equivalent - : (value_ < 0 ? weak_ordering::less : weak_ordering::greater); - } - // Comparisons - friend constexpr bool operator==( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ == 0; - } - friend constexpr bool operator!=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ != 0; - } - friend constexpr bool operator<( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ < 0; - } - friend constexpr bool operator<=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ <= 0; - } - friend constexpr bool operator>( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ > 0; - } - friend constexpr bool operator>=( - strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { - return v.value_ >= 0; - } - friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 == v.value_; - } - friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 != v.value_; - } - friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 < v.value_; - } - friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 <= v.value_; - } - friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 > v.value_; - } - friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, - strong_ordering v) noexcept { - return 0 >= v.value_; - } - - private: - compare_internal::value_type value_; - }; - PHMAP_COMPARE_INLINE_INIT(strong_ordering, less, compare_internal::ord::less); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, equal, compare_internal::eq::equal); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, equivalent, - compare_internal::eq::equivalent); - PHMAP_COMPARE_INLINE_INIT(strong_ordering, greater, - compare_internal::ord::greater); +// These template base classes allow for defining the values of the constants +// in the header file (for performance) without using inline variables (which +// aren't available in C++11). +template +struct weak_equality_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) +}; + +template +struct strong_equality_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequal) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) +}; + +template +struct partial_ordering_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(unordered) +}; + +template +struct weak_ordering_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) +}; + +template +struct strong_ordering_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) +}; + +} // namespace compare_internal + +class weak_equality : public compare_internal::weak_equality_base { + explicit constexpr weak_equality(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::weak_equality_base; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, nonequivalent) + + // Comparisons + friend constexpr bool operator==(weak_equality v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=(weak_equality v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + weak_equality v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + weak_equality v) noexcept { + return 0 != v.value_; + } + + private: + compare_internal::value_type value_; +}; +PHMAP_COMPARE_INLINE_INIT(weak_equality, equivalent, compare_internal::eq::equivalent); +PHMAP_COMPARE_INLINE_INIT(weak_equality, nonequivalent, + compare_internal::eq::nonequivalent); + +class strong_equality : public compare_internal::strong_equality_base { + explicit constexpr strong_equality(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::strong_equality_base; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equal) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequal) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequivalent) + + // Conversion + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent : weak_equality::nonequivalent; + } + // Comparisons + friend constexpr bool operator==(strong_equality v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=(strong_equality v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + strong_equality v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + strong_equality v) noexcept { + return 0 != v.value_; + } + + private: + compare_internal::value_type value_; +}; + +PHMAP_COMPARE_INLINE_INIT(strong_equality, equal, compare_internal::eq::equal); +PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequal, compare_internal::eq::nonequal); +PHMAP_COMPARE_INLINE_INIT(strong_equality, equivalent, compare_internal::eq::equivalent); +PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequivalent, + compare_internal::eq::nonequivalent); + +class partial_ordering + : public compare_internal::partial_ordering_base { + explicit constexpr partial_ordering(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + explicit constexpr partial_ordering(compare_internal::ord v) noexcept + : value_(static_cast(v)) {} + explicit constexpr partial_ordering(compare_internal::ncmp v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::partial_ordering_base; + + constexpr bool is_ordered() const noexcept { + return value_ != compare_internal::value_type(compare_internal::ncmp::unordered); + } + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, less) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, greater) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, unordered) + + // Conversion + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent : weak_equality::nonequivalent; + } + // Comparisons + friend constexpr bool operator==(partial_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ == 0; + } + friend constexpr bool operator!=(partial_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return !v.is_ordered() || v.value_ != 0; + } + friend constexpr bool operator<(partial_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ < 0; + } + friend constexpr bool operator<=(partial_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ <= 0; + } + friend constexpr bool operator>(partial_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ > 0; + } + friend constexpr bool operator>=(partial_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return !v.is_ordered() || 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 >= v.value_; + } + + private: + compare_internal::value_type value_; +}; + +PHMAP_COMPARE_INLINE_INIT(partial_ordering, less, compare_internal::ord::less); +PHMAP_COMPARE_INLINE_INIT(partial_ordering, equivalent, compare_internal::eq::equivalent); +PHMAP_COMPARE_INLINE_INIT(partial_ordering, greater, compare_internal::ord::greater); +PHMAP_COMPARE_INLINE_INIT(partial_ordering, unordered, compare_internal::ncmp::unordered); + +class weak_ordering : public compare_internal::weak_ordering_base { + explicit constexpr weak_ordering(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + explicit constexpr weak_ordering(compare_internal::ord v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::weak_ordering_base; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, less) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, greater) + + // Conversions + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent : weak_equality::nonequivalent; + } + constexpr operator partial_ordering() const noexcept { // NOLINT + return value_ == 0 + ? partial_ordering::equivalent + : (value_ < 0 ? partial_ordering::less : partial_ordering::greater); + } + // Comparisons + friend constexpr bool operator==(weak_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=(weak_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator<(weak_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ < 0; + } + friend constexpr bool operator<=(weak_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ <= 0; + } + friend constexpr bool operator>(weak_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ > 0; + } + friend constexpr bool operator>=(weak_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 >= v.value_; + } + + private: + compare_internal::value_type value_; +}; + +PHMAP_COMPARE_INLINE_INIT(weak_ordering, less, compare_internal::ord::less); +PHMAP_COMPARE_INLINE_INIT(weak_ordering, equivalent, compare_internal::eq::equivalent); +PHMAP_COMPARE_INLINE_INIT(weak_ordering, greater, compare_internal::ord::greater); + +class strong_ordering : public compare_internal::strong_ordering_base { + explicit constexpr strong_ordering(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + explicit constexpr strong_ordering(compare_internal::ord v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::strong_ordering_base; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, less) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equal) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, greater) + + // Conversions + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent : weak_equality::nonequivalent; + } + constexpr operator strong_equality() const noexcept { // NOLINT + return value_ == 0 ? strong_equality::equal : strong_equality::nonequal; + } + constexpr operator partial_ordering() const noexcept { // NOLINT + return value_ == 0 + ? partial_ordering::equivalent + : (value_ < 0 ? partial_ordering::less : partial_ordering::greater); + } + constexpr operator weak_ordering() const noexcept { // NOLINT + return value_ == 0 ? weak_ordering::equivalent + : (value_ < 0 ? weak_ordering::less : weak_ordering::greater); + } + // Comparisons + friend constexpr bool operator==(strong_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=(strong_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator<(strong_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ < 0; + } + friend constexpr bool operator<=(strong_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ <= 0; + } + friend constexpr bool operator>(strong_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ > 0; + } + friend constexpr bool operator>=(strong_ordering v, + compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 >= v.value_; + } + + private: + compare_internal::value_type value_; +}; +PHMAP_COMPARE_INLINE_INIT(strong_ordering, less, compare_internal::ord::less); +PHMAP_COMPARE_INLINE_INIT(strong_ordering, equal, compare_internal::eq::equal); +PHMAP_COMPARE_INLINE_INIT(strong_ordering, equivalent, compare_internal::eq::equivalent); +PHMAP_COMPARE_INLINE_INIT(strong_ordering, greater, compare_internal::ord::greater); #undef PHMAP_COMPARE_INLINE_BASECLASS_DECL #undef PHMAP_COMPARE_INLINE_SUBCLASS_DECL #undef PHMAP_COMPARE_INLINE_INIT - namespace compare_internal { - // We also provide these comparator adapter functions for internal phmap use. - - // Helper functions to do a boolean comparison of two keys given a boolean - // or three-way comparator. - // SFINAE prevents implicit conversions to bool (such as from int). - template ::value, int> = 0> - constexpr bool compare_result_as_less_than(const BoolType r) { return r; } - constexpr bool compare_result_as_less_than(const phmap::weak_ordering r) { - return r < 0; - } +namespace compare_internal { +// We also provide these comparator adapter functions for internal phmap use. - template - constexpr bool do_less_than_comparison(const Compare &compare, const K &x, - const LK &y) { - return compare_result_as_less_than(compare(x, y)); - } +// Helper functions to do a boolean comparison of two keys given a boolean +// or three-way comparator. +// SFINAE prevents implicit conversions to bool (such as from int). +template ::value, int> = 0> +constexpr bool compare_result_as_less_than(const BoolType r) { + return r; +} +constexpr bool compare_result_as_less_than(const phmap::weak_ordering r) { return r < 0; } - // Helper functions to do a three-way comparison of two keys given a boolean or - // three-way comparator. - // SFINAE prevents implicit conversions to int (such as from bool). - template ::value, int> = 0> - constexpr phmap::weak_ordering compare_result_as_ordering(const Int c) { - return c < 0 ? phmap::weak_ordering::less - : c == 0 ? phmap::weak_ordering::equivalent - : phmap::weak_ordering::greater; - } - constexpr phmap::weak_ordering compare_result_as_ordering( - const phmap::weak_ordering c) { - return c; - } +template +constexpr bool do_less_than_comparison(const Compare& compare, const K& x, const LK& y) { + return compare_result_as_less_than(compare(x, y)); +} - template < - typename Compare, typename K, typename LK, - phmap::enable_if_t>::value, - int> = 0> - constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, - const K &x, const LK &y) { - return compare_result_as_ordering(compare(x, y)); - } - template < - typename Compare, typename K, typename LK, - phmap::enable_if_t>::value, - int> = 0> - constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, - const K &x, const LK &y) { - return compare(x, y) ? phmap::weak_ordering::less - : compare(y, x) ? phmap::weak_ordering::greater - : phmap::weak_ordering::equivalent; - } +// Helper functions to do a three-way comparison of two keys given a boolean or +// three-way comparator. +// SFINAE prevents implicit conversions to int (such as from bool). +template ::value, int> = 0> +constexpr phmap::weak_ordering compare_result_as_ordering(const Int c) { + return c < 0 + ? phmap::weak_ordering::less + : c == 0 ? phmap::weak_ordering::equivalent : phmap::weak_ordering::greater; +} +constexpr phmap::weak_ordering compare_result_as_ordering(const phmap::weak_ordering c) { + return c; +} - } // namespace compare_internal +template < + typename Compare, typename K, typename LK, + phmap::enable_if_t< + !std::is_same>::value, + int> = 0> +constexpr phmap::weak_ordering do_three_way_comparison(const Compare& compare, const K& x, + const LK& y) { + return compare_result_as_ordering(compare(x, y)); +} +template < + typename Compare, typename K, typename LK, + phmap::enable_if_t< + std::is_same>::value, + int> = 0> +constexpr phmap::weak_ordering do_three_way_comparison(const Compare& compare, const K& x, + const LK& y) { + return compare(x, y) ? phmap::weak_ordering::less + : compare(y, x) ? phmap::weak_ordering::greater + : phmap::weak_ordering::equivalent; } +} // namespace compare_internal +} // namespace phmap namespace phmap { namespace priv { - // A helper class that indicates if the Compare parameter is a key-compare-to - // comparator. - template - using btree_is_key_compare_to = - std::is_convertible, - phmap::weak_ordering>; +// A helper class that indicates if the Compare parameter is a key-compare-to +// comparator. +template +using btree_is_key_compare_to = + std::is_convertible, + phmap::weak_ordering>; - struct StringBtreeDefaultLess { - using is_transparent = void; +struct StringBtreeDefaultLess { + using is_transparent = void; - StringBtreeDefaultLess() = default; + StringBtreeDefaultLess() = default; - // Compatibility constructor. - StringBtreeDefaultLess(std::less) {} // NOLINT + // Compatibility constructor. + StringBtreeDefaultLess(std::less) {} // NOLINT #if PHMAP_HAVE_STD_STRING_VIEW - StringBtreeDefaultLess(std::less) {} // NOLINT - StringBtreeDefaultLess(phmap::Less) {} // NOLINT + StringBtreeDefaultLess(std::less) {} // NOLINT + StringBtreeDefaultLess(phmap::Less) {} // NOLINT - phmap::weak_ordering operator()(std::string_view lhs, - std::string_view rhs) const { - return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); - } + phmap::weak_ordering operator()(std::string_view lhs, std::string_view rhs) const { + return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); + } #else - phmap::weak_ordering operator()(std::string lhs, - std::string rhs) const { - return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); - } + phmap::weak_ordering operator()(std::string lhs, std::string rhs) const { + return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); + } #endif - }; +}; - struct StringBtreeDefaultGreater { - using is_transparent = void; +struct StringBtreeDefaultGreater { + using is_transparent = void; - StringBtreeDefaultGreater() = default; + StringBtreeDefaultGreater() = default; - StringBtreeDefaultGreater(std::greater) {} // NOLINT + StringBtreeDefaultGreater(std::greater) {} // NOLINT #if PHMAP_HAVE_STD_STRING_VIEW - StringBtreeDefaultGreater(std::greater) {} // NOLINT + StringBtreeDefaultGreater(std::greater) {} // NOLINT - phmap::weak_ordering operator()(std::string_view lhs, - std::string_view rhs) const { - return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); - } + phmap::weak_ordering operator()(std::string_view lhs, std::string_view rhs) const { + return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); + } #else - phmap::weak_ordering operator()(std::string lhs, - std::string rhs) const { - return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); - } + phmap::weak_ordering operator()(std::string lhs, std::string rhs) const { + return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); + } #endif - }; - - // A helper class to convert a boolean comparison into a three-way "compare-to" - // comparison that returns a negative value to indicate less-than, zero to - // indicate equality and a positive value to indicate greater-than. This helper - // class is specialized for less, greater, - // less, and greater. - // - // key_compare_to_adapter is provided so that btree users - // automatically get the more efficient compare-to code when using common - // google string types with common comparison functors. - // These string-like specializations also turn on heterogeneous lookup by - // default. - template - struct key_compare_to_adapter { - using type = Compare; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultGreater; - }; - -#if PHMAP_HAVE_STD_STRING_VIEW - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultLess; - }; - - template <> - struct key_compare_to_adapter> { - using type = StringBtreeDefaultGreater; - }; -#endif - - template - struct common_params { - // If Compare is a common comparator for a std::string-like type, then we adapt it - // to use heterogeneous lookup and to be a key-compare-to comparator. - using key_compare = typename key_compare_to_adapter::type; - // A type which indicates if we have a key-compare-to functor or a plain old - // key-compare functor. - using is_key_compare_to = btree_is_key_compare_to; - - using allocator_type = Alloc; - using key_type = Key; - using size_type = std::size_t ; - using difference_type = ptrdiff_t; - - // True if this is a multiset or multimap. - using is_multi_container = std::integral_constant; - - using slot_policy = SlotPolicy; - using slot_type = typename slot_policy::slot_type; - using value_type = typename slot_policy::value_type; - using init_type = typename slot_policy::mutable_value_type; - using pointer = value_type *; - using const_pointer = const value_type *; - using reference = value_type &; - using const_reference = const value_type &; - - enum { - kTargetNodeSize = TargetNodeSize, - - // Upper bound for the available space for values. This is largest for leaf - // nodes, which have overhead of at least a pointer + 4 bytes (for storing - // 3 field_types and an enum). - kNodeValueSpace = - TargetNodeSize - /*minimum overhead=*/(sizeof(void *) + 4), - }; - - // This is an integral type large enough to hold as many - // ValueSize-values as will fit a node of TargetNodeSize bytes. - using node_count_type = - phmap::conditional_t<(kNodeValueSpace / sizeof(value_type) > - (std::numeric_limits::max)()), - uint16_t, uint8_t>; // NOLINT - - // The following methods are necessary for passing this struct as PolicyTraits - // for node_handle and/or are used within btree. - static value_type &element(slot_type *slot) { - return slot_policy::element(slot); - } - static const value_type &element(const slot_type *slot) { - return slot_policy::element(slot); - } - template - static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { - slot_policy::construct(alloc, slot, std::forward(args)...); - } - static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { - slot_policy::construct(alloc, slot, other); - } - static void destroy(Alloc *alloc, slot_type *slot) { - slot_policy::destroy(alloc, slot); - } - static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) { - construct(alloc, new_slot, old_slot); - destroy(alloc, old_slot); - } - static void swap(Alloc *alloc, slot_type *a, slot_type *b) { - slot_policy::swap(alloc, a, b); - } - static void move(Alloc *alloc, slot_type *src, slot_type *dest) { - slot_policy::move(alloc, src, dest); - } - static void move(Alloc *alloc, slot_type *first, slot_type *last, - slot_type *result) { - slot_policy::move(alloc, first, last, result); - } - }; - - // A parameters structure for holding the type parameters for a btree_map. - // Compare and Alloc should be nothrow copy-constructible. - template - struct map_params : common_params> { - using super_type = typename map_params::common_params; - using mapped_type = Data; - // This type allows us to move keys when it is safe to do so. It is safe - // for maps in which value_type and mutable_value_type are layout compatible. - using slot_policy = typename super_type::slot_policy; - using slot_type = typename super_type::slot_type; - using value_type = typename super_type::value_type; - using init_type = typename super_type::init_type; - - using key_compare = typename super_type::key_compare; - // Inherit from key_compare for empty base class optimization. - struct value_compare : private key_compare { - value_compare() = default; - explicit value_compare(const key_compare &cmp) : key_compare(cmp) {} - - template - auto operator()(const T &left, const U &right) const - -> decltype(std::declval()(left.first, right.first)) { - return key_compare::operator()(left.first, right.first); - } - }; - using is_map_container = std::true_type; - - static const Key &key(const value_type &x) { return x.first; } - static const Key &key(const init_type &x) { return x.first; } - static const Key &key(const slot_type *x) { return slot_policy::key(x); } - static mapped_type &value(value_type *value) { return value->second; } - }; - - // This type implements the necessary functions from the - // btree::priv::slot_type interface. - template - struct set_slot_policy { - using slot_type = Key; - using value_type = Key; - using mutable_value_type = Key; - - static value_type &element(slot_type *slot) { return *slot; } - static const value_type &element(const slot_type *slot) { return *slot; } - - template - static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { - phmap::allocator_traits::construct(*alloc, slot, - std::forward(args)...); - } - - template - static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { - phmap::allocator_traits::construct(*alloc, slot, std::move(*other)); - } - - template - static void destroy(Alloc *alloc, slot_type *slot) { - phmap::allocator_traits::destroy(*alloc, slot); - } - - template - static void swap(Alloc * /*alloc*/, slot_type *a, slot_type *b) { - using std::swap; - swap(*a, *b); - } - - template - static void move(Alloc * /*alloc*/, slot_type *src, slot_type *dest) { - *dest = std::move(*src); - } - - template - static void move(Alloc *alloc, slot_type *first, slot_type *last, - slot_type *result) { - for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) - move(alloc, src, dest); - } - }; - - // A parameters structure for holding the type parameters for a btree_set. - // Compare and Alloc should be nothrow copy-constructible. - template - struct set_params : common_params> { - using value_type = Key; - using slot_type = typename set_params::common_params::slot_type; - using value_compare = typename set_params::common_params::key_compare; - using is_map_container = std::false_type; - - static const Key &key(const value_type &x) { return x; } - static const Key &key(const slot_type *x) { return *x; } - }; - - // An adapter class that converts a lower-bound compare into an upper-bound - // compare. Note: there is no need to make a version of this adapter specialized - // for key-compare-to functors because the upper-bound (the first value greater - // than the input) is never an exact match. - template - struct upper_bound_adapter { - explicit upper_bound_adapter(const Compare &c) : comp(c) {} - template - bool operator()(const K &a, const LK &b) const { - // Returns true when a is not greater than b. - return !phmap::compare_internal::compare_result_as_less_than(comp(b, a)); - } - - private: - Compare comp; - }; - - enum class MatchKind : uint8_t { kEq, kNe }; - - template - struct SearchResult { - V value; - MatchKind match; - - static constexpr bool HasMatch() { return true; } - bool IsEq() const { return match == MatchKind::kEq; } - }; - - // When we don't use CompareTo, `match` is not present. - // This ensures that callers can't use it accidentally when it provides no - // useful information. - template - struct SearchResult { - V value; - - static constexpr bool HasMatch() { return false; } - static constexpr bool IsEq() { return false; } - }; - - // A node in the btree holding. The same node type is used for both internal - // and leaf nodes in the btree, though the nodes are allocated in such a way - // that the children array is only valid in internal nodes. - template - class btree_node { - using is_key_compare_to = typename Params::is_key_compare_to; - using is_multi_container = typename Params::is_multi_container; - using field_type = typename Params::node_count_type; - using allocator_type = typename Params::allocator_type; - using slot_type = typename Params::slot_type; - - public: - using params_type = Params; - using key_type = typename Params::key_type; - using value_type = typename Params::value_type; - using pointer = typename Params::pointer; - using const_pointer = typename Params::const_pointer; - using reference = typename Params::reference; - using const_reference = typename Params::const_reference; - using key_compare = typename Params::key_compare; - using size_type = typename Params::size_type; - using difference_type = typename Params::difference_type; - - // Btree decides whether to use linear node search as follows: - // - If the key is arithmetic and the comparator is std::less or - // std::greater, choose linear. - // - Otherwise, choose binary. - // TODO(ezb): Might make sense to add condition(s) based on node-size. - using use_linear_search = std::integral_constant< - bool, - std::is_arithmetic::value && - (std::is_same, key_compare>::value || - std::is_same, key_compare>::value || - std::is_same, key_compare>::value)>; - - - ~btree_node() = default; - btree_node(btree_node const &) = delete; - btree_node &operator=(btree_node const &) = delete; - - // Public for EmptyNodeType. - constexpr static size_type Alignment() { - static_assert(LeafLayout(1).Alignment() == InternalLayout().Alignment(), - "Alignment of all nodes must be equal."); - return (size_type)InternalLayout().Alignment(); - } - - protected: - btree_node() = default; - - private: - using layout_type = phmap::priv::Layout; - constexpr static size_type SizeWithNValues(size_type n) { - return (size_type)layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ (size_t)n, - /*children*/ 0) - .AllocSize(); - } - // A lower bound for the overhead of fields other than values in a leaf node. - constexpr static size_type MinimumOverhead() { - return (size_type)(SizeWithNValues(1) - sizeof(value_type)); - } - - // Compute how many values we can fit onto a leaf node taking into account - // padding. - constexpr static size_type NodeTargetValues(const int begin, const int end) { - return begin == end ? begin - : SizeWithNValues((begin + end) / 2 + 1) > - params_type::kTargetNodeSize - ? NodeTargetValues(begin, (begin + end) / 2) - : NodeTargetValues((begin + end) / 2 + 1, end); - } - - enum { - kTargetNodeSize = params_type::kTargetNodeSize, - kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize), - - // We need a minimum of 3 values per internal node in order to perform - // splitting (1 value for the two nodes involved in the split and 1 value - // propagated to the parent as the delimiter for the split). - kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, - - // The node is internal (i.e. is not a leaf node) if and only if `max_count` - // has this value. - kInternalNodeMaxCount = 0, - }; - - // Leaves can have less than kNodeValues values. - constexpr static layout_type LeafLayout(const int max_values = kNodeValues) { - return layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ (size_t)max_values, - /*children*/ 0); - } - constexpr static layout_type InternalLayout() { - return layout_type(/*parent*/ 1, - /*position, start, count, max_count*/ 4, - /*values*/ kNodeValues, - /*children*/ kNodeValues + 1); - } - constexpr static size_type LeafSize(const int max_values = kNodeValues) { - return (size_type)LeafLayout(max_values).AllocSize(); - } - constexpr static size_type InternalSize() { - return (size_type)InternalLayout().AllocSize(); - } - - // N is the index of the type in the Layout definition. - // ElementType is the Nth type in the Layout definition. - template - inline typename layout_type::template ElementType *GetField() { - // We assert that we don't read from values that aren't there. - assert(N < 3 || !leaf()); - return InternalLayout().template Pointer(reinterpret_cast(this)); - } - - template - inline const typename layout_type::template ElementType *GetField() const { - assert(N < 3 || !leaf()); - return InternalLayout().template Pointer( - reinterpret_cast(this)); - } - - void set_parent(btree_node *p) { *GetField<0>() = p; } - field_type &mutable_count() { return GetField<1>()[2]; } - slot_type *slot(size_type i) { return &GetField<2>()[i]; } - const slot_type *slot(size_type i) const { return &GetField<2>()[i]; } - void set_position(field_type v) { GetField<1>()[0] = v; } - void set_start(field_type v) { GetField<1>()[1] = v; } - void set_count(field_type v) { GetField<1>()[2] = v; } - void set_max_count(field_type v) { GetField<1>()[3] = v; } - - public: - // Whether this is a leaf node or not. This value doesn't change after the - // node is created. - bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; } - - // Getter for the position of this node in its parent. - field_type position() const { return GetField<1>()[0]; } - - // Getter for the offset of the first value in the `values` array. - field_type start() const { return GetField<1>()[1]; } - - // Getters for the number of values stored in this node. - field_type count() const { return GetField<1>()[2]; } - field_type max_count() const { - // Internal nodes have max_count==kInternalNodeMaxCount. - // Leaf nodes have max_count in [1, kNodeValues]. - const field_type max_cnt = GetField<1>()[3]; - return max_cnt == field_type{kInternalNodeMaxCount} - ? field_type{kNodeValues} - : max_cnt; - } - - // Getter for the parent of this node. - btree_node *parent() const { return *GetField<0>(); } - // Getter for whether the node is the root of the tree. The parent of the - // root of the tree is the leftmost node in the tree which is guaranteed to - // be a leaf. - bool is_root() const { return parent()->leaf(); } - void make_root() { - assert(parent()->is_root()); - set_parent(parent()->parent()); - } - - // Getters for the key/value at position i in the node. - const key_type &key(size_type i) const { return params_type::key(slot(i)); } - reference value(size_type i) { return params_type::element(slot(i)); } - const_reference value(size_type i) const { return params_type::element(slot(i)); } - - // Getters/setter for the child at position i in the node. - btree_node *child(size_type i) const { return GetField<3>()[i]; } - btree_node *&mutable_child(size_type i) { return GetField<3>()[i]; } - void clear_child(size_type i) { - phmap::priv::SanitizerPoisonObject(&mutable_child(i)); - } - void set_child(size_type i, btree_node *c) { - phmap::priv::SanitizerUnpoisonObject(&mutable_child(i)); - mutable_child(i) = c; - c->set_position((field_type)i); - } - void init_child(int i, btree_node *c) { - set_child(i, c); - c->set_parent(this); - } - - // Returns the position of the first value whose key is not less than k. - template - SearchResult lower_bound( - const K &k, const key_compare &comp) const { - return use_linear_search::value ? linear_search(k, comp) - : binary_search(k, comp); - } - // Returns the position of the first value whose key is greater than k. - template - int upper_bound(const K &k, const key_compare &comp) const { - auto upper_compare = upper_bound_adapter(comp); - return use_linear_search::value ? linear_search(k, upper_compare).value - : binary_search(k, upper_compare).value; - } - - template - SearchResult::value> - linear_search(const K &k, const Compare &comp) const { - return linear_search_impl(k, 0, count(), comp, - btree_is_key_compare_to()); - } - - template - SearchResult::value> - binary_search(const K &k, const Compare &comp) const { - return binary_search_impl(k, 0, count(), comp, - btree_is_key_compare_to()); - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using plain compare. - template - SearchResult linear_search_impl( - const K &k, int s, const int e, const Compare &comp, - std::false_type /* IsCompareTo */) const { - while (s < e) { - if (!comp(key(s), k)) { - break; - } - ++s; - } - return {s}; - } - - // Returns the position of the first value whose key is not less than k using - // linear search performed using compare-to. - template - SearchResult linear_search_impl( - const K &k, int s, const int e, const Compare &comp, - std::true_type /* IsCompareTo */) const { - while (s < e) { - const phmap::weak_ordering c = comp(key(s), k); - if (c == 0) { - return {s, MatchKind::kEq}; - } else if (c > 0) { - break; - } - ++s; - } - return {s, MatchKind::kNe}; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using plain compare. - template - SearchResult binary_search_impl( - const K &k, int s, int e, const Compare &comp, - std::false_type /* IsCompareTo */) const { - while (s != e) { - const int mid = (s + e) >> 1; - if (comp(key(mid), k)) { - s = mid + 1; - } else { - e = mid; - } - } - return {s}; - } - - // Returns the position of the first value whose key is not less than k using - // binary search performed using compare-to. - template - SearchResult binary_search_impl( - const K &k, int s, int e, const CompareTo &comp, - std::true_type /* IsCompareTo */) const { - if (is_multi_container::value) { - MatchKind exact_match = MatchKind::kNe; - while (s != e) { - const int mid = (s + e) >> 1; - const phmap::weak_ordering c = comp(key(mid), k); - if (c < 0) { - s = mid + 1; - } else { - e = mid; - if (c == 0) { - // Need to return the first value whose key is not less than k, - // which requires continuing the binary search if this is a - // multi-container. - exact_match = MatchKind::kEq; - } - } - } - return {s, exact_match}; - } else { // Not a multi-container. - while (s != e) { - const int mid = (s + e) >> 1; - const phmap::weak_ordering c = comp(key(mid), k); - if (c < 0) { - s = mid + 1; - } else if (c > 0) { - e = mid; - } else { - return {mid, MatchKind::kEq}; - } - } - return {s, MatchKind::kNe}; - } - } - - // Emplaces a value at position i, shifting all existing values and - // children at positions >= i to the right by 1. - template - void emplace_value(size_type i, allocator_type *alloc, Args &&... args); - - // Removes the value at position i, shifting all existing values and children - // at positions > i to the left by 1. - void remove_value(int i, allocator_type *alloc); - - // Removes the values at positions [i, i + to_erase), shifting all values - // after that range to the left by to_erase. Does not change children at all. - void remove_values_ignore_children(int i, size_type to_erase, - allocator_type *alloc); - - // Rebalances a node with its right sibling. - void rebalance_right_to_left(int to_move, btree_node *right, - allocator_type *alloc); - void rebalance_left_to_right(int to_move, btree_node *right, - allocator_type *alloc); - - // Splits a node, moving a portion of the node's values to its right sibling. - void split(int insert_position, btree_node *dest, allocator_type *alloc); - - // Merges a node with its right sibling, moving all of the values and the - // delimiting key in the parent node onto itself. - void merge(btree_node *sibling, allocator_type *alloc); - - // Swap the contents of "this" and "src". - void swap(btree_node *src, allocator_type *alloc); - - // Node allocation/deletion routines. - static btree_node *init_leaf(btree_node *n, btree_node *parent, - int max_cnt) { - n->set_parent(parent); - n->set_position(0); - n->set_start(0); - n->set_count(0); - n->set_max_count((field_type)max_cnt); - phmap::priv::SanitizerPoisonMemoryRegion( - n->slot(0), max_cnt * sizeof(slot_type)); - return n; - } - static btree_node *init_internal(btree_node *n, btree_node *parent) { - init_leaf(n, parent, kNodeValues); - // Set `max_count` to a sentinel value to indicate that this node is - // internal. - n->set_max_count(kInternalNodeMaxCount); - phmap::priv::SanitizerPoisonMemoryRegion( - &n->mutable_child(0), (kNodeValues + 1) * sizeof(btree_node *)); - return n; - } - void destroy(allocator_type *alloc) { - for (int i = 0; i < count(); ++i) { - value_destroy(i, alloc); - } - } - - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return use_linear_search::value; - } - - private: - template - void value_init(const size_type i, allocator_type *alloc, Args &&... args) { - phmap::priv::SanitizerUnpoisonObject(slot(i)); - params_type::construct(alloc, slot(i), std::forward(args)...); - } - void value_destroy(const size_type i, allocator_type *alloc) { - params_type::destroy(alloc, slot(i)); - phmap::priv::SanitizerPoisonObject(slot(i)); - } - - // Move n values starting at value i in this node into the values starting at - // value j in node x. - void uninitialized_move_n(const size_type n, const size_type i, - const size_type j, btree_node *x, - allocator_type *alloc) { - phmap::priv::SanitizerUnpoisonMemoryRegion( - x->slot(j), n * sizeof(slot_type)); - for (slot_type *src = slot(i), *end = src + n, *dest = x->slot(j); - src != end; ++src, ++dest) { - params_type::construct(alloc, dest, src); - } - } - - // Destroys a range of n values, starting at index i. - void value_destroy_n(const size_type i, const size_type n, - allocator_type *alloc) { - for (size_type j = 0; j < n; ++j) { - value_destroy(i + j, alloc); - } - } - - template - friend class btree; - template - friend struct btree_iterator; - friend class BtreeNodePeer; - }; - - template - struct btree_iterator { - private: - using key_type = typename Node::key_type; - using size_type = typename Node::size_type; - using params_type = typename Node::params_type; - - using node_type = Node; - using normal_node = typename std::remove_const::type; - using const_node = const Node; - using normal_pointer = typename params_type::pointer; - using normal_reference = typename params_type::reference; - using const_pointer = typename params_type::const_pointer; - using const_reference = typename params_type::const_reference; - using slot_type = typename params_type::slot_type; - - using iterator = - btree_iterator; - using const_iterator = - btree_iterator; - - public: - // These aliases are public for std::iterator_traits. - using difference_type = typename Node::difference_type; - using value_type = typename params_type::value_type; - using pointer = Pointer; - using reference = Reference; - using iterator_category = std::bidirectional_iterator_tag; - - btree_iterator() : node(nullptr), position(-1) {} - btree_iterator(Node *n, int p) : node(n), position(p) {} - - // NOTE: this SFINAE allows for implicit conversions from iterator to - // const_iterator, but it specifically avoids defining copy constructors so - // that btree_iterator can be trivially copyable. This is for performance and - // binary size reasons. - template , iterator>::value && - std::is_same::value, - int> = 0> - btree_iterator(const btree_iterator &x) // NOLINT - : node(x.node), position(x.position) {} - - private: - // This SFINAE allows explicit conversions from const_iterator to - // iterator, but also avoids defining a copy constructor. - // NOTE: the const_cast is safe because this constructor is only called by - // non-const methods and the container owns the nodes. - template , const_iterator>::value && - std::is_same::value, - int> = 0> - explicit btree_iterator(const btree_iterator &x) - : node(const_cast(x.node)), position(x.position) {} - - // Increment/decrement the iterator. - void increment() { - if (node->leaf() && ++position < node->count()) { - return; - } - increment_slow(); - } - void increment_slow(); - - void decrement() { - if (node->leaf() && --position >= 0) { - return; - } - decrement_slow(); - } - void decrement_slow(); - - public: - bool operator==(const const_iterator &x) const { - return node == x.node && position == x.position; - } - bool operator!=(const const_iterator &x) const { - return node != x.node || position != x.position; - } - - // Accessors for the key/value the iterator is pointing at. - reference operator*() const { - return node->value(position); - } - pointer operator->() const { - return &node->value(position); - } - - btree_iterator& operator++() { - increment(); - return *this; - } - btree_iterator& operator--() { - decrement(); - return *this; - } - btree_iterator operator++(int) { - btree_iterator tmp = *this; - ++*this; - return tmp; - } - btree_iterator operator--(int) { - btree_iterator tmp = *this; - --*this; - return tmp; - } - - private: - template - friend class btree; - template - friend class btree_container; - template - friend class btree_set_container; - template - friend class btree_map_container; - template - friend class btree_multiset_container; - template - friend struct btree_iterator; - template - friend class base_checker; - - const key_type &key() const { return node->key(position); } - slot_type *slot() { return node->slot(position); } - - // The node in the tree the iterator is pointing at. - Node *node; - // The position within the node of the tree the iterator is pointing at. - // TODO(ezb): make this a field_type - int position; - }; - - template - class btree { - using node_type = btree_node; - using is_key_compare_to = typename Params::is_key_compare_to; - - // We use a static empty node for the root/leftmost/rightmost of empty btrees - // in order to avoid branching in begin()/end(). - struct alignas(node_type::Alignment()) EmptyNodeType : node_type { - using field_type = typename node_type::field_type; - node_type *parent; - field_type position = 0; - field_type start = 0; - field_type count = 0; - // max_count must be != kInternalNodeMaxCount (so that this node is regarded - // as a leaf node). max_count() is never called when the tree is empty. - field_type max_count = node_type::kInternalNodeMaxCount + 1; +}; -#ifdef _MSC_VER - // MSVC has constexpr code generations bugs here. - EmptyNodeType() : parent(this) {} -#else - constexpr EmptyNodeType(node_type *p) : parent(p) {} -#endif - }; +// A helper class to convert a boolean comparison into a three-way "compare-to" +// comparison that returns a negative value to indicate less-than, zero to +// indicate equality and a positive value to indicate greater-than. This helper +// class is specialized for less, greater, +// less, and greater. +// +// key_compare_to_adapter is provided so that btree users +// automatically get the more efficient compare-to code when using common +// google string types with common comparison functors. +// These string-like specializations also turn on heterogeneous lookup by +// default. +template +struct key_compare_to_adapter { + using type = Compare; +}; + +template <> +struct key_compare_to_adapter> { + using type = StringBtreeDefaultLess; +}; + +template <> +struct key_compare_to_adapter> { + using type = StringBtreeDefaultLess; +}; + +template <> +struct key_compare_to_adapter> { + using type = StringBtreeDefaultGreater; +}; - static node_type *EmptyNode() { -#ifdef _MSC_VER - static EmptyNodeType empty_node; - // This assert fails on some other construction methods. - assert(empty_node.parent == &empty_node); - return &empty_node; -#else - static constexpr EmptyNodeType empty_node( - const_cast(&empty_node)); - return const_cast(&empty_node); +#if PHMAP_HAVE_STD_STRING_VIEW +template <> +struct key_compare_to_adapter> { + using type = StringBtreeDefaultLess; +}; + +template <> +struct key_compare_to_adapter> { + using type = StringBtreeDefaultLess; +}; + +template <> +struct key_compare_to_adapter> { + using type = StringBtreeDefaultGreater; +}; #endif - } - - enum { - kNodeValues = node_type::kNodeValues, - kMinNodeValues = kNodeValues / 2, - }; - - struct node_stats { - using size_type = typename Params::size_type; - - node_stats(size_type l, size_type i) - : leaf_nodes(l), - internal_nodes(i) { - } - - node_stats& operator+=(const node_stats &x) { - leaf_nodes += x.leaf_nodes; - internal_nodes += x.internal_nodes; - return *this; - } - - size_type leaf_nodes; - size_type internal_nodes; - }; - - public: - using key_type = typename Params::key_type; - using value_type = typename Params::value_type; - using size_type = typename Params::size_type; - using difference_type = typename Params::difference_type; - using key_compare = typename Params::key_compare; - using value_compare = typename Params::value_compare; - using allocator_type = typename Params::allocator_type; - using reference = typename Params::reference; - using const_reference = typename Params::const_reference; - using pointer = typename Params::pointer; - using const_pointer = typename Params::const_pointer; - using iterator = btree_iterator; - using const_iterator = typename iterator::const_iterator; - using reverse_iterator = std::reverse_iterator; - using const_reverse_iterator = std::reverse_iterator; - using node_handle_type = node_handle; - - // Internal types made public for use by btree_container types. - using params_type = Params; - using slot_type = typename Params::slot_type; - - private: - // For use in copy_or_move_values_in_order. - const value_type &maybe_move_from_iterator(const_iterator x) { return *x; } - value_type &&maybe_move_from_iterator(iterator x) { return std::move(*x); } - - // Copies or moves (depending on the template parameter) the values in - // x into this btree in their order in x. This btree must be empty before this - // method is called. This method is used in copy construction, copy - // assignment, and move assignment. - template - void copy_or_move_values_in_order(Btree *x); - - // Validates that various assumptions/requirements are true at compile time. - constexpr static bool static_assert_validation(); - - public: - btree(const key_compare &comp, const allocator_type &alloc); - - btree(const btree &x); - btree(btree &&x) noexcept - : root_(std::move(x.root_)), - rightmost_(phmap::exchange(x.rightmost_, EmptyNode())), - size_(phmap::exchange(x.size_, 0)) { - x.mutable_root() = EmptyNode(); - } - - ~btree() { - // Put static_asserts in destructor to avoid triggering them before the type - // is complete. - static_assert(static_assert_validation(), "This call must be elided."); - clear(); - } - - // Assign the contents of x to *this. - btree &operator=(const btree &x); - btree &operator=(btree &&x) noexcept; - - iterator begin() { - return iterator(leftmost(), 0); - } - const_iterator begin() const { - return const_iterator(leftmost(), 0); - } - iterator end() { return iterator(rightmost_, rightmost_->count()); } - const_iterator end() const { - return const_iterator(rightmost_, rightmost_->count()); - } - reverse_iterator rbegin() { - return reverse_iterator(end()); - } - const_reverse_iterator rbegin() const { - return const_reverse_iterator(end()); - } - reverse_iterator rend() { - return reverse_iterator(begin()); - } - const_reverse_iterator rend() const { - return const_reverse_iterator(begin()); - } - - // Finds the first element whose key is not less than key. - template - iterator lower_bound(const K &key) { - return internal_end(internal_lower_bound(key)); - } - template - const_iterator lower_bound(const K &key) const { - return internal_end(internal_lower_bound(key)); - } - - // Finds the first element whose key is greater than key. - template - iterator upper_bound(const K &key) { - return internal_end(internal_upper_bound(key)); - } - template - const_iterator upper_bound(const K &key) const { - return internal_end(internal_upper_bound(key)); - } - - // Finds the range of values which compare equal to key. The first member of - // the returned pair is equal to lower_bound(key). The second member pair of - // the pair is equal to upper_bound(key). - template - std::pair equal_range(const K &key) { - return {lower_bound(key), upper_bound(key)}; - } - template - std::pair equal_range(const K &key) const { - return {lower_bound(key), upper_bound(key)}; - } - - // Inserts a value into the btree only if it does not already exist. The - // boolean return value indicates whether insertion succeeded or failed. - // Requirement: if `key` already exists in the btree, does not consume `args`. - // Requirement: `key` is never referenced after consuming `args`. - template - std::pair insert_unique(const key_type &key, Args &&... args); - - // Inserts with hint. Checks to see if the value should be placed immediately - // before `position` in the tree. If so, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_unique() were made. - // Requirement: if `key` already exists in the btree, does not consume `args`. - // Requirement: `key` is never referenced after consuming `args`. - template - std::pair insert_hint_unique(iterator position, - const key_type &key, - Args &&... args); - - // Insert a range of values into the btree. - template - void insert_iterator_unique(InputIterator b, InputIterator e); - - // Inserts a value into the btree. - template - iterator insert_multi(const key_type &key, ValueType &&v); - - // Inserts a value into the btree. - template - iterator insert_multi(ValueType &&v) { - return insert_multi(params_type::key(v), std::forward(v)); - } - // Insert with hint. Check to see if the value should be placed immediately - // before position in the tree. If it does, then the insertion will take - // amortized constant time. If not, the insertion will take amortized - // logarithmic time as if a call to insert_multi(v) were made. - template - iterator insert_hint_multi(iterator position, ValueType &&v); - - // Insert a range of values into the btree. - template - void insert_iterator_multi(InputIterator b, InputIterator e); - - // Erase the specified iterator from the btree. The iterator must be valid - // (i.e. not equal to end()). Return an iterator pointing to the node after - // the one that was erased (or end() if none exists). - // Requirement: does not read the value at `*iter`. - iterator erase(iterator iter); - - // Erases range. Returns the number of keys erased and an iterator pointing - // to the element after the last erased element. - std::pair erase(iterator begin, iterator end); - - // Erases the specified key from the btree. Returns 1 if an element was - // erased and 0 otherwise. - template - size_type erase_unique(const K &key); - - // Erases all of the entries matching the specified key from the - // btree. Returns the number of elements erased. - template - size_type erase_multi(const K &key); - - // Finds the iterator corresponding to a key or returns end() if the key is - // not present. - template - iterator find(const K &key) { - return internal_end(internal_find(key)); - } - template - const_iterator find(const K &key) const { - return internal_end(internal_find(key)); - } - - // Returns a count of the number of times the key appears in the btree. - template - size_type count_unique(const K &key) const { - const iterator beg = internal_find(key); - if (beg.node == nullptr) { - // The key doesn't exist in the tree. - return 0; - } - return 1; - } - // Returns a count of the number of times the key appears in the btree. - template - size_type count_multi(const K &key) const { - const auto range = equal_range(key); - return std::distance(range.first, range.second); - } - - // Clear the btree, deleting all of the values it contains. - void clear(); - - // Swap the contents of *this and x. - void swap(btree &x); - - const key_compare &key_comp() const noexcept { - return root_.template get<0>(); - } - template - bool compare_keys(const K &x, const LK &y) const { - return compare_internal::compare_result_as_less_than(key_comp()(x, y)); - } - - value_compare value_comp() const { return value_compare(key_comp()); } - - // Verifies the structure of the btree. - void verify() const; - - // Size routines. - size_type size() const { return size_; } - size_type max_size() const { return (std::numeric_limits::max)(); } - bool empty() const { return size_ == 0; } - - // The height of the btree. An empty tree will have height 0. - size_type height() const { - size_type h = 0; - if (!empty()) { - // Count the length of the chain from the leftmost node up to the - // root. We actually count from the root back around to the level below - // the root, but the calculation is the same because of the circularity - // of that traversal. - const node_type *n = root(); - do { - ++h; - n = n->parent(); - } while (n != root()); - } - return h; - } - - // The number of internal, leaf and total nodes used by the btree. - size_type leaf_nodes() const { - return internal_stats(root()).leaf_nodes; - } - size_type internal_nodes() const { - return internal_stats(root()).internal_nodes; - } - size_type nodes() const { - node_stats stats = internal_stats(root()); - return stats.leaf_nodes + stats.internal_nodes; - } - - // The total number of bytes used by the btree. - size_type bytes_used() const { - node_stats stats = internal_stats(root()); - if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { - return sizeof(*this) + - node_type::LeafSize(root()->max_count()); - } else { - return sizeof(*this) + - stats.leaf_nodes * node_type::LeafSize() + - stats.internal_nodes * node_type::InternalSize(); - } - } - - // The average number of bytes used per value stored in the btree. - static double average_bytes_per_value() { - // Returns the number of bytes per value on a leaf node that is 75% - // full. Experimentally, this matches up nicely with the computed number of - // bytes per value in trees that had their values inserted in random order. - return node_type::LeafSize() / (kNodeValues * 0.75); - } - - // The fullness of the btree. Computed as the number of elements in the btree - // divided by the maximum number of elements a tree with the current number - // of nodes could hold. A value of 1 indicates perfect space - // utilization. Smaller values indicate space wastage. - // Returns 0 for empty trees. - double fullness() const { - if (empty()) return 0.0; - return static_cast(size()) / (nodes() * kNodeValues); - } - // The overhead of the btree structure in bytes per node. Computed as the - // total number of bytes used by the btree minus the number of bytes used for - // storing elements divided by the number of elements. - // Returns 0 for empty trees. - double overhead() const { - if (empty()) return 0.0; - return (bytes_used() - size() * sizeof(value_type)) / - static_cast(size()); - } - - // The allocator used by the btree. - allocator_type get_allocator() const { - return allocator(); - } - - private: - // Internal accessor routines. - node_type *root() { return root_.template get<2>(); } - const node_type *root() const { return root_.template get<2>(); } - node_type *&mutable_root() noexcept { return root_.template get<2>(); } - key_compare *mutable_key_comp() noexcept { return &root_.template get<0>(); } - - // The leftmost node is stored as the parent of the root node. - node_type *leftmost() { return root()->parent(); } - const node_type *leftmost() const { return root()->parent(); } - - // Allocator routines. - allocator_type *mutable_allocator() noexcept { - return &root_.template get<1>(); - } - const allocator_type &allocator() const noexcept { - return root_.template get<1>(); - } - - // Allocates a correctly aligned node of at least size bytes using the - // allocator. - node_type *allocate(const size_type sz) { - return reinterpret_cast( - phmap::priv::Allocate( - mutable_allocator(), (size_t)sz)); - } - - // Node creation/deletion routines. - node_type* new_internal_node(node_type *parent) { - node_type *p = allocate(node_type::InternalSize()); - return node_type::init_internal(p, parent); - } - node_type* new_leaf_node(node_type *parent) { - node_type *p = allocate(node_type::LeafSize()); - return node_type::init_leaf(p, parent, kNodeValues); - } - node_type *new_leaf_root_node(const int max_count) { - node_type *p = allocate(node_type::LeafSize(max_count)); - return node_type::init_leaf(p, p, max_count); - } - - // Deletion helper routines. - void erase_same_node(iterator begin, iterator end); - iterator erase_from_leaf_node(iterator begin, size_type to_erase); - iterator rebalance_after_delete(iterator iter); - - // Deallocates a node of a certain size in bytes using the allocator. - void deallocate(const size_type sz, node_type *node) { - phmap::priv::Deallocate( - mutable_allocator(), node, (size_t)sz); - } - - void delete_internal_node(node_type *node) { - node->destroy(mutable_allocator()); - deallocate(node_type::InternalSize(), node); - } - void delete_leaf_node(node_type *node) { - node->destroy(mutable_allocator()); - deallocate(node_type::LeafSize(node->max_count()), node); - } - - // Rebalances or splits the node iter points to. - void rebalance_or_split(iterator *iter); - - // Merges the values of left, right and the delimiting key on their parent - // onto left, removing the delimiting key and deleting right. - void merge_nodes(node_type *left, node_type *right); - - // Tries to merge node with its left or right sibling, and failing that, - // rebalance with its left or right sibling. Returns true if a merge - // occurred, at which point it is no longer valid to access node. Returns - // false if no merging took place. - bool try_merge_or_rebalance(iterator *iter); - - // Tries to shrink the height of the tree by 1. - void try_shrink(); - - iterator internal_end(iterator iter) { - return iter.node != nullptr ? iter : end(); - } - const_iterator internal_end(const_iterator iter) const { - return iter.node != nullptr ? iter : end(); - } - - // Emplaces a value into the btree immediately before iter. Requires that - // key(v) <= iter.key() and (--iter).key() <= key(v). - template - iterator internal_emplace(iterator iter, Args &&... args); - - // Returns an iterator pointing to the first value >= the value "iter" is - // pointing at. Note that "iter" might be pointing to an invalid location as - // iter.position == iter.node->count(). This routine simply moves iter up in - // the tree to a valid location. - // Requires: iter.node is non-null. - template - static IterType internal_last(IterType iter); - - // Returns an iterator pointing to the leaf position at which key would - // reside in the tree. We provide 2 versions of internal_locate. The first - // version uses a less-than comparator and is incapable of distinguishing when - // there is an exact match. The second version is for the key-compare-to - // specialization and distinguishes exact matches. The key-compare-to - // specialization allows the caller to avoid a subsequent comparison to - // determine if an exact match was made, which is important for keys with - // expensive comparison, such as strings. - template - SearchResult internal_locate( - const K &key) const; - - template - SearchResult internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const; - - template - SearchResult internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const; - - // Internal routine which implements lower_bound(). - template - iterator internal_lower_bound(const K &key) const; - - // Internal routine which implements upper_bound(). - template - iterator internal_upper_bound(const K &key) const; - - // Internal routine which implements find(). - template - iterator internal_find(const K &key) const; - - // Deletes a node and all of its children. - void internal_clear(node_type *node); - - // Verifies the tree structure of node. - int internal_verify(const node_type *node, - const key_type *lo, const key_type *hi) const; - - node_stats internal_stats(const node_type *node) const { - // The root can be a static empty node. - if (node == nullptr || (node == root() && empty())) { - return node_stats(0, 0); - } - if (node->leaf()) { - return node_stats(1, 0); - } - node_stats res(0, 1); - for (int i = 0; i <= node->count(); ++i) { - res += internal_stats(node->child(i)); - } - return res; - } - - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return node_type::testonly_uses_linear_node_search(); - } - - private: - // We use compressed tuple in order to save space because key_compare and - // allocator_type are usually empty. - phmap::priv::CompressedTuple - root_; - - // A pointer to the rightmost node. Note that the leftmost node is stored as - // the root's parent. - node_type *rightmost_; - - // Number of values. - size_type size_; - }; - - //// - // btree_node methods - template - template - inline void btree_node

::emplace_value(const size_type i, - allocator_type *alloc, - Args &&... args) { - assert(i <= count()); - // Shift old values to create space for new value and then construct it in - // place. - if (i < count()) { - value_init(count(), alloc, slot(count() - 1)); - for (size_type j = count() - 1; j > i; --j) - params_type::move(alloc, slot(j - 1), slot(j)); - value_destroy(i, alloc); - } - value_init(i, alloc, std::forward(args)...); - set_count((field_type)(count() + 1)); - - if (!leaf() && count() > i + 1) { - for (int j = count(); j > (int)(i + 1); --j) { - set_child(j, child(j - 1)); - } - clear_child(i + 1); - } +template +struct common_params { + // If Compare is a common comparator for a std::string-like type, then we adapt it + // to use heterogeneous lookup and to be a key-compare-to comparator. + using key_compare = typename key_compare_to_adapter::type; + // A type which indicates if we have a key-compare-to functor or a plain old + // key-compare functor. + using is_key_compare_to = btree_is_key_compare_to; + + using allocator_type = Alloc; + using key_type = Key; + using size_type = std::size_t; + using difference_type = ptrdiff_t; + + // True if this is a multiset or multimap. + using is_multi_container = std::integral_constant; + + using slot_policy = SlotPolicy; + using slot_type = typename slot_policy::slot_type; + using value_type = typename slot_policy::value_type; + using init_type = typename slot_policy::mutable_value_type; + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + + enum { + kTargetNodeSize = TargetNodeSize, + + // Upper bound for the available space for values. This is largest for leaf + // nodes, which have overhead of at least a pointer + 4 bytes (for storing + // 3 field_types and an enum). + kNodeValueSpace = TargetNodeSize - /*minimum overhead=*/(sizeof(void*) + 4), + }; + + // This is an integral type large enough to hold as many + // ValueSize-values as will fit a node of TargetNodeSize bytes. + using node_count_type = phmap::conditional_t<(kNodeValueSpace / sizeof(value_type) > + (std::numeric_limits::max)()), + uint16_t, uint8_t>; // NOLINT + + // The following methods are necessary for passing this struct as PolicyTraits + // for node_handle and/or are used within btree. + static value_type& element(slot_type* slot) { return slot_policy::element(slot); } + static const value_type& element(const slot_type* slot) { + return slot_policy::element(slot); + } + template + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + slot_policy::construct(alloc, slot, std::forward(args)...); + } + static void construct(Alloc* alloc, slot_type* slot, slot_type* other) { + slot_policy::construct(alloc, slot, other); + } + static void destroy(Alloc* alloc, slot_type* slot) { + slot_policy::destroy(alloc, slot); + } + static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { + construct(alloc, new_slot, old_slot); + destroy(alloc, old_slot); + } + static void swap(Alloc* alloc, slot_type* a, slot_type* b) { + slot_policy::swap(alloc, a, b); + } + static void move(Alloc* alloc, slot_type* src, slot_type* dest) { + slot_policy::move(alloc, src, dest); + } + static void move(Alloc* alloc, slot_type* first, slot_type* last, slot_type* result) { + slot_policy::move(alloc, first, last, result); + } +}; + +// A parameters structure for holding the type parameters for a btree_map. +// Compare and Alloc should be nothrow copy-constructible. +template +struct map_params : common_params> { + using super_type = typename map_params::common_params; + using mapped_type = Data; + // This type allows us to move keys when it is safe to do so. It is safe + // for maps in which value_type and mutable_value_type are layout compatible. + using slot_policy = typename super_type::slot_policy; + using slot_type = typename super_type::slot_type; + using value_type = typename super_type::value_type; + using init_type = typename super_type::init_type; + + using key_compare = typename super_type::key_compare; + // Inherit from key_compare for empty base class optimization. + struct value_compare : private key_compare { + value_compare() = default; + explicit value_compare(const key_compare& cmp) : key_compare(cmp) {} + + template + auto operator()(const T& left, const U& right) const + -> decltype(std::declval()(left.first, right.first)) { + return key_compare::operator()(left.first, right.first); } - - template - inline void btree_node

::remove_value(const int i, allocator_type *alloc) { - if (!leaf() && count() > i + 1) { - assert(child(i + 1)->count() == 0); - for (size_type j = i + 1; j < count(); ++j) { - set_child(j, child(j + 1)); - } - clear_child(count()); - } - - remove_values_ignore_children(i, /*to_erase=*/1, alloc); - } - - template - inline void btree_node

::remove_values_ignore_children( - int i, size_type to_erase, allocator_type *alloc) { - params_type::move(alloc, slot(i + to_erase), slot(count()), slot(i)); - value_destroy_n(count() - to_erase, to_erase, alloc); - set_count((field_type)(count() - to_erase)); - } - - template - void btree_node

::rebalance_right_to_left(const int to_move, - btree_node *right, - allocator_type *alloc) { - assert(parent() == right->parent()); - assert(position() + 1 == right->position()); - assert(right->count() >= count()); - assert(to_move >= 1); - assert(to_move <= right->count()); - - // 1) Move the delimiting value in the parent to the left node. - value_init(count(), alloc, parent()->slot(position())); - - // 2) Move the (to_move - 1) values from the right node to the left node. - right->uninitialized_move_n(to_move - 1, 0, count() + 1, this, alloc); - - // 3) Move the new delimiting value to the parent from the right node. - params_type::move(alloc, right->slot(to_move - 1), - parent()->slot(position())); - - // 4) Shift the values in the right node to their correct position. - params_type::move(alloc, right->slot(to_move), right->slot(right->count()), - right->slot(0)); - - // 5) Destroy the now-empty to_move entries in the right node. - right->value_destroy_n(right->count() - to_move, to_move, alloc); - - if (!leaf()) { - // Move the child pointers from the right to the left node. - for (int i = 0; i < to_move; ++i) { - init_child(count() + i + 1, right->child(i)); - } - for (int i = 0; i <= right->count() - to_move; ++i) { - assert(i + to_move <= right->max_count()); - right->init_child(i, right->child(i + to_move)); - right->clear_child(i + to_move); - } - } - - // Fixup the counts on the left and right nodes. - set_count((field_type)(count() + to_move)); - right->set_count((field_type)(right->count() - to_move)); - } - - template - void btree_node

::rebalance_left_to_right(const int to_move, - btree_node *right, - allocator_type *alloc) { - assert(parent() == right->parent()); - assert(position() + 1 == right->position()); - assert(count() >= right->count()); - assert(to_move >= 1); - assert(to_move <= count()); - - // Values in the right node are shifted to the right to make room for the - // new to_move values. Then, the delimiting value in the parent and the - // other (to_move - 1) values in the left node are moved into the right node. - // Lastly, a new delimiting value is moved from the left node into the - // parent, and the remaining empty left node entries are destroyed. - - if (right->count() >= to_move) { - // The original location of the right->count() values are sufficient to hold - // the new to_move entries from the parent and left node. - - // 1) Shift existing values in the right node to their correct positions. - right->uninitialized_move_n(to_move, right->count() - to_move, - right->count(), right, alloc); - for (slot_type *src = right->slot(right->count() - to_move - 1), - *dest = right->slot(right->count() - 1), - *end = right->slot(0); - src >= end; --src, --dest) { - params_type::move(alloc, src, dest); - } - - // 2) Move the delimiting value in the parent to the right node. - params_type::move(alloc, parent()->slot(position()), - right->slot(to_move - 1)); - - // 3) Move the (to_move - 1) values from the left node to the right node. - params_type::move(alloc, slot(count() - (to_move - 1)), slot(count()), - right->slot(0)); + }; + using is_map_container = std::true_type; + + static const Key& key(const value_type& x) { return x.first; } + static const Key& key(const init_type& x) { return x.first; } + static const Key& key(const slot_type* x) { return slot_policy::key(x); } + static mapped_type& value(value_type* value) { return value->second; } +}; + +// This type implements the necessary functions from the +// btree::priv::slot_type interface. +template +struct set_slot_policy { + using slot_type = Key; + using value_type = Key; + using mutable_value_type = Key; + + static value_type& element(slot_type* slot) { return *slot; } + static const value_type& element(const slot_type* slot) { return *slot; } + + template + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + phmap::allocator_traits::construct(*alloc, slot, std::forward(args)...); + } + + template + static void construct(Alloc* alloc, slot_type* slot, slot_type* other) { + phmap::allocator_traits::construct(*alloc, slot, std::move(*other)); + } + + template + static void destroy(Alloc* alloc, slot_type* slot) { + phmap::allocator_traits::destroy(*alloc, slot); + } + + template + static void swap(Alloc* /*alloc*/, slot_type* a, slot_type* b) { + using std::swap; + swap(*a, *b); + } + + template + static void move(Alloc* /*alloc*/, slot_type* src, slot_type* dest) { + *dest = std::move(*src); + } + + template + static void move(Alloc* alloc, slot_type* first, slot_type* last, slot_type* result) { + for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) + move(alloc, src, dest); + } +}; + +// A parameters structure for holding the type parameters for a btree_set. +// Compare and Alloc should be nothrow copy-constructible. +template +struct set_params + : common_params> { + using value_type = Key; + using slot_type = typename set_params::common_params::slot_type; + using value_compare = typename set_params::common_params::key_compare; + using is_map_container = std::false_type; + + static const Key& key(const value_type& x) { return x; } + static const Key& key(const slot_type* x) { return *x; } +}; + +// An adapter class that converts a lower-bound compare into an upper-bound +// compare. Note: there is no need to make a version of this adapter specialized +// for key-compare-to functors because the upper-bound (the first value greater +// than the input) is never an exact match. +template +struct upper_bound_adapter { + explicit upper_bound_adapter(const Compare& c) : comp(c) {} + template + bool operator()(const K& a, const LK& b) const { + // Returns true when a is not greater than b. + return !phmap::compare_internal::compare_result_as_less_than(comp(b, a)); + } + + private: + Compare comp; +}; + +enum class MatchKind : uint8_t { kEq, kNe }; + +template +struct SearchResult { + V value; + MatchKind match; + + static constexpr bool HasMatch() { return true; } + bool IsEq() const { return match == MatchKind::kEq; } +}; + +// When we don't use CompareTo, `match` is not present. +// This ensures that callers can't use it accidentally when it provides no +// useful information. +template +struct SearchResult { + V value; + + static constexpr bool HasMatch() { return false; } + static constexpr bool IsEq() { return false; } +}; + +// A node in the btree holding. The same node type is used for both internal +// and leaf nodes in the btree, though the nodes are allocated in such a way +// that the children array is only valid in internal nodes. +template +class btree_node { + using is_key_compare_to = typename Params::is_key_compare_to; + using is_multi_container = typename Params::is_multi_container; + using field_type = typename Params::node_count_type; + using allocator_type = typename Params::allocator_type; + using slot_type = typename Params::slot_type; + + public: + using params_type = Params; + using key_type = typename Params::key_type; + using value_type = typename Params::value_type; + using pointer = typename Params::pointer; + using const_pointer = typename Params::const_pointer; + using reference = typename Params::reference; + using const_reference = typename Params::const_reference; + using key_compare = typename Params::key_compare; + using size_type = typename Params::size_type; + using difference_type = typename Params::difference_type; + + // Btree decides whether to use linear node search as follows: + // - If the key is arithmetic and the comparator is std::less or + // std::greater, choose linear. + // - Otherwise, choose binary. + // TODO(ezb): Might make sense to add condition(s) based on node-size. + using use_linear_search = std::integral_constant< + bool, std::is_arithmetic::value && + (std::is_same, key_compare>::value || + std::is_same, key_compare>::value || + std::is_same, key_compare>::value)>; + + ~btree_node() = default; + btree_node(btree_node const&) = delete; + btree_node& operator=(btree_node const&) = delete; + + // Public for EmptyNodeType. + constexpr static size_type Alignment() { + static_assert(LeafLayout(1).Alignment() == InternalLayout().Alignment(), + "Alignment of all nodes must be equal."); + return (size_type)InternalLayout().Alignment(); + } + + protected: + btree_node() = default; + + private: + using layout_type = + phmap::priv::Layout; + constexpr static size_type SizeWithNValues(size_type n) { + return (size_type)layout_type(/*parent*/ 1, + /*position, start, count, max_count*/ 4, + /*values*/ (size_t)n, + /*children*/ 0) + .AllocSize(); + } + // A lower bound for the overhead of fields other than values in a leaf node. + constexpr static size_type MinimumOverhead() { + return (size_type)(SizeWithNValues(1) - sizeof(value_type)); + } + + // Compute how many values we can fit onto a leaf node taking into account + // padding. + constexpr static size_type NodeTargetValues(const int begin, const int end) { + return begin == end + ? begin + : SizeWithNValues((begin + end) / 2 + 1) > params_type::kTargetNodeSize + ? NodeTargetValues(begin, (begin + end) / 2) + : NodeTargetValues((begin + end) / 2 + 1, end); + } + + enum { + kTargetNodeSize = params_type::kTargetNodeSize, + kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize), + + // We need a minimum of 3 values per internal node in order to perform + // splitting (1 value for the two nodes involved in the split and 1 value + // propagated to the parent as the delimiter for the split). + kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, + + // The node is internal (i.e. is not a leaf node) if and only if `max_count` + // has this value. + kInternalNodeMaxCount = 0, + }; + + // Leaves can have less than kNodeValues values. + constexpr static layout_type LeafLayout(const int max_values = kNodeValues) { + return layout_type(/*parent*/ 1, + /*position, start, count, max_count*/ 4, + /*values*/ (size_t)max_values, + /*children*/ 0); + } + constexpr static layout_type InternalLayout() { + return layout_type(/*parent*/ 1, + /*position, start, count, max_count*/ 4, + /*values*/ kNodeValues, + /*children*/ kNodeValues + 1); + } + constexpr static size_type LeafSize(const int max_values = kNodeValues) { + return (size_type)LeafLayout(max_values).AllocSize(); + } + constexpr static size_type InternalSize() { + return (size_type)InternalLayout().AllocSize(); + } + + // N is the index of the type in the Layout definition. + // ElementType is the Nth type in the Layout definition. + template + inline typename layout_type::template ElementType* GetField() { + // We assert that we don't read from values that aren't there. + assert(N < 3 || !leaf()); + return InternalLayout().template Pointer(reinterpret_cast(this)); + } + + template + inline const typename layout_type::template ElementType* GetField() const { + assert(N < 3 || !leaf()); + return InternalLayout().template Pointer(reinterpret_cast(this)); + } + + void set_parent(btree_node* p) { *GetField<0>() = p; } + field_type& mutable_count() { return GetField<1>()[2]; } + slot_type* slot(size_type i) { return &GetField<2>()[i]; } + const slot_type* slot(size_type i) const { return &GetField<2>()[i]; } + void set_position(field_type v) { GetField<1>()[0] = v; } + void set_start(field_type v) { GetField<1>()[1] = v; } + void set_count(field_type v) { GetField<1>()[2] = v; } + void set_max_count(field_type v) { GetField<1>()[3] = v; } + + public: + // Whether this is a leaf node or not. This value doesn't change after the + // node is created. + bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; } + + // Getter for the position of this node in its parent. + field_type position() const { return GetField<1>()[0]; } + + // Getter for the offset of the first value in the `values` array. + field_type start() const { return GetField<1>()[1]; } + + // Getters for the number of values stored in this node. + field_type count() const { return GetField<1>()[2]; } + field_type max_count() const { + // Internal nodes have max_count==kInternalNodeMaxCount. + // Leaf nodes have max_count in [1, kNodeValues]. + const field_type max_cnt = GetField<1>()[3]; + return max_cnt == field_type{kInternalNodeMaxCount} ? field_type{kNodeValues} + : max_cnt; + } + + // Getter for the parent of this node. + btree_node* parent() const { return *GetField<0>(); } + // Getter for whether the node is the root of the tree. The parent of the + // root of the tree is the leftmost node in the tree which is guaranteed to + // be a leaf. + bool is_root() const { return parent()->leaf(); } + void make_root() { + assert(parent()->is_root()); + set_parent(parent()->parent()); + } + + // Getters for the key/value at position i in the node. + const key_type& key(size_type i) const { return params_type::key(slot(i)); } + reference value(size_type i) { return params_type::element(slot(i)); } + const_reference value(size_type i) const { return params_type::element(slot(i)); } + + // Getters/setter for the child at position i in the node. + btree_node* child(size_type i) const { return GetField<3>()[i]; } + btree_node*& mutable_child(size_type i) { return GetField<3>()[i]; } + void clear_child(size_type i) { phmap::priv::SanitizerPoisonObject(&mutable_child(i)); } + void set_child(size_type i, btree_node* c) { + phmap::priv::SanitizerUnpoisonObject(&mutable_child(i)); + mutable_child(i) = c; + c->set_position((field_type)i); + } + void init_child(int i, btree_node* c) { + set_child(i, c); + c->set_parent(this); + } + + // Returns the position of the first value whose key is not less than k. + template + SearchResult lower_bound(const K& k, + const key_compare& comp) const { + return use_linear_search::value ? linear_search(k, comp) : binary_search(k, comp); + } + // Returns the position of the first value whose key is greater than k. + template + int upper_bound(const K& k, const key_compare& comp) const { + auto upper_compare = upper_bound_adapter(comp); + return use_linear_search::value ? linear_search(k, upper_compare).value + : binary_search(k, upper_compare).value; + } + + template + SearchResult::value> linear_search( + const K& k, const Compare& comp) const { + return linear_search_impl(k, 0, count(), comp, + btree_is_key_compare_to()); + } + + template + SearchResult::value> binary_search( + const K& k, const Compare& comp) const { + return binary_search_impl(k, 0, count(), comp, + btree_is_key_compare_to()); + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using plain compare. + template + SearchResult linear_search_impl(const K& k, int s, const int e, + const Compare& comp, + std::false_type /* IsCompareTo */) const { + while (s < e) { + if (!comp(key(s), k)) { + break; + } + ++s; + } + return {s}; + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using compare-to. + template + SearchResult linear_search_impl(const K& k, int s, const int e, + const Compare& comp, + std::true_type /* IsCompareTo */) const { + while (s < e) { + const phmap::weak_ordering c = comp(key(s), k); + if (c == 0) { + return {s, MatchKind::kEq}; + } else if (c > 0) { + break; + } + ++s; + } + return {s, MatchKind::kNe}; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using plain compare. + template + SearchResult binary_search_impl(const K& k, int s, int e, + const Compare& comp, + std::false_type /* IsCompareTo */) const { + while (s != e) { + const int mid = (s + e) >> 1; + if (comp(key(mid), k)) { + s = mid + 1; + } else { + e = mid; + } + } + return {s}; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using compare-to. + template + SearchResult binary_search_impl(const K& k, int s, int e, + const CompareTo& comp, + std::true_type /* IsCompareTo */) const { + if (is_multi_container::value) { + MatchKind exact_match = MatchKind::kNe; + while (s != e) { + const int mid = (s + e) >> 1; + const phmap::weak_ordering c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; } else { - // The right node does not have enough initialized space to hold the new - // to_move entries, so part of them will move to uninitialized space. - - // 1) Shift existing values in the right node to their correct positions. - right->uninitialized_move_n(right->count(), 0, to_move, right, alloc); - - // 2) Move the delimiting value in the parent to the right node. - right->value_init(to_move - 1, alloc, parent()->slot(position())); - - // 3) Move the (to_move - 1) values from the left node to the right node. - const size_type uninitialized_remaining = to_move - right->count() - 1; - uninitialized_move_n(uninitialized_remaining, - count() - uninitialized_remaining, right->count(), - right, alloc); - params_type::move(alloc, slot(count() - (to_move - 1)), - slot(count() - uninitialized_remaining), right->slot(0)); - } - - // 4) Move the new delimiting value to the parent from the left node. - params_type::move(alloc, slot(count() - to_move), parent()->slot(position())); - - // 5) Destroy the now-empty to_move entries in the left node. - value_destroy_n(count() - to_move, to_move, alloc); - - if (!leaf()) { - // Move the child pointers from the left to the right node. - for (int i = right->count(); i >= 0; --i) { - right->init_child(i + to_move, right->child(i)); - right->clear_child(i); - } - for (int i = 1; i <= to_move; ++i) { - right->init_child(i - 1, child(count() - to_move + i)); - clear_child(count() - to_move + i); - } + e = mid; + if (c == 0) { + // Need to return the first value whose key is not less than k, + // which requires continuing the binary search if this is a + // multi-container. + exact_match = MatchKind::kEq; + } + } + } + return {s, exact_match}; + } else { // Not a multi-container. + while (s != e) { + const int mid = (s + e) >> 1; + const phmap::weak_ordering c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; + } else if (c > 0) { + e = mid; + } else { + return {mid, MatchKind::kEq}; } + } + return {s, MatchKind::kNe}; + } + } + + // Emplaces a value at position i, shifting all existing values and + // children at positions >= i to the right by 1. + template + void emplace_value(size_type i, allocator_type* alloc, Args&&... args); + + // Removes the value at position i, shifting all existing values and children + // at positions > i to the left by 1. + void remove_value(int i, allocator_type* alloc); + + // Removes the values at positions [i, i + to_erase), shifting all values + // after that range to the left by to_erase. Does not change children at all. + void remove_values_ignore_children(int i, size_type to_erase, allocator_type* alloc); + + // Rebalances a node with its right sibling. + void rebalance_right_to_left(int to_move, btree_node* right, allocator_type* alloc); + void rebalance_left_to_right(int to_move, btree_node* right, allocator_type* alloc); + + // Splits a node, moving a portion of the node's values to its right sibling. + void split(int insert_position, btree_node* dest, allocator_type* alloc); + + // Merges a node with its right sibling, moving all of the values and the + // delimiting key in the parent node onto itself. + void merge(btree_node* sibling, allocator_type* alloc); + + // Swap the contents of "this" and "src". + void swap(btree_node* src, allocator_type* alloc); + + // Node allocation/deletion routines. + static btree_node* init_leaf(btree_node* n, btree_node* parent, int max_cnt) { + n->set_parent(parent); + n->set_position(0); + n->set_start(0); + n->set_count(0); + n->set_max_count((field_type)max_cnt); + phmap::priv::SanitizerPoisonMemoryRegion(n->slot(0), max_cnt * sizeof(slot_type)); + return n; + } + static btree_node* init_internal(btree_node* n, btree_node* parent) { + init_leaf(n, parent, kNodeValues); + // Set `max_count` to a sentinel value to indicate that this node is + // internal. + n->set_max_count(kInternalNodeMaxCount); + phmap::priv::SanitizerPoisonMemoryRegion(&n->mutable_child(0), + (kNodeValues + 1) * sizeof(btree_node*)); + return n; + } + void destroy(allocator_type* alloc) { + for (int i = 0; i < count(); ++i) { + value_destroy(i, alloc); + } + } + + public: + // Exposed only for tests. + static bool testonly_uses_linear_node_search() { return use_linear_search::value; } + + private: + template + void value_init(const size_type i, allocator_type* alloc, Args&&... args) { + phmap::priv::SanitizerUnpoisonObject(slot(i)); + params_type::construct(alloc, slot(i), std::forward(args)...); + } + void value_destroy(const size_type i, allocator_type* alloc) { + params_type::destroy(alloc, slot(i)); + phmap::priv::SanitizerPoisonObject(slot(i)); + } + + // Move n values starting at value i in this node into the values starting at + // value j in node x. + void uninitialized_move_n(const size_type n, const size_type i, const size_type j, + btree_node* x, allocator_type* alloc) { + phmap::priv::SanitizerUnpoisonMemoryRegion(x->slot(j), n * sizeof(slot_type)); + for (slot_type *src = slot(i), *end = src + n, *dest = x->slot(j); src != end; + ++src, ++dest) { + params_type::construct(alloc, dest, src); + } + } - // Fixup the counts on the left and right nodes. - set_count((field_type)(count() - to_move)); - right->set_count((field_type)(right->count() + to_move)); - } - - template - void btree_node

::split(const int insert_position, btree_node *dest, - allocator_type *alloc) { - assert(dest->count() == 0); - assert(max_count() == kNodeValues); - - // We bias the split based on the position being inserted. If we're - // inserting at the beginning of the left node then bias the split to put - // more values on the right node. If we're inserting at the end of the - // right node then bias the split to put more values on the left node. - if (insert_position == 0) { - dest->set_count((field_type)(count() - 1)); - } else if (insert_position == kNodeValues) { - dest->set_count(0); - } else { - dest->set_count((field_type)(count() / 2)); - } - set_count((field_type)(count() - dest->count())); - assert(count() >= 1); - - // Move values from the left sibling to the right sibling. - uninitialized_move_n(dest->count(), count(), 0, dest, alloc); - - // Destroy the now-empty entries in the left node. - value_destroy_n(count(), dest->count(), alloc); - - // The split key is the largest value in the left sibling. - set_count((field_type)(count() - 1)); - parent()->emplace_value(position(), alloc, slot(count())); - value_destroy(count(), alloc); - parent()->init_child(position() + 1, dest); - - if (!leaf()) { - for (int i = 0; i <= dest->count(); ++i) { - assert(child(count() + i + 1) != nullptr); - dest->init_child(i, child(count() + i + 1)); - clear_child(count() + i + 1); - } - } + // Destroys a range of n values, starting at index i. + void value_destroy_n(const size_type i, const size_type n, allocator_type* alloc) { + for (size_type j = 0; j < n; ++j) { + value_destroy(i + j, alloc); + } + } + + template + friend class btree; + template + friend struct btree_iterator; + friend class BtreeNodePeer; +}; + +template +struct btree_iterator { + private: + using key_type = typename Node::key_type; + using size_type = typename Node::size_type; + using params_type = typename Node::params_type; + + using node_type = Node; + using normal_node = typename std::remove_const::type; + using const_node = const Node; + using normal_pointer = typename params_type::pointer; + using normal_reference = typename params_type::reference; + using const_pointer = typename params_type::const_pointer; + using const_reference = typename params_type::const_reference; + using slot_type = typename params_type::slot_type; + + using iterator = btree_iterator; + using const_iterator = btree_iterator; + + public: + // These aliases are public for std::iterator_traits. + using difference_type = typename Node::difference_type; + using value_type = typename params_type::value_type; + using pointer = Pointer; + using reference = Reference; + using iterator_category = std::bidirectional_iterator_tag; + + btree_iterator() : node(nullptr), position(-1) {} + btree_iterator(Node* n, int p) : node(n), position(p) {} + + // NOTE: this SFINAE allows for implicit conversions from iterator to + // const_iterator, but it specifically avoids defining copy constructors so + // that btree_iterator can be trivially copyable. This is for performance and + // binary size reasons. + template , iterator>::value && + std::is_same::value, + int> = 0> + btree_iterator(const btree_iterator& x) // NOLINT + : node(x.node), position(x.position) {} + + private: + // This SFINAE allows explicit conversions from const_iterator to + // iterator, but also avoids defining a copy constructor. + // NOTE: the const_cast is safe because this constructor is only called by + // non-const methods and the container owns the nodes. + template < + typename N, typename R, typename P, + phmap::enable_if_t, const_iterator>::value && + std::is_same::value, + int> = 0> + explicit btree_iterator(const btree_iterator& x) + : node(const_cast(x.node)), position(x.position) {} + + // Increment/decrement the iterator. + void increment() { + if (node->leaf() && ++position < node->count()) { + return; } + increment_slow(); + } + void increment_slow(); - template - void btree_node

::merge(btree_node *src, allocator_type *alloc) { - assert(parent() == src->parent()); - assert(position() + 1 == src->position()); + void decrement() { + if (node->leaf() && --position >= 0) { + return; + } + decrement_slow(); + } + void decrement_slow(); + + public: + bool operator==(const const_iterator& x) const { + return node == x.node && position == x.position; + } + bool operator!=(const const_iterator& x) const { + return node != x.node || position != x.position; + } + + // Accessors for the key/value the iterator is pointing at. + reference operator*() const { return node->value(position); } + pointer operator->() const { return &node->value(position); } + + btree_iterator& operator++() { + increment(); + return *this; + } + btree_iterator& operator--() { + decrement(); + return *this; + } + btree_iterator operator++(int) { + btree_iterator tmp = *this; + ++*this; + return tmp; + } + btree_iterator operator--(int) { + btree_iterator tmp = *this; + --*this; + return tmp; + } + + private: + template + friend class btree; + template + friend class btree_container; + template + friend class btree_set_container; + template + friend class btree_map_container; + template + friend class btree_multiset_container; + template + friend struct btree_iterator; + template + friend class base_checker; + + const key_type& key() const { return node->key(position); } + slot_type* slot() { return node->slot(position); } + + // The node in the tree the iterator is pointing at. + Node* node; + // The position within the node of the tree the iterator is pointing at. + // TODO(ezb): make this a field_type + int position; +}; + +template +class btree { + using node_type = btree_node; + using is_key_compare_to = typename Params::is_key_compare_to; + + // We use a static empty node for the root/leftmost/rightmost of empty btrees + // in order to avoid branching in begin()/end(). + struct alignas(node_type::Alignment()) EmptyNodeType : node_type { + using field_type = typename node_type::field_type; + node_type* parent; + field_type position = 0; + field_type start = 0; + field_type count = 0; + // max_count must be != kInternalNodeMaxCount (so that this node is regarded + // as a leaf node). max_count() is never called when the tree is empty. + field_type max_count = node_type::kInternalNodeMaxCount + 1; - // Move the delimiting value to the left node. - value_init(count(), alloc, parent()->slot(position())); +#ifdef _MSC_VER + // MSVC has constexpr code generations bugs here. + EmptyNodeType() : parent(this) {} +#else + constexpr EmptyNodeType(node_type* p) : parent(p) {} +#endif + }; - // Move the values from the right to the left node. - src->uninitialized_move_n(src->count(), 0, count() + 1, this, alloc); + static node_type* EmptyNode() { +#ifdef _MSC_VER + static EmptyNodeType empty_node; + // This assert fails on some other construction methods. + assert(empty_node.parent == &empty_node); + return &empty_node; +#else + static constexpr EmptyNodeType empty_node(const_cast(&empty_node)); + return const_cast(&empty_node); +#endif + } - // Destroy the now-empty entries in the right node. - src->value_destroy_n(0, src->count(), alloc); + enum { + kNodeValues = node_type::kNodeValues, + kMinNodeValues = kNodeValues / 2, + }; - if (!leaf()) { - // Move the child pointers from the right to the left node. - for (int i = 0; i <= src->count(); ++i) { - init_child(count() + i + 1, src->child(i)); - src->clear_child(i); - } - } + struct node_stats { + using size_type = typename Params::size_type; - // Fixup the counts on the src and dest nodes. - set_count((field_type)(1 + count() + src->count())); - src->set_count(0); + node_stats(size_type l, size_type i) : leaf_nodes(l), internal_nodes(i) {} - // Remove the value on the parent node. - parent()->remove_value(position(), alloc); + node_stats& operator+=(const node_stats& x) { + leaf_nodes += x.leaf_nodes; + internal_nodes += x.internal_nodes; + return *this; } - template - void btree_node

::swap(btree_node *x, allocator_type *alloc) { - using std::swap; - assert(leaf() == x->leaf()); - - // Determine which is the smaller/larger node. - btree_node *smaller = this, *larger = x; - if (smaller->count() > larger->count()) { - swap(smaller, larger); - } - - // Swap the values. - for (slot_type *a = smaller->slot(0), *b = larger->slot(0), - *end = a + smaller->count(); - a != end; ++a, ++b) { - params_type::swap(alloc, a, b); - } - - // Move values that can't be swapped. - const size_type to_move = larger->count() - smaller->count(); - larger->uninitialized_move_n(to_move, smaller->count(), smaller->count(), - smaller, alloc); - larger->value_destroy_n(smaller->count(), to_move, alloc); - - if (!leaf()) { - // Swap the child pointers. - std::swap_ranges(&smaller->mutable_child(0), - &smaller->mutable_child(smaller->count() + 1), - &larger->mutable_child(0)); - // Update swapped children's parent pointers. - int i = 0; - for (; i <= smaller->count(); ++i) { - smaller->child(i)->set_parent(smaller); - larger->child(i)->set_parent(larger); - } - // Move the child pointers that couldn't be swapped. - for (; i <= larger->count(); ++i) { - smaller->init_child(i, larger->child(i)); - larger->clear_child(i); - } - } - - // Swap the counts. - swap(mutable_count(), x->mutable_count()); - } - - //// - // btree_iterator methods - template - void btree_iterator::increment_slow() { - if (node->leaf()) { - assert(position >= node->count()); - btree_iterator save(*this); - while (position == node->count() && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position(); - node = node->parent(); - } - if (position == node->count()) { - *this = save; - } - } else { - assert(position < node->count()); - node = node->child(position + 1); - while (!node->leaf()) { - node = node->child(0); - } - position = 0; - } + size_type leaf_nodes; + size_type internal_nodes; + }; + + public: + using key_type = typename Params::key_type; + using value_type = typename Params::value_type; + using size_type = typename Params::size_type; + using difference_type = typename Params::difference_type; + using key_compare = typename Params::key_compare; + using value_compare = typename Params::value_compare; + using allocator_type = typename Params::allocator_type; + using reference = typename Params::reference; + using const_reference = typename Params::const_reference; + using pointer = typename Params::pointer; + using const_pointer = typename Params::const_pointer; + using iterator = btree_iterator; + using const_iterator = typename iterator::const_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using node_handle_type = node_handle; + + // Internal types made public for use by btree_container types. + using params_type = Params; + using slot_type = typename Params::slot_type; + + private: + // For use in copy_or_move_values_in_order. + const value_type& maybe_move_from_iterator(const_iterator x) { return *x; } + value_type&& maybe_move_from_iterator(iterator x) { return std::move(*x); } + + // Copies or moves (depending on the template parameter) the values in + // x into this btree in their order in x. This btree must be empty before this + // method is called. This method is used in copy construction, copy + // assignment, and move assignment. + template + void copy_or_move_values_in_order(Btree* x); + + // Validates that various assumptions/requirements are true at compile time. + constexpr static bool static_assert_validation(); + + public: + btree(const key_compare& comp, const allocator_type& alloc); + + btree(const btree& x); + btree(btree&& x) noexcept + : root_(std::move(x.root_)), + rightmost_(phmap::exchange(x.rightmost_, EmptyNode())), + size_(phmap::exchange(x.size_, 0)) { + x.mutable_root() = EmptyNode(); + } + + ~btree() { + // Put static_asserts in destructor to avoid triggering them before the type + // is complete. + static_assert(static_assert_validation(), "This call must be elided."); + clear(); + } + + // Assign the contents of x to *this. + btree& operator=(const btree& x); + btree& operator=(btree&& x) noexcept; + + iterator begin() { return iterator(leftmost(), 0); } + const_iterator begin() const { return const_iterator(leftmost(), 0); } + iterator end() { return iterator(rightmost_, rightmost_->count()); } + const_iterator end() const { return const_iterator(rightmost_, rightmost_->count()); } + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } + + // Finds the first element whose key is not less than key. + template + iterator lower_bound(const K& key) { + return internal_end(internal_lower_bound(key)); + } + template + const_iterator lower_bound(const K& key) const { + return internal_end(internal_lower_bound(key)); + } + + // Finds the first element whose key is greater than key. + template + iterator upper_bound(const K& key) { + return internal_end(internal_upper_bound(key)); + } + template + const_iterator upper_bound(const K& key) const { + return internal_end(internal_upper_bound(key)); + } + + // Finds the range of values which compare equal to key. The first member of + // the returned pair is equal to lower_bound(key). The second member pair of + // the pair is equal to upper_bound(key). + template + std::pair equal_range(const K& key) { + return {lower_bound(key), upper_bound(key)}; + } + template + std::pair equal_range(const K& key) const { + return {lower_bound(key), upper_bound(key)}; + } + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. + // Requirement: if `key` already exists in the btree, does not consume `args`. + // Requirement: `key` is never referenced after consuming `args`. + template + std::pair insert_unique(const key_type& key, Args&&... args); + + // Inserts with hint. Checks to see if the value should be placed immediately + // before `position` in the tree. If so, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_unique() were made. + // Requirement: if `key` already exists in the btree, does not consume `args`. + // Requirement: `key` is never referenced after consuming `args`. + template + std::pair insert_hint_unique(iterator position, const key_type& key, + Args&&... args); + + // Insert a range of values into the btree. + template + void insert_iterator_unique(InputIterator b, InputIterator e); + + // Inserts a value into the btree. + template + iterator insert_multi(const key_type& key, ValueType&& v); + + // Inserts a value into the btree. + template + iterator insert_multi(ValueType&& v) { + return insert_multi(params_type::key(v), std::forward(v)); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_multi(v) were made. + template + iterator insert_hint_multi(iterator position, ValueType&& v); + + // Insert a range of values into the btree. + template + void insert_iterator_multi(InputIterator b, InputIterator e); + + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + // Requirement: does not read the value at `*iter`. + iterator erase(iterator iter); + + // Erases range. Returns the number of keys erased and an iterator pointing + // to the element after the last erased element. + std::pair erase(iterator begin, iterator end); + + // Erases the specified key from the btree. Returns 1 if an element was + // erased and 0 otherwise. + template + size_type erase_unique(const K& key); + + // Erases all of the entries matching the specified key from the + // btree. Returns the number of elements erased. + template + size_type erase_multi(const K& key); + + // Finds the iterator corresponding to a key or returns end() if the key is + // not present. + template + iterator find(const K& key) { + return internal_end(internal_find(key)); + } + template + const_iterator find(const K& key) const { + return internal_end(internal_find(key)); + } + + // Returns a count of the number of times the key appears in the btree. + template + size_type count_unique(const K& key) const { + const iterator beg = internal_find(key); + if (beg.node == nullptr) { + // The key doesn't exist in the tree. + return 0; } - - template - void btree_iterator::decrement_slow() { - if (node->leaf()) { - assert(position <= -1); - btree_iterator save(*this); - while (position < 0 && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position() - 1; - node = node->parent(); - } - if (position < 0) { - *this = save; - } - } else { - assert(position >= 0); - node = node->child(position); - while (!node->leaf()) { - node = node->child(node->count()); - } - position = node->count() - 1; - } + return 1; + } + // Returns a count of the number of times the key appears in the btree. + template + size_type count_multi(const K& key) const { + const auto range = equal_range(key); + return std::distance(range.first, range.second); + } + + // Clear the btree, deleting all of the values it contains. + void clear(); + + // Swap the contents of *this and x. + void swap(btree& x); + + const key_compare& key_comp() const noexcept { return root_.template get<0>(); } + template + bool compare_keys(const K& x, const LK& y) const { + return compare_internal::compare_result_as_less_than(key_comp()(x, y)); + } + + value_compare value_comp() const { return value_compare(key_comp()); } + + // Verifies the structure of the btree. + void verify() const; + + // Size routines. + size_type size() const { return size_; } + size_type max_size() const { return (std::numeric_limits::max)(); } + bool empty() const { return size_ == 0; } + + // The height of the btree. An empty tree will have height 0. + size_type height() const { + size_type h = 0; + if (!empty()) { + // Count the length of the chain from the leftmost node up to the + // root. We actually count from the root back around to the level below + // the root, but the calculation is the same because of the circularity + // of that traversal. + const node_type* n = root(); + do { + ++h; + n = n->parent(); + } while (n != root()); + } + return h; + } + + // The number of internal, leaf and total nodes used by the btree. + size_type leaf_nodes() const { return internal_stats(root()).leaf_nodes; } + size_type internal_nodes() const { return internal_stats(root()).internal_nodes; } + size_type nodes() const { + node_stats stats = internal_stats(root()); + return stats.leaf_nodes + stats.internal_nodes; + } + + // The total number of bytes used by the btree. + size_type bytes_used() const { + node_stats stats = internal_stats(root()); + if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { + return sizeof(*this) + node_type::LeafSize(root()->max_count()); + } else { + return sizeof(*this) + stats.leaf_nodes * node_type::LeafSize() + + stats.internal_nodes * node_type::InternalSize(); + } + } + + // The average number of bytes used per value stored in the btree. + static double average_bytes_per_value() { + // Returns the number of bytes per value on a leaf node that is 75% + // full. Experimentally, this matches up nicely with the computed number of + // bytes per value in trees that had their values inserted in random order. + return node_type::LeafSize() / (kNodeValues * 0.75); + } + + // The fullness of the btree. Computed as the number of elements in the btree + // divided by the maximum number of elements a tree with the current number + // of nodes could hold. A value of 1 indicates perfect space + // utilization. Smaller values indicate space wastage. + // Returns 0 for empty trees. + double fullness() const { + if (empty()) return 0.0; + return static_cast(size()) / (nodes() * kNodeValues); + } + // The overhead of the btree structure in bytes per node. Computed as the + // total number of bytes used by the btree minus the number of bytes used for + // storing elements divided by the number of elements. + // Returns 0 for empty trees. + double overhead() const { + if (empty()) return 0.0; + return (bytes_used() - size() * sizeof(value_type)) / static_cast(size()); + } + + // The allocator used by the btree. + allocator_type get_allocator() const { return allocator(); } + + private: + // Internal accessor routines. + node_type* root() { return root_.template get<2>(); } + const node_type* root() const { return root_.template get<2>(); } + node_type*& mutable_root() noexcept { return root_.template get<2>(); } + key_compare* mutable_key_comp() noexcept { return &root_.template get<0>(); } + + // The leftmost node is stored as the parent of the root node. + node_type* leftmost() { return root()->parent(); } + const node_type* leftmost() const { return root()->parent(); } + + // Allocator routines. + allocator_type* mutable_allocator() noexcept { return &root_.template get<1>(); } + const allocator_type& allocator() const noexcept { return root_.template get<1>(); } + + // Allocates a correctly aligned node of at least size bytes using the + // allocator. + node_type* allocate(const size_type sz) { + return reinterpret_cast( + phmap::priv::Allocate(mutable_allocator(), (size_t)sz)); + } + + // Node creation/deletion routines. + node_type* new_internal_node(node_type* parent) { + node_type* p = allocate(node_type::InternalSize()); + return node_type::init_internal(p, parent); + } + node_type* new_leaf_node(node_type* parent) { + node_type* p = allocate(node_type::LeafSize()); + return node_type::init_leaf(p, parent, kNodeValues); + } + node_type* new_leaf_root_node(const int max_count) { + node_type* p = allocate(node_type::LeafSize(max_count)); + return node_type::init_leaf(p, p, max_count); + } + + // Deletion helper routines. + void erase_same_node(iterator begin, iterator end); + iterator erase_from_leaf_node(iterator begin, size_type to_erase); + iterator rebalance_after_delete(iterator iter); + + // Deallocates a node of a certain size in bytes using the allocator. + void deallocate(const size_type sz, node_type* node) { + phmap::priv::Deallocate(mutable_allocator(), node, + (size_t)sz); + } + + void delete_internal_node(node_type* node) { + node->destroy(mutable_allocator()); + deallocate(node_type::InternalSize(), node); + } + void delete_leaf_node(node_type* node) { + node->destroy(mutable_allocator()); + deallocate(node_type::LeafSize(node->max_count()), node); + } + + // Rebalances or splits the node iter points to. + void rebalance_or_split(iterator* iter); + + // Merges the values of left, right and the delimiting key on their parent + // onto left, removing the delimiting key and deleting right. + void merge_nodes(node_type* left, node_type* right); + + // Tries to merge node with its left or right sibling, and failing that, + // rebalance with its left or right sibling. Returns true if a merge + // occurred, at which point it is no longer valid to access node. Returns + // false if no merging took place. + bool try_merge_or_rebalance(iterator* iter); + + // Tries to shrink the height of the tree by 1. + void try_shrink(); + + iterator internal_end(iterator iter) { return iter.node != nullptr ? iter : end(); } + const_iterator internal_end(const_iterator iter) const { + return iter.node != nullptr ? iter : end(); + } + + // Emplaces a value into the btree immediately before iter. Requires that + // key(v) <= iter.key() and (--iter).key() <= key(v). + template + iterator internal_emplace(iterator iter, Args&&... args); + + // Returns an iterator pointing to the first value >= the value "iter" is + // pointing at. Note that "iter" might be pointing to an invalid location as + // iter.position == iter.node->count(). This routine simply moves iter up in + // the tree to a valid location. + // Requires: iter.node is non-null. + template + static IterType internal_last(IterType iter); + + // Returns an iterator pointing to the leaf position at which key would + // reside in the tree. We provide 2 versions of internal_locate. The first + // version uses a less-than comparator and is incapable of distinguishing when + // there is an exact match. The second version is for the key-compare-to + // specialization and distinguishes exact matches. The key-compare-to + // specialization allows the caller to avoid a subsequent comparison to + // determine if an exact match was made, which is important for keys with + // expensive comparison, such as strings. + template + SearchResult internal_locate(const K& key) const; + + template + SearchResult internal_locate_impl( + const K& key, std::false_type /* IsCompareTo */) const; + + template + SearchResult internal_locate_impl( + const K& key, std::true_type /* IsCompareTo */) const; + + // Internal routine which implements lower_bound(). + template + iterator internal_lower_bound(const K& key) const; + + // Internal routine which implements upper_bound(). + template + iterator internal_upper_bound(const K& key) const; + + // Internal routine which implements find(). + template + iterator internal_find(const K& key) const; + + // Deletes a node and all of its children. + void internal_clear(node_type* node); + + // Verifies the tree structure of node. + int internal_verify(const node_type* node, const key_type* lo, + const key_type* hi) const; + + node_stats internal_stats(const node_type* node) const { + // The root can be a static empty node. + if (node == nullptr || (node == root() && empty())) { + return node_stats(0, 0); + } + if (node->leaf()) { + return node_stats(1, 0); } + node_stats res(0, 1); + for (int i = 0; i <= node->count(); ++i) { + res += internal_stats(node->child(i)); + } + return res; + } + + public: + // Exposed only for tests. + static bool testonly_uses_linear_node_search() { + return node_type::testonly_uses_linear_node_search(); + } + + private: + // We use compressed tuple in order to save space because key_compare and + // allocator_type are usually empty. + phmap::priv::CompressedTuple root_; + + // A pointer to the rightmost node. Note that the leftmost node is stored as + // the root's parent. + node_type* rightmost_; + + // Number of values. + size_type size_; +}; + +//// +// btree_node methods +template +template +inline void btree_node

::emplace_value(const size_type i, allocator_type* alloc, + Args&&... args) { + assert(i <= count()); + // Shift old values to create space for new value and then construct it in + // place. + if (i < count()) { + value_init(count(), alloc, slot(count() - 1)); + for (size_type j = count() - 1; j > i; --j) + params_type::move(alloc, slot(j - 1), slot(j)); + value_destroy(i, alloc); + } + value_init(i, alloc, std::forward(args)...); + set_count((field_type)(count() + 1)); + + if (!leaf() && count() > i + 1) { + for (int j = count(); j > (int)(i + 1); --j) { + set_child(j, child(j - 1)); + } + clear_child(i + 1); + } +} - //// - // btree methods - template - template - void btree

::copy_or_move_values_in_order(Btree *x) { - static_assert(std::is_same::value || - std::is_same::value, - "Btree type must be same or const."); - assert(empty()); - - // We can avoid key comparisons because we know the order of the - // values is the same order we'll store them in. - auto iter = x->begin(); - if (iter == x->end()) return; - insert_multi(maybe_move_from_iterator(iter)); - ++iter; - for (; iter != x->end(); ++iter) { - // If the btree is not empty, we can just insert the new value at the end - // of the tree. - internal_emplace(end(), maybe_move_from_iterator(iter)); - } +template +inline void btree_node

::remove_value(const int i, allocator_type* alloc) { + if (!leaf() && count() > i + 1) { + assert(child(i + 1)->count() == 0); + for (size_type j = i + 1; j < count(); ++j) { + set_child(j, child(j + 1)); } + clear_child(count()); + } - template - constexpr bool btree

::static_assert_validation() { - static_assert(std::is_nothrow_copy_constructible::value, - "Key comparison must be nothrow copy constructible"); - static_assert(std::is_nothrow_copy_constructible::value, - "Allocator must be nothrow copy constructible"); - static_assert(type_traits_internal::is_trivially_copyable::value, - "iterator not trivially copyable."); + remove_values_ignore_children(i, /*to_erase=*/1, alloc); +} - // Note: We assert that kTargetValues, which is computed from - // Params::kTargetNodeSize, must fit the node_type::field_type. - static_assert( - kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))), - "target node size too large"); +template +inline void btree_node

::remove_values_ignore_children(int i, size_type to_erase, + allocator_type* alloc) { + params_type::move(alloc, slot(i + to_erase), slot(count()), slot(i)); + value_destroy_n(count() - to_erase, to_erase, alloc); + set_count((field_type)(count() - to_erase)); +} - // Verify that key_compare returns an phmap::{weak,strong}_ordering or bool. - using compare_result_type = - phmap::invoke_result_t; - static_assert( - std::is_same::value || - std::is_convertible::value, - "key comparison function must return phmap::{weak,strong}_ordering or " - "bool."); +template +void btree_node

::rebalance_right_to_left(const int to_move, btree_node* right, + allocator_type* alloc) { + assert(parent() == right->parent()); + assert(position() + 1 == right->position()); + assert(right->count() >= count()); + assert(to_move >= 1); + assert(to_move <= right->count()); - // Test the assumption made in setting kNodeValueSpace. - static_assert(node_type::MinimumOverhead() >= sizeof(void *) + 4, - "node space assumption incorrect"); + // 1) Move the delimiting value in the parent to the left node. + value_init(count(), alloc, parent()->slot(position())); - return true; - } + // 2) Move the (to_move - 1) values from the right node to the left node. + right->uninitialized_move_n(to_move - 1, 0, count() + 1, this, alloc); - template - btree

::btree(const key_compare &comp, const allocator_type &alloc) - : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} + // 3) Move the new delimiting value to the parent from the right node. + params_type::move(alloc, right->slot(to_move - 1), parent()->slot(position())); - template - btree

::btree(const btree &x) : btree(x.key_comp(), x.allocator()) { - copy_or_move_values_in_order(&x); - } + // 4) Shift the values in the right node to their correct position. + params_type::move(alloc, right->slot(to_move), right->slot(right->count()), + right->slot(0)); - template - template - auto btree

::insert_unique(const key_type &key, Args &&... args) - -> std::pair { - if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); - } + // 5) Destroy the now-empty to_move entries in the right node. + right->value_destroy_n(right->count() - to_move, to_move, alloc); + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i < to_move; ++i) { + init_child(count() + i + 1, right->child(i)); + } + for (int i = 0; i <= right->count() - to_move; ++i) { + assert(i + to_move <= right->max_count()); + right->init_child(i, right->child(i + to_move)); + right->clear_child(i + to_move); + } + } - auto res = internal_locate(key); - iterator &iter = res.value; + // Fixup the counts on the left and right nodes. + set_count((field_type)(count() + to_move)); + right->set_count((field_type)(right->count() - to_move)); +} - if (res.HasMatch()) { - if (res.IsEq()) { - // The key already exists in the tree, do nothing. - return {iter, false}; - } - } else { - iterator last = internal_last(iter); - if (last.node && !compare_keys(key, last.key())) { - // The key already exists in the tree, do nothing. - return {last, false}; - } - } - return {internal_emplace(iter, std::forward(args)...), true}; - } - - template - template - inline auto btree

::insert_hint_unique(iterator position, const key_type &key, - Args &&... args) - -> std::pair { - if (!empty()) { - if (position == end() || compare_keys(key, position.key())) { - iterator prev = position; - if (position == begin() || compare_keys((--prev).key(), key)) { - // prev.key() < key < position.key() - return {internal_emplace(position, std::forward(args)...), true}; - } - } else if (compare_keys(position.key(), key)) { - ++position; - if (position == end() || compare_keys(key, position.key())) { - // {original `position`}.key() < key < {current `position`}.key() - return {internal_emplace(position, std::forward(args)...), true}; - } - } else { - // position.key() == key - return {position, false}; - } - } - return insert_unique(key, std::forward(args)...); +template +void btree_node

::rebalance_left_to_right(const int to_move, btree_node* right, + allocator_type* alloc) { + assert(parent() == right->parent()); + assert(position() + 1 == right->position()); + assert(count() >= right->count()); + assert(to_move >= 1); + assert(to_move <= count()); + + // Values in the right node are shifted to the right to make room for the + // new to_move values. Then, the delimiting value in the parent and the + // other (to_move - 1) values in the left node are moved into the right node. + // Lastly, a new delimiting value is moved from the left node into the + // parent, and the remaining empty left node entries are destroyed. + + if (right->count() >= to_move) { + // The original location of the right->count() values are sufficient to hold + // the new to_move entries from the parent and left node. + + // 1) Shift existing values in the right node to their correct positions. + right->uninitialized_move_n(to_move, right->count() - to_move, right->count(), right, + alloc); + for (slot_type *src = right->slot(right->count() - to_move - 1), + *dest = right->slot(right->count() - 1), *end = right->slot(0); + src >= end; --src, --dest) { + params_type::move(alloc, src, dest); } - template - template - void btree

::insert_iterator_unique(InputIterator b, InputIterator e) { - for (; b != e; ++b) { - insert_hint_unique(end(), params_type::key(*b), *b); - } + // 2) Move the delimiting value in the parent to the right node. + params_type::move(alloc, parent()->slot(position()), right->slot(to_move - 1)); + + // 3) Move the (to_move - 1) values from the left node to the right node. + params_type::move(alloc, slot(count() - (to_move - 1)), slot(count()), + right->slot(0)); + } else { + // The right node does not have enough initialized space to hold the new + // to_move entries, so part of them will move to uninitialized space. + + // 1) Shift existing values in the right node to their correct positions. + right->uninitialized_move_n(right->count(), 0, to_move, right, alloc); + + // 2) Move the delimiting value in the parent to the right node. + right->value_init(to_move - 1, alloc, parent()->slot(position())); + + // 3) Move the (to_move - 1) values from the left node to the right node. + const size_type uninitialized_remaining = to_move - right->count() - 1; + uninitialized_move_n(uninitialized_remaining, count() - uninitialized_remaining, + right->count(), right, alloc); + params_type::move(alloc, slot(count() - (to_move - 1)), + slot(count() - uninitialized_remaining), right->slot(0)); + } + + // 4) Move the new delimiting value to the parent from the left node. + params_type::move(alloc, slot(count() - to_move), parent()->slot(position())); + + // 5) Destroy the now-empty to_move entries in the left node. + value_destroy_n(count() - to_move, to_move, alloc); + + if (!leaf()) { + // Move the child pointers from the left to the right node. + for (int i = right->count(); i >= 0; --i) { + right->init_child(i + to_move, right->child(i)); + right->clear_child(i); } + for (int i = 1; i <= to_move; ++i) { + right->init_child(i - 1, child(count() - to_move + i)); + clear_child(count() - to_move + i); + } + } - template - template - auto btree

::insert_multi(const key_type &key, ValueType &&v) -> iterator { - if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); - } + // Fixup the counts on the left and right nodes. + set_count((field_type)(count() - to_move)); + right->set_count((field_type)(right->count() + to_move)); +} - iterator iter = internal_upper_bound(key); - if (iter.node == nullptr) { - iter = end(); - } - return internal_emplace(iter, std::forward(v)); - } - - template - template - auto btree

::insert_hint_multi(iterator position, ValueType &&v) -> iterator { - if (!empty()) { - const key_type &key = params_type::key(v); - if (position == end() || !compare_keys(position.key(), key)) { - iterator prev = position; - if (position == begin() || !compare_keys(key, (--prev).key())) { - // prev.key() <= key <= position.key() - return internal_emplace(position, std::forward(v)); - } - } else { - iterator next = position; - ++next; - if (next == end() || !compare_keys(next.key(), key)) { - // position.key() < key <= next.key() - return internal_emplace(next, std::forward(v)); - } - } - } - return insert_multi(std::forward(v)); +template +void btree_node

::split(const int insert_position, btree_node* dest, + allocator_type* alloc) { + assert(dest->count() == 0); + assert(max_count() == kNodeValues); + + // We bias the split based on the position being inserted. If we're + // inserting at the beginning of the left node then bias the split to put + // more values on the right node. If we're inserting at the end of the + // right node then bias the split to put more values on the left node. + if (insert_position == 0) { + dest->set_count((field_type)(count() - 1)); + } else if (insert_position == kNodeValues) { + dest->set_count(0); + } else { + dest->set_count((field_type)(count() / 2)); + } + set_count((field_type)(count() - dest->count())); + assert(count() >= 1); + + // Move values from the left sibling to the right sibling. + uninitialized_move_n(dest->count(), count(), 0, dest, alloc); + + // Destroy the now-empty entries in the left node. + value_destroy_n(count(), dest->count(), alloc); + + // The split key is the largest value in the left sibling. + set_count((field_type)(count() - 1)); + parent()->emplace_value(position(), alloc, slot(count())); + value_destroy(count(), alloc); + parent()->init_child(position() + 1, dest); + + if (!leaf()) { + for (int i = 0; i <= dest->count(); ++i) { + assert(child(count() + i + 1) != nullptr); + dest->init_child(i, child(count() + i + 1)); + clear_child(count() + i + 1); } + } +} - template - template - void btree

::insert_iterator_multi(InputIterator b, InputIterator e) { - for (; b != e; ++b) { - insert_hint_multi(end(), *b); - } - } +template +void btree_node

::merge(btree_node* src, allocator_type* alloc) { + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); - template - auto btree

::operator=(const btree &x) -> btree & { - if (this != &x) { - clear(); + // Move the delimiting value to the left node. + value_init(count(), alloc, parent()->slot(position())); - *mutable_key_comp() = x.key_comp(); - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_copy_assignment::value) { - *mutable_allocator() = x.allocator(); - } + // Move the values from the right to the left node. + src->uninitialized_move_n(src->count(), 0, count() + 1, this, alloc); - copy_or_move_values_in_order(&x); - } - return *this; - } - - template - auto btree

::operator=(btree &&x) noexcept -> btree & { - if (this != &x) { - clear(); - - using std::swap; - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_copy_assignment::value) { - // Note: `root_` also contains the allocator and the key comparator. - swap(root_, x.root_); - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } else { - if (allocator() == x.allocator()) { - swap(mutable_root(), x.mutable_root()); - swap(*mutable_key_comp(), *x.mutable_key_comp()); - swap(rightmost_, x.rightmost_); - swap(size_, x.size_); - } else { - // We aren't allowed to propagate the allocator and the allocator is - // different so we can't take over its memory. We must move each element - // individually. We need both `x` and `this` to have `x`s key comparator - // while moving the values so we can't swap the key comparators. - *mutable_key_comp() = x.key_comp(); - copy_or_move_values_in_order(&x); - } - } - } - return *this; - } - - template - auto btree

::erase(iterator iter) -> iterator { - bool internal_delete = false; - if (!iter.node->leaf()) { - // Deletion of a value on an internal node. First, move the largest value - // from our left child here, then delete that position (in remove_value() - // below). We can get to the largest value from our left child by - // decrementing iter. - iterator internal_iter(iter); - --iter; - assert(iter.node->leaf()); - params_type::move(mutable_allocator(), iter.node->slot(iter.position), - internal_iter.node->slot(internal_iter.position)); - internal_delete = true; - } + // Destroy the now-empty entries in the right node. + src->value_destroy_n(0, src->count(), alloc); - // Delete the key from the leaf. - iter.node->remove_value(iter.position, mutable_allocator()); - --size_; + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i <= src->count(); ++i) { + init_child(count() + i + 1, src->child(i)); + src->clear_child(i); + } + } - // We want to return the next value after the one we just erased. If we - // erased from an internal node (internal_delete == true), then the next - // value is ++(++iter). If we erased from a leaf node (internal_delete == - // false) then the next value is ++iter. Note that ++iter may point to an - // internal node and the value in the internal node may move to a leaf node - // (iter.node) when rebalancing is performed at the leaf level. + // Fixup the counts on the src and dest nodes. + set_count((field_type)(1 + count() + src->count())); + src->set_count(0); - iterator res = rebalance_after_delete(iter); + // Remove the value on the parent node. + parent()->remove_value(position(), alloc); +} - // If we erased from an internal node, advance the iterator. - if (internal_delete) { - ++res; - } - return res; - } - - template - auto btree

::rebalance_after_delete(iterator iter) -> iterator { - // Merge/rebalance as we walk back up the tree. - iterator res(iter); - bool first_iteration = true; - for (;;) { - if (iter.node == root()) { - try_shrink(); - if (empty()) { - return end(); - } - break; - } - if (iter.node->count() >= kMinNodeValues) { - break; - } - bool merged = try_merge_or_rebalance(&iter); - // On the first iteration, we should update `res` with `iter` because `res` - // may have been invalidated. - if (first_iteration) { - res = iter; - first_iteration = false; - } - if (!merged) { - break; - } - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - } +template +void btree_node

::swap(btree_node* x, allocator_type* alloc) { + using std::swap; + assert(leaf() == x->leaf()); + + // Determine which is the smaller/larger node. + btree_node *smaller = this, *larger = x; + if (smaller->count() > larger->count()) { + swap(smaller, larger); + } + + // Swap the values. + for (slot_type *a = smaller->slot(0), *b = larger->slot(0), *end = a + smaller->count(); + a != end; ++a, ++b) { + params_type::swap(alloc, a, b); + } + + // Move values that can't be swapped. + const size_type to_move = larger->count() - smaller->count(); + larger->uninitialized_move_n(to_move, smaller->count(), smaller->count(), smaller, + alloc); + larger->value_destroy_n(smaller->count(), to_move, alloc); + + if (!leaf()) { + // Swap the child pointers. + std::swap_ranges(&smaller->mutable_child(0), + &smaller->mutable_child(smaller->count() + 1), + &larger->mutable_child(0)); + // Update swapped children's parent pointers. + int i = 0; + for (; i <= smaller->count(); ++i) { + smaller->child(i)->set_parent(smaller); + larger->child(i)->set_parent(larger); + } + // Move the child pointers that couldn't be swapped. + for (; i <= larger->count(); ++i) { + smaller->init_child(i, larger->child(i)); + larger->clear_child(i); + } + } - // Adjust our return value. If we're pointing at the end of a node, advance - // the iterator. - if (res.position == res.node->count()) { - res.position = res.node->count() - 1; - ++res; - } + // Swap the counts. + swap(mutable_count(), x->mutable_count()); +} - return res; +//// +// btree_iterator methods +template +void btree_iterator::increment_slow() { + if (node->leaf()) { + assert(position >= node->count()); + btree_iterator save(*this); + while (position == node->count() && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position(); + node = node->parent(); + } + if (position == node->count()) { + *this = save; } + } else { + assert(position < node->count()); + node = node->child(position + 1); + while (!node->leaf()) { + node = node->child(0); + } + position = 0; + } +} - template - auto btree

::erase(iterator _begin, iterator _end) - -> std::pair { - difference_type count = std::distance(_begin, _end); - assert(count >= 0); +template +void btree_iterator::decrement_slow() { + if (node->leaf()) { + assert(position <= -1); + btree_iterator save(*this); + while (position < 0 && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position() - 1; + node = node->parent(); + } + if (position < 0) { + *this = save; + } + } else { + assert(position >= 0); + node = node->child(position); + while (!node->leaf()) { + node = node->child(node->count()); + } + position = node->count() - 1; + } +} - if (count == 0) { - return {0, _begin}; - } +//// +// btree methods +template +template +void btree

::copy_or_move_values_in_order(Btree* x) { + static_assert( + std::is_same::value || std::is_same::value, + "Btree type must be same or const."); + assert(empty()); + + // We can avoid key comparisons because we know the order of the + // values is the same order we'll store them in. + auto iter = x->begin(); + if (iter == x->end()) return; + insert_multi(maybe_move_from_iterator(iter)); + ++iter; + for (; iter != x->end(); ++iter) { + // If the btree is not empty, we can just insert the new value at the end + // of the tree. + internal_emplace(end(), maybe_move_from_iterator(iter)); + } +} - if (count == (difference_type)size_) { - clear(); - return {count, this->end()}; - } +template +constexpr bool btree

::static_assert_validation() { + static_assert(std::is_nothrow_copy_constructible::value, + "Key comparison must be nothrow copy constructible"); + static_assert(std::is_nothrow_copy_constructible::value, + "Allocator must be nothrow copy constructible"); + static_assert(type_traits_internal::is_trivially_copyable::value, + "iterator not trivially copyable."); + + // Note: We assert that kTargetValues, which is computed from + // Params::kTargetNodeSize, must fit the node_type::field_type. + static_assert(kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))), + "target node size too large"); + + // Verify that key_compare returns an phmap::{weak,strong}_ordering or bool. + using compare_result_type = phmap::invoke_result_t; + static_assert(std::is_same::value || + std::is_convertible::value, + "key comparison function must return phmap::{weak,strong}_ordering or " + "bool."); + + // Test the assumption made in setting kNodeValueSpace. + static_assert(node_type::MinimumOverhead() >= sizeof(void*) + 4, + "node space assumption incorrect"); + + return true; +} - if (_begin.node == _end.node) { - erase_same_node(_begin, _end); - size_ -= count; - return {count, rebalance_after_delete(_begin)}; - } +template +btree

::btree(const key_compare& comp, const allocator_type& alloc) + : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} - const size_type target_size = size_ - count; - while (size_ > target_size) { - if (_begin.node->leaf()) { - const size_type remaining_to_erase = size_ - target_size; - const size_type remaining_in_node = _begin.node->count() - _begin.position; - _begin = erase_from_leaf_node( - _begin, (std::min)(remaining_to_erase, remaining_in_node)); - } else { - _begin = erase(_begin); - } - } - return {count, _begin}; - } - - template - void btree

::erase_same_node(iterator _begin, iterator _end) { - assert(_begin.node == _end.node); - assert(_end.position > _begin.position); - - node_type *node = _begin.node; - size_type to_erase = _end.position - _begin.position; - if (!node->leaf()) { - // Delete all children between _begin and _end. - for (size_type i = 0; i < to_erase; ++i) { - internal_clear(node->child(_begin.position + i + 1)); - } - // Rotate children after _end into new positions. - for (size_type i = _begin.position + to_erase + 1; i <= node->count(); ++i) { - node->set_child(i - to_erase, node->child(i)); - node->clear_child(i); - } - } - node->remove_values_ignore_children(_begin.position, to_erase, - mutable_allocator()); +template +btree

::btree(const btree& x) : btree(x.key_comp(), x.allocator()) { + copy_or_move_values_in_order(&x); +} - // Do not need to update rightmost_, because - // * either _end == this->end(), and therefore node == rightmost_, and still - // exists - // * or _end != this->end(), and therefore rightmost_ hasn't been erased, since - // it wasn't covered in [_begin, _end) +template +template +auto btree

::insert_unique(const key_type& key, Args&&... args) + -> std::pair { + if (empty()) { + mutable_root() = rightmost_ = new_leaf_root_node(1); + } + + auto res = internal_locate(key); + iterator& iter = res.value; + + if (res.HasMatch()) { + if (res.IsEq()) { + // The key already exists in the tree, do nothing. + return {iter, false}; } + } else { + iterator last = internal_last(iter); + if (last.node && !compare_keys(key, last.key())) { + // The key already exists in the tree, do nothing. + return {last, false}; + } + } + return {internal_emplace(iter, std::forward(args)...), true}; +} - template - auto btree

::erase_from_leaf_node(iterator _begin, size_type to_erase) - -> iterator { - node_type *node = _begin.node; - assert(node->leaf()); - assert(node->count() > _begin.position); - assert(_begin.position + to_erase <= node->count()); +template +template +inline auto btree

::insert_hint_unique(iterator position, const key_type& key, + Args&&... args) -> std::pair { + if (!empty()) { + if (position == end() || compare_keys(key, position.key())) { + iterator prev = position; + if (position == begin() || compare_keys((--prev).key(), key)) { + // prev.key() < key < position.key() + return {internal_emplace(position, std::forward(args)...), true}; + } + } else if (compare_keys(position.key(), key)) { + ++position; + if (position == end() || compare_keys(key, position.key())) { + // {original `position`}.key() < key < {current `position`}.key() + return {internal_emplace(position, std::forward(args)...), true}; + } + } else { + // position.key() == key + return {position, false}; + } + } + return insert_unique(key, std::forward(args)...); +} - node->remove_values_ignore_children(_begin.position, to_erase, - mutable_allocator()); +template +template +void btree

::insert_iterator_unique(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_hint_unique(end(), params_type::key(*b), *b); + } +} - size_ -= to_erase; +template +template +auto btree

::insert_multi(const key_type& key, ValueType&& v) -> iterator { + if (empty()) { + mutable_root() = rightmost_ = new_leaf_root_node(1); + } + + iterator iter = internal_upper_bound(key); + if (iter.node == nullptr) { + iter = end(); + } + return internal_emplace(iter, std::forward(v)); +} - return rebalance_after_delete(_begin); +template +template +auto btree

::insert_hint_multi(iterator position, ValueType&& v) -> iterator { + if (!empty()) { + const key_type& key = params_type::key(v); + if (position == end() || !compare_keys(position.key(), key)) { + iterator prev = position; + if (position == begin() || !compare_keys(key, (--prev).key())) { + // prev.key() <= key <= position.key() + return internal_emplace(position, std::forward(v)); + } + } else { + iterator next = position; + ++next; + if (next == end() || !compare_keys(next.key(), key)) { + // position.key() < key <= next.key() + return internal_emplace(next, std::forward(v)); + } } + } + return insert_multi(std::forward(v)); +} - template - template - auto btree

::erase_unique(const K &key) -> size_type { - const iterator iter = internal_find(key); - if (iter.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - erase(iter); - return 1; - } +template +template +void btree

::insert_iterator_multi(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_hint_multi(end(), *b); + } +} - template - template - auto btree

::erase_multi(const K &key) -> size_type { - const iterator _begin = internal_lower_bound(key); - if (_begin.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - // Delete all of the keys between _begin and upper_bound(key). - const iterator _end = internal_end(internal_upper_bound(key)); - return erase(_begin, _end).first; +template +auto btree

::operator=(const btree& x) -> btree& { + if (this != &x) { + clear(); + + *mutable_key_comp() = x.key_comp(); + if (phmap::allocator_traits< + allocator_type>::propagate_on_container_copy_assignment::value) { + *mutable_allocator() = x.allocator(); } - template - void btree

::clear() { - if (!empty()) { - internal_clear(root()); - } - mutable_root() = EmptyNode(); - rightmost_ = EmptyNode(); - size_ = 0; - } - - template - void btree

::swap(btree &x) { - using std::swap; - if (phmap::allocator_traits< - allocator_type>::propagate_on_container_swap::value) { - // Note: `root_` also contains the allocator and the key comparator. - swap(root_, x.root_); - } else { - // It's undefined behavior if the allocators are unequal here. - assert(allocator() == x.allocator()); - swap(mutable_root(), x.mutable_root()); - swap(*mutable_key_comp(), *x.mutable_key_comp()); - } + copy_or_move_values_in_order(&x); + } + return *this; +} + +template +auto btree

::operator=(btree&& x) noexcept -> btree& { + if (this != &x) { + clear(); + + using std::swap; + if (phmap::allocator_traits< + allocator_type>::propagate_on_container_copy_assignment::value) { + // Note: `root_` also contains the allocator and the key comparator. + swap(root_, x.root_); + swap(rightmost_, x.rightmost_); + swap(size_, x.size_); + } else { + if (allocator() == x.allocator()) { + swap(mutable_root(), x.mutable_root()); + swap(*mutable_key_comp(), *x.mutable_key_comp()); swap(rightmost_, x.rightmost_); swap(size_, x.size_); + } else { + // We aren't allowed to propagate the allocator and the allocator is + // different so we can't take over its memory. We must move each element + // individually. We need both `x` and `this` to have `x`s key comparator + // while moving the values so we can't swap the key comparators. + *mutable_key_comp() = x.key_comp(); + copy_or_move_values_in_order(&x); + } } + } + return *this; +} - template - void btree

::verify() const { - assert(root() != nullptr); - assert(leftmost() != nullptr); - assert(rightmost_ != nullptr); - assert(empty() || size() == internal_verify(root(), nullptr, nullptr)); - assert(leftmost() == (++const_iterator(root(), -1)).node); - assert(rightmost_ == (--const_iterator(root(), root()->count())).node); - assert(leftmost()->leaf()); - assert(rightmost_->leaf()); - } - - template - void btree

::rebalance_or_split(iterator *iter) { - node_type *&node = iter->node; - int &insert_position = iter->position; - assert(node->count() == node->max_count()); - assert(kNodeValues == node->max_count()); - - // First try to make room on the node by rebalancing. - node_type *parent = node->parent(); - if (node != root()) { - if (node->position() > 0) { - // Try rebalancing with our left sibling. - node_type *left = parent->child(node->position() - 1); - assert(left->max_count() == kNodeValues); - if (left->count() < kNodeValues) { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the end of the right node then we bias rebalancing to - // fill up the left node. - int to_move = (kNodeValues - left->count()) / - (1 + (insert_position < kNodeValues)); - to_move = (std::max)(1, to_move); - - if (((insert_position - to_move) >= 0) || - ((left->count() + to_move) < kNodeValues)) { - left->rebalance_right_to_left(to_move, node, mutable_allocator()); - - assert(node->max_count() - node->count() == to_move); - insert_position = insert_position - to_move; - if (insert_position < 0) { - insert_position = insert_position + left->count() + 1; - node = left; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - if (node->position() < parent->count()) { - // Try rebalancing with our right sibling. - node_type *right = parent->child(node->position() + 1); - assert(right->max_count() == kNodeValues); - if (right->count() < kNodeValues) { - // We bias rebalancing based on the position being inserted. If we're - // inserting at the _beginning of the left node then we bias rebalancing - // to fill up the right node. - int to_move = - (kNodeValues - right->count()) / (1 + (insert_position > 0)); - to_move = (std::max)(1, to_move); - - if ((insert_position <= (node->count() - to_move)) || - ((right->count() + to_move) < kNodeValues)) { - node->rebalance_left_to_right(to_move, right, mutable_allocator()); - - if (insert_position > node->count()) { - insert_position = insert_position - node->count() - 1; - node = right; - } - - assert(node->count() < node->max_count()); - return; - } - } - } - - // Rebalancing failed, make sure there is room on the parent node for a new - // value. - assert(parent->max_count() == kNodeValues); - if (parent->count() == kNodeValues) { - iterator parent_iter(node->parent(), node->position()); - rebalance_or_split(&parent_iter); - } - } else { - // Rebalancing not possible because this is the root node. - // Create a new root node and set the current root node as the child of the - // new root. - parent = new_internal_node(parent); - parent->init_child(0, root()); - mutable_root() = parent; - // If the former root was a leaf node, then it's now the rightmost node. - assert(!parent->child(0)->leaf() || parent->child(0) == rightmost_); - } - - // Split the node. - node_type *split_node; - if (node->leaf()) { - split_node = new_leaf_node(parent); - node->split(insert_position, split_node, mutable_allocator()); - if (rightmost_ == node) rightmost_ = split_node; - } else { - split_node = new_internal_node(parent); - node->split(insert_position, split_node, mutable_allocator()); - } +template +auto btree

::erase(iterator iter) -> iterator { + bool internal_delete = false; + if (!iter.node->leaf()) { + // Deletion of a value on an internal node. First, move the largest value + // from our left child here, then delete that position (in remove_value() + // below). We can get to the largest value from our left child by + // decrementing iter. + iterator internal_iter(iter); + --iter; + assert(iter.node->leaf()); + params_type::move(mutable_allocator(), iter.node->slot(iter.position), + internal_iter.node->slot(internal_iter.position)); + internal_delete = true; + } + + // Delete the key from the leaf. + iter.node->remove_value(iter.position, mutable_allocator()); + --size_; + + // We want to return the next value after the one we just erased. If we + // erased from an internal node (internal_delete == true), then the next + // value is ++(++iter). If we erased from a leaf node (internal_delete == + // false) then the next value is ++iter. Note that ++iter may point to an + // internal node and the value in the internal node may move to a leaf node + // (iter.node) when rebalancing is performed at the leaf level. + + iterator res = rebalance_after_delete(iter); + + // If we erased from an internal node, advance the iterator. + if (internal_delete) { + ++res; + } + return res; +} - if (insert_position > node->count()) { - insert_position = insert_position - node->count() - 1; - node = split_node; - } +template +auto btree

::rebalance_after_delete(iterator iter) -> iterator { + // Merge/rebalance as we walk back up the tree. + iterator res(iter); + bool first_iteration = true; + for (;;) { + if (iter.node == root()) { + try_shrink(); + if (empty()) { + return end(); + } + break; } - - template - void btree

::merge_nodes(node_type *left, node_type *right) { - left->merge(right, mutable_allocator()); - if (right->leaf()) { - if (rightmost_ == right) rightmost_ = left; - delete_leaf_node(right); - } else { - delete_internal_node(right); - } + if (iter.node->count() >= kMinNodeValues) { + break; } - - template - bool btree

::try_merge_or_rebalance(iterator *iter) { - node_type *parent = iter->node->parent(); - if (iter->node->position() > 0) { - // Try merging with our left sibling. - node_type *left = parent->child(iter->node->position() - 1); - assert(left->max_count() == kNodeValues); - if ((1 + left->count() + iter->node->count()) <= kNodeValues) { - iter->position += 1 + left->count(); - merge_nodes(left, iter->node); - iter->node = left; - return true; - } - } - if (iter->node->position() < parent->count()) { - // Try merging with our right sibling. - node_type *right = parent->child(iter->node->position() + 1); - assert(right->max_count() == kNodeValues); - if ((1 + iter->node->count() + right->count()) <= kNodeValues) { - merge_nodes(iter->node, right); - return true; - } - // Try rebalancing with our right sibling. We don't perform rebalancing if - // we deleted the first element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the front of the tree. - if ((right->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position > 0))) { - int to_move = (right->count() - iter->node->count()) / 2; - to_move = (std::min)(to_move, right->count() - 1); - iter->node->rebalance_right_to_left(to_move, right, mutable_allocator()); - return false; - } - } - if (iter->node->position() > 0) { - // Try rebalancing with our left sibling. We don't perform rebalancing if - // we deleted the last element from iter->node and the node is not - // empty. This is a small optimization for the common pattern of deleting - // from the back of the tree. - node_type *left = parent->child(iter->node->position() - 1); - if ((left->count() > kMinNodeValues) && - ((iter->node->count() == 0) || - (iter->position < iter->node->count()))) { - int to_move = (left->count() - iter->node->count()) / 2; - to_move = (std::min)(to_move, left->count() - 1); - left->rebalance_left_to_right(to_move, iter->node, mutable_allocator()); - iter->position += to_move; - return false; - } - } - return false; + bool merged = try_merge_or_rebalance(&iter); + // On the first iteration, we should update `res` with `iter` because `res` + // may have been invalidated. + if (first_iteration) { + res = iter; + first_iteration = false; } - - template - void btree

::try_shrink() { - if (root()->count() > 0) { - return; - } - // Deleted the last item on the root node, shrink the height of the tree. - if (root()->leaf()) { - assert(size() == 0); - delete_leaf_node(root()); - mutable_root() = EmptyNode(); - rightmost_ = EmptyNode(); - } else { - node_type *child = root()->child(0); - child->make_root(); - delete_internal_node(root()); - mutable_root() = child; - } + if (!merged) { + break; } + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + } + + // Adjust our return value. If we're pointing at the end of a node, advance + // the iterator. + if (res.position == res.node->count()) { + res.position = res.node->count() - 1; + ++res; + } + + return res; +} - template - template - inline IterType btree

::internal_last(IterType iter) { - assert(iter.node != nullptr); - while (iter.position == iter.node->count()) { - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - if (iter.node->leaf()) { - iter.node = nullptr; - break; - } - } - return iter; - } - - template - template - inline auto btree

::internal_emplace(iterator iter, Args &&... args) - -> iterator { - if (!iter.node->leaf()) { - // We can't insert on an internal node. Instead, we'll insert after the - // previous value which is guaranteed to be on a leaf node. - --iter; - ++iter.position; - } - const int max_count = iter.node->max_count(); - if (iter.node->count() == max_count) { - // Make room in the leaf for the new item. - if (max_count < kNodeValues) { - // Insertion into the root where the root is smaller than the full node - // size. Simply grow the size of the root node. - assert(iter.node == root()); - iter.node = - new_leaf_root_node((std::min)(kNodeValues, 2 * max_count)); - iter.node->swap(root(), mutable_allocator()); - delete_leaf_node(root()); - mutable_root() = iter.node; - rightmost_ = iter.node; - } else { - rebalance_or_split(&iter); - } - } - iter.node->emplace_value(iter.position, mutable_allocator(), - std::forward(args)...); - ++size_; - return iter; - } - - template - template - inline auto btree

::internal_locate(const K &key) const - -> SearchResult { - return internal_locate_impl(key, is_key_compare_to()); - } - - template - template - inline auto btree

::internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const - -> SearchResult { - iterator iter(const_cast(root()), 0); - for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; - // NOTE: we don't need to walk all the way down the tree if the keys are - // equal, but determining equality would require doing an extra comparison - // on each node on the way down, and we will need to go all the way to the - // leaf node in the expected case. - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return {iter}; - } - - template - template - inline auto btree

::internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const - -> SearchResult { - iterator iter(const_cast(root()), 0); - for (;;) { - SearchResult res = iter.node->lower_bound(key, key_comp()); - iter.position = res.value; - if (res.match == MatchKind::kEq) { - return {iter, MatchKind::kEq}; - } - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return {iter, MatchKind::kNe}; - } - - template - template - auto btree

::internal_lower_bound(const K &key) const -> iterator { - iterator iter(const_cast(root()), 0); - for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return internal_last(iter); - } - - template - template - auto btree

::internal_upper_bound(const K &key) const -> iterator { - iterator iter(const_cast(root()), 0); - for (;;) { - iter.position = iter.node->upper_bound(key, key_comp()); - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return internal_last(iter); +template +auto btree

::erase(iterator _begin, iterator _end) -> std::pair { + difference_type count = std::distance(_begin, _end); + assert(count >= 0); + + if (count == 0) { + return {0, _begin}; + } + + if (count == (difference_type)size_) { + clear(); + return {count, this->end()}; + } + + if (_begin.node == _end.node) { + erase_same_node(_begin, _end); + size_ -= count; + return {count, rebalance_after_delete(_begin)}; + } + + const size_type target_size = size_ - count; + while (size_ > target_size) { + if (_begin.node->leaf()) { + const size_type remaining_to_erase = size_ - target_size; + const size_type remaining_in_node = _begin.node->count() - _begin.position; + _begin = + erase_from_leaf_node(_begin, (std::min)(remaining_to_erase, remaining_in_node)); + } else { + _begin = erase(_begin); } + } + return {count, _begin}; +} - template - template - auto btree

::internal_find(const K &key) const -> iterator { - auto res = internal_locate(key); - if (res.HasMatch()) { - if (res.IsEq()) { - return res.value; - } - } else { - const iterator iter = internal_last(res.value); - if (iter.node != nullptr && !compare_keys(key, iter.key())) { - return iter; - } - } - return {nullptr, 0}; +template +void btree

::erase_same_node(iterator _begin, iterator _end) { + assert(_begin.node == _end.node); + assert(_end.position > _begin.position); + + node_type* node = _begin.node; + size_type to_erase = _end.position - _begin.position; + if (!node->leaf()) { + // Delete all children between _begin and _end. + for (size_type i = 0; i < to_erase; ++i) { + internal_clear(node->child(_begin.position + i + 1)); } - - template - void btree

::internal_clear(node_type *node) { - if (!node->leaf()) { - for (int i = 0; i <= node->count(); ++i) { - internal_clear(node->child(i)); - } - delete_internal_node(node); - } else { - delete_leaf_node(node); - } + // Rotate children after _end into new positions. + for (size_type i = _begin.position + to_erase + 1; i <= node->count(); ++i) { + node->set_child(i - to_erase, node->child(i)); + node->clear_child(i); } + } + node->remove_values_ignore_children(_begin.position, to_erase, mutable_allocator()); + + // Do not need to update rightmost_, because + // * either _end == this->end(), and therefore node == rightmost_, and still + // exists + // * or _end != this->end(), and therefore rightmost_ hasn't been erased, since + // it wasn't covered in [_begin, _end) +} - template - int btree

::internal_verify( - const node_type *node, const key_type *lo, const key_type *hi) const { - assert(node->count() > 0); - assert(node->count() <= node->max_count()); - if (lo) { - assert(!compare_keys(node->key(0), *lo)); - } - if (hi) { - assert(!compare_keys(*hi, node->key(node->count() - 1))); - } - for (int i = 1; i < node->count(); ++i) { - assert(!compare_keys(node->key(i), node->key(i - 1))); - } - int count = node->count(); - if (!node->leaf()) { - for (int i = 0; i <= node->count(); ++i) { - assert(node->child(i) != nullptr); - assert(node->child(i)->parent() == node); - assert(node->child(i)->position() == i); - count += internal_verify( - node->child(i), - (i == 0) ? lo : &node->key(i - 1), - (i == node->count()) ? hi : &node->key(i)); - } - } - return count; - } - - // A common base class for btree_set, btree_map, btree_multiset, and btree_multimap. - // --------------------------------------------------------------------------------- - template - class btree_container { - using params_type = typename Tree::params_type; - - protected: - // Alias used for heterogeneous lookup functions. - // `key_arg` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - template - using key_arg = - typename KeyArg::value>:: - template type; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using difference_type = typename Tree::difference_type; - using key_compare = typename Tree::key_compare; - using value_compare = typename Tree::value_compare; - using allocator_type = typename Tree::allocator_type; - using reference = typename Tree::reference; - using const_reference = typename Tree::const_reference; - using pointer = typename Tree::pointer; - using const_pointer = typename Tree::const_pointer; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using reverse_iterator = typename Tree::reverse_iterator; - using const_reverse_iterator = typename Tree::const_reverse_iterator; - using node_type = typename Tree::node_handle_type; - - // Constructors/assignments. - btree_container() : tree_(key_compare(), allocator_type()) {} - explicit btree_container(const key_compare &comp, - const allocator_type &alloc = allocator_type()) - : tree_(comp, alloc) {} - btree_container(const btree_container &x) = default; - btree_container(btree_container &&x) noexcept = default; - btree_container &operator=(const btree_container &x) = default; - btree_container &operator=(btree_container &&x) noexcept( - std::is_nothrow_move_assignable::value) = default; - - // Iterator routines. - iterator begin() { return tree_.begin(); } - const_iterator begin() const { return tree_.begin(); } - const_iterator cbegin() const { return tree_.begin(); } - iterator end() { return tree_.end(); } - const_iterator end() const { return tree_.end(); } - const_iterator cend() const { return tree_.end(); } - reverse_iterator rbegin() { return tree_.rbegin(); } - const_reverse_iterator rbegin() const { return tree_.rbegin(); } - const_reverse_iterator crbegin() const { return tree_.rbegin(); } - reverse_iterator rend() { return tree_.rend(); } - const_reverse_iterator rend() const { return tree_.rend(); } - const_reverse_iterator crend() const { return tree_.rend(); } - - // Lookup routines. - template - iterator find(const key_arg &key) { - return tree_.find(key); - } - template - const_iterator find(const key_arg &key) const { return tree_.find(key); } - - template - bool contains(const key_arg &key) const { return find(key) != end(); } - - template - iterator lower_bound(const key_arg &key) { return tree_.lower_bound(key); } - - template - const_iterator lower_bound(const key_arg &key) const { return tree_.lower_bound(key); } - - template - iterator upper_bound(const key_arg &key) { return tree_.upper_bound(key); } +template +auto btree

::erase_from_leaf_node(iterator _begin, size_type to_erase) -> iterator { + node_type* node = _begin.node; + assert(node->leaf()); + assert(node->count() > _begin.position); + assert(_begin.position + to_erase <= node->count()); - template - const_iterator upper_bound(const key_arg &key) const { return tree_.upper_bound(key); } + node->remove_values_ignore_children(_begin.position, to_erase, mutable_allocator()); - template - std::pair equal_range(const key_arg &key) { return tree_.equal_range(key); } + size_ -= to_erase; - template - std::pair equal_range( - const key_arg &key) const { - return tree_.equal_range(key); - } + return rebalance_after_delete(_begin); +} - iterator erase(const_iterator iter) { return tree_.erase(iterator(iter)); } - iterator erase(iterator iter) { return tree_.erase(iter); } - iterator erase(const_iterator first, const_iterator last) { - return tree_.erase(iterator(first), iterator(last)).second; - } +template +template +auto btree

::erase_unique(const K& key) -> size_type { + const iterator iter = internal_find(key); + if (iter.node == nullptr) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + erase(iter); + return 1; +} - node_type extract(iterator position) { - // Use Move instead of Transfer, because the rebalancing code expects to - // have a valid object to scribble metadata bits on top of. - auto node = CommonAccess::Move(get_allocator(), position.slot()); - erase(position); - return node; - } +template +template +auto btree

::erase_multi(const K& key) -> size_type { + const iterator _begin = internal_lower_bound(key); + if (_begin.node == nullptr) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + // Delete all of the keys between _begin and upper_bound(key). + const iterator _end = internal_end(internal_upper_bound(key)); + return erase(_begin, _end).first; +} - node_type extract(const_iterator position) { - return extract(iterator(position)); - } +template +void btree

::clear() { + if (!empty()) { + internal_clear(root()); + } + mutable_root() = EmptyNode(); + rightmost_ = EmptyNode(); + size_ = 0; +} - public: - void clear() { tree_.clear(); } - void swap(btree_container &x) { tree_.swap(x.tree_); } - void verify() const { tree_.verify(); } +template +void btree

::swap(btree& x) { + using std::swap; + if (phmap::allocator_traits::propagate_on_container_swap::value) { + // Note: `root_` also contains the allocator and the key comparator. + swap(root_, x.root_); + } else { + // It's undefined behavior if the allocators are unequal here. + assert(allocator() == x.allocator()); + swap(mutable_root(), x.mutable_root()); + swap(*mutable_key_comp(), *x.mutable_key_comp()); + } + swap(rightmost_, x.rightmost_); + swap(size_, x.size_); +} - size_type size() const { return tree_.size(); } - size_type max_size() const { return tree_.max_size(); } - bool empty() const { return tree_.empty(); } +template +void btree

::verify() const { + assert(root() != nullptr); + assert(leftmost() != nullptr); + assert(rightmost_ != nullptr); + assert(empty() || size() == internal_verify(root(), nullptr, nullptr)); + assert(leftmost() == (++const_iterator(root(), -1)).node); + assert(rightmost_ == (--const_iterator(root(), root()->count())).node); + assert(leftmost()->leaf()); + assert(rightmost_->leaf()); +} - friend bool operator==(const btree_container &x, const btree_container &y) { - if (x.size() != y.size()) return false; - return std::equal(x.begin(), x.end(), y.begin()); - } +template +void btree

::rebalance_or_split(iterator* iter) { + node_type*& node = iter->node; + int& insert_position = iter->position; + assert(node->count() == node->max_count()); + assert(kNodeValues == node->max_count()); + + // First try to make room on the node by rebalancing. + node_type* parent = node->parent(); + if (node != root()) { + if (node->position() > 0) { + // Try rebalancing with our left sibling. + node_type* left = parent->child(node->position() - 1); + assert(left->max_count() == kNodeValues); + if (left->count() < kNodeValues) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the end of the right node then we bias rebalancing to + // fill up the left node. + int to_move = + (kNodeValues - left->count()) / (1 + (insert_position < kNodeValues)); + to_move = (std::max)(1, to_move); + + if (((insert_position - to_move) >= 0) || + ((left->count() + to_move) < kNodeValues)) { + left->rebalance_right_to_left(to_move, node, mutable_allocator()); + + assert(node->max_count() - node->count() == to_move); + insert_position = insert_position - to_move; + if (insert_position < 0) { + insert_position = insert_position + left->count() + 1; + node = left; + } + + assert(node->count() < node->max_count()); + return; + } + } + } - friend bool operator!=(const btree_container &x, const btree_container &y) { return !(x == y); } + if (node->position() < parent->count()) { + // Try rebalancing with our right sibling. + node_type* right = parent->child(node->position() + 1); + assert(right->max_count() == kNodeValues); + if (right->count() < kNodeValues) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the _beginning of the left node then we bias rebalancing + // to fill up the right node. + int to_move = (kNodeValues - right->count()) / (1 + (insert_position > 0)); + to_move = (std::max)(1, to_move); + + if ((insert_position <= (node->count() - to_move)) || + ((right->count() + to_move) < kNodeValues)) { + node->rebalance_left_to_right(to_move, right, mutable_allocator()); + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = right; + } - friend bool operator<(const btree_container &x, const btree_container &y) { - return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end()); + assert(node->count() < node->max_count()); + return; } + } + } - friend bool operator>(const btree_container &x, const btree_container &y) { return y < x; } - - friend bool operator<=(const btree_container &x, const btree_container &y) { return !(y < x); } - - friend bool operator>=(const btree_container &x, const btree_container &y) { return !(x < y); } - - // The allocator used by the btree. - allocator_type get_allocator() const { return tree_.get_allocator(); } - - // The key comparator used by the btree. - key_compare key_comp() const { return tree_.key_comp(); } - value_compare value_comp() const { return tree_.value_comp(); } + // Rebalancing failed, make sure there is room on the parent node for a new + // value. + assert(parent->max_count() == kNodeValues); + if (parent->count() == kNodeValues) { + iterator parent_iter(node->parent(), node->position()); + rebalance_or_split(&parent_iter); + } + } else { + // Rebalancing not possible because this is the root node. + // Create a new root node and set the current root node as the child of the + // new root. + parent = new_internal_node(parent); + parent->init_child(0, root()); + mutable_root() = parent; + // If the former root was a leaf node, then it's now the rightmost node. + assert(!parent->child(0)->leaf() || parent->child(0) == rightmost_); + } + + // Split the node. + node_type* split_node; + if (node->leaf()) { + split_node = new_leaf_node(parent); + node->split(insert_position, split_node, mutable_allocator()); + if (rightmost_ == node) rightmost_ = split_node; + } else { + split_node = new_internal_node(parent); + node->split(insert_position, split_node, mutable_allocator()); + } + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = split_node; + } +} - // Support absl::Hash. - template - friend State AbslHashValue(State h, const btree_container &b) { - for (const auto &v : b) { - h = State::combine(std::move(h), v); - } - return State::combine(std::move(h), b.size()); - } +template +void btree

::merge_nodes(node_type* left, node_type* right) { + left->merge(right, mutable_allocator()); + if (right->leaf()) { + if (rightmost_ == right) rightmost_ = left; + delete_leaf_node(right); + } else { + delete_internal_node(right); + } +} - protected: - Tree tree_; - }; - - // A common base class for btree_set and btree_map. - // ----------------------------------------------- - template - class btree_set_container : public btree_container { - using super_type = btree_container; - using params_type = typename Tree::params_type; - using init_type = typename params_type::init_type; - using is_key_compare_to = typename params_type::is_key_compare_to; - friend class BtreeNodePeer; - - protected: - template - using key_arg = typename super_type::template key_arg; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using node_type = typename super_type::node_type; - using insert_return_type = InsertReturnType; - using super_type::super_type; - btree_set_container() {} - - template - btree_set_container(InputIterator b, InputIterator e, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); - } +template +bool btree

::try_merge_or_rebalance(iterator* iter) { + node_type* parent = iter->node->parent(); + if (iter->node->position() > 0) { + // Try merging with our left sibling. + node_type* left = parent->child(iter->node->position() - 1); + assert(left->max_count() == kNodeValues); + if ((1 + left->count() + iter->node->count()) <= kNodeValues) { + iter->position += 1 + left->count(); + merge_nodes(left, iter->node); + iter->node = left; + return true; + } + } + if (iter->node->position() < parent->count()) { + // Try merging with our right sibling. + node_type* right = parent->child(iter->node->position() + 1); + assert(right->max_count() == kNodeValues); + if ((1 + iter->node->count() + right->count()) <= kNodeValues) { + merge_nodes(iter->node, right); + return true; + } + // Try rebalancing with our right sibling. We don't perform rebalancing if + // we deleted the first element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the front of the tree. + if ((right->count() > kMinNodeValues) && + ((iter->node->count() == 0) || (iter->position > 0))) { + int to_move = (right->count() - iter->node->count()) / 2; + to_move = (std::min)(to_move, right->count() - 1); + iter->node->rebalance_right_to_left(to_move, right, mutable_allocator()); + return false; + } + } + if (iter->node->position() > 0) { + // Try rebalancing with our left sibling. We don't perform rebalancing if + // we deleted the last element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the back of the tree. + node_type* left = parent->child(iter->node->position() - 1); + if ((left->count() > kMinNodeValues) && + ((iter->node->count() == 0) || (iter->position < iter->node->count()))) { + int to_move = (left->count() - iter->node->count()) / 2; + to_move = (std::min)(to_move, left->count() - 1); + left->rebalance_left_to_right(to_move, iter->node, mutable_allocator()); + iter->position += to_move; + return false; + } + } + return false; +} - btree_set_container(std::initializer_list init, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : btree_set_container(init.begin(), init.end(), comp, alloc) {} +template +void btree

::try_shrink() { + if (root()->count() > 0) { + return; + } + // Deleted the last item on the root node, shrink the height of the tree. + if (root()->leaf()) { + assert(size() == 0); + delete_leaf_node(root()); + mutable_root() = EmptyNode(); + rightmost_ = EmptyNode(); + } else { + node_type* child = root()->child(0); + child->make_root(); + delete_internal_node(root()); + mutable_root() = child; + } +} - // Lookup routines. - template - size_type count(const key_arg &key) const { - return this->tree_.count_unique(key); - } +template +template +inline IterType btree

::internal_last(IterType iter) { + assert(iter.node != nullptr); + while (iter.position == iter.node->count()) { + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + if (iter.node->leaf()) { + iter.node = nullptr; + break; + } + } + return iter; +} - // Insertion routines. - std::pair insert(const value_type &x) { - return this->tree_.insert_unique(params_type::key(x), x); - } - std::pair insert(value_type &&x) { - return this->tree_.insert_unique(params_type::key(x), std::move(x)); - } - template - std::pair emplace(Args &&... args) { - init_type v(std::forward(args)...); - return this->tree_.insert_unique(params_type::key(v), std::move(v)); - } - iterator insert(const_iterator position, const value_type &x) { - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(x), x) - .first; - } - iterator insert(const_iterator position, value_type &&x) { - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(x), - std::move(x)) - .first; - } +template +template +inline auto btree

::internal_emplace(iterator iter, Args&&... args) -> iterator { + if (!iter.node->leaf()) { + // We can't insert on an internal node. Instead, we'll insert after the + // previous value which is guaranteed to be on a leaf node. + --iter; + ++iter.position; + } + const int max_count = iter.node->max_count(); + if (iter.node->count() == max_count) { + // Make room in the leaf for the new item. + if (max_count < kNodeValues) { + // Insertion into the root where the root is smaller than the full node + // size. Simply grow the size of the root node. + assert(iter.node == root()); + iter.node = new_leaf_root_node((std::min)(kNodeValues, 2 * max_count)); + iter.node->swap(root(), mutable_allocator()); + delete_leaf_node(root()); + mutable_root() = iter.node; + rightmost_ = iter.node; + } else { + rebalance_or_split(&iter); + } + } + iter.node->emplace_value(iter.position, mutable_allocator(), + std::forward(args)...); + ++size_; + return iter; +} - template - iterator emplace_hint(const_iterator position, Args &&... args) { - init_type v(std::forward(args)...); - return this->tree_ - .insert_hint_unique(iterator(position), params_type::key(v), - std::move(v)) - .first; - } +template +template +inline auto btree

::internal_locate(const K& key) const + -> SearchResult { + return internal_locate_impl(key, is_key_compare_to()); +} - template - void insert(InputIterator b, InputIterator e) { - this->tree_.insert_iterator_unique(b, e); - } +template +template +inline auto btree

::internal_locate_impl(const K& key, + std::false_type /* IsCompareTo */) const + -> SearchResult { + iterator iter(const_cast(root()), 0); + for (;;) { + iter.position = iter.node->lower_bound(key, key_comp()).value; + // NOTE: we don't need to walk all the way down the tree if the keys are + // equal, but determining equality would require doing an extra comparison + // on each node on the way down, and we will need to go all the way to the + // leaf node in the expected case. + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return {iter}; +} - void insert(std::initializer_list init) { - this->tree_.insert_iterator_unique(init.begin(), init.end()); - } +template +template +inline auto btree

::internal_locate_impl(const K& key, + std::true_type /* IsCompareTo */) const + -> SearchResult { + iterator iter(const_cast(root()), 0); + for (;;) { + SearchResult res = iter.node->lower_bound(key, key_comp()); + iter.position = res.value; + if (res.match == MatchKind::kEq) { + return {iter, MatchKind::kEq}; + } + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return {iter, MatchKind::kNe}; +} - insert_return_type insert(node_type &&node) { - if (!node) return {this->end(), false, node_type()}; - std::pair res = - this->tree_.insert_unique(params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - if (res.second) { - CommonAccess::Destroy(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } +template +template +auto btree

::internal_lower_bound(const K& key) const -> iterator { + iterator iter(const_cast(root()), 0); + for (;;) { + iter.position = iter.node->lower_bound(key, key_comp()).value; + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return internal_last(iter); +} - iterator insert(const_iterator hint, node_type &&node) { - if (!node) return this->end(); - std::pair res = this->tree_.insert_hint_unique( - iterator(hint), params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - if (res.second) CommonAccess::Destroy(&node); - return res.first; - } +template +template +auto btree

::internal_upper_bound(const K& key) const -> iterator { + iterator iter(const_cast(root()), 0); + for (;;) { + iter.position = iter.node->upper_bound(key, key_comp()); + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return internal_last(iter); +} - template - size_type erase(const key_arg &key) { return this->tree_.erase_unique(key); } - using super_type::erase; +template +template +auto btree

::internal_find(const K& key) const -> iterator { + auto res = internal_locate(key); + if (res.HasMatch()) { + if (res.IsEq()) { + return res.value; + } + } else { + const iterator iter = internal_last(res.value); + if (iter.node != nullptr && !compare_keys(key, iter.key())) { + return iter; + } + } + return {nullptr, 0}; +} - template - node_type extract(const key_arg &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); - } +template +void btree

::internal_clear(node_type* node) { + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + internal_clear(node->child(i)); + } + delete_internal_node(node); + } else { + delete_leaf_node(node); + } +} - using super_type::extract; +template +int btree

::internal_verify(const node_type* node, const key_type* lo, + const key_type* hi) const { + assert(node->count() > 0); + assert(node->count() <= node->max_count()); + if (lo) { + assert(!compare_keys(node->key(0), *lo)); + } + if (hi) { + assert(!compare_keys(*hi, node->key(node->count() - 1))); + } + for (int i = 1; i < node->count(); ++i) { + assert(!compare_keys(node->key(i), node->key(i - 1))); + } + int count = node->count(); + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + assert(node->child(i) != nullptr); + assert(node->child(i)->parent() == node); + assert(node->child(i)->position() == i); + count += internal_verify(node->child(i), (i == 0) ? lo : &node->key(i - 1), + (i == node->count()) ? hi : &node->key(i)); + } + } + return count; +} - // Merge routines. - // Moves elements from `src` into `this`. If the element already exists in - // `this`, it is left unmodified in `src`. - template < - typename T, +// A common base class for btree_set, btree_map, btree_multiset, and btree_multimap. +// --------------------------------------------------------------------------------- +template +class btree_container { + using params_type = typename Tree::params_type; + + protected: + // Alias used for heterogeneous lookup functions. + // `key_arg` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + template + using key_arg = + typename KeyArg::value>::template type< + K, typename Tree::key_type>; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using difference_type = typename Tree::difference_type; + using key_compare = typename Tree::key_compare; + using value_compare = typename Tree::value_compare; + using allocator_type = typename Tree::allocator_type; + using reference = typename Tree::reference; + using const_reference = typename Tree::const_reference; + using pointer = typename Tree::pointer; + using const_pointer = typename Tree::const_pointer; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using reverse_iterator = typename Tree::reverse_iterator; + using const_reverse_iterator = typename Tree::const_reverse_iterator; + using node_type = typename Tree::node_handle_type; + + // Constructors/assignments. + btree_container() : tree_(key_compare(), allocator_type()) {} + explicit btree_container(const key_compare& comp, + const allocator_type& alloc = allocator_type()) + : tree_(comp, alloc) {} + btree_container(const btree_container& x) = default; + btree_container(btree_container&& x) noexcept = default; + btree_container& operator=(const btree_container& x) = default; + btree_container& operator=(btree_container&& x) noexcept( + std::is_nothrow_move_assignable::value) = default; + + // Iterator routines. + iterator begin() { return tree_.begin(); } + const_iterator begin() const { return tree_.begin(); } + const_iterator cbegin() const { return tree_.begin(); } + iterator end() { return tree_.end(); } + const_iterator end() const { return tree_.end(); } + const_iterator cend() const { return tree_.end(); } + reverse_iterator rbegin() { return tree_.rbegin(); } + const_reverse_iterator rbegin() const { return tree_.rbegin(); } + const_reverse_iterator crbegin() const { return tree_.rbegin(); } + reverse_iterator rend() { return tree_.rend(); } + const_reverse_iterator rend() const { return tree_.rend(); } + const_reverse_iterator crend() const { return tree_.rend(); } + + // Lookup routines. + template + iterator find(const key_arg& key) { + return tree_.find(key); + } + template + const_iterator find(const key_arg& key) const { + return tree_.find(key); + } + + template + bool contains(const key_arg& key) const { + return find(key) != end(); + } + + template + iterator lower_bound(const key_arg& key) { + return tree_.lower_bound(key); + } + + template + const_iterator lower_bound(const key_arg& key) const { + return tree_.lower_bound(key); + } + + template + iterator upper_bound(const key_arg& key) { + return tree_.upper_bound(key); + } + + template + const_iterator upper_bound(const key_arg& key) const { + return tree_.upper_bound(key); + } + + template + std::pair equal_range(const key_arg& key) { + return tree_.equal_range(key); + } + + template + std::pair equal_range(const key_arg& key) const { + return tree_.equal_range(key); + } + + iterator erase(const_iterator iter) { return tree_.erase(iterator(iter)); } + iterator erase(iterator iter) { return tree_.erase(iter); } + iterator erase(const_iterator first, const_iterator last) { + return tree_.erase(iterator(first), iterator(last)).second; + } + + node_type extract(iterator position) { + // Use Move instead of Transfer, because the rebalancing code expects to + // have a valid object to scribble metadata bits on top of. + auto node = CommonAccess::Move(get_allocator(), position.slot()); + erase(position); + return node; + } + + node_type extract(const_iterator position) { return extract(iterator(position)); } + + public: + void clear() { tree_.clear(); } + void swap(btree_container& x) { tree_.swap(x.tree_); } + void verify() const { tree_.verify(); } + + size_type size() const { return tree_.size(); } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { return tree_.empty(); } + + friend bool operator==(const btree_container& x, const btree_container& y) { + if (x.size() != y.size()) return false; + return std::equal(x.begin(), x.end(), y.begin()); + } + + friend bool operator!=(const btree_container& x, const btree_container& y) { + return !(x == y); + } + + friend bool operator<(const btree_container& x, const btree_container& y) { + return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end()); + } + + friend bool operator>(const btree_container& x, const btree_container& y) { + return y < x; + } + + friend bool operator<=(const btree_container& x, const btree_container& y) { + return !(y < x); + } + + friend bool operator>=(const btree_container& x, const btree_container& y) { + return !(x < y); + } + + // The allocator used by the btree. + allocator_type get_allocator() const { return tree_.get_allocator(); } + + // The key comparator used by the btree. + key_compare key_comp() const { return tree_.key_comp(); } + value_compare value_comp() const { return tree_.value_comp(); } + + // Support absl::Hash. + template + friend State AbslHashValue(State h, const btree_container& b) { + for (const auto& v : b) { + h = State::combine(std::move(h), v); + } + return State::combine(std::move(h), b.size()); + } + + protected: + Tree tree_; +}; + +// A common base class for btree_set and btree_map. +// ----------------------------------------------- +template +class btree_set_container : public btree_container { + using super_type = btree_container; + using params_type = typename Tree::params_type; + using init_type = typename params_type::init_type; + using is_key_compare_to = typename params_type::is_key_compare_to; + friend class BtreeNodePeer; + + protected: + template + using key_arg = typename super_type::template key_arg; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using node_type = typename super_type::node_type; + using insert_return_type = InsertReturnType; + using super_type::super_type; + btree_set_container() {} + + template + btree_set_container(InputIterator b, InputIterator e, + const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + btree_set_container(std::initializer_list init, + const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : btree_set_container(init.begin(), init.end(), comp, alloc) {} + + // Lookup routines. + template + size_type count(const key_arg& key) const { + return this->tree_.count_unique(key); + } + + // Insertion routines. + std::pair insert(const value_type& x) { + return this->tree_.insert_unique(params_type::key(x), x); + } + std::pair insert(value_type&& x) { + return this->tree_.insert_unique(params_type::key(x), std::move(x)); + } + template + std::pair emplace(Args&&... args) { + init_type v(std::forward(args)...); + return this->tree_.insert_unique(params_type::key(v), std::move(v)); + } + iterator insert(const_iterator position, const value_type& x) { + return this->tree_.insert_hint_unique(iterator(position), params_type::key(x), x) + .first; + } + iterator insert(const_iterator position, value_type&& x) { + return this->tree_ + .insert_hint_unique(iterator(position), params_type::key(x), std::move(x)) + .first; + } + + template + iterator emplace_hint(const_iterator position, Args&&... args) { + init_type v(std::forward(args)...); + return this->tree_ + .insert_hint_unique(iterator(position), params_type::key(v), std::move(v)) + .first; + } + + template + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_iterator_unique(b, e); + } + + void insert(std::initializer_list init) { + this->tree_.insert_iterator_unique(init.begin(), init.end()); + } + + insert_return_type insert(node_type&& node) { + if (!node) return {this->end(), false, node_type()}; + std::pair res = this->tree_.insert_unique( + params_type::key(CommonAccess::GetSlot(node)), CommonAccess::GetSlot(node)); + if (res.second) { + CommonAccess::Destroy(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + + iterator insert(const_iterator hint, node_type&& node) { + if (!node) return this->end(); + std::pair res = this->tree_.insert_hint_unique( + iterator(hint), params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + if (res.second) CommonAccess::Destroy(&node); + return res.first; + } + + template + size_type erase(const key_arg& key) { + return this->tree_.erase_unique(key); + } + using super_type::erase; + + template + node_type extract(const key_arg& key) { + auto it = this->find(key); + return it == this->end() ? node_type() : extract(it); + } + + using super_type::extract; + + // Merge routines. + // Moves elements from `src` into `this`. If the element already exists in + // `this`, it is left unmodified in `src`. + template , @@ -3543,18 +3430,17 @@ namespace priv { std::is_same>::value, int> = 0> - void merge(btree_container &src) { // NOLINT - for (auto src_it = src.begin(); src_it != src.end();) { - if (insert(std::move(*src_it)).second) { - src_it = src.erase(src_it); - } else { - ++src_it; - } - } - } + void merge(btree_container& src) { // NOLINT + for (auto src_it = src.begin(); src_it != src.end();) { + if (insert(std::move(*src_it)).second) { + src_it = src.erase(src_it); + } else { + ++src_it; + } + } + } - template < - typename T, + template , @@ -3562,207 +3448,197 @@ namespace priv { std::is_same>::value, int> = 0> - void merge(btree_container &&src) { - merge(src); - } - }; - - // Base class for btree_map. - // ------------------------- - template - class btree_map_container : public btree_set_container { - using super_type = btree_set_container; - using params_type = typename Tree::params_type; - - protected: - template - using key_arg = typename super_type::template key_arg; - - public: - using key_type = typename Tree::key_type; - using mapped_type = typename params_type::mapped_type; - using value_type = typename Tree::value_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - - // Inherit constructors. - using super_type::super_type; - btree_map_container() {} - - // Insertion routines. - template - std::pair try_emplace(const key_type &k, Args &&... args) { - return this->tree_.insert_unique( - k, std::piecewise_construct, std::forward_as_tuple(k), - std::forward_as_tuple(std::forward(args)...)); - } - template - std::pair try_emplace(key_type &&k, Args &&... args) { - // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` - // and then using `k` unsequenced. This is safe because the move is into a - // forwarding reference and insert_unique guarantees that `key` is never - // referenced after consuming `args`. - const key_type& key_ref = k; - return this->tree_.insert_unique( - key_ref, std::piecewise_construct, std::forward_as_tuple(std::move(k)), - std::forward_as_tuple(std::forward(args)...)); - } - template - iterator try_emplace(const_iterator hint, const key_type &k, - Args &&... args) { - return this->tree_ - .insert_hint_unique(iterator(hint), k, std::piecewise_construct, - std::forward_as_tuple(k), - std::forward_as_tuple(std::forward(args)...)) - .first; - } - template - iterator try_emplace(const_iterator hint, key_type &&k, Args &&... args) { - // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` - // and then using `k` unsequenced. This is safe because the move is into a - // forwarding reference and insert_hint_unique guarantees that `key` is - // never referenced after consuming `args`. - const key_type& key_ref = k; - return this->tree_ - .insert_hint_unique(iterator(hint), key_ref, std::piecewise_construct, - std::forward_as_tuple(std::move(k)), - std::forward_as_tuple(std::forward(args)...)) - .first; - } - mapped_type &operator[](const key_type &k) { - return try_emplace(k).first->second; - } - mapped_type &operator[](key_type &&k) { - return try_emplace(std::move(k)).first->second; - } - - template - mapped_type &at(const key_arg &key) { - auto it = this->find(key); - if (it == this->end()) - base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); - return it->second; - } - template - const mapped_type &at(const key_arg &key) const { - auto it = this->find(key); - if (it == this->end()) - base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); - return it->second; - } - }; - - // A common base class for btree_multiset and btree_multimap. - template - class btree_multiset_container : public btree_container { - using super_type = btree_container; - using params_type = typename Tree::params_type; - using init_type = typename params_type::init_type; - using is_key_compare_to = typename params_type::is_key_compare_to; - - template - using key_arg = typename super_type::template key_arg; - - public: - using key_type = typename Tree::key_type; - using value_type = typename Tree::value_type; - using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; - using allocator_type = typename Tree::allocator_type; - using iterator = typename Tree::iterator; - using const_iterator = typename Tree::const_iterator; - using node_type = typename super_type::node_type; - - // Inherit constructors. - using super_type::super_type; - btree_multiset_container() {} - - // Range constructor. - template - btree_multiset_container(InputIterator b, InputIterator e, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); - } - - // Initializer list constructor. - btree_multiset_container(std::initializer_list init, - const key_compare &comp = key_compare(), - const allocator_type &alloc = allocator_type()) - : btree_multiset_container(init.begin(), init.end(), comp, alloc) {} - - // Lookup routines. - template - size_type count(const key_arg &key) const { - return this->tree_.count_multi(key); - } - - // Insertion routines. - iterator insert(const value_type &x) { return this->tree_.insert_multi(x); } - iterator insert(value_type &&x) { - return this->tree_.insert_multi(std::move(x)); - } - iterator insert(const_iterator position, const value_type &x) { - return this->tree_.insert_hint_multi(iterator(position), x); - } - iterator insert(const_iterator position, value_type &&x) { - return this->tree_.insert_hint_multi(iterator(position), std::move(x)); - } - template - void insert(InputIterator b, InputIterator e) { - this->tree_.insert_iterator_multi(b, e); - } - void insert(std::initializer_list init) { - this->tree_.insert_iterator_multi(init.begin(), init.end()); - } - template - iterator emplace(Args &&... args) { - return this->tree_.insert_multi(init_type(std::forward(args)...)); - } - template - iterator emplace_hint(const_iterator position, Args &&... args) { - return this->tree_.insert_hint_multi( - iterator(position), init_type(std::forward(args)...)); - } - iterator insert(node_type &&node) { - if (!node) return this->end(); - iterator res = - this->tree_.insert_multi(params_type::key(CommonAccess::GetSlot(node)), - CommonAccess::GetSlot(node)); - CommonAccess::Destroy(&node); - return res; - } - iterator insert(const_iterator hint, node_type &&node) { - if (!node) return this->end(); - iterator res = this->tree_.insert_hint_multi( - iterator(hint), - std::move(params_type::element(CommonAccess::GetSlot(node)))); - CommonAccess::Destroy(&node); - return res; - } - - // Deletion routines. - template - size_type erase(const key_arg &key) { - return this->tree_.erase_multi(key); - } - using super_type::erase; - - // Node extraction routines. - template - node_type extract(const key_arg &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); - } - using super_type::extract; - - // Merge routines. - // Moves all elements from `src` into `this`. - template < - typename T, + void merge(btree_container&& src) { + merge(src); + } +}; + +// Base class for btree_map. +// ------------------------- +template +class btree_map_container : public btree_set_container { + using super_type = btree_set_container; + using params_type = typename Tree::params_type; + + protected: + template + using key_arg = typename super_type::template key_arg; + + public: + using key_type = typename Tree::key_type; + using mapped_type = typename params_type::mapped_type; + using value_type = typename Tree::value_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + + // Inherit constructors. + using super_type::super_type; + btree_map_container() {} + + // Insertion routines. + template + std::pair try_emplace(const key_type& k, Args&&... args) { + return this->tree_.insert_unique(k, std::piecewise_construct, + std::forward_as_tuple(k), + std::forward_as_tuple(std::forward(args)...)); + } + template + std::pair try_emplace(key_type&& k, Args&&... args) { + // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` + // and then using `k` unsequenced. This is safe because the move is into a + // forwarding reference and insert_unique guarantees that `key` is never + // referenced after consuming `args`. + const key_type& key_ref = k; + return this->tree_.insert_unique(key_ref, std::piecewise_construct, + std::forward_as_tuple(std::move(k)), + std::forward_as_tuple(std::forward(args)...)); + } + template + iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args) { + return this->tree_ + .insert_hint_unique(iterator(hint), k, std::piecewise_construct, + std::forward_as_tuple(k), + std::forward_as_tuple(std::forward(args)...)) + .first; + } + template + iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args) { + // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` + // and then using `k` unsequenced. This is safe because the move is into a + // forwarding reference and insert_hint_unique guarantees that `key` is + // never referenced after consuming `args`. + const key_type& key_ref = k; + return this->tree_ + .insert_hint_unique(iterator(hint), key_ref, std::piecewise_construct, + std::forward_as_tuple(std::move(k)), + std::forward_as_tuple(std::forward(args)...)) + .first; + } + mapped_type& operator[](const key_type& k) { return try_emplace(k).first->second; } + mapped_type& operator[](key_type&& k) { + return try_emplace(std::move(k)).first->second; + } + + template + mapped_type& at(const key_arg& key) { + auto it = this->find(key); + if (it == this->end()) base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); + return it->second; + } + template + const mapped_type& at(const key_arg& key) const { + auto it = this->find(key); + if (it == this->end()) base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); + return it->second; + } +}; + +// A common base class for btree_multiset and btree_multimap. +template +class btree_multiset_container : public btree_container { + using super_type = btree_container; + using params_type = typename Tree::params_type; + using init_type = typename params_type::init_type; + using is_key_compare_to = typename params_type::is_key_compare_to; + + template + using key_arg = typename super_type::template key_arg; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using node_type = typename super_type::node_type; + + // Inherit constructors. + using super_type::super_type; + btree_multiset_container() {} + + // Range constructor. + template + btree_multiset_container(InputIterator b, InputIterator e, + const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + // Initializer list constructor. + btree_multiset_container(std::initializer_list init, + const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : btree_multiset_container(init.begin(), init.end(), comp, alloc) {} + + // Lookup routines. + template + size_type count(const key_arg& key) const { + return this->tree_.count_multi(key); + } + + // Insertion routines. + iterator insert(const value_type& x) { return this->tree_.insert_multi(x); } + iterator insert(value_type&& x) { return this->tree_.insert_multi(std::move(x)); } + iterator insert(const_iterator position, const value_type& x) { + return this->tree_.insert_hint_multi(iterator(position), x); + } + iterator insert(const_iterator position, value_type&& x) { + return this->tree_.insert_hint_multi(iterator(position), std::move(x)); + } + template + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_iterator_multi(b, e); + } + void insert(std::initializer_list init) { + this->tree_.insert_iterator_multi(init.begin(), init.end()); + } + template + iterator emplace(Args&&... args) { + return this->tree_.insert_multi(init_type(std::forward(args)...)); + } + template + iterator emplace_hint(const_iterator position, Args&&... args) { + return this->tree_.insert_hint_multi(iterator(position), + init_type(std::forward(args)...)); + } + iterator insert(node_type&& node) { + if (!node) return this->end(); + iterator res = this->tree_.insert_multi(params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + CommonAccess::Destroy(&node); + return res; + } + iterator insert(const_iterator hint, node_type&& node) { + if (!node) return this->end(); + iterator res = this->tree_.insert_hint_multi( + iterator(hint), std::move(params_type::element(CommonAccess::GetSlot(node)))); + CommonAccess::Destroy(&node); + return res; + } + + // Deletion routines. + template + size_type erase(const key_arg& key) { + return this->tree_.erase_multi(key); + } + using super_type::erase; + + // Node extraction routines. + template + node_type extract(const key_arg& key) { + auto it = this->find(key); + return it == this->end() ? node_type() : extract(it); + } + using super_type::extract; + + // Merge routines. + // Moves all elements from `src` into `this`. + template , @@ -3770,14 +3646,12 @@ namespace priv { std::is_same>::value, int> = 0> - void merge(btree_container &src) { // NOLINT - insert(std::make_move_iterator(src.begin()), - std::make_move_iterator(src.end())); - src.clear(); - } + void merge(btree_container& src) { // NOLINT + insert(std::make_move_iterator(src.begin()), std::make_move_iterator(src.end())); + src.clear(); + } - template < - typename T, + template , @@ -3785,266 +3659,256 @@ namespace priv { std::is_same>::value, int> = 0> - void merge(btree_container &&src) { - merge(src); - } - }; + void merge(btree_container&& src) { + merge(src); + } +}; - // A base class for btree_multimap. - template - class btree_multimap_container : public btree_multiset_container { - using super_type = btree_multiset_container; - using params_type = typename Tree::params_type; +// A base class for btree_multimap. +template +class btree_multimap_container : public btree_multiset_container { + using super_type = btree_multiset_container; + using params_type = typename Tree::params_type; - public: - using mapped_type = typename params_type::mapped_type; + public: + using mapped_type = typename params_type::mapped_type; - // Inherit constructors. - using super_type::super_type; - btree_multimap_container() {} - }; + // Inherit constructors. + using super_type::super_type; + btree_multimap_container() {} +}; } // namespace priv +// ---------------------------------------------------------------------- +// btree_set - default values in phmap_fwd_decl.h +// ---------------------------------------------------------------------- +template +class btree_set : public priv::btree_set_container>> { + using Base = typename btree_set::btree_set_container; + + public: + btree_set() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::get_allocator; + using Base::insert; + using Base::key_comp; + using Base::max_size; + using Base::merge; + using Base::size; + using Base::swap; + using Base::value_comp; +}; + +// Swaps the contents of two `phmap::btree_set` containers. +// ------------------------------------------------------- +template +void swap(btree_set& x, btree_set& y) { + return x.swap(y); +} - - // ---------------------------------------------------------------------- - // btree_set - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template - class btree_set : public priv::btree_set_container< - priv::btree>> - { - using Base = typename btree_set::btree_set_container; - - public: - btree_set() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_set` containers. - // ------------------------------------------------------- - template - void swap(btree_set &x, btree_set &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template - void erase_if(btree_set &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } +// Erases all elements that satisfy the predicate pred from the container. +// ---------------------------------------------------------------------- +template +void erase_if(btree_set& set, Pred pred) { + for (auto it = set.begin(); it != set.end();) { + if (pred(*it)) { + it = set.erase(it); + } else { + ++it; } + } +} - // ---------------------------------------------------------------------- - // btree_multiset - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template - class btree_multiset : public priv::btree_multiset_container< - priv::btree>> - { - using Base = typename btree_multiset::btree_multiset_container; - - public: - btree_multiset() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_multiset` containers. - // ------------------------------------------------------------ - template - void swap(btree_multiset &x, btree_multiset &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template - void erase_if(btree_multiset &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } +// ---------------------------------------------------------------------- +// btree_multiset - default values in phmap_fwd_decl.h +// ---------------------------------------------------------------------- +template +class btree_multiset + : public priv::btree_multiset_container>> { + using Base = typename btree_multiset::btree_multiset_container; + + public: + btree_multiset() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::get_allocator; + using Base::insert; + using Base::key_comp; + using Base::max_size; + using Base::merge; + using Base::size; + using Base::swap; + using Base::value_comp; +}; + +// Swaps the contents of two `phmap::btree_multiset` containers. +// ------------------------------------------------------------ +template +void swap(btree_multiset& x, btree_multiset& y) { + return x.swap(y); +} + +// Erases all elements that satisfy the predicate pred from the container. +// ---------------------------------------------------------------------- +template +void erase_if(btree_multiset& set, Pred pred) { + for (auto it = set.begin(); it != set.end();) { + if (pred(*it)) { + it = set.erase(it); + } else { + ++it; } + } +} +// ---------------------------------------------------------------------- +// btree_map - default values in phmap_fwd_decl.h +// ---------------------------------------------------------------------- +template +class btree_map + : public priv::btree_map_container>> { + using Base = typename btree_map::btree_map_container; + + public: + btree_map() {} + using Base::at; + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::insert; + using Base::max_size; + using Base::merge; + using Base::size; + using Base::swap; + using Base::try_emplace; + using Base::operator[]; + using Base::get_allocator; + using Base::key_comp; + using Base::value_comp; +}; + +// Swaps the contents of two `phmap::btree_map` containers. +// ------------------------------------------------------- +template +void swap(btree_map& x, btree_map& y) { + return x.swap(y); +} - // ---------------------------------------------------------------------- - // btree_map - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template - class btree_map : public priv::btree_map_container< - priv::btree>> - { - using Base = typename btree_map::btree_map_container; - - public: - btree_map() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_map` containers. - // ------------------------------------------------------- - template - void swap(btree_map &x, btree_map &y) { - return x.swap(y); - } - - // ---------------------------------------------------------------------- - template - void erase_if(btree_map &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } +// ---------------------------------------------------------------------- +template +void erase_if(btree_map& map, Pred pred) { + for (auto it = map.begin(); it != map.end();) { + if (pred(*it)) { + it = map.erase(it); + } else { + ++it; } + } +} - // ---------------------------------------------------------------------- - // btree_multimap - default values in phmap_fwd_decl.h - // ---------------------------------------------------------------------- - template - class btree_multimap : public priv::btree_multimap_container< - priv::btree>> - { - using Base = typename btree_multimap::btree_multimap_container; - - public: - btree_multimap() {} - using Base::Base; - using Base::begin; - using Base::cbegin; - using Base::end; - using Base::cend; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::get_allocator; - using Base::key_comp; - using Base::value_comp; - }; - - // Swaps the contents of two `phmap::btree_multimap` containers. - // ------------------------------------------------------------ - template - void swap(btree_multimap &x, btree_multimap &y) { - return x.swap(y); - } - - // Erases all elements that satisfy the predicate pred from the container. - // ---------------------------------------------------------------------- - template - void erase_if(btree_multimap &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } - } +// ---------------------------------------------------------------------- +// btree_multimap - default values in phmap_fwd_decl.h +// ---------------------------------------------------------------------- +template +class btree_multimap + : public priv::btree_multimap_container>> { + using Base = typename btree_multimap::btree_multimap_container; + + public: + btree_multimap() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::get_allocator; + using Base::insert; + using Base::key_comp; + using Base::max_size; + using Base::merge; + using Base::size; + using Base::swap; + using Base::value_comp; +}; + +// Swaps the contents of two `phmap::btree_multimap` containers. +// ------------------------------------------------------------ +template +void swap(btree_multimap& x, btree_multimap& y) { + return x.swap(y); +} +// Erases all elements that satisfy the predicate pred from the container. +// ---------------------------------------------------------------------- +template +void erase_if(btree_multimap& map, Pred pred) { + for (auto it = map.begin(); it != map.end();) { + if (pred(*it)) { + it = map.erase(it); + } else { + ++it; + } + } +} -} // namespace btree +} // namespace phmap #ifdef _MSC_VER - #pragma warning(pop) +#pragma warning(pop) #endif - #endif // PHMAP_BTREE_BTREE_CONTAINER_H_ diff --git a/native-sql-engine/cpp/src/third_party/parallel_hashmap/meminfo.h b/native-sql-engine/cpp/src/third_party/parallel_hashmap/meminfo.h index 872f3c698..47feb26cd 100644 --- a/native-sql-engine/cpp/src/third_party/parallel_hashmap/meminfo.h +++ b/native-sql-engine/cpp/src/third_party/parallel_hashmap/meminfo.h @@ -2,194 +2,185 @@ #define spp_memory_h_guard #include -#include #include +#include -#if defined(_WIN32) || defined( __CYGWIN__) - #define SPP_WIN +#if defined(_WIN32) || defined(__CYGWIN__) +#define SPP_WIN #endif #ifdef SPP_WIN - #include - #include - #undef min - #undef max +#include +#include +#undef min +#undef max #elif defined(__linux__) - #include - #include +#include +#include #elif defined(__FreeBSD__) - #include - #include - #include - #include - #include - #include +#include +#include +#include +#include +#include +#include #endif -namespace spp -{ - uint64_t GetSystemMemory(); - uint64_t GetTotalMemoryUsed(); - uint64_t GetProcessMemoryUsed(); - uint64_t GetPhysicalMemory(); +namespace spp { +uint64_t GetSystemMemory(); +uint64_t GetTotalMemoryUsed(); +uint64_t GetProcessMemoryUsed(); +uint64_t GetPhysicalMemory(); - uint64_t GetSystemMemory() - { +uint64_t GetSystemMemory() { #ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast(memInfo.ullTotalPageFile); + MEMORYSTATUSEX memInfo; + memInfo.dwLength = sizeof(MEMORYSTATUSEX); + GlobalMemoryStatusEx(&memInfo); + return static_cast(memInfo.ullTotalPageFile); #elif defined(__linux__) - struct sysinfo memInfo; - sysinfo (&memInfo); - auto totalVirtualMem = memInfo.totalram; + struct sysinfo memInfo; + sysinfo(&memInfo); + auto totalVirtualMem = memInfo.totalram; - totalVirtualMem += memInfo.totalswap; - totalVirtualMem *= memInfo.mem_unit; - return static_cast(totalVirtualMem); + totalVirtualMem += memInfo.totalswap; + totalVirtualMem *= memInfo.mem_unit; + return static_cast(totalVirtualMem); #elif defined(__FreeBSD__) - kvm_t *kd; - u_int pageCnt; - size_t pageCntLen = sizeof(pageCnt); - u_int pageSize; - struct kvm_swap kswap; - uint64_t totalVirtualMem; + kvm_t* kd; + u_int pageCnt; + size_t pageCntLen = sizeof(pageCnt); + u_int pageSize; + struct kvm_swap kswap; + uint64_t totalVirtualMem; - pageSize = static_cast(getpagesize()); + pageSize = static_cast(getpagesize()); - sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); - totalVirtualMem = pageCnt * pageSize; + sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); + totalVirtualMem = pageCnt * pageSize; - kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); - kvm_getswapinfo(kd, &kswap, 1, 0); - kvm_close(kd); - totalVirtualMem += kswap.ksw_total * pageSize; + kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); + kvm_getswapinfo(kd, &kswap, 1, 0); + kvm_close(kd); + totalVirtualMem += kswap.ksw_total * pageSize; - return totalVirtualMem; + return totalVirtualMem; #else - return 0; + return 0; #endif - } +} - uint64_t GetTotalMemoryUsed() - { +uint64_t GetTotalMemoryUsed() { #ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast(memInfo.ullTotalPageFile - memInfo.ullAvailPageFile); + MEMORYSTATUSEX memInfo; + memInfo.dwLength = sizeof(MEMORYSTATUSEX); + GlobalMemoryStatusEx(&memInfo); + return static_cast(memInfo.ullTotalPageFile - memInfo.ullAvailPageFile); #elif defined(__linux__) - struct sysinfo memInfo; - sysinfo(&memInfo); - auto virtualMemUsed = memInfo.totalram - memInfo.freeram; + struct sysinfo memInfo; + sysinfo(&memInfo); + auto virtualMemUsed = memInfo.totalram - memInfo.freeram; - virtualMemUsed += memInfo.totalswap - memInfo.freeswap; - virtualMemUsed *= memInfo.mem_unit; + virtualMemUsed += memInfo.totalswap - memInfo.freeswap; + virtualMemUsed *= memInfo.mem_unit; - return static_cast(virtualMemUsed); + return static_cast(virtualMemUsed); #elif defined(__FreeBSD__) - kvm_t *kd; - u_int pageSize; - u_int pageCnt, freeCnt; - size_t pageCntLen = sizeof(pageCnt); - size_t freeCntLen = sizeof(freeCnt); - struct kvm_swap kswap; - uint64_t virtualMemUsed; - - pageSize = static_cast(getpagesize()); - - sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); - sysctlbyname("vm.stats.vm.v_free_count", &freeCnt, &freeCntLen, NULL, 0); - virtualMemUsed = (pageCnt - freeCnt) * pageSize; - - kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); - kvm_getswapinfo(kd, &kswap, 1, 0); - kvm_close(kd); - virtualMemUsed += kswap.ksw_used * pageSize; - - return virtualMemUsed; + kvm_t* kd; + u_int pageSize; + u_int pageCnt, freeCnt; + size_t pageCntLen = sizeof(pageCnt); + size_t freeCntLen = sizeof(freeCnt); + struct kvm_swap kswap; + uint64_t virtualMemUsed; + + pageSize = static_cast(getpagesize()); + + sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); + sysctlbyname("vm.stats.vm.v_free_count", &freeCnt, &freeCntLen, NULL, 0); + virtualMemUsed = (pageCnt - freeCnt) * pageSize; + + kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); + kvm_getswapinfo(kd, &kswap, 1, 0); + kvm_close(kd); + virtualMemUsed += kswap.ksw_used * pageSize; + + return virtualMemUsed; #else - return 0; + return 0; #endif - } +} - uint64_t GetProcessMemoryUsed() - { +uint64_t GetProcessMemoryUsed() { #ifdef SPP_WIN - PROCESS_MEMORY_COUNTERS_EX pmc; - GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast(&pmc), sizeof(pmc)); - return static_cast(pmc.PrivateUsage); + PROCESS_MEMORY_COUNTERS_EX pmc; + GetProcessMemoryInfo(GetCurrentProcess(), + reinterpret_cast(&pmc), sizeof(pmc)); + return static_cast(pmc.PrivateUsage); #elif defined(__linux__) - auto parseLine = - [](char* line)->int - { - auto i = strlen(line); - - while(*line < '0' || *line > '9') - { - line++; - } - - line[i-3] = '\0'; - i = atoi(line); - return i; - }; - - auto file = fopen("/proc/self/status", "r"); - auto result = -1; - char line[128]; - - while(fgets(line, 128, file) != nullptr) - { - if(strncmp(line, "VmSize:", 7) == 0) - { - result = parseLine(line); - break; - } - } - - fclose(file); - return static_cast(result) * 1024; + auto parseLine = [](char* line) -> int { + auto i = strlen(line); + + while (*line < '0' || *line > '9') { + line++; + } + + line[i - 3] = '\0'; + i = atoi(line); + return i; + }; + + auto file = fopen("/proc/self/status", "r"); + auto result = -1; + char line[128]; + + while (fgets(line, 128, file) != nullptr) { + if (strncmp(line, "VmSize:", 7) == 0) { + result = parseLine(line); + break; + } + } + + fclose(file); + return static_cast(result) * 1024; #elif defined(__FreeBSD__) - struct kinfo_proc info; - size_t infoLen = sizeof(info); - int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid() }; + struct kinfo_proc info; + size_t infoLen = sizeof(info); + int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()}; - sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &infoLen, NULL, 0); - return static_cast(info.ki_rssize * getpagesize()); + sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &infoLen, NULL, 0); + return static_cast(info.ki_rssize * getpagesize()); #else - return 0; + return 0; #endif - } +} - uint64_t GetPhysicalMemory() - { +uint64_t GetPhysicalMemory() { #ifdef SPP_WIN - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - GlobalMemoryStatusEx(&memInfo); - return static_cast(memInfo.ullTotalPhys); + MEMORYSTATUSEX memInfo; + memInfo.dwLength = sizeof(MEMORYSTATUSEX); + GlobalMemoryStatusEx(&memInfo); + return static_cast(memInfo.ullTotalPhys); #elif defined(__linux__) - struct sysinfo memInfo; - sysinfo(&memInfo); + struct sysinfo memInfo; + sysinfo(&memInfo); - auto totalPhysMem = memInfo.totalram; + auto totalPhysMem = memInfo.totalram; - totalPhysMem *= memInfo.mem_unit; - return static_cast(totalPhysMem); + totalPhysMem *= memInfo.mem_unit; + return static_cast(totalPhysMem); #elif defined(__FreeBSD__) - u_long physMem; - size_t physMemLen = sizeof(physMem); - int mib[] = { CTL_HW, HW_PHYSMEM }; + u_long physMem; + size_t physMemLen = sizeof(physMem); + int mib[] = {CTL_HW, HW_PHYSMEM}; - sysctl(mib, sizeof(mib) / sizeof(*mib), &physMem, &physMemLen, NULL, 0); - return physMem; + sysctl(mib, sizeof(mib) / sizeof(*mib), &physMem, &physMemLen, NULL, 0); + return physMem; #else - return 0; + return 0; #endif - } - } -#endif // spp_memory_h_guard +} // namespace spp + +#endif // spp_memory_h_guard diff --git a/native-sql-engine/cpp/src/third_party/parallel_hashmap/phmap.h b/native-sql-engine/cpp/src/third_party/parallel_hashmap/phmap.h index f8de2394a..4628cca30 100644 --- a/native-sql-engine/cpp/src/third_party/parallel_hashmap/phmap.h +++ b/native-sql-engine/cpp/src/third_party/parallel_hashmap/phmap.h @@ -18,7 +18,7 @@ // // Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) // with modifications. -// +// // Copyright 2018 The Abseil Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,23 +35,29 @@ // --------------------------------------------------------------------------- #ifdef _MSC_VER - #pragma warning(push) - - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4324) // structure was padded due to alignment specifier - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member - #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list - #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted - #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified +#pragma warning(push) + +#pragma warning(disable : 4127) // conditional expression is constant +#pragma warning(disable : 4324) // structure was padded due to alignment specifier +#pragma warning(disable : 4514) // unreferenced inline function has been removed +#pragma warning(disable : 4623) // default constructor was implicitly defined as deleted +#pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted +#pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted +#pragma warning(disable : 4710) // function not inlined +#pragma warning(disable : 4711) // selected for automatic inline expansion +#pragma warning(disable : 4820) // '6' bytes padding added after data member +#pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation + // order in braced initializer list +#pragma warning( \ + disable : 5027) // move assignment operator was implicitly defined as deleted +#pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory + // load if /Qspectre switch specified #endif #include +#include +#include +#include #include #include #include @@ -60,16 +66,13 @@ #include #include #include -#include -#include -#include +#include "phmap_base.h" #include "phmap_fwd_decl.h" #include "phmap_utils.h" -#include "phmap_base.h" #if PHMAP_HAVE_STD_STRING_VIEW - #include +#include #endif namespace phmap { @@ -78,41 +81,38 @@ namespace priv { // -------------------------------------------------------------------------- template -class probe_seq -{ -public: - probe_seq(size_t hashval, size_t mask) { - assert(((mask + 1) & mask) == 0 && "not a mask"); - mask_ = mask; - offset_ = hashval & mask_; - } - size_t offset() const { return offset_; } - size_t offset(size_t i) const { return (offset_ + i) & mask_; } +class probe_seq { + public: + probe_seq(size_t hashval, size_t mask) { + assert(((mask + 1) & mask) == 0 && "not a mask"); + mask_ = mask; + offset_ = hashval & mask_; + } + size_t offset() const { return offset_; } + size_t offset(size_t i) const { return (offset_ + i) & mask_; } - void next() { - index_ += Width; - offset_ += index_; - offset_ &= mask_; - } - // 0-based probe index. The i-th probe in the probe sequence. - size_t getindex() const { return index_; } + void next() { + index_ += Width; + offset_ += index_; + offset_ &= mask_; + } + // 0-based probe index. The i-th probe in the probe sequence. + size_t getindex() const { return index_; } -private: - size_t mask_; - size_t offset_; - size_t index_ = 0; + private: + size_t mask_; + size_t offset_; + size_t index_ = 0; }; // -------------------------------------------------------------------------- template -struct RequireUsableKey -{ - template - std::pair< - decltype(std::declval()(std::declval())), - decltype(std::declval()(std::declval(), - std::declval()))>* - operator()(const PassedKey&, const Args&...) const; +struct RequireUsableKey { + template + std::pair()(std::declval())), + decltype(std::declval()(std::declval(), + std::declval()))>* + operator()(const PassedKey&, const Args&...) const; }; // -------------------------------------------------------------------------- @@ -121,35 +121,32 @@ struct IsDecomposable : std::false_type {}; template struct IsDecomposable< - phmap::void_t(), - std::declval()...))>, + phmap::void_t(), std::declval()...))>, Policy, Hash, Eq, Ts...> : std::true_type {}; // TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. // -------------------------------------------------------------------------- template constexpr bool IsNoThrowSwappable() { - using std::swap; - return noexcept(swap(std::declval(), std::declval())); + using std::swap; + return noexcept(swap(std::declval(), std::declval())); } // -------------------------------------------------------------------------- template int TrailingZeros(T x) { - PHMAP_IF_CONSTEXPR(sizeof(T) == 8) - return base_internal::CountTrailingZerosNonZero64(static_cast(x)); - else - return base_internal::CountTrailingZerosNonZero32(static_cast(x)); + PHMAP_IF_CONSTEXPR(sizeof(T) == 8) + return base_internal::CountTrailingZerosNonZero64(static_cast(x)); + else return base_internal::CountTrailingZerosNonZero32(static_cast(x)); } // -------------------------------------------------------------------------- template int LeadingZeros(T x) { - PHMAP_IF_CONSTEXPR(sizeof(T) == 8) - return base_internal::CountLeadingZeros64(static_cast(x)); - else - return base_internal::CountLeadingZeros32(static_cast(x)); + PHMAP_IF_CONSTEXPR(sizeof(T) == 8) + return base_internal::CountLeadingZeros64(static_cast(x)); + else return base_internal::CountLeadingZeros32(static_cast(x)); } // -------------------------------------------------------------------------- @@ -164,55 +161,48 @@ int LeadingZeros(T x) { // for (int i : BitMask(0x0000000080800000)) -> yields 2, 3 // -------------------------------------------------------------------------- template -class BitMask -{ - static_assert(std::is_unsigned::value, ""); - static_assert(Shift == 0 || Shift == 3, ""); - -public: - // These are useful for unit tests (gunit). - using value_type = int; - using iterator = BitMask; - using const_iterator = BitMask; - - explicit BitMask(T mask) : mask_(mask) {} - BitMask& operator++() { - mask_ &= (mask_ - 1); - return *this; - } - explicit operator bool() const { return mask_ != 0; } - int operator*() const { return LowestBitSet(); } - int LowestBitSet() const { - return priv::TrailingZeros(mask_) >> Shift; - } - int HighestBitSet() const { - return (sizeof(T) * CHAR_BIT - priv::LeadingZeros(mask_) - - 1) >> - Shift; - } +class BitMask { + static_assert(std::is_unsigned::value, ""); + static_assert(Shift == 0 || Shift == 3, ""); + + public: + // These are useful for unit tests (gunit). + using value_type = int; + using iterator = BitMask; + using const_iterator = BitMask; + + explicit BitMask(T mask) : mask_(mask) {} + BitMask& operator++() { + mask_ &= (mask_ - 1); + return *this; + } + explicit operator bool() const { return mask_ != 0; } + int operator*() const { return LowestBitSet(); } + int LowestBitSet() const { return priv::TrailingZeros(mask_) >> Shift; } + int HighestBitSet() const { + return (sizeof(T) * CHAR_BIT - priv::LeadingZeros(mask_) - 1) >> Shift; + } - BitMask begin() const { return *this; } - BitMask end() const { return BitMask(0); } + BitMask begin() const { return *this; } + BitMask end() const { return BitMask(0); } - int TrailingZeros() const { - return priv::TrailingZeros(mask_) >> Shift; - } + int TrailingZeros() const { return priv::TrailingZeros(mask_) >> Shift; } - int LeadingZeros() const { - constexpr int total_significant_bits = SignificantBits << Shift; - constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; - return priv::LeadingZeros(mask_ << extra_bits) >> Shift; - } + int LeadingZeros() const { + constexpr int total_significant_bits = SignificantBits << Shift; + constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; + return priv::LeadingZeros(mask_ << extra_bits) >> Shift; + } -private: - friend bool operator==(const BitMask& a, const BitMask& b) { - return a.mask_ == b.mask_; - } - friend bool operator!=(const BitMask& a, const BitMask& b) { - return a.mask_ != b.mask_; - } + private: + friend bool operator==(const BitMask& a, const BitMask& b) { + return a.mask_ == b.mask_; + } + friend bool operator!=(const BitMask& a, const BitMask& b) { + return a.mask_ != b.mask_; + } - T mask_; + T mask_; }; // -------------------------------------------------------------------------- @@ -223,16 +213,14 @@ using h2_t = uint8_t; // The values here are selected for maximum performance. See the static asserts // below for details. // -------------------------------------------------------------------------- -enum Ctrl : ctrl_t -{ - kEmpty = -128, // 0b10000000 - kDeleted = -2, // 0b11111110 - kSentinel = -1, // 0b11111111 +enum Ctrl : ctrl_t { + kEmpty = -128, // 0b10000000 + kDeleted = -2, // 0b11111110 + kSentinel = -1, // 0b11111111 }; -static_assert( - kEmpty & kDeleted & kSentinel & 0x80, - "Special markers need to have the MSB to make checking for them efficient"); +static_assert(kEmpty & kDeleted & kSentinel & 0x80, + "Special markers need to have the MSB to make checking for them efficient"); static_assert(kEmpty < kSentinel && kDeleted < kSentinel, "kEmpty and kDeleted must be smaller than kSentinel to make the " "SIMD test of IsEmptyOrDeleted() efficient"); @@ -272,32 +260,29 @@ inline size_t HashSeed(const ctrl_t* ctrl) { #ifdef PHMAP_NON_DETERMINISTIC inline size_t H1(size_t hashval, const ctrl_t* ctrl) { - // use ctrl_ pointer to add entropy to ensure - // non-deterministic iteration order. - return (hashval >> 7) ^ HashSeed(ctrl); + // use ctrl_ pointer to add entropy to ensure + // non-deterministic iteration order. + return (hashval >> 7) ^ HashSeed(ctrl); } #else -inline size_t H1(size_t hashval, const ctrl_t* ) { - return (hashval >> 7); -} +inline size_t H1(size_t hashval, const ctrl_t*) { return (hashval >> 7); } #endif +inline ctrl_t H2(size_t hashval) { return (ctrl_t)(hashval & 0x7F); } -inline ctrl_t H2(size_t hashval) { return (ctrl_t)(hashval & 0x7F); } - -inline bool IsEmpty(ctrl_t c) { return c == kEmpty; } -inline bool IsFull(ctrl_t c) { return c >= 0; } -inline bool IsDeleted(ctrl_t c) { return c == kDeleted; } +inline bool IsEmpty(ctrl_t c) { return c == kEmpty; } +inline bool IsFull(ctrl_t c) { return c >= 0; } +inline bool IsDeleted(ctrl_t c) { return c == kDeleted; } inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; } #if PHMAP_HAVE_SSE2 #ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4365) // conversion from 'int' to 'T', signed/unsigned mismatch +#pragma warning(push) +#pragma warning(disable : 4365) // conversion from 'int' to 'T', signed/unsigned mismatch #endif // -------------------------------------------------------------------------- @@ -309,8 +294,8 @@ inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; } // -------------------------------------------------------------------------- inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { #if defined(__GNUC__) && !defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Woverflow" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverflow" if (std::is_unsigned::value) { const __m128i mask = _mm_set1_epi8(static_cast(0x80)); @@ -318,139 +303,133 @@ inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); } - #pragma GCC diagnostic pop +#pragma GCC diagnostic pop #endif return _mm_cmpgt_epi8(a, b); } // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- -struct GroupSse2Impl -{ - enum { kWidth = 16 }; // the number of slots per group +struct GroupSse2Impl { + enum { kWidth = 16 }; // the number of slots per group - explicit GroupSse2Impl(const ctrl_t* pos) { - ctrl = _mm_loadu_si128(reinterpret_cast(pos)); - } + explicit GroupSse2Impl(const ctrl_t* pos) { + ctrl = _mm_loadu_si128(reinterpret_cast(pos)); + } - // Returns a bitmask representing the positions of slots that match hash. - // ---------------------------------------------------------------------- - BitMask Match(h2_t hash) const { - auto match = _mm_set1_epi8((char)hash); - return BitMask( - _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); - } + // Returns a bitmask representing the positions of slots that match hash. + // ---------------------------------------------------------------------- + BitMask Match(h2_t hash) const { + auto match = _mm_set1_epi8((char)hash); + return BitMask(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); + } - // Returns a bitmask representing the positions of empty slots. - // ------------------------------------------------------------ - BitMask MatchEmpty() const { + // Returns a bitmask representing the positions of empty slots. + // ------------------------------------------------------------ + BitMask MatchEmpty() const { #if PHMAP_HAVE_SSSE3 - // This only works because kEmpty is -128. - return BitMask( - _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); + // This only works because kEmpty is -128. + return BitMask(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); #else - return Match(static_cast(kEmpty)); + return Match(static_cast(kEmpty)); #endif - } + } - // Returns a bitmask representing the positions of empty or deleted slots. - // ----------------------------------------------------------------------- - BitMask MatchEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); - return BitMask( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); - } + // Returns a bitmask representing the positions of empty or deleted slots. + // ----------------------------------------------------------------------- + BitMask MatchEmptyOrDeleted() const { + auto special = _mm_set1_epi8(kSentinel); + return BitMask( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); + } - // Returns the number of trailing empty or deleted elements in the group. - // ---------------------------------------------------------------------- - uint32_t CountLeadingEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); - return TrailingZeros( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1); - } + // Returns the number of trailing empty or deleted elements in the group. + // ---------------------------------------------------------------------- + uint32_t CountLeadingEmptyOrDeleted() const { + auto special = _mm_set1_epi8(kSentinel); + return TrailingZeros(_mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1); + } - // ---------------------------------------------------------------------- - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - auto msbs = _mm_set1_epi8(static_cast(-128)); - auto x126 = _mm_set1_epi8(126); + // ---------------------------------------------------------------------- + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + auto msbs = _mm_set1_epi8(static_cast(-128)); + auto x126 = _mm_set1_epi8(126); #if PHMAP_HAVE_SSSE3 - auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); + auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); #else - auto zero = _mm_setzero_si128(); - auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); - auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); + auto zero = _mm_setzero_si128(); + auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); + auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); #endif - _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); - } + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); + } - __m128i ctrl; + __m128i ctrl; }; #ifdef _MSC_VER - #pragma warning(pop) +#pragma warning(pop) #endif #endif // PHMAP_HAVE_SSE2 // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- -struct GroupPortableImpl -{ - enum { kWidth = 8 }; - - explicit GroupPortableImpl(const ctrl_t* pos) - : ctrl(little_endian::Load64(pos)) {} - - BitMask Match(h2_t hash) const { - // For the technique, see: - // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord - // (Determine if a word has a byte equal to n). - // - // Caveat: there are false positives but: - // - they only occur if there is a real match - // - they never occur on kEmpty, kDeleted, kSentinel - // - they will be handled gracefully by subsequent checks in code - // - // Example: - // v = 0x1716151413121110 - // hash = 0x12 - // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl ^ (lsbs * hash); - return BitMask((x - lsbs) & ~x & msbs); - } +struct GroupPortableImpl { + enum { kWidth = 8 }; - BitMask MatchEmpty() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask((ctrl & (~ctrl << 6)) & msbs); - } + explicit GroupPortableImpl(const ctrl_t* pos) : ctrl(little_endian::Load64(pos)) {} - BitMask MatchEmptyOrDeleted() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask((ctrl & (~ctrl << 7)) & msbs); - } + BitMask Match(h2_t hash) const { + // For the technique, see: + // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord + // (Determine if a word has a byte equal to n). + // + // Caveat: there are false positives but: + // - they only occur if there is a real match + // - they never occur on kEmpty, kDeleted, kSentinel + // - they will be handled gracefully by subsequent checks in code + // + // Example: + // v = 0x1716151413121110 + // hash = 0x12 + // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl ^ (lsbs * hash); + return BitMask((x - lsbs) & ~x & msbs); + } - uint32_t CountLeadingEmptyOrDeleted() const { - constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; - return (uint32_t)((TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3); - } + BitMask MatchEmpty() const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask((ctrl & (~ctrl << 6)) & msbs); + } - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl & msbs; - auto res = (~x + (x >> 7)) & ~lsbs; - little_endian::Store64(dst, res); - } + BitMask MatchEmptyOrDeleted() const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask((ctrl & (~ctrl << 7)) & msbs); + } + + uint32_t CountLeadingEmptyOrDeleted() const { + constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; + return (uint32_t)((TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3); + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl & msbs; + auto res = (~x + (x >> 7)) & ~lsbs; + little_endian::Store64(dst, res); + } - uint64_t ctrl; + uint64_t ctrl; }; -#if PHMAP_HAVE_SSE2 - using Group = GroupSse2Impl; +#if PHMAP_HAVE_SSE2 +using Group = GroupSse2Impl; #else - using Group = GroupPortableImpl; +using Group = GroupPortableImpl; #endif template @@ -468,60 +447,51 @@ inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } // EMPTY -> EMPTY // FULL -> DELETED // -------------------------------------------------------------------------- -inline void ConvertDeletedToEmptyAndFullToDeleted( - ctrl_t* ctrl, size_t capacity) -{ - assert(ctrl[capacity] == kSentinel); - assert(IsValidCapacity(capacity)); - for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { - Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); - } - // Copy the cloned ctrl bytes. - std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); - ctrl[capacity] = kSentinel; +inline void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) { + assert(ctrl[capacity] == kSentinel); + assert(IsValidCapacity(capacity)); + for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { + Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); + } + // Copy the cloned ctrl bytes. + std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); + ctrl[capacity] = kSentinel; } // -------------------------------------------------------------------------- // Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. // -------------------------------------------------------------------------- -inline size_t NormalizeCapacity(size_t n) -{ - return n ? ~size_t{} >> LeadingZeros(n) : 1; -} +inline size_t NormalizeCapacity(size_t n) { return n ? ~size_t{} >> LeadingZeros(n) : 1; } // -------------------------------------------------------------------------- // We use 7/8th as maximum load factor. // For 16-wide groups, that gives an average of two empty slots per group. // -------------------------------------------------------------------------- -inline size_t CapacityToGrowth(size_t capacity) -{ - assert(IsValidCapacity(capacity)); - // `capacity*7/8` - PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { - if (capacity == 7) - { - // x-x/8 does not work when x==7. - return 6; - } +inline size_t CapacityToGrowth(size_t capacity) { + assert(IsValidCapacity(capacity)); + // `capacity*7/8` + PHMAP_IF_CONSTEXPR(Group::kWidth == 8) { + if (capacity == 7) { + // x-x/8 does not work when x==7. + return 6; } - return capacity - capacity / 8; + } + return capacity - capacity / 8; } // -------------------------------------------------------------------------- // From desired "growth" to a lowerbound of the necessary capacity. // Might not be a valid one and required NormalizeCapacity(). // -------------------------------------------------------------------------- -inline size_t GrowthToLowerboundCapacity(size_t growth) -{ - // `growth*8/7` - PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { - if (growth == 7) - { - // x+(x-1)/7 does not work when x==7. - return 8; - } +inline size_t GrowthToLowerboundCapacity(size_t growth) { + // `growth*8/7` + PHMAP_IF_CONSTEXPR(Group::kWidth == 8) { + if (growth == 7) { + // x+(x-1)/7 does not work when x==7. + return 8; } - return growth + static_cast((static_cast(growth) - 1) / 7); + } + return growth + static_cast((static_cast(growth) - 1) / 7); } namespace hashtable_debug_internal { @@ -530,13 +500,13 @@ namespace hashtable_debug_internal { using std::get; template auto GetKey(const typename T::value_type& pair, int) -> decltype(get<0>(pair)) { - return get<0>(pair); + return get<0>(pair); } // If it is not a map, return the value directly. template const typename T::key_type& GetKey(const typename T::key_type& key, char) { - return key; + return key; } // -------------------------------------------------------------------------- @@ -544,27 +514,26 @@ const typename T::key_type& GetKey(const typename T::key_type& key, char) { // container. // -------------------------------------------------------------------------- template -struct HashtableDebugAccess -{ - // Returns the number of probes required to find `key` in `c`. The "number of - // probes" is a concept that can vary by container. Implementations should - // return 0 when `key` was found in the minimum number of operations and - // should increment the result for each non-trivial operation required to find - // `key`. - // - // The default implementation uses the bucket api from the standard and thus - // works for `std::unordered_*` containers. - // -------------------------------------------------------------------------- - static size_t GetNumProbes(const Container& c, - const typename Container::key_type& key) { - if (!c.bucket_count()) return {}; - size_t num_probes = 0; - size_t bucket = c.bucket(key); - for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) { - if (it == e) return num_probes; - if (c.key_eq()(key, GetKey(*it, 0))) return num_probes; - } +struct HashtableDebugAccess { + // Returns the number of probes required to find `key` in `c`. The "number of + // probes" is a concept that can vary by container. Implementations should + // return 0 when `key` was found in the minimum number of operations and + // should increment the result for each non-trivial operation required to find + // `key`. + // + // The default implementation uses the bucket api from the standard and thus + // works for `std::unordered_*` containers. + // -------------------------------------------------------------------------- + static size_t GetNumProbes(const Container& c, + const typename Container::key_type& key) { + if (!c.bucket_count()) return {}; + size_t num_probes = 0; + size_t bucket = c.bucket(key); + for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) { + if (it == e) return num_probes; + if (c.key_eq()(key, GetKey(*it, 0))) return num_probes; } + } }; } // namespace hashtable_debug_internal @@ -572,50 +541,51 @@ struct HashtableDebugAccess // ---------------------------------------------------------------------------- // I N F O Z S T U B S // ---------------------------------------------------------------------------- -struct HashtablezInfo -{ - void PrepareForSampling() {} +struct HashtablezInfo { + void PrepareForSampling() {} }; -inline void RecordRehashSlow(HashtablezInfo*, size_t ) {} +inline void RecordRehashSlow(HashtablezInfo*, size_t) {} -static inline void RecordInsertSlow(HashtablezInfo* , size_t, size_t ) {} +static inline void RecordInsertSlow(HashtablezInfo*, size_t, size_t) {} static inline void RecordEraseSlow(HashtablezInfo*) {} static inline HashtablezInfo* SampleSlow(int64_t*) { return nullptr; } -static inline void UnsampleSlow(HashtablezInfo* ) {} - -class HashtablezInfoHandle -{ -public: - inline void RecordStorageChanged(size_t , size_t ) {} - inline void RecordRehash(size_t ) {} - inline void RecordInsert(size_t , size_t ) {} - inline void RecordErase() {} - friend inline void swap(HashtablezInfoHandle& , - HashtablezInfoHandle& ) noexcept {} +static inline void UnsampleSlow(HashtablezInfo*) {} + +class HashtablezInfoHandle { + public: + inline void RecordStorageChanged(size_t, size_t) {} + inline void RecordRehash(size_t) {} + inline void RecordInsert(size_t, size_t) {} + inline void RecordErase() {} + friend inline void swap(HashtablezInfoHandle&, HashtablezInfoHandle&) noexcept {} }; static inline HashtablezInfoHandle Sample() { return HashtablezInfoHandle(); } -class HashtablezSampler -{ -public: - // Returns a global Sampler. - static HashtablezSampler& Global() { static HashtablezSampler hzs; return hzs; } - HashtablezInfo* Register() { static HashtablezInfo info; return &info; } - void Unregister(HashtablezInfo* ) {} - - using DisposeCallback = void (*)(const HashtablezInfo&); - DisposeCallback SetDisposeCallback(DisposeCallback ) { return nullptr; } - int64_t Iterate(const std::function& ) { return 0; } -}; +class HashtablezSampler { + public: + // Returns a global Sampler. + static HashtablezSampler& Global() { + static HashtablezSampler hzs; + return hzs; + } + HashtablezInfo* Register() { + static HashtablezInfo info; + return &info; + } + void Unregister(HashtablezInfo*) {} -static inline void SetHashtablezEnabled(bool ) {} -static inline void SetHashtablezSampleParameter(int32_t ) {} -static inline void SetHashtablezMaxSamples(int32_t ) {} + using DisposeCallback = void (*)(const HashtablezInfo&); + DisposeCallback SetDisposeCallback(DisposeCallback) { return nullptr; } + int64_t Iterate(const std::function&) { return 0; } +}; +static inline void SetHashtablezEnabled(bool) {} +static inline void SetHashtablezSampleParameter(int32_t) {} +static inline void SetHashtablezMaxSamples(int32_t) {} namespace memory_internal { @@ -625,25 +595,24 @@ namespace memory_internal { template void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t, phmap::index_sequence) { - phmap::allocator_traits::construct( - *alloc, ptr, std::get(std::forward(t))...); + phmap::allocator_traits::construct(*alloc, ptr, + std::get(std::forward(t))...); } template struct WithConstructedImplF { - template - decltype(std::declval()(std::declval())) operator()( - Args&&... args) const { - return std::forward(f)(T(std::forward(args)...)); - } - F&& f; + template + decltype(std::declval()(std::declval())) operator()(Args&&... args) const { + return std::forward(f)(T(std::forward(args)...)); + } + F&& f; }; template decltype(std::declval()(std::declval())) WithConstructedImpl( Tuple&& t, phmap::index_sequence, F&& f) { - return WithConstructedImplF{std::forward(f)}( - std::get(std::forward(t))...); + return WithConstructedImplF{std::forward(f)}( + std::get(std::forward(t))...); } template @@ -656,28 +625,25 @@ auto TupleRefImpl(T&& t, phmap::index_sequence) // tuple. // ---------------------------------------------------------------------------- template -auto TupleRef(T&& t) -> decltype( - TupleRefImpl(std::forward(t), - phmap::make_index_sequence< - std::tuple_size::type>::value>())) { +auto TupleRef(T&& t) -> decltype(TupleRefImpl( + std::forward(t), + phmap::make_index_sequence::type>::value>())) { return TupleRefImpl( std::forward(t), - phmap::make_index_sequence< - std::tuple_size::type>::value>()); + phmap::make_index_sequence::type>::value>()); } template decltype(std::declval()(std::declval(), std::piecewise_construct, std::declval>(), std::declval())) DecomposePairImpl(F&& f, std::pair, V> p) { - const auto& key = std::get<0>(p.first); - return std::forward(f)(key, std::piecewise_construct, std::move(p.first), - std::move(p.second)); + const auto& key = std::get<0>(p.first); + return std::forward(f)(key, std::piecewise_construct, std::move(p.first), + std::move(p.second)); } } // namespace memory_internal - // ---------------------------------------------------------------------------- // R A W _ H A S H _ S E T // ---------------------------------------------------------------------------- @@ -758,3104 +724,3013 @@ DecomposePairImpl(F&& f, std::pair, V> p) { // table will be probed exactly once. // ---------------------------------------------------------------------------- template -class raw_hash_set -{ - using PolicyTraits = hash_policy_traits; - using KeyArgImpl = - KeyArg::value && IsTransparent::value>; - -public: - using init_type = typename PolicyTraits::init_type; - using key_type = typename PolicyTraits::key_type; - // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user - // code fixes! - using slot_type = typename PolicyTraits::slot_type; - using allocator_type = Alloc; - using size_type = size_t; - using difference_type = ptrdiff_t; - using hasher = Hash; - using key_equal = Eq; - using policy_type = Policy; - using value_type = typename PolicyTraits::value_type; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits::pointer; - using const_pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits::const_pointer; - - // Alias used for heterogeneous lookup functions. - // `key_arg` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - template - using key_arg = typename KeyArgImpl::template type; - -private: - // Give an early error when key_type is not hashable/eq. - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - - using Layout = phmap::priv::Layout; - - static Layout MakeLayout(size_t capacity) { - assert(IsValidCapacity(capacity)); - return Layout(capacity + Group::kWidth + 1, capacity); - } +class raw_hash_set { + using PolicyTraits = hash_policy_traits; + using KeyArgImpl = KeyArg::value && IsTransparent::value>; + + public: + using init_type = typename PolicyTraits::init_type; + using key_type = typename PolicyTraits::key_type; + // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user + // code fixes! + using slot_type = typename PolicyTraits::slot_type; + using allocator_type = Alloc; + using size_type = size_t; + using difference_type = ptrdiff_t; + using hasher = Hash; + using key_equal = Eq; + using policy_type = Policy; + using value_type = typename PolicyTraits::value_type; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits::pointer; + using const_pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits::const_pointer; + + // Alias used for heterogeneous lookup functions. + // `key_arg` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + template + using key_arg = typename KeyArgImpl::template type; + + private: + // Give an early error when key_type is not hashable/eq. + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); + + using Layout = phmap::priv::Layout; + + static Layout MakeLayout(size_t capacity) { + assert(IsValidCapacity(capacity)); + return Layout(capacity + Group::kWidth + 1, capacity); + } - using AllocTraits = phmap::allocator_traits; - using SlotAlloc = typename phmap::allocator_traits< - allocator_type>::template rebind_alloc; - using SlotAllocTraits = typename phmap::allocator_traits< - allocator_type>::template rebind_traits; - - static_assert(std::is_lvalue_reference::value, - "Policy::element() must return a reference"); - - template - struct SameAsElementReference - : std::is_same::type>::type, - typename std::remove_cv< - typename std::remove_reference::type>::type> {}; - - // An enabler for insert(T&&): T must be convertible to init_type or be the - // same as [cv] value_type [ref]. - // Note: we separate SameAsElementReference into its own type to avoid using - // reference unless we need to. MSVC doesn't seem to like it in some - // cases. - template - using RequiresInsertable = typename std::enable_if< - phmap::disjunction, - SameAsElementReference>::value, - int>::type; - - // RequiresNotInit is a workaround for gcc prior to 7.1. - // See https://godbolt.org/g/Y4xsUh. - template - using RequiresNotInit = - typename std::enable_if::value, int>::type; - - template - using IsDecomposable = IsDecomposable; - -public: - static_assert(std::is_same::value, - "Allocators with custom pointer types are not supported"); - static_assert(std::is_same::value, - "Allocators with custom pointer types are not supported"); - - class iterator - { - friend class raw_hash_set; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename raw_hash_set::value_type; - using reference = - phmap::conditional_t; - using pointer = phmap::remove_reference_t*; - using difference_type = typename raw_hash_set::difference_type; - - iterator() {} - - // PRECONDITION: not an end() iterator. - reference operator*() const { return PolicyTraits::element(slot_); } - - // PRECONDITION: not an end() iterator. - pointer operator->() const { return &operator*(); } - - // PRECONDITION: not an end() iterator. - iterator& operator++() { - ++ctrl_; - ++slot_; - skip_empty_or_deleted(); - return *this; - } - // PRECONDITION: not an end() iterator. - iterator operator++(int) { - auto tmp = *this; - ++*this; - return tmp; - } + using AllocTraits = phmap::allocator_traits; + using SlotAlloc = + typename phmap::allocator_traits::template rebind_alloc; + using SlotAllocTraits = + typename phmap::allocator_traits::template rebind_traits; + + static_assert(std::is_lvalue_reference::value, + "Policy::element() must return a reference"); + + template + struct SameAsElementReference + : std::is_same< + typename std::remove_cv< + typename std::remove_reference::type>::type, + typename std::remove_cv::type>::type> {}; + + // An enabler for insert(T&&): T must be convertible to init_type or be the + // same as [cv] value_type [ref]. + // Note: we separate SameAsElementReference into its own type to avoid using + // reference unless we need to. MSVC doesn't seem to like it in some + // cases. + template + using RequiresInsertable = + typename std::enable_if, + SameAsElementReference>::value, + int>::type; + + // RequiresNotInit is a workaround for gcc prior to 7.1. + // See https://godbolt.org/g/Y4xsUh. + template + using RequiresNotInit = + typename std::enable_if::value, int>::type; + + template + using IsDecomposable = IsDecomposable; + + public: + static_assert(std::is_same::value, + "Allocators with custom pointer types are not supported"); + static_assert(std::is_same::value, + "Allocators with custom pointer types are not supported"); + + class iterator { + friend class raw_hash_set; + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename raw_hash_set::value_type; + using reference = phmap::conditional_t; + using pointer = phmap::remove_reference_t*; + using difference_type = typename raw_hash_set::difference_type; + + iterator() {} + + // PRECONDITION: not an end() iterator. + reference operator*() const { return PolicyTraits::element(slot_); } + + // PRECONDITION: not an end() iterator. + pointer operator->() const { return &operator*(); } + + // PRECONDITION: not an end() iterator. + iterator& operator++() { + ++ctrl_; + ++slot_; + skip_empty_or_deleted(); + return *this; + } + // PRECONDITION: not an end() iterator. + iterator operator++(int) { + auto tmp = *this; + ++*this; + return tmp; + } #if PHMAP_BIDIRECTIONAL - // PRECONDITION: not a begin() iterator. - iterator& operator--() { - assert(ctrl_); - do { - --ctrl_; - --slot_; - } while (IsEmptyOrDeleted(*ctrl_)); - return *this; - } - - // PRECONDITION: not a begin() iterator. - iterator operator--(int) { - auto tmp = *this; - --*this; - return tmp; - } + // PRECONDITION: not a begin() iterator. + iterator& operator--() { + assert(ctrl_); + do { + --ctrl_; + --slot_; + } while (IsEmptyOrDeleted(*ctrl_)); + return *this; + } + + // PRECONDITION: not a begin() iterator. + iterator operator--(int) { + auto tmp = *this; + --*this; + return tmp; + } #endif - friend bool operator==(const iterator& a, const iterator& b) { - return a.ctrl_ == b.ctrl_; - } - friend bool operator!=(const iterator& a, const iterator& b) { - return !(a == b); - } - - private: - iterator(ctrl_t* ctrl) : ctrl_(ctrl) {} // for end() - iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {} - - void skip_empty_or_deleted() { - while (IsEmptyOrDeleted(*ctrl_)) { - // ctrl is not necessarily aligned to Group::kWidth. It is also likely - // to read past the space for ctrl bytes and into slots. This is ok - // because ctrl has sizeof() == 1 and slot has sizeof() >= 1 so there - // is no way to read outside the combined slot array. - uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); - ctrl_ += shift; - slot_ += shift; - } - } - - ctrl_t* ctrl_ = nullptr; - // To avoid uninitialized member warnings, put slot_ in an anonymous union. - // The member is not initialized on singleton and end iterators. - union { - slot_type* slot_; - }; - }; + friend bool operator==(const iterator& a, const iterator& b) { + return a.ctrl_ == b.ctrl_; + } + friend bool operator!=(const iterator& a, const iterator& b) { return !(a == b); } - class const_iterator - { - friend class raw_hash_set; + private: + iterator(ctrl_t* ctrl) : ctrl_(ctrl) {} // for end() + iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {} - public: - using iterator_category = typename iterator::iterator_category; - using value_type = typename raw_hash_set::value_type; - using reference = typename raw_hash_set::const_reference; - using pointer = typename raw_hash_set::const_pointer; - using difference_type = typename raw_hash_set::difference_type; + void skip_empty_or_deleted() { + while (IsEmptyOrDeleted(*ctrl_)) { + // ctrl is not necessarily aligned to Group::kWidth. It is also likely + // to read past the space for ctrl bytes and into slots. This is ok + // because ctrl has sizeof() == 1 and slot has sizeof() >= 1 so there + // is no way to read outside the combined slot array. + uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); + ctrl_ += shift; + slot_ += shift; + } + } - const_iterator() {} - // Implicit construction from iterator. - const_iterator(iterator i) : inner_(std::move(i)) {} + ctrl_t* ctrl_ = nullptr; + // To avoid uninitialized member warnings, put slot_ in an anonymous union. + // The member is not initialized on singleton and end iterators. + union { + slot_type* slot_; + }; + }; - reference operator*() const { return *inner_; } - pointer operator->() const { return inner_.operator->(); } + class const_iterator { + friend class raw_hash_set; - const_iterator& operator++() { - ++inner_; - return *this; - } - const_iterator operator++(int) { return inner_++; } + public: + using iterator_category = typename iterator::iterator_category; + using value_type = typename raw_hash_set::value_type; + using reference = typename raw_hash_set::const_reference; + using pointer = typename raw_hash_set::const_pointer; + using difference_type = typename raw_hash_set::difference_type; - friend bool operator==(const const_iterator& a, const const_iterator& b) { - return a.inner_ == b.inner_; - } - friend bool operator!=(const const_iterator& a, const const_iterator& b) { - return !(a == b); - } + const_iterator() {} + // Implicit construction from iterator. + const_iterator(iterator i) : inner_(std::move(i)) {} - private: - const_iterator(const ctrl_t* ctrl, const slot_type* slot) - : inner_(const_cast(ctrl), const_cast(slot)) {} + reference operator*() const { return *inner_; } + pointer operator->() const { return inner_.operator->(); } - iterator inner_; - }; + const_iterator& operator++() { + ++inner_; + return *this; + } + const_iterator operator++(int) { return inner_++; } - using node_type = node_handle, Alloc>; - using insert_return_type = InsertReturnType; - - raw_hash_set() noexcept( - std::is_nothrow_default_constructible::value&& - std::is_nothrow_default_constructible::value&& - std::is_nothrow_default_constructible::value) {} - - explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) { - if (bucket_cnt) { - size_t new_capacity = NormalizeCapacity(bucket_cnt); - reset_growth_left(new_capacity); - initialize_slots(new_capacity); - capacity_ = new_capacity; - } + friend bool operator==(const const_iterator& a, const const_iterator& b) { + return a.inner_ == b.inner_; + } + friend bool operator!=(const const_iterator& a, const const_iterator& b) { + return !(a == b); } - raw_hash_set(size_t bucket_cnt, const hasher& hashfn, - const allocator_type& alloc) - : raw_hash_set(bucket_cnt, hashfn, key_equal(), alloc) {} + private: + const_iterator(const ctrl_t* ctrl, const slot_type* slot) + : inner_(const_cast(ctrl), const_cast(slot)) {} - raw_hash_set(size_t bucket_cnt, const allocator_type& alloc) - : raw_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} + iterator inner_; + }; - explicit raw_hash_set(const allocator_type& alloc) - : raw_hash_set(0, hasher(), key_equal(), alloc) {} + using node_type = node_handle, Alloc>; + using insert_return_type = InsertReturnType; - template - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(bucket_cnt, hashfn, eq, alloc) { - insert(first, last); + raw_hash_set() noexcept( + std::is_nothrow_default_constructible::value&& + std::is_nothrow_default_constructible::value&& + std::is_nothrow_default_constructible::value) {} + + explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) { + if (bucket_cnt) { + size_t new_capacity = NormalizeCapacity(bucket_cnt); + reset_growth_left(new_capacity); + initialize_slots(new_capacity); + capacity_ = new_capacity; } + } - template - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(first, last, bucket_cnt, hashfn, key_equal(), alloc) {} + raw_hash_set(size_t bucket_cnt, const hasher& hashfn, const allocator_type& alloc) + : raw_hash_set(bucket_cnt, hashfn, key_equal(), alloc) {} - template - raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} + raw_hash_set(size_t bucket_cnt, const allocator_type& alloc) + : raw_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} - template - raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) - : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + explicit raw_hash_set(const allocator_type& alloc) + : raw_hash_set(0, hasher(), key_equal(), alloc) {} - // Instead of accepting std::initializer_list as the first - // argument like std::unordered_set does, we have two overloads - // that accept std::initializer_list and std::initializer_list. - // This is advantageous for performance. - // - // // Turns {"abc", "def"} into std::initializer_list, then - // // copies the strings into the set. - // std::unordered_set s = {"abc", "def"}; - // - // // Turns {"abc", "def"} into std::initializer_list, then - // // copies the strings into the set. - // phmap::flat_hash_set s = {"abc", "def"}; - // - // The same trick is used in insert(). - // - // The enabler is necessary to prevent this constructor from triggering where - // the copy constructor is meant to be called. - // - // phmap::flat_hash_set a, b{a}; - // - // RequiresNotInit is a workaround for gcc prior to 7.1. - template = 0, RequiresInsertable = 0> - raw_hash_set(std::initializer_list init, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} - - raw_hash_set(std::initializer_list init, size_t bucket_cnt = 0, - const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} - - template = 0, RequiresInsertable = 0> - raw_hash_set(std::initializer_list init, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} - - raw_hash_set(std::initializer_list init, size_t bucket_cnt, - const hasher& hashfn, const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} - - template = 0, RequiresInsertable = 0> - raw_hash_set(std::initializer_list init, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list init, size_t bucket_cnt, - const allocator_type& alloc) - : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - template = 0, RequiresInsertable = 0> - raw_hash_set(std::initializer_list init, const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list init, - const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(const raw_hash_set& that) - : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} - - raw_hash_set(const raw_hash_set& that, const allocator_type& a) - : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - reserve(that.size()); - // Because the table is guaranteed to be empty, we can do something faster - // than a full `insert`. - for (const auto& v : that) { - const size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, v); - auto target = find_first_non_full(hashval); - set_ctrl(target.offset, H2(hashval)); - emplace_at(target.offset, v); - infoz_.RecordInsert(hashval, target.probe_length); - } - size_ = that.size(); - growth_left() -= that.size(); - } + template + raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, + const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(bucket_cnt, hashfn, eq, alloc) { + insert(first, last); + } + + template + raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, const hasher& hashfn, + const allocator_type& alloc) + : raw_hash_set(first, last, bucket_cnt, hashfn, key_equal(), alloc) {} + + template + raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const allocator_type& alloc) + : raw_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} + + template + raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) + : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + + // Instead of accepting std::initializer_list as the first + // argument like std::unordered_set does, we have two overloads + // that accept std::initializer_list and std::initializer_list. + // This is advantageous for performance. + // + // // Turns {"abc", "def"} into std::initializer_list, then + // // copies the strings into the set. + // std::unordered_set s = {"abc", "def"}; + // + // // Turns {"abc", "def"} into std::initializer_list, then + // // copies the strings into the set. + // phmap::flat_hash_set s = {"abc", "def"}; + // + // The same trick is used in insert(). + // + // The enabler is necessary to prevent this constructor from triggering where + // the copy constructor is meant to be called. + // + // phmap::flat_hash_set a, b{a}; + // + // RequiresNotInit is a workaround for gcc prior to 7.1. + template = 0, RequiresInsertable = 0> + raw_hash_set(std::initializer_list init, size_t bucket_cnt = 0, + const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} + + raw_hash_set(std::initializer_list init, size_t bucket_cnt = 0, + const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} + + template = 0, RequiresInsertable = 0> + raw_hash_set(std::initializer_list init, size_t bucket_cnt, const hasher& hashfn, + const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} + + raw_hash_set(std::initializer_list init, size_t bucket_cnt, + const hasher& hashfn, const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} + + template = 0, RequiresInsertable = 0> + raw_hash_set(std::initializer_list init, size_t bucket_cnt, + const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list init, size_t bucket_cnt, + const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + template = 0, RequiresInsertable = 0> + raw_hash_set(std::initializer_list init, const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list init, const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(const raw_hash_set& that) + : raw_hash_set( + that, AllocTraits::select_on_container_copy_construction(that.alloc_ref())) {} + + raw_hash_set(const raw_hash_set& that, const allocator_type& a) + : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + reserve(that.size()); + // Because the table is guaranteed to be empty, we can do something faster + // than a full `insert`. + for (const auto& v : that) { + const size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, v); + auto target = find_first_non_full(hashval); + set_ctrl(target.offset, H2(hashval)); + emplace_at(target.offset, v); + infoz_.RecordInsert(hashval, target.probe_length); + } + size_ = that.size(); + growth_left() -= that.size(); + } - raw_hash_set(raw_hash_set&& that) noexcept( - std::is_nothrow_copy_constructible::value&& - std::is_nothrow_copy_constructible::value&& - std::is_nothrow_copy_constructible::value) - : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())), + raw_hash_set(raw_hash_set&& that) noexcept( + std::is_nothrow_copy_constructible::value&& + std::is_nothrow_copy_constructible::value&& + std::is_nothrow_copy_constructible::value) + : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())), slots_(phmap::exchange(that.slots_, nullptr)), size_(phmap::exchange(that.size_, 0)), capacity_(phmap::exchange(that.capacity_, 0)), infoz_(phmap::exchange(that.infoz_, HashtablezInfoHandle())), - // Hash, equality and allocator are copied instead of moved because - // `that` must be left valid. If Hash is std::function, moving it - // would create a nullptr functor that cannot be called. + // Hash, equality and allocator are copied instead of moved because + // `that` must be left valid. If Hash is std::function, moving it + // would create a nullptr functor that cannot be called. settings_(that.settings_) { - // growth_left was copied above, reset the one from `that`. - that.growth_left() = 0; - } + // growth_left was copied above, reset the one from `that`. + that.growth_left() = 0; + } - raw_hash_set(raw_hash_set&& that, const allocator_type& a) - : ctrl_(EmptyGroup()), - slots_(nullptr), - size_(0), - capacity_(0), - settings_(0, that.hash_ref(), that.eq_ref(), a) { - if (a == that.alloc_ref()) { - std::swap(ctrl_, that.ctrl_); - std::swap(slots_, that.slots_); - std::swap(size_, that.size_); - std::swap(capacity_, that.capacity_); - std::swap(growth_left(), that.growth_left()); - std::swap(infoz_, that.infoz_); - } else { - reserve(that.size()); - // Note: this will copy elements of dense_set and unordered_set instead of - // moving them. This can be fixed if it ever becomes an issue. - for (auto& elem : that) insert(std::move(elem)); - } + raw_hash_set(raw_hash_set&& that, const allocator_type& a) + : ctrl_(EmptyGroup()), + slots_(nullptr), + size_(0), + capacity_(0), + settings_(0, that.hash_ref(), that.eq_ref(), a) { + if (a == that.alloc_ref()) { + std::swap(ctrl_, that.ctrl_); + std::swap(slots_, that.slots_); + std::swap(size_, that.size_); + std::swap(capacity_, that.capacity_); + std::swap(growth_left(), that.growth_left()); + std::swap(infoz_, that.infoz_); + } else { + reserve(that.size()); + // Note: this will copy elements of dense_set and unordered_set instead of + // moving them. This can be fixed if it ever becomes an issue. + for (auto& elem : that) insert(std::move(elem)); } + } - raw_hash_set& operator=(const raw_hash_set& that) { - raw_hash_set tmp(that, - AllocTraits::propagate_on_container_copy_assignment::value - ? that.alloc_ref() - : alloc_ref()); - swap(tmp); - return *this; - } + raw_hash_set& operator=(const raw_hash_set& that) { + raw_hash_set tmp(that, AllocTraits::propagate_on_container_copy_assignment::value + ? that.alloc_ref() + : alloc_ref()); + swap(tmp); + return *this; + } - raw_hash_set& operator=(raw_hash_set&& that) noexcept( - phmap::allocator_traits::is_always_equal::value&& - std::is_nothrow_move_assignable::value&& - std::is_nothrow_move_assignable::value) { - // TODO(sbenza): We should only use the operations from the noexcept clause - // to make sure we actually adhere to that contract. - return move_assign( - std::move(that), - typename AllocTraits::propagate_on_container_move_assignment()); - } + raw_hash_set& operator=(raw_hash_set&& that) noexcept( + phmap::allocator_traits::is_always_equal::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_assignable::value) { + // TODO(sbenza): We should only use the operations from the noexcept clause + // to make sure we actually adhere to that contract. + return move_assign(std::move(that), + typename AllocTraits::propagate_on_container_move_assignment()); + } - ~raw_hash_set() { destroy_slots(); } + ~raw_hash_set() { destroy_slots(); } - iterator begin() { - auto it = iterator_at(0); - it.skip_empty_or_deleted(); - return it; - } - iterator end() - { + iterator begin() { + auto it = iterator_at(0); + it.skip_empty_or_deleted(); + return it; + } + iterator end() { #if PHMAP_BIDIRECTIONAL - return iterator_at(capacity_); + return iterator_at(capacity_); #else - return {ctrl_ + capacity_}; + return {ctrl_ + capacity_}; #endif - } + } - const_iterator begin() const { - return const_cast(this)->begin(); - } - const_iterator end() const { return const_cast(this)->end(); } - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } - - bool empty() const { return !size(); } - size_t size() const { return size_; } - size_t capacity() const { return capacity_; } - size_t max_size() const { return (std::numeric_limits::max)(); } - - PHMAP_ATTRIBUTE_REINITIALIZES void clear() { - // Iterating over this container is O(bucket_count()). When bucket_count() - // is much greater than size(), iteration becomes prohibitively expensive. - // For clear() it is more important to reuse the allocated array when the - // container is small because allocation takes comparatively long time - // compared to destruction of the elements of the container. So we pick the - // largest bucket_count() threshold for which iteration is still fast and - // past that we simply deallocate the array. - if (empty()) - return; - if (capacity_ > 127) { - destroy_slots(); - } else if (capacity_) { - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - size_ = 0; - reset_ctrl(capacity_); - reset_growth_left(capacity_); + const_iterator begin() const { return const_cast(this)->begin(); } + const_iterator end() const { return const_cast(this)->end(); } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + + bool empty() const { return !size(); } + size_t size() const { return size_; } + size_t capacity() const { return capacity_; } + size_t max_size() const { return (std::numeric_limits::max)(); } + + PHMAP_ATTRIBUTE_REINITIALIZES void clear() { + // Iterating over this container is O(bucket_count()). When bucket_count() + // is much greater than size(), iteration becomes prohibitively expensive. + // For clear() it is more important to reuse the allocated array when the + // container is small because allocation takes comparatively long time + // compared to destruction of the elements of the container. So we pick the + // largest bucket_count() threshold for which iteration is still fast and + // past that we simply deallocate the array. + if (empty()) return; + if (capacity_ > 127) { + destroy_slots(); + } else if (capacity_) { + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); } - assert(empty()); - infoz_.RecordStorageChanged(0, capacity_); + } + size_ = 0; + reset_ctrl(capacity_); + reset_growth_left(capacity_); } + assert(empty()); + infoz_.RecordStorageChanged(0, capacity_); + } - // This overload kicks in when the argument is an rvalue of insertable and - // decomposable type other than init_type. - // - // flat_hash_map m; - // m.insert(std::make_pair("abc", 42)); - template = 0, - typename std::enable_if::value, int>::type = 0, - T* = nullptr> - std::pair insert(T&& value) { - return emplace(std::forward(value)); - } + // This overload kicks in when the argument is an rvalue of insertable and + // decomposable type other than init_type. + // + // flat_hash_map m; + // m.insert(std::make_pair("abc", 42)); + template = 0, + typename std::enable_if::value, int>::type = 0, + T* = nullptr> + std::pair insert(T&& value) { + return emplace(std::forward(value)); + } - // This overload kicks in when the argument is a bitfield or an lvalue of - // insertable and decomposable type. - // - // union { int n : 1; }; - // flat_hash_set s; - // s.insert(n); - // - // flat_hash_set s; - // const char* p = "hello"; - // s.insert(p); - // - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable with RequiresInsertable. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template = 0, - typename std::enable_if::value, int>::type = 0> - std::pair insert(const T& value) { - return emplace(value); - } + // This overload kicks in when the argument is a bitfield or an lvalue of + // insertable and decomposable type. + // + // union { int n : 1; }; + // flat_hash_set s; + // s.insert(n); + // + // flat_hash_set s; + // const char* p = "hello"; + // s.insert(p); + // + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable with RequiresInsertable. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template = 0, + typename std::enable_if::value, int>::type = 0> + std::pair insert(const T& value) { + return emplace(value); + } - // This overload kicks in when the argument is an rvalue of init_type. Its - // purpose is to handle brace-init-list arguments. - // - // flat_hash_set s; - // s.insert({"abc", 42}); - std::pair insert(init_type&& value) { - return emplace(std::move(value)); - } + // This overload kicks in when the argument is an rvalue of init_type. Its + // purpose is to handle brace-init-list arguments. + // + // flat_hash_set s; + // s.insert({"abc", 42}); + std::pair insert(init_type&& value) { + return emplace(std::move(value)); + } - template = 0, - typename std::enable_if::value, int>::type = 0, - T* = nullptr> - iterator insert(const_iterator, T&& value) { - return insert(std::forward(value)).first; - } + template = 0, + typename std::enable_if::value, int>::type = 0, + T* = nullptr> + iterator insert(const_iterator, T&& value) { + return insert(std::forward(value)).first; + } - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable with RequiresInsertable. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template = 0, - typename std::enable_if::value, int>::type = 0> - iterator insert(const_iterator, const T& value) { - return insert(value).first; - } + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable with RequiresInsertable. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template = 0, + typename std::enable_if::value, int>::type = 0> + iterator insert(const_iterator, const T& value) { + return insert(value).first; + } - iterator insert(const_iterator, init_type&& value) { - return insert(std::move(value)).first; - } + iterator insert(const_iterator, init_type&& value) { + return insert(std::move(value)).first; + } - template - using IsRandomAccess = std::is_same::iterator_category, - std::random_access_iterator_tag>; - - - template - struct has_difference_operator - { - private: - using yes = std::true_type; - using no = std::false_type; - - template static auto test(int) -> decltype(std::declval() - std::declval() == 1, yes()); - template static no test(...); - - public: - static constexpr bool value = std::is_same(0)), yes>::value; - }; + template + using IsRandomAccess = + std::is_same::iterator_category, + std::random_access_iterator_tag>; + + template + struct has_difference_operator { + private: + using yes = std::true_type; + using no = std::false_type; + + template + static auto test(int) -> decltype(std::declval() - std::declval() == 1, yes()); + template + static no test(...); + + public: + static constexpr bool value = std::is_same(0)), yes>::value; + }; + + template ::value, int> = 0> + void insert(InputIt first, InputIt last) { + this->reserve(this->size() + (last - first)); + for (; first != last; ++first) emplace(*first); + } - template ::value, int> = 0> - void insert(InputIt first, InputIt last) { - this->reserve(this->size() + (last - first)); - for (; first != last; ++first) - emplace(*first); - } + template ::value, int> = 0> + void insert(InputIt first, InputIt last) { + for (; first != last; ++first) emplace(*first); + } - template ::value, int> = 0> - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) - emplace(*first); - } + template = 0, RequiresInsertable = 0> + void insert(std::initializer_list ilist) { + insert(ilist.begin(), ilist.end()); + } - template = 0, RequiresInsertable = 0> - void insert(std::initializer_list ilist) { - insert(ilist.begin(), ilist.end()); - } + void insert(std::initializer_list ilist) { + insert(ilist.begin(), ilist.end()); + } - void insert(std::initializer_list ilist) { - insert(ilist.begin(), ilist.end()); + insert_return_type insert(node_type&& node) { + if (!node) return {end(), false, node_type()}; + const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); + auto res = PolicyTraits::apply( + InsertSlot{*this, std::move(*CommonAccess::GetSlot(node))}, elem); + if (res.second) { + CommonAccess::Reset(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; } + } - insert_return_type insert(node_type&& node) { - if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); - auto res = PolicyTraits::apply( - InsertSlot{*this, std::move(*CommonAccess::GetSlot(node))}, - elem); - if (res.second) { - CommonAccess::Reset(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } + insert_return_type insert(node_type&& node, size_t hashval) { + if (!node) return {end(), false, node_type()}; + const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); + auto res = PolicyTraits::apply( + InsertSlotWithHash{*this, std::move(*CommonAccess::GetSlot(node)), + hashval}, + elem); + if (res.second) { + CommonAccess::Reset(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; } + } - insert_return_type insert(node_type&& node, size_t hashval) { - if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); - auto res = PolicyTraits::apply( - InsertSlotWithHash{*this, std::move(*CommonAccess::GetSlot(node)), hashval}, - elem); - if (res.second) { - CommonAccess::Reset(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } + iterator insert(const_iterator, node_type&& node) { + return insert(std::move(node)).first; + } - iterator insert(const_iterator, node_type&& node) { - return insert(std::move(node)).first; - } + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + template ::value, int>::type = 0> + std::pair emplace(Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposable{*this}, std::forward(args)...); + } - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - template ::value, int>::type = 0> - std::pair emplace(Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposable{*this}, - std::forward(args)...); - } + template ::value, int>::type = 0> + std::pair emplace_with_hash(size_t hashval, Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, + std::forward(args)...); + } - template ::value, int>::type = 0> - std::pair emplace_with_hash(size_t hashval, Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, std::forward(args)...); - } + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + template ::value, int>::type = 0> + std::pair emplace(Args&&... args) { + typename std::aligned_storage::type raw; + slot_type* slot = reinterpret_cast(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); + const auto& elem = PolicyTraits::element(slot); + return PolicyTraits::apply(InsertSlot{*this, std::move(*slot)}, elem); + } - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - template ::value, int>::type = 0> - std::pair emplace(Args&&... args) { - typename std::aligned_storage::type - raw; - slot_type* slot = reinterpret_cast(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); - const auto& elem = PolicyTraits::element(slot); - return PolicyTraits::apply(InsertSlot{*this, std::move(*slot)}, elem); - } + template ::value, int>::type = 0> + std::pair emplace_with_hash(size_t hashval, Args&&... args) { + typename std::aligned_storage::type raw; + slot_type* slot = reinterpret_cast(&raw); - template ::value, int>::type = 0> - std::pair emplace_with_hash(size_t hashval, Args&&... args) { - typename std::aligned_storage::type raw; - slot_type* slot = reinterpret_cast(&raw); + PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); + const auto& elem = PolicyTraits::element(slot); + return PolicyTraits::apply(InsertSlotWithHash{*this, std::move(*slot), hashval}, + elem); + } - PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); - const auto& elem = PolicyTraits::element(slot); - return PolicyTraits::apply(InsertSlotWithHash{*this, std::move(*slot), hashval}, elem); - } + template + iterator emplace_hint(const_iterator, Args&&... args) { + return emplace(std::forward(args)...).first; + } - template - iterator emplace_hint(const_iterator, Args&&... args) { - return emplace(std::forward(args)...).first; - } + template + iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { + return emplace_with_hash(hashval, std::forward(args)...).first; + } + // Extension API: support for lazy emplace. + // + // Looks up key in the table. If found, returns the iterator to the element. + // Otherwise calls f with one argument of type raw_hash_set::constructor. f + // MUST call raw_hash_set::constructor with arguments as if a + // raw_hash_set::value_type is constructed, otherwise the behavior is + // undefined. + // + // For example: + // + // std::unordered_set s; + // // Makes ArenaStr even if "abc" is in the map. + // s.insert(ArenaString(&arena, "abc")); + // + // flat_hash_set s; + // // Makes ArenaStr only if "abc" is not in the map. + // s.lazy_emplace("abc", [&](const constructor& ctor) { + // ctor(&arena, "abc"); + // }); + // + // WARNING: This API is currently experimental. If there is a way to implement + // the same thing with the rest of the API, prefer that. + class constructor { + friend class raw_hash_set; + + public: template - iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { - return emplace_with_hash(hashval, std::forward(args)...).first; + void operator()(Args&&... args) const { + assert(*slot_); + PolicyTraits::construct(alloc_, *slot_, std::forward(args)...); + *slot_ = nullptr; } - // Extension API: support for lazy emplace. - // - // Looks up key in the table. If found, returns the iterator to the element. - // Otherwise calls f with one argument of type raw_hash_set::constructor. f - // MUST call raw_hash_set::constructor with arguments as if a - // raw_hash_set::value_type is constructed, otherwise the behavior is - // undefined. - // - // For example: - // - // std::unordered_set s; - // // Makes ArenaStr even if "abc" is in the map. - // s.insert(ArenaString(&arena, "abc")); - // - // flat_hash_set s; - // // Makes ArenaStr only if "abc" is not in the map. - // s.lazy_emplace("abc", [&](const constructor& ctor) { - // ctor(&arena, "abc"); - // }); - // - // WARNING: This API is currently experimental. If there is a way to implement - // the same thing with the rest of the API, prefer that. - class constructor - { - friend class raw_hash_set; - - public: - template - void operator()(Args&&... args) const { - assert(*slot_); - PolicyTraits::construct(alloc_, *slot_, std::forward(args)...); - *slot_ = nullptr; - } - - private: - constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} - - allocator_type* alloc_; - slot_type** slot_; - }; + private: + constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} - template - iterator lazy_emplace(const key_arg& key, F&& f) { - auto res = find_or_prepare_insert(key); - if (res.second) { - lazy_emplace_at(res.first, std::forward(f)); - } - return iterator_at(res.first); - } + allocator_type* alloc_; + slot_type** slot_; + }; - template - iterator lazy_emplace_with_hash(const key_arg& key, size_t &hashval, F&& f) { - auto res = find_or_prepare_insert(key, hashval); - if (res.second) { - lazy_emplace_at(res.first, std::forward(f)); - } - return iterator_at(res.first); + template + iterator lazy_emplace(const key_arg& key, F&& f) { + auto res = find_or_prepare_insert(key); + if (res.second) { + lazy_emplace_at(res.first, std::forward(f)); } + return iterator_at(res.first); + } - template - void lazy_emplace_at(size_t& idx, F&& f) { - slot_type* slot = slots_ + idx; - std::forward(f)(constructor(&alloc_ref(), &slot)); - assert(!slot); + template + iterator lazy_emplace_with_hash(const key_arg& key, size_t& hashval, F&& f) { + auto res = find_or_prepare_insert(key, hashval); + if (res.second) { + lazy_emplace_at(res.first, std::forward(f)); } + return iterator_at(res.first); + } + template + void lazy_emplace_at(size_t& idx, F&& f) { + slot_type* slot = slots_ + idx; + std::forward(f)(constructor(&alloc_ref(), &slot)); + assert(!slot); + } - // Extension API: support for heterogeneous keys. - // - // std::unordered_set s; - // // Turns "abc" into std::string. - // s.erase("abc"); - // - // flat_hash_set s; - // // Uses "abc" directly without copying it into std::string. - // s.erase("abc"); - template - size_type erase(const key_arg& key) { - auto it = find(key); - if (it == end()) return 0; - _erase(it); - return 1; - } - + // Extension API: support for heterogeneous keys. + // + // std::unordered_set s; + // // Turns "abc" into std::string. + // s.erase("abc"); + // + // flat_hash_set s; + // // Uses "abc" directly without copying it into std::string. + // s.erase("abc"); + template + size_type erase(const key_arg& key) { + auto it = find(key); + if (it == end()) return 0; + _erase(it); + return 1; + } - iterator erase(const_iterator cit) { return erase(cit.inner_); } - - // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, - // this method returns void to reduce algorithmic complexity to O(1). In - // order to erase while iterating across a map, use the following idiom (which - // also works for standard containers): - // - // for (auto it = m.begin(), end = m.end(); it != end;) { - // if () { - // m._erase(it++); - // } else { - // ++it; - // } - // } - void _erase(iterator it) { - assert(it != end()); - PolicyTraits::destroy(&alloc_ref(), it.slot_); - erase_meta_only(it); - } - void _erase(const_iterator cit) { _erase(cit.inner_); } - - // This overload is necessary because otherwise erase(const K&) would be - // a better match if non-const iterator is passed as an argument. - iterator erase(iterator it) { - auto res = it; - ++res; - _erase(it); - return res; - } + iterator erase(const_iterator cit) { return erase(cit.inner_); } + + // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, + // this method returns void to reduce algorithmic complexity to O(1). In + // order to erase while iterating across a map, use the following idiom (which + // also works for standard containers): + // + // for (auto it = m.begin(), end = m.end(); it != end;) { + // if () { + // m._erase(it++); + // } else { + // ++it; + // } + // } + void _erase(iterator it) { + assert(it != end()); + PolicyTraits::destroy(&alloc_ref(), it.slot_); + erase_meta_only(it); + } + void _erase(const_iterator cit) { _erase(cit.inner_); } + + // This overload is necessary because otherwise erase(const K&) would be + // a better match if non-const iterator is passed as an argument. + iterator erase(iterator it) { + auto res = it; + ++res; + _erase(it); + return res; + } - iterator erase(const_iterator first, const_iterator last) { - while (first != last) { - _erase(first++); - } - return last.inner_; + iterator erase(const_iterator first, const_iterator last) { + while (first != last) { + _erase(first++); } + return last.inner_; + } - // Moves elements from `src` into `this`. - // If the element already exists in `this`, it is left unmodified in `src`. - template - void merge(raw_hash_set& src) { // NOLINT - assert(this != &src); - for (auto it = src.begin(), e = src.end(); it != e; ++it) { - if (PolicyTraits::apply(InsertSlot{*this, std::move(*it.slot_)}, - PolicyTraits::element(it.slot_)) - .second) { - src.erase_meta_only(it); - } - } + // Moves elements from `src` into `this`. + // If the element already exists in `this`, it is left unmodified in `src`. + template + void merge(raw_hash_set& src) { // NOLINT + assert(this != &src); + for (auto it = src.begin(), e = src.end(); it != e; ++it) { + if (PolicyTraits::apply(InsertSlot{*this, std::move(*it.slot_)}, + PolicyTraits::element(it.slot_)) + .second) { + src.erase_meta_only(it); + } } + } - template - void merge(raw_hash_set&& src) { - merge(src); - } + template + void merge(raw_hash_set&& src) { + merge(src); + } - node_type extract(const_iterator position) { - auto node = - CommonAccess::Make(alloc_ref(), position.inner_.slot_); - erase_meta_only(position); - return node; - } + node_type extract(const_iterator position) { + auto node = CommonAccess::Make(alloc_ref(), position.inner_.slot_); + erase_meta_only(position); + return node; + } - template < - class K = key_type, - typename std::enable_if::value, int>::type = 0> - node_type extract(const key_arg& key) { - auto it = find(key); - return it == end() ? node_type() : extract(const_iterator{it}); - } + template ::value, int>::type = 0> + node_type extract(const key_arg& key) { + auto it = find(key); + return it == end() ? node_type() : extract(const_iterator{it}); + } - void swap(raw_hash_set& that) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - using std::swap; - swap(ctrl_, that.ctrl_); - swap(slots_, that.slots_); - swap(size_, that.size_); - swap(capacity_, that.capacity_); - swap(growth_left(), that.growth_left()); - swap(hash_ref(), that.hash_ref()); - swap(eq_ref(), that.eq_ref()); - swap(infoz_, that.infoz_); - if (AllocTraits::propagate_on_container_swap::value) { - swap(alloc_ref(), that.alloc_ref()); - } else { - // If the allocators do not compare equal it is officially undefined - // behavior. We choose to do nothing. - } + void swap(raw_hash_set& that) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + using std::swap; + swap(ctrl_, that.ctrl_); + swap(slots_, that.slots_); + swap(size_, that.size_); + swap(capacity_, that.capacity_); + swap(growth_left(), that.growth_left()); + swap(hash_ref(), that.hash_ref()); + swap(eq_ref(), that.eq_ref()); + swap(infoz_, that.infoz_); + if (AllocTraits::propagate_on_container_swap::value) { + swap(alloc_ref(), that.alloc_ref()); + } else { + // If the allocators do not compare equal it is officially undefined + // behavior. We choose to do nothing. } + } #if !defined(PHMAP_NON_DETERMINISTIC) - template - bool phmap_dump(OutputArchive&) const; + template + bool phmap_dump(OutputArchive&) const; - template - bool phmap_load(InputArchive&); + template + bool phmap_load(InputArchive&); #endif - void rehash(size_t n) { - if (n == 0 && capacity_ == 0) return; - if (n == 0 && size_ == 0) { - destroy_slots(); - infoz_.RecordStorageChanged(0, 0); - return; - } - // bitor is a faster way of doing `max` here. We will round up to the next - // power-of-2-minus-1, so bitor is good enough. - auto m = NormalizeCapacity((std::max)(n, size())); - // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > capacity_) { - resize(m); - } + void rehash(size_t n) { + if (n == 0 && capacity_ == 0) return; + if (n == 0 && size_ == 0) { + destroy_slots(); + infoz_.RecordStorageChanged(0, 0); + return; } - - void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set s; - // // Turns "abc" into std::string. - // s.count("abc"); - // - // ch_set s; - // // Uses "abc" directly without copying it into std::string. - // s.count("abc"); - template - size_t count(const key_arg& key) const { - return find(key) == end() ? size_t(0) : size_t(1); + // bitor is a faster way of doing `max` here. We will round up to the next + // power-of-2-minus-1, so bitor is good enough. + auto m = NormalizeCapacity((std::max)(n, size())); + // n == 0 unconditionally rehashes as per the standard. + if (n == 0 || m > capacity_) { + resize(m); } + } - // Issues CPU prefetch instructions for the memory needed to find or insert - // a key. Like all lookup functions, this support heterogeneous keys. - // - // NOTE: This is a very low level operation and should not be used without - // specific benchmarks indicating its importance. - void prefetch_hash(size_t hashval) const { - (void)hashval; + void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set s; + // // Turns "abc" into std::string. + // s.count("abc"); + // + // ch_set s; + // // Uses "abc" directly without copying it into std::string. + // s.count("abc"); + template + size_t count(const key_arg& key) const { + return find(key) == end() ? size_t(0) : size_t(1); + } + + // Issues CPU prefetch instructions for the memory needed to find or insert + // a key. Like all lookup functions, this support heterogeneous keys. + // + // NOTE: This is a very low level operation and should not be used without + // specific benchmarks indicating its importance. + void prefetch_hash(size_t hashval) const { + (void)hashval; #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) - auto seq = probe(hashval); - _mm_prefetch((const char *)(ctrl_ + seq.offset()), _MM_HINT_NTA); - _mm_prefetch((const char *)(slots_ + seq.offset()), _MM_HINT_NTA); + auto seq = probe(hashval); + _mm_prefetch((const char*)(ctrl_ + seq.offset()), _MM_HINT_NTA); + _mm_prefetch((const char*)(slots_ + seq.offset()), _MM_HINT_NTA); #elif defined(__GNUC__) - auto seq = probe(hashval); - __builtin_prefetch(static_cast(ctrl_ + seq.offset())); - __builtin_prefetch(static_cast(slots_ + seq.offset())); + auto seq = probe(hashval); + __builtin_prefetch(static_cast(ctrl_ + seq.offset())); + __builtin_prefetch(static_cast(slots_ + seq.offset())); #endif // __GNUC__ - } + } - template - void prefetch(const key_arg& key) const { - prefetch_hash(this->hash(key)); - } + template + void prefetch(const key_arg& key) const { + prefetch_hash(this->hash(key)); + } - // The API of find() has two extensions. - // - // 1. The hash can be passed by the user. It must be equal to the hash of the - // key. - // - // 2. The type of the key argument doesn't have to be key_type. This is so - // called heterogeneous key support. - template - iterator find(const key_arg& key, size_t hashval) { - size_t offset; - if (find_impl(key, hashval, offset)) - return iterator_at(offset); - else - return end(); - } + // The API of find() has two extensions. + // + // 1. The hash can be passed by the user. It must be equal to the hash of the + // key. + // + // 2. The type of the key argument doesn't have to be key_type. This is so + // called heterogeneous key support. + template + iterator find(const key_arg& key, size_t hashval) { + size_t offset; + if (find_impl(key, hashval, offset)) + return iterator_at(offset); + else + return end(); + } - template - pointer find_ptr(const key_arg& key, size_t hashval) { - size_t offset; - if (find_impl(key, hashval, offset)) - return &PolicyTraits::element(slots_ + offset); - else - return nullptr; - } + template + pointer find_ptr(const key_arg& key, size_t hashval) { + size_t offset; + if (find_impl(key, hashval, offset)) + return &PolicyTraits::element(slots_ + offset); + else + return nullptr; + } - template - iterator find(const key_arg& key) { - return find(key, this->hash(key)); - } + template + iterator find(const key_arg& key) { + return find(key, this->hash(key)); + } - template - const_iterator find(const key_arg& key, size_t hashval) const { - return const_cast(this)->find(key, hashval); - } - template - const_iterator find(const key_arg& key) const { - return find(key, this->hash(key)); - } + template + const_iterator find(const key_arg& key, size_t hashval) const { + return const_cast(this)->find(key, hashval); + } + template + const_iterator find(const key_arg& key) const { + return find(key, this->hash(key)); + } - template - bool contains(const key_arg& key) const { - return find(key) != end(); - } + template + bool contains(const key_arg& key) const { + return find(key) != end(); + } - template - bool contains(const key_arg& key, size_t hashval) const { - return find(key, hashval) != end(); - } + template + bool contains(const key_arg& key, size_t hashval) const { + return find(key, hashval) != end(); + } - template - std::pair equal_range(const key_arg& key) { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - template - std::pair equal_range( - const key_arg& key) const { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } + template + std::pair equal_range(const key_arg& key) { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + template + std::pair equal_range(const key_arg& key) const { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } - size_t bucket_count() const { return capacity_; } - float load_factor() const { - return capacity_ ? static_cast(size()) / capacity_ : 0.0; - } - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { - // Does nothing. - } + size_t bucket_count() const { return capacity_; } + float load_factor() const { + return capacity_ ? static_cast(size()) / capacity_ : 0.0; + } + float max_load_factor() const { return 1.0f; } + void max_load_factor(float) { + // Does nothing. + } - hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function - key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } - - friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { - if (a.size() != b.size()) return false; - const raw_hash_set* outer = &a; - const raw_hash_set* inner = &b; - if (outer->capacity() > inner->capacity()) - std::swap(outer, inner); - for (const value_type& elem : *outer) - if (!inner->has_element(elem)) return false; - return true; - } + hasher hash_function() const { + return hash_ref(); + } // warning: doesn't match internal hash - use hash() member function + key_equal key_eq() const { return eq_ref(); } + allocator_type get_allocator() const { return alloc_ref(); } + + friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { + if (a.size() != b.size()) return false; + const raw_hash_set* outer = &a; + const raw_hash_set* inner = &b; + if (outer->capacity() > inner->capacity()) std::swap(outer, inner); + for (const value_type& elem : *outer) + if (!inner->has_element(elem)) return false; + return true; + } - friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { - return !(a == b); - } + friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { + return !(a == b); + } - friend void swap(raw_hash_set& a, - raw_hash_set& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); - } + friend void swap(raw_hash_set& a, raw_hash_set& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); + } - template - size_t hash(const K& key) const { - return HashElement{hash_ref()}(key); - } + template + size_t hash(const K& key) const { + return HashElement{hash_ref()}(key); + } -private: - template - friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; - - template - bool find_impl(const key_arg& key, size_t hashval, size_t& offset) { - auto seq = probe(hashval); - while (true) { - Group g{ ctrl_ + seq.offset() }; - for (int i : g.Match((h2_t)H2(hashval))) { - offset = seq.offset((size_t)i); - if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( - EqualElement{key, eq_ref()}, - PolicyTraits::element(slots_ + offset)))) - return true; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) - return false; - seq.next(); - } + private: + template + friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; + + template + bool find_impl(const key_arg& key, size_t hashval, size_t& offset) { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match((h2_t)H2(hashval))) { + offset = seq.offset((size_t)i); + if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( + EqualElement{key, eq_ref()}, PolicyTraits::element(slots_ + offset)))) + return true; + } + if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) return false; + seq.next(); } + } - struct FindElement - { - template - const_iterator operator()(const K& key, Args&&...) const { - return s.find(key); - } - const raw_hash_set& s; - }; - - struct HashElement - { - template - size_t operator()(const K& key, Args&&...) const { - return phmap_mix()(h(key)); - } - const hasher& h; - }; - - template - struct EqualElement - { - template - bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); - } - const K1& rhs; - const key_equal& eq; - }; - + struct FindElement { template - std::pair emplace_decomposable(const K& key, size_t hashval, - Args&&... args) - { - auto res = find_or_prepare_insert(key, hashval); - if (res.second) { - emplace_at(res.first, std::forward(args)...); - } - return {iterator_at(res.first), res.second}; + const_iterator operator()(const K& key, Args&&...) const { + return s.find(key); } + const raw_hash_set& s; + }; - struct EmplaceDecomposable - { - template - std::pair operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, s.hash(key), std::forward(args)...); - } - raw_hash_set& s; - }; - - struct EmplaceDecomposableHashval { - template - std::pair operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, hashval, std::forward(args)...); - } - raw_hash_set& s; - size_t hashval; - }; - - template - struct InsertSlot - { - template - std::pair operator()(const K& key, Args&&...) && { - auto res = s.find_or_prepare_insert(key); - if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); - } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - // Constructed slot. Either moved into place or destroyed. - slot_type&& slot; - }; - - template - struct InsertSlotWithHash - { - template - std::pair operator()(const K& key, Args&&...) && { - auto res = s.find_or_prepare_insert(key, hashval); - if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); - } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - // Constructed slot. Either moved into place or destroyed. - slot_type&& slot; - size_t &hashval; - }; - - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. - void erase_meta_only(const_iterator it) { - assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); - --size_; - const size_t index = (size_t)(it.inner_.ctrl_ - ctrl_); - const size_t index_before = (index - Group::kWidth) & capacity_; - const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); - const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); - - // We count how many consecutive non empties we have to the right and to the - // left of `it`. If the sum is >= kWidth then there is at least one probe - // window that might have seen a full group. - bool was_never_full = - empty_before && empty_after && - static_cast(empty_after.TrailingZeros() + - empty_before.LeadingZeros()) < Group::kWidth; - - set_ctrl(index, was_never_full ? kEmpty : kDeleted); - growth_left() += was_never_full; - infoz_.RecordErase(); - } - - void initialize_slots(size_t new_capacity) { - assert(new_capacity); - if (std::is_same>::value && - slots_ == nullptr) { - infoz_ = Sample(); - } - - auto layout = MakeLayout(new_capacity); - char* mem = static_cast( - Allocate(&alloc_ref(), layout.AllocSize())); - ctrl_ = reinterpret_cast(layout.template Pointer<0>(mem)); - slots_ = layout.template Pointer<1>(mem); - reset_ctrl(new_capacity); - reset_growth_left(new_capacity); - infoz_.RecordStorageChanged(size_, new_capacity); - } - - void destroy_slots() { - if (!capacity_) return; - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - auto layout = MakeLayout(capacity_); - // Unpoison before returning the memory to the allocator. - SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - Deallocate(&alloc_ref(), ctrl_, layout.AllocSize()); - ctrl_ = EmptyGroup(); - slots_ = nullptr; - size_ = 0; - capacity_ = 0; - growth_left() = 0; - } + struct HashElement { + template + size_t operator()(const K& key, Args&&...) const { + return phmap_mix()(h(key)); + } + const hasher& h; + }; + + template + struct EqualElement { + template + bool operator()(const K2& lhs, Args&&...) const { + return eq(lhs, rhs); + } + const K1& rhs; + const key_equal& eq; + }; + + template + std::pair emplace_decomposable(const K& key, size_t hashval, + Args&&... args) { + auto res = find_or_prepare_insert(key, hashval); + if (res.second) { + emplace_at(res.first, std::forward(args)...); + } + return {iterator_at(res.first), res.second}; + } - void resize(size_t new_capacity) { - assert(IsValidCapacity(new_capacity)); - auto* old_ctrl = ctrl_; - auto* old_slots = slots_; - const size_t old_capacity = capacity_; - initialize_slots(new_capacity); - capacity_ = new_capacity; - - for (size_t i = 0; i != old_capacity; ++i) { - if (IsFull(old_ctrl[i])) { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(old_slots + i)); - auto target = find_first_non_full(hashval); - size_t new_i = target.offset; - set_ctrl(new_i, H2(hashval)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); - } - } - if (old_capacity) { - SanitizerUnpoisonMemoryRegion(old_slots, - sizeof(slot_type) * old_capacity); - auto layout = MakeLayout(old_capacity); - Deallocate(&alloc_ref(), old_ctrl, - layout.AllocSize()); - } + struct EmplaceDecomposable { + template + std::pair operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable(key, s.hash(key), std::forward(args)...); } + raw_hash_set& s; + }; - void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { - assert(IsValidCapacity(capacity_)); - assert(!is_small()); - // Algorithm: - // - mark all DELETED slots as EMPTY - // - mark all FULL slots as DELETED - // - for each slot marked as DELETED - // hash = Hash(element) - // target = find_first_non_full(hash) - // if target is in the same group - // mark slot as FULL - // else if target is EMPTY - // transfer element to target - // mark slot as EMPTY - // mark target as FULL - // else if target is DELETED - // swap current element with target element - // mark target as FULL - // repeat procedure for current slot with moved from element (target) - ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); - typename std::aligned_storage::type - raw; - slot_type* slot = reinterpret_cast(&raw); - for (size_t i = 0; i != capacity_; ++i) { - if (!IsDeleted(ctrl_[i])) continue; - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(slots_ + i)); - auto target = find_first_non_full(hashval); - size_t new_i = target.offset; - - // Verify if the old and new i fall within the same group wrt the hashval. - // If they do, we don't need to move the object as it falls already in the - // best probe we can. - const auto probe_index = [&](size_t pos) { - return ((pos - probe(hashval).offset()) & capacity_) / Group::kWidth; - }; - - // Element doesn't move. - if (PHMAP_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { - set_ctrl(i, H2(hashval)); - continue; - } - if (IsEmpty(ctrl_[new_i])) { - // Transfer element to the empty spot. - // set_ctrl poisons/unpoisons the slots so we have to call it at the - // right time. - set_ctrl(new_i, H2(hashval)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); - set_ctrl(i, kEmpty); - } else { - assert(IsDeleted(ctrl_[new_i])); - set_ctrl(new_i, H2(hashval)); - // Until we are done rehashing, DELETED marks previously FULL slots. - // Swap i and new_i elements. - PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); - PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); - --i; // repeat - } - } - reset_growth_left(capacity_); + struct EmplaceDecomposableHashval { + template + std::pair operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable(key, hashval, std::forward(args)...); } + raw_hash_set& s; + size_t hashval; + }; - void rehash_and_grow_if_necessary() { - if (capacity_ == 0) { - resize(1); - } else if (size() <= CapacityToGrowth(capacity()) / 2) { - // Squash DELETED without growing if there is enough capacity. - drop_deletes_without_resize(); - } else { - // Otherwise grow the container. - resize(capacity_ * 2 + 1); - } - } + template + struct InsertSlot { + template + std::pair operator()(const K& key, Args&&...) && { + auto res = s.find_or_prepare_insert(key); + if (res.second) { + PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); + } else if (do_destroy) { + PolicyTraits::destroy(&s.alloc_ref(), &slot); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + // Constructed slot. Either moved into place or destroyed. + slot_type&& slot; + }; + + template + struct InsertSlotWithHash { + template + std::pair operator()(const K& key, Args&&...) && { + auto res = s.find_or_prepare_insert(key, hashval); + if (res.second) { + PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); + } else if (do_destroy) { + PolicyTraits::destroy(&s.alloc_ref(), &slot); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + // Constructed slot. Either moved into place or destroyed. + slot_type&& slot; + size_t& hashval; + }; + + // "erases" the object from the container, except that it doesn't actually + // destroy the object. It only updates all the metadata of the class. + // This can be used in conjunction with Policy::transfer to move the object to + // another place. + void erase_meta_only(const_iterator it) { + assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); + --size_; + const size_t index = (size_t)(it.inner_.ctrl_ - ctrl_); + const size_t index_before = (index - Group::kWidth) & capacity_; + const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); + const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); + + // We count how many consecutive non empties we have to the right and to the + // left of `it`. If the sum is >= kWidth then there is at least one probe + // window that might have seen a full group. + bool was_never_full = + empty_before && empty_after && + static_cast(empty_after.TrailingZeros() + empty_before.LeadingZeros()) < + Group::kWidth; + + set_ctrl(index, was_never_full ? kEmpty : kDeleted); + growth_left() += was_never_full; + infoz_.RecordErase(); + } - bool has_element(const value_type& elem, size_t hashval) const { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match((h2_t)H2(hashval))) { - if (PHMAP_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset((size_t)i)) == - elem)) - return true; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) return false; - seq.next(); - assert(seq.getindex() < capacity_ && "full table!"); - } - return false; + void initialize_slots(size_t new_capacity) { + assert(new_capacity); + if (std::is_same>::value && slots_ == nullptr) { + infoz_ = Sample(); } - bool has_element(const value_type& elem) const { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); - return has_element(elem, hashval); - } + auto layout = MakeLayout(new_capacity); + char* mem = static_cast( + Allocate(&alloc_ref(), layout.AllocSize())); + ctrl_ = reinterpret_cast(layout.template Pointer<0>(mem)); + slots_ = layout.template Pointer<1>(mem); + reset_ctrl(new_capacity); + reset_growth_left(new_capacity); + infoz_.RecordStorageChanged(size_, new_capacity); + } - // Probes the raw_hash_set with the probe sequence for hash and returns the - // pointer to the first empty or deleted slot. - // NOTE: this function must work with tables having both kEmpty and kDelete - // in one group. Such tables appears during drop_deletes_without_resize. - // - // This function is very useful when insertions happen and: - // - the input is already a set - // - there are enough slots - // - the element with the hash is not in the table - struct FindInfo - { - size_t offset; - size_t probe_length; - }; - FindInfo find_first_non_full(size_t hashval) { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - auto mask = g.MatchEmptyOrDeleted(); - if (mask) { - return {seq.offset((size_t)mask.LowestBitSet()), seq.getindex()}; - } - assert(seq.getindex() < capacity_ && "full table!"); - seq.next(); - } - } + void destroy_slots() { + if (!capacity_) return; + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); + } + } + auto layout = MakeLayout(capacity_); + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); + Deallocate(&alloc_ref(), ctrl_, layout.AllocSize()); + ctrl_ = EmptyGroup(); + slots_ = nullptr; + size_ = 0; + capacity_ = 0; + growth_left() = 0; + } - // TODO(alkis): Optimize this assuming *this and that don't overlap. - raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { - raw_hash_set tmp(std::move(that)); - swap(tmp); - return *this; + void resize(size_t new_capacity) { + assert(IsValidCapacity(new_capacity)); + auto* old_ctrl = ctrl_; + auto* old_slots = slots_; + const size_t old_capacity = capacity_; + initialize_slots(new_capacity); + capacity_ = new_capacity; + + for (size_t i = 0; i != old_capacity; ++i) { + if (IsFull(old_ctrl[i])) { + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(old_slots + i)); + auto target = find_first_non_full(hashval); + size_t new_i = target.offset; + set_ctrl(new_i, H2(hashval)); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); + } } - raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { - raw_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); - return *this; + if (old_capacity) { + SanitizerUnpoisonMemoryRegion(old_slots, sizeof(slot_type) * old_capacity); + auto layout = MakeLayout(old_capacity); + Deallocate(&alloc_ref(), old_ctrl, layout.AllocSize()); } + } -protected: - template - std::pair find_or_prepare_insert(const K& key, size_t hashval) { - auto seq = probe(hashval); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match((h2_t)H2(hashval))) { - if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( - EqualElement{key, eq_ref()}, - PolicyTraits::element(slots_ + seq.offset((size_t)i))))) - return {seq.offset((size_t)i), false}; - } - if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) break; - seq.next(); - } - return {prepare_insert(hashval), true}; - } + void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { + assert(IsValidCapacity(capacity_)); + assert(!is_small()); + // Algorithm: + // - mark all DELETED slots as EMPTY + // - mark all FULL slots as DELETED + // - for each slot marked as DELETED + // hash = Hash(element) + // target = find_first_non_full(hash) + // if target is in the same group + // mark slot as FULL + // else if target is EMPTY + // transfer element to target + // mark slot as EMPTY + // mark target as FULL + // else if target is DELETED + // swap current element with target element + // mark target as FULL + // repeat procedure for current slot with moved from element (target) + ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); + typename std::aligned_storage::type raw; + slot_type* slot = reinterpret_cast(&raw); + for (size_t i = 0; i != capacity_; ++i) { + if (!IsDeleted(ctrl_[i])) continue; + size_t hashval = + PolicyTraits::apply(HashElement{hash_ref()}, PolicyTraits::element(slots_ + i)); + auto target = find_first_non_full(hashval); + size_t new_i = target.offset; + + // Verify if the old and new i fall within the same group wrt the hashval. + // If they do, we don't need to move the object as it falls already in the + // best probe we can. + const auto probe_index = [&](size_t pos) { + return ((pos - probe(hashval).offset()) & capacity_) / Group::kWidth; + }; + + // Element doesn't move. + if (PHMAP_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { + set_ctrl(i, H2(hashval)); + continue; + } + if (IsEmpty(ctrl_[new_i])) { + // Transfer element to the empty spot. + // set_ctrl poisons/unpoisons the slots so we have to call it at the + // right time. + set_ctrl(new_i, H2(hashval)); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); + set_ctrl(i, kEmpty); + } else { + assert(IsDeleted(ctrl_[new_i])); + set_ctrl(new_i, H2(hashval)); + // Until we are done rehashing, DELETED marks previously FULL slots. + // Swap i and new_i elements. + PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); + PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); + --i; // repeat + } + } + reset_growth_left(capacity_); + } - template - std::pair find_or_prepare_insert(const K& key) { - return find_or_prepare_insert(key, this->hash(key)); + void rehash_and_grow_if_necessary() { + if (capacity_ == 0) { + resize(1); + } else if (size() <= CapacityToGrowth(capacity()) / 2) { + // Squash DELETED without growing if there is enough capacity. + drop_deletes_without_resize(); + } else { + // Otherwise grow the container. + resize(capacity_ * 2 + 1); } + } - size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE { - auto target = find_first_non_full(hashval); - if (PHMAP_PREDICT_FALSE(growth_left() == 0 && - !IsDeleted(ctrl_[target.offset]))) { - rehash_and_grow_if_necessary(); - target = find_first_non_full(hashval); - } - ++size_; - growth_left() -= IsEmpty(ctrl_[target.offset]); - set_ctrl(target.offset, H2(hashval)); - infoz_.RecordInsert(hashval, target.probe_length); - return target.offset; - } + bool has_element(const value_type& elem, size_t hashval) const { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match((h2_t)H2(hashval))) { + if (PHMAP_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset((size_t)i)) == + elem)) + return true; + } + if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) return false; + seq.next(); + assert(seq.getindex() < capacity_ && "full table!"); + } + return false; + } - // Constructs the value in the space pointed by the iterator. This only works - // after an unsuccessful find_or_prepare_insert() and before any other - // modifications happen in the raw_hash_set. - // - // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where - // k is the key decomposed from `forward(args)...`, and the bool - // returned by find_or_prepare_insert(k) was true. - // POSTCONDITION: *m.iterator_at(i) == value_type(forward(args)...). - template - void emplace_at(size_t i, Args&&... args) { - PolicyTraits::construct(&alloc_ref(), slots_ + i, - std::forward(args)...); + bool has_element(const value_type& elem) const { + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); + return has_element(elem, hashval); + } - assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == - iterator_at(i) && - "constructed value does not match the lookup key"); + // Probes the raw_hash_set with the probe sequence for hash and returns the + // pointer to the first empty or deleted slot. + // NOTE: this function must work with tables having both kEmpty and kDelete + // in one group. Such tables appears during drop_deletes_without_resize. + // + // This function is very useful when insertions happen and: + // - the input is already a set + // - there are enough slots + // - the element with the hash is not in the table + struct FindInfo { + size_t offset; + size_t probe_length; + }; + FindInfo find_first_non_full(size_t hashval) { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + auto mask = g.MatchEmptyOrDeleted(); + if (mask) { + return {seq.offset((size_t)mask.LowestBitSet()), seq.getindex()}; + } + assert(seq.getindex() < capacity_ && "full table!"); + seq.next(); } + } - iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } - const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } + // TODO(alkis): Optimize this assuming *this and that don't overlap. + raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { + raw_hash_set tmp(std::move(that)); + swap(tmp); + return *this; + } + raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { + raw_hash_set tmp(std::move(that), alloc_ref()); + swap(tmp); + return *this; + } -private: - friend struct RawHashSetTestOnlyAccess; + protected: + template + std::pair find_or_prepare_insert(const K& key, size_t hashval) { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match((h2_t)H2(hashval))) { + if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( + EqualElement{key, eq_ref()}, + PolicyTraits::element(slots_ + seq.offset((size_t)i))))) + return {seq.offset((size_t)i), false}; + } + if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) break; + seq.next(); + } + return {prepare_insert(hashval), true}; + } - probe_seq probe(size_t hashval) const { - return probe_seq(H1(hashval, ctrl_), capacity_); - } + template + std::pair find_or_prepare_insert(const K& key) { + return find_or_prepare_insert(key, this->hash(key)); + } - // Reset all ctrl bytes back to kEmpty, except the sentinel. - void reset_ctrl(size_t capacity) { - std::memset(ctrl_, kEmpty, capacity + Group::kWidth); - ctrl_[capacity] = kSentinel; - SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity); - } + size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE { + auto target = find_first_non_full(hashval); + if (PHMAP_PREDICT_FALSE(growth_left() == 0 && !IsDeleted(ctrl_[target.offset]))) { + rehash_and_grow_if_necessary(); + target = find_first_non_full(hashval); + } + ++size_; + growth_left() -= IsEmpty(ctrl_[target.offset]); + set_ctrl(target.offset, H2(hashval)); + infoz_.RecordInsert(hashval, target.probe_length); + return target.offset; + } - void reset_growth_left(size_t capacity) { - growth_left() = CapacityToGrowth(capacity) - size_; - } + // Constructs the value in the space pointed by the iterator. This only works + // after an unsuccessful find_or_prepare_insert() and before any other + // modifications happen in the raw_hash_set. + // + // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where + // k is the key decomposed from `forward(args)...`, and the bool + // returned by find_or_prepare_insert(k) was true. + // POSTCONDITION: *m.iterator_at(i) == value_type(forward(args)...). + template + void emplace_at(size_t i, Args&&... args) { + PolicyTraits::construct(&alloc_ref(), slots_ + i, std::forward(args)...); + + assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == iterator_at(i) && + "constructed value does not match the lookup key"); + } - // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at - // the end too. - void set_ctrl(size_t i, ctrl_t h) { - assert(i < capacity_); + iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } + const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } - if (IsFull(h)) { - SanitizerUnpoisonObject(slots_ + i); - } else { - SanitizerPoisonObject(slots_ + i); - } + private: + friend struct RawHashSetTestOnlyAccess; - ctrl_[i] = h; - ctrl_[((i - Group::kWidth) & capacity_) + 1 + - ((Group::kWidth - 1) & capacity_)] = h; - } + probe_seq probe(size_t hashval) const { + return probe_seq(H1(hashval, ctrl_), capacity_); + } - size_t& growth_left() { return settings_.template get<0>(); } + // Reset all ctrl bytes back to kEmpty, except the sentinel. + void reset_ctrl(size_t capacity) { + std::memset(ctrl_, kEmpty, capacity + Group::kWidth); + ctrl_[capacity] = kSentinel; + SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity); + } - template class RefSet, - class M, class P, class H, class E, class A> - friend class parallel_hash_set; + void reset_growth_left(size_t capacity) { + growth_left() = CapacityToGrowth(capacity) - size_; + } - template class RefSet, - class M, class P, class H, class E, class A> - friend class parallel_hash_map; + // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at + // the end too. + void set_ctrl(size_t i, ctrl_t h) { + assert(i < capacity_); - // The representation of the object has two modes: - // - small: For capacities < kWidth-1 - // - large: For the rest. - // - // Differences: - // - In small mode we are able to use the whole capacity. The extra control - // bytes give us at least one "empty" control byte to stop the iteration. - // This is important to make 1 a valid capacity. - // - // - In small mode only the first `capacity()` control bytes after the - // sentinel are valid. The rest contain dummy kEmpty values that do not - // represent a real slot. This is important to take into account on - // find_first_non_full(), where we never try ShouldInsertBackwards() for - // small tables. - bool is_small() const { return capacity_ < Group::kWidth - 1; } - - hasher& hash_ref() { return settings_.template get<1>(); } - const hasher& hash_ref() const { return settings_.template get<1>(); } - key_equal& eq_ref() { return settings_.template get<2>(); } - const key_equal& eq_ref() const { return settings_.template get<2>(); } - allocator_type& alloc_ref() { return settings_.template get<3>(); } - const allocator_type& alloc_ref() const { - return settings_.template get<3>(); + if (IsFull(h)) { + SanitizerUnpoisonObject(slots_ + i); + } else { + SanitizerPoisonObject(slots_ + i); } - // TODO(alkis): Investigate removing some of these fields: - // - ctrl/slots can be derived from each other - // - size can be moved into the slot array - ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t] - slot_type* slots_ = nullptr; // [capacity * slot_type] - size_t size_ = 0; // number of full slots - size_t capacity_ = 0; // total number of slots - HashtablezInfoHandle infoz_; - phmap::priv::CompressedTuple - settings_{0, hasher{}, key_equal{}, allocator_type{}}; -}; + ctrl_[i] = h; + ctrl_[((i - Group::kWidth) & capacity_) + 1 + ((Group::kWidth - 1) & capacity_)] = h; + } + size_t& growth_left() { return settings_.template get<0>(); } + + template class RefSet, class M, + class P, class H, class E, class A> + friend class parallel_hash_set; + + template class RefSet, class M, + class P, class H, class E, class A> + friend class parallel_hash_map; + + // The representation of the object has two modes: + // - small: For capacities < kWidth-1 + // - large: For the rest. + // + // Differences: + // - In small mode we are able to use the whole capacity. The extra control + // bytes give us at least one "empty" control byte to stop the iteration. + // This is important to make 1 a valid capacity. + // + // - In small mode only the first `capacity()` control bytes after the + // sentinel are valid. The rest contain dummy kEmpty values that do not + // represent a real slot. This is important to take into account on + // find_first_non_full(), where we never try ShouldInsertBackwards() for + // small tables. + bool is_small() const { return capacity_ < Group::kWidth - 1; } + + hasher& hash_ref() { return settings_.template get<1>(); } + const hasher& hash_ref() const { return settings_.template get<1>(); } + key_equal& eq_ref() { return settings_.template get<2>(); } + const key_equal& eq_ref() const { return settings_.template get<2>(); } + allocator_type& alloc_ref() { return settings_.template get<3>(); } + const allocator_type& alloc_ref() const { return settings_.template get<3>(); } + + // TODO(alkis): Investigate removing some of these fields: + // - ctrl/slots can be derived from each other + // - size can be moved into the slot array + ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t] + slot_type* slots_ = nullptr; // [capacity * slot_type] + size_t size_ = 0; // number of full slots + size_t capacity_ = 0; // total number of slots + HashtablezInfoHandle infoz_; + phmap::priv::CompressedTuple + settings_{0, hasher{}, key_equal{}, allocator_type{}}; +}; // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- template -class raw_hash_map : public raw_hash_set -{ - // P is Policy. It's passed as a template argument to support maps that have - // incomplete types as values, as in unordered_map. - // MappedReference<> may be a non-reference type. - template - using MappedReference = decltype(P::value( - std::addressof(std::declval()))); - - // MappedConstReference<> may be a non-reference type. - template - using MappedConstReference = decltype(P::value( - std::addressof(std::declval()))); - - using KeyArgImpl = - KeyArg::value && IsTransparent::value>; - - using Base = raw_hash_set; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - template - using key_arg = typename KeyArgImpl::template type; - - static_assert(!std::is_reference::value, ""); - // TODO(alkis): remove this assertion and verify that reference mapped_type is - // supported. - static_assert(!std::is_reference::value, ""); - - using iterator = typename raw_hash_map::raw_hash_set::iterator; - using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator; - - raw_hash_map() {} - using Base::raw_hash_set; // use raw_hash_set constructor - - // The last two template parameters ensure that both arguments are rvalues - // (lvalue arguments are handled by the overloads below). This is necessary - // for supporting bitfield arguments. - // - // union { int n : 1; }; - // flat_hash_map m; - // m.insert_or_assign(n, n); - template - std::pair insert_or_assign(key_arg&& k, V&& v) { - return insert_or_assign_impl(std::forward(k), std::forward(v)); - } +class raw_hash_map : public raw_hash_set { + // P is Policy. It's passed as a template argument to support maps that have + // incomplete types as values, as in unordered_map. + // MappedReference<> may be a non-reference type. + template + using MappedReference = decltype( + P::value(std::addressof(std::declval()))); + + // MappedConstReference<> may be a non-reference type. + template + using MappedConstReference = decltype( + P::value(std::addressof(std::declval()))); + + using KeyArgImpl = KeyArg::value && IsTransparent::value>; + + using Base = raw_hash_set; + + public: + using key_type = typename Policy::key_type; + using mapped_type = typename Policy::mapped_type; + template + using key_arg = typename KeyArgImpl::template type; + + static_assert(!std::is_reference::value, ""); + // TODO(alkis): remove this assertion and verify that reference mapped_type is + // supported. + static_assert(!std::is_reference::value, ""); + + using iterator = typename raw_hash_map::raw_hash_set::iterator; + using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator; + + raw_hash_map() {} + using Base::raw_hash_set; // use raw_hash_set constructor + + // The last two template parameters ensure that both arguments are rvalues + // (lvalue arguments are handled by the overloads below). This is necessary + // for supporting bitfield arguments. + // + // union { int n : 1; }; + // flat_hash_map m; + // m.insert_or_assign(n, n); + template + std::pair insert_or_assign(key_arg&& k, V&& v) { + return insert_or_assign_impl(std::forward(k), std::forward(v)); + } - template - std::pair insert_or_assign(key_arg&& k, const V& v) { - return insert_or_assign_impl(std::forward(k), v); - } + template + std::pair insert_or_assign(key_arg&& k, const V& v) { + return insert_or_assign_impl(std::forward(k), v); + } - template - std::pair insert_or_assign(const key_arg& k, V&& v) { - return insert_or_assign_impl(k, std::forward(v)); - } + template + std::pair insert_or_assign(const key_arg& k, V&& v) { + return insert_or_assign_impl(k, std::forward(v)); + } - template - std::pair insert_or_assign(const key_arg& k, const V& v) { - return insert_or_assign_impl(k, v); - } + template + std::pair insert_or_assign(const key_arg& k, const V& v) { + return insert_or_assign_impl(k, v); + } - template - iterator insert_or_assign(const_iterator, key_arg&& k, V&& v) { - return insert_or_assign(std::forward(k), std::forward(v)).first; - } + template + iterator insert_or_assign(const_iterator, key_arg&& k, V&& v) { + return insert_or_assign(std::forward(k), std::forward(v)).first; + } - template - iterator insert_or_assign(const_iterator, key_arg&& k, const V& v) { - return insert_or_assign(std::forward(k), v).first; - } + template + iterator insert_or_assign(const_iterator, key_arg&& k, const V& v) { + return insert_or_assign(std::forward(k), v).first; + } - template - iterator insert_or_assign(const_iterator, const key_arg& k, V&& v) { - return insert_or_assign(k, std::forward(v)).first; - } + template + iterator insert_or_assign(const_iterator, const key_arg& k, V&& v) { + return insert_or_assign(k, std::forward(v)).first; + } - template - iterator insert_or_assign(const_iterator, const key_arg& k, const V& v) { - return insert_or_assign(k, v).first; - } + template + iterator insert_or_assign(const_iterator, const key_arg& k, const V& v) { + return insert_or_assign(k, v).first; + } - template ::value, int>::type = 0, - K* = nullptr> - std::pair try_emplace(key_arg&& k, Args&&... args) { - return try_emplace_impl(std::forward(k), std::forward(args)...); - } + template ::value, + int>::type = 0, + K* = nullptr> + std::pair try_emplace(key_arg&& k, Args&&... args) { + return try_emplace_impl(std::forward(k), std::forward(args)...); + } - template ::value, int>::type = 0> - std::pair try_emplace(const key_arg& k, Args&&... args) { - return try_emplace_impl(k, std::forward(args)...); - } + template ::value, + int>::type = 0> + std::pair try_emplace(const key_arg& k, Args&&... args) { + return try_emplace_impl(k, std::forward(args)...); + } - template - iterator try_emplace(const_iterator, key_arg&& k, Args&&... args) { - return try_emplace(std::forward(k), std::forward(args)...).first; - } + template + iterator try_emplace(const_iterator, key_arg&& k, Args&&... args) { + return try_emplace(std::forward(k), std::forward(args)...).first; + } - template - iterator try_emplace(const_iterator, const key_arg& k, Args&&... args) { - return try_emplace(k, std::forward(args)...).first; - } + template + iterator try_emplace(const_iterator, const key_arg& k, Args&&... args) { + return try_emplace(k, std::forward(args)...).first; + } - template - MappedReference

at(const key_arg& key) { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } + template + MappedReference

at(const key_arg& key) { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } - template - MappedConstReference

at(const key_arg& key) const { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } + template + MappedConstReference

at(const key_arg& key) const { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } - template - MappedReference

operator[](key_arg&& key) { - return Policy::value(&*try_emplace(std::forward(key)).first); - } + template + MappedReference

operator[](key_arg&& key) { + return Policy::value(&*try_emplace(std::forward(key)).first); + } - template - MappedReference

operator[](const key_arg& key) { - return Policy::value(&*try_emplace(key).first); - } + template + MappedReference

operator[](const key_arg& key) { + return Policy::value(&*try_emplace(key).first); + } -private: - template - std::pair insert_or_assign_impl(K&& k, V&& v) { - auto res = this->find_or_prepare_insert(k); - if (res.second) - this->emplace_at(res.first, std::forward(k), std::forward(v)); - else - Policy::value(&*this->iterator_at(res.first)) = std::forward(v); - return {this->iterator_at(res.first), res.second}; - } + private: + template + std::pair insert_or_assign_impl(K&& k, V&& v) { + auto res = this->find_or_prepare_insert(k); + if (res.second) + this->emplace_at(res.first, std::forward(k), std::forward(v)); + else + Policy::value(&*this->iterator_at(res.first)) = std::forward(v); + return {this->iterator_at(res.first), res.second}; + } - template - std::pair try_emplace_impl(K&& k, Args&&... args) { - auto res = this->find_or_prepare_insert(k); - if (res.second) - this->emplace_at(res.first, std::piecewise_construct, - std::forward_as_tuple(std::forward(k)), - std::forward_as_tuple(std::forward(args)...)); - return {this->iterator_at(res.first), res.second}; - } + template + std::pair try_emplace_impl(K&& k, Args&&... args) { + auto res = this->find_or_prepare_insert(k); + if (res.second) + this->emplace_at(res.first, std::piecewise_construct, + std::forward_as_tuple(std::forward(k)), + std::forward_as_tuple(std::forward(args)...)); + return {this->iterator_at(res.first), res.second}; + } }; // ---------------------------------------------------------------------------- // ---------------------------------------------------------------------------- // Returns "random" seed. -inline size_t RandomSeed() -{ +inline size_t RandomSeed() { #if PHMAP_HAVE_THREAD_LOCAL - static thread_local size_t counter = 0; - size_t value = ++counter; + static thread_local size_t counter = 0; + size_t value = ++counter; #else // PHMAP_HAVE_THREAD_LOCAL - static std::atomic counter(0); - size_t value = counter.fetch_add(1, std::memory_order_relaxed); + static std::atomic counter(0); + size_t value = counter.fetch_add(1, std::memory_order_relaxed); #endif // PHMAP_HAVE_THREAD_LOCAL - return value ^ static_cast(reinterpret_cast(&counter)); + return value ^ static_cast(reinterpret_cast(&counter)); } // ---------------------------------------------------------------------------- // ---------------------------------------------------------------------------- -template class RefSet, - class Mtx_, +template class RefSet, class Mtx_, class Policy, class Hash, class Eq, class Alloc> -class parallel_hash_set -{ - using PolicyTraits = hash_policy_traits; - using KeyArgImpl = - KeyArg::value && IsTransparent::value>; - - static_assert(N <= 12, "N = 12 means 4096 hash tables!"); - constexpr static size_t num_tables = 1 << N; - constexpr static size_t mask = num_tables - 1; - -public: - using EmbeddedSet = RefSet; - using EmbeddedIterator= typename EmbeddedSet::iterator; - using EmbeddedConstIterator= typename EmbeddedSet::const_iterator; - using constructor = typename EmbeddedSet::constructor; - using init_type = typename PolicyTraits::init_type; - using key_type = typename PolicyTraits::key_type; - using slot_type = typename PolicyTraits::slot_type; - using allocator_type = Alloc; - using size_type = size_t; - using difference_type = ptrdiff_t; - using hasher = Hash; - using key_equal = Eq; - using policy_type = Policy; - using value_type = typename PolicyTraits::value_type; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits::pointer; - using const_pointer = typename phmap::allocator_traits< - allocator_type>::template rebind_traits::const_pointer; - - // Alias used for heterogeneous lookup functions. - // `key_arg` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - // -------------------------------------------------------------------- - template - using key_arg = typename KeyArgImpl::template type; - -protected: - using Lockable = phmap::LockableImpl; - - // -------------------------------------------------------------------- - struct Inner : public Lockable - { - bool operator==(const Inner& o) const - { - typename Lockable::SharedLocks l(const_cast(*this), const_cast(o)); - return set_ == o.set_; - } +class parallel_hash_set { + using PolicyTraits = hash_policy_traits; + using KeyArgImpl = KeyArg::value && IsTransparent::value>; + + static_assert(N <= 12, "N = 12 means 4096 hash tables!"); + constexpr static size_t num_tables = 1 << N; + constexpr static size_t mask = num_tables - 1; + + public: + using EmbeddedSet = RefSet; + using EmbeddedIterator = typename EmbeddedSet::iterator; + using EmbeddedConstIterator = typename EmbeddedSet::const_iterator; + using constructor = typename EmbeddedSet::constructor; + using init_type = typename PolicyTraits::init_type; + using key_type = typename PolicyTraits::key_type; + using slot_type = typename PolicyTraits::slot_type; + using allocator_type = Alloc; + using size_type = size_t; + using difference_type = ptrdiff_t; + using hasher = Hash; + using key_equal = Eq; + using policy_type = Policy; + using value_type = typename PolicyTraits::value_type; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits::pointer; + using const_pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits::const_pointer; + + // Alias used for heterogeneous lookup functions. + // `key_arg` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + // -------------------------------------------------------------------- + template + using key_arg = typename KeyArgImpl::template type; + + protected: + using Lockable = phmap::LockableImpl; + + // -------------------------------------------------------------------- + struct Inner : public Lockable { + bool operator==(const Inner& o) const { + typename Lockable::SharedLocks l(const_cast(*this), const_cast(o)); + return set_ == o.set_; + } + + EmbeddedSet set_; + }; + + private: + // Give an early error when key_type is not hashable/eq. + // -------------------------------------------------------------------- + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); + + using AllocTraits = phmap::allocator_traits; + + static_assert(std::is_lvalue_reference::value, + "Policy::element() must return a reference"); + + template + struct SameAsElementReference + : std::is_same< + typename std::remove_cv< + typename std::remove_reference::type>::type, + typename std::remove_cv::type>::type> {}; + + // An enabler for insert(T&&): T must be convertible to init_type or be the + // same as [cv] value_type [ref]. + // Note: we separate SameAsElementReference into its own type to avoid using + // reference unless we need to. MSVC doesn't seem to like it in some + // cases. + // -------------------------------------------------------------------- + template + using RequiresInsertable = + typename std::enable_if, + SameAsElementReference>::value, + int>::type; + + // RequiresNotInit is a workaround for gcc prior to 7.1. + // See https://godbolt.org/g/Y4xsUh. + template + using RequiresNotInit = + typename std::enable_if::value, int>::type; + + template + using IsDecomposable = IsDecomposable; + + public: + static_assert(std::is_same::value, + "Allocators with custom pointer types are not supported"); + static_assert(std::is_same::value, + "Allocators with custom pointer types are not supported"); + + // --------------------- i t e r a t o r ------------------------------ + class iterator { + friend class parallel_hash_set; - EmbeddedSet set_; - }; + public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename parallel_hash_set::value_type; + using reference = phmap::conditional_t; + using pointer = phmap::remove_reference_t*; + using difference_type = typename parallel_hash_set::difference_type; + using Inner = typename parallel_hash_set::Inner; + using EmbeddedSet = typename parallel_hash_set::EmbeddedSet; + using EmbeddedIterator = typename EmbeddedSet::iterator; -private: - // Give an early error when key_type is not hashable/eq. - // -------------------------------------------------------------------- - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - - using AllocTraits = phmap::allocator_traits; - - static_assert(std::is_lvalue_reference::value, - "Policy::element() must return a reference"); - - template - struct SameAsElementReference : std::is_same< - typename std::remove_cv::type>::type, - typename std::remove_cv::type>::type> {}; - - // An enabler for insert(T&&): T must be convertible to init_type or be the - // same as [cv] value_type [ref]. - // Note: we separate SameAsElementReference into its own type to avoid using - // reference unless we need to. MSVC doesn't seem to like it in some - // cases. - // -------------------------------------------------------------------- - template - using RequiresInsertable = typename std::enable_if< - phmap::disjunction, - SameAsElementReference>::value, - int>::type; - - // RequiresNotInit is a workaround for gcc prior to 7.1. - // See https://godbolt.org/g/Y4xsUh. - template - using RequiresNotInit = - typename std::enable_if::value, int>::type; - - template - using IsDecomposable = IsDecomposable; - -public: - static_assert(std::is_same::value, - "Allocators with custom pointer types are not supported"); - static_assert(std::is_same::value, - "Allocators with custom pointer types are not supported"); - - // --------------------- i t e r a t o r ------------------------------ - class iterator - { - friend class parallel_hash_set; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename parallel_hash_set::value_type; - using reference = - phmap::conditional_t; - using pointer = phmap::remove_reference_t*; - using difference_type = typename parallel_hash_set::difference_type; - using Inner = typename parallel_hash_set::Inner; - using EmbeddedSet = typename parallel_hash_set::EmbeddedSet; - using EmbeddedIterator = typename EmbeddedSet::iterator; - - iterator() {} - - reference operator*() const { return *it_; } - pointer operator->() const { return &operator*(); } - - iterator& operator++() { - assert(inner_); // null inner means we are already at the end - ++it_; - skip_empty(); - return *this; - } - - iterator operator++(int) { - assert(inner_); // null inner means we are already at the end - auto tmp = *this; - ++*this; - return tmp; - } + iterator() {} - friend bool operator==(const iterator& a, const iterator& b) { - return a.inner_ == b.inner_ && (!a.inner_ || a.it_ == b.it_); - } + reference operator*() const { return *it_; } + pointer operator->() const { return &operator*(); } - friend bool operator!=(const iterator& a, const iterator& b) { - return !(a == b); - } + iterator& operator++() { + assert(inner_); // null inner means we are already at the end + ++it_; + skip_empty(); + return *this; + } - private: - iterator(Inner *inner, Inner *inner_end, const EmbeddedIterator& it) : - inner_(inner), inner_end_(inner_end), it_(it) { // for begin() and end() - if (inner) - it_end_ = inner->set_.end(); - } + iterator operator++(int) { + assert(inner_); // null inner means we are already at the end + auto tmp = *this; + ++*this; + return tmp; + } - void skip_empty() { - while (it_ == it_end_) { - ++inner_; - if (inner_ == inner_end_) { - inner_ = nullptr; // marks end() - break; - } - else { - it_ = inner_->set_.begin(); - it_end_ = inner_->set_.end(); - } - } - } + friend bool operator==(const iterator& a, const iterator& b) { + return a.inner_ == b.inner_ && (!a.inner_ || a.it_ == b.it_); + } - Inner *inner_ = nullptr; - Inner *inner_end_ = nullptr; - EmbeddedIterator it_, it_end_; - }; + friend bool operator!=(const iterator& a, const iterator& b) { return !(a == b); } - // --------------------- c o n s t i t e r a t o r ----------------- - class const_iterator - { - friend class parallel_hash_set; - - public: - using iterator_category = typename iterator::iterator_category; - using value_type = typename parallel_hash_set::value_type; - using reference = typename parallel_hash_set::const_reference; - using pointer = typename parallel_hash_set::const_pointer; - using difference_type = typename parallel_hash_set::difference_type; - using Inner = typename parallel_hash_set::Inner; - - const_iterator() {} - // Implicit construction from iterator. - const_iterator(iterator i) : iter_(std::move(i)) {} - - reference operator*() const { return *(iter_); } - pointer operator->() const { return iter_.operator->(); } - - const_iterator& operator++() { - ++iter_; - return *this; - } - const_iterator operator++(int) { return iter_++; } + private: + iterator(Inner* inner, Inner* inner_end, const EmbeddedIterator& it) + : inner_(inner), inner_end_(inner_end), it_(it) { // for begin() and end() + if (inner) it_end_ = inner->set_.end(); + } - friend bool operator==(const const_iterator& a, const const_iterator& b) { - return a.iter_ == b.iter_; - } - friend bool operator!=(const const_iterator& a, const const_iterator& b) { - return !(a == b); + void skip_empty() { + while (it_ == it_end_) { + ++inner_; + if (inner_ == inner_end_) { + inner_ = nullptr; // marks end() + break; + } else { + it_ = inner_->set_.begin(); + it_end_ = inner_->set_.end(); } + } + } - private: - const_iterator(const Inner *inner, const Inner *inner_end, const EmbeddedIterator& it) - : iter_(const_cast(inner), - const_cast(inner_end), - const_cast(it)) {} + Inner* inner_ = nullptr; + Inner* inner_end_ = nullptr; + EmbeddedIterator it_, it_end_; + }; - iterator iter_; - }; + // --------------------- c o n s t i t e r a t o r ----------------- + class const_iterator { + friend class parallel_hash_set; - using node_type = node_handle, Alloc>; - using insert_return_type = InsertReturnType; + public: + using iterator_category = typename iterator::iterator_category; + using value_type = typename parallel_hash_set::value_type; + using reference = typename parallel_hash_set::const_reference; + using pointer = typename parallel_hash_set::const_pointer; + using difference_type = typename parallel_hash_set::difference_type; + using Inner = typename parallel_hash_set::Inner; - // ------------------------- c o n s t r u c t o r s ------------------ + const_iterator() {} + // Implicit construction from iterator. + const_iterator(iterator i) : iter_(std::move(i)) {} - parallel_hash_set() noexcept( - std::is_nothrow_default_constructible::value&& - std::is_nothrow_default_constructible::value&& - std::is_nothrow_default_constructible::value) {} + reference operator*() const { return *(iter_); } + pointer operator->() const { return iter_.operator->(); } - explicit parallel_hash_set(size_t bucket_cnt, - const hasher& hash_param = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) { - for (auto& inner : sets_) - inner.set_ = EmbeddedSet(bucket_cnt / N, hash_param, eq, alloc); + const_iterator& operator++() { + ++iter_; + return *this; } + const_iterator operator++(int) { return iter_++; } - parallel_hash_set(size_t bucket_cnt, - const hasher& hash_param, - const allocator_type& alloc) - : parallel_hash_set(bucket_cnt, hash_param, key_equal(), alloc) {} + friend bool operator==(const const_iterator& a, const const_iterator& b) { + return a.iter_ == b.iter_; + } + friend bool operator!=(const const_iterator& a, const const_iterator& b) { + return !(a == b); + } - parallel_hash_set(size_t bucket_cnt, const allocator_type& alloc) - : parallel_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} + private: + const_iterator(const Inner* inner, const Inner* inner_end, const EmbeddedIterator& it) + : iter_(const_cast(inner), const_cast(inner_end), + const_cast(it)) {} - explicit parallel_hash_set(const allocator_type& alloc) - : parallel_hash_set(0, hasher(), key_equal(), alloc) {} + iterator iter_; + }; - template - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(bucket_cnt, hash_param, eq, alloc) { - insert(first, last); - } + using node_type = node_handle, Alloc>; + using insert_return_type = InsertReturnType; - template - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(first, last, bucket_cnt, hash_param, key_equal(), alloc) {} + // ------------------------- c o n s t r u c t o r s ------------------ - template - parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} + parallel_hash_set() noexcept( + std::is_nothrow_default_constructible::value&& + std::is_nothrow_default_constructible::value&& + std::is_nothrow_default_constructible::value) {} - template - parallel_hash_set(InputIter first, InputIter last, const allocator_type& alloc) - : parallel_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + explicit parallel_hash_set(size_t bucket_cnt, const hasher& hash_param = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) { + for (auto& inner : sets_) + inner.set_ = EmbeddedSet(bucket_cnt / N, hash_param, eq, alloc); + } - // Instead of accepting std::initializer_list as the first - // argument like std::unordered_set does, we have two overloads - // that accept std::initializer_list and std::initializer_list. - // This is advantageous for performance. - // - // // Turns {"abc", "def"} into std::initializer_list, then copies - // // the strings into the set. - // std::unordered_set s = {"abc", "def"}; - // - // // Turns {"abc", "def"} into std::initializer_list, then - // // copies the strings into the set. - // phmap::flat_hash_set s = {"abc", "def"}; - // - // The same trick is used in insert(). - // - // The enabler is necessary to prevent this constructor from triggering where - // the copy constructor is meant to be called. - // - // phmap::flat_hash_set a, b{a}; - // - // RequiresNotInit is a workaround for gcc prior to 7.1. - // -------------------------------------------------------------------- - template = 0, RequiresInsertable = 0> - parallel_hash_set(std::initializer_list init, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} - - parallel_hash_set(std::initializer_list init, size_t bucket_cnt = 0, - const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} - - template = 0, RequiresInsertable = 0> - parallel_hash_set(std::initializer_list init, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list init, size_t bucket_cnt, - const hasher& hash_param, const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} - - template = 0, RequiresInsertable = 0> - parallel_hash_set(std::initializer_list init, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list init, size_t bucket_cnt, - const allocator_type& alloc) - : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} - - template = 0, RequiresInsertable = 0> - parallel_hash_set(std::initializer_list init, const allocator_type& alloc) - : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - parallel_hash_set(std::initializer_list init, - const allocator_type& alloc) - : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - parallel_hash_set(const parallel_hash_set& that) - : parallel_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} - - parallel_hash_set(const parallel_hash_set& that, const allocator_type& a) - : parallel_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - for (size_t i=0; i::value&& - std::is_nothrow_copy_constructible::value&& - std::is_nothrow_copy_constructible::value) - : parallel_hash_set(std::move(that), that.alloc_ref()) { - } + parallel_hash_set(size_t bucket_cnt, const hasher& hash_param, + const allocator_type& alloc) + : parallel_hash_set(bucket_cnt, hash_param, key_equal(), alloc) {} - parallel_hash_set(parallel_hash_set&& that, const allocator_type& a) - { - for (size_t i=0; i::is_always_equal::value && - std::is_nothrow_move_assignable::value && - std::is_nothrow_move_assignable::value) { - for (size_t i=0; i + parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, + const hasher& hash_param = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : parallel_hash_set(bucket_cnt, hash_param, eq, alloc) { + insert(first, last); + } - iterator begin() { - auto it = iterator(&sets_[0], &sets_[0] + num_tables, sets_[0].set_.begin()); - it.skip_empty(); - return it; - } + template + parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const hasher& hash_param, const allocator_type& alloc) + : parallel_hash_set(first, last, bucket_cnt, hash_param, key_equal(), alloc) {} + + template + parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const allocator_type& alloc) + : parallel_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} + + template + parallel_hash_set(InputIter first, InputIter last, const allocator_type& alloc) + : parallel_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + + // Instead of accepting std::initializer_list as the first + // argument like std::unordered_set does, we have two overloads + // that accept std::initializer_list and std::initializer_list. + // This is advantageous for performance. + // + // // Turns {"abc", "def"} into std::initializer_list, then copies + // // the strings into the set. + // std::unordered_set s = {"abc", "def"}; + // + // // Turns {"abc", "def"} into std::initializer_list, then + // // copies the strings into the set. + // phmap::flat_hash_set s = {"abc", "def"}; + // + // The same trick is used in insert(). + // + // The enabler is necessary to prevent this constructor from triggering where + // the copy constructor is meant to be called. + // + // phmap::flat_hash_set a, b{a}; + // + // RequiresNotInit is a workaround for gcc prior to 7.1. + // -------------------------------------------------------------------- + template = 0, RequiresInsertable = 0> + parallel_hash_set(std::initializer_list init, size_t bucket_cnt = 0, + const hasher& hash_param = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} + + parallel_hash_set(std::initializer_list init, size_t bucket_cnt = 0, + const hasher& hash_param = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} + + template = 0, RequiresInsertable = 0> + parallel_hash_set(std::initializer_list init, size_t bucket_cnt, + const hasher& hash_param, const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} + + parallel_hash_set(std::initializer_list init, size_t bucket_cnt, + const hasher& hash_param, const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} + + template = 0, RequiresInsertable = 0> + parallel_hash_set(std::initializer_list init, size_t bucket_cnt, + const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + parallel_hash_set(std::initializer_list init, size_t bucket_cnt, + const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + template = 0, RequiresInsertable = 0> + parallel_hash_set(std::initializer_list init, const allocator_type& alloc) + : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + parallel_hash_set(std::initializer_list init, const allocator_type& alloc) + : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + parallel_hash_set(const parallel_hash_set& that) + : parallel_hash_set( + that, AllocTraits::select_on_container_copy_construction(that.alloc_ref())) {} + + parallel_hash_set(const parallel_hash_set& that, const allocator_type& a) + : parallel_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + for (size_t i = 0; i < num_tables; ++i) sets_[i].set_ = {that.sets_[i].set_, a}; + } - iterator end() { return iterator(); } - const_iterator begin() const { return const_cast(this)->begin(); } - const_iterator end() const { return const_cast(this)->end(); } - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } + parallel_hash_set(parallel_hash_set&& that) noexcept( + std::is_nothrow_copy_constructible::value&& + std::is_nothrow_copy_constructible::value&& + std::is_nothrow_copy_constructible::value) + : parallel_hash_set(std::move(that), that.alloc_ref()) {} - bool empty() const { return !size(); } + parallel_hash_set(parallel_hash_set&& that, const allocator_type& a) { + for (size_t i = 0; i < num_tables; ++i) + sets_[i].set_ = {std::move(that.sets_[i]).set_, a}; + } - size_t size() const { - size_t sz = 0; - for (const auto& inner : sets_) - sz += inner.set_.size(); - return sz; - } - - size_t capacity() const { - size_t c = 0; - for (const auto& inner : sets_) - c += inner.set_.capacity(); - return c; - } + parallel_hash_set& operator=(const parallel_hash_set& that) { + for (size_t i = 0; i < num_tables; ++i) sets_[i].set_ = that.sets_[i].set_; + return *this; + } - size_t max_size() const { return (std::numeric_limits::max)(); } + parallel_hash_set& operator=(parallel_hash_set&& that) noexcept( + phmap::allocator_traits::is_always_equal::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_assignable::value) { + for (size_t i = 0; i < num_tables; ++i) sets_[i].set_ = std::move(that.sets_[i].set_); + return *this; + } - PHMAP_ATTRIBUTE_REINITIALIZES void clear() { - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.clear(); - } - } + ~parallel_hash_set() {} - // extension - clears only soecified submap - // ---------------------------------------- - void clear(std::size_t submap_index) { - Inner& inner = sets_[submap_index]; - typename Lockable::UniqueLock m(inner); - inner.set_.clear(); - } + iterator begin() { + auto it = iterator(&sets_[0], &sets_[0] + num_tables, sets_[0].set_.begin()); + it.skip_empty(); + return it; + } - // This overload kicks in when the argument is an rvalue of insertable and - // decomposable type other than init_type. - // - // flat_hash_map m; - // m.insert(std::make_pair("abc", 42)); - // -------------------------------------------------------------------- - template = 0, - typename std::enable_if::value, int>::type = 0, - T* = nullptr> - std::pair insert(T&& value) { - return emplace(std::forward(value)); - } + iterator end() { return iterator(); } + const_iterator begin() const { return const_cast(this)->begin(); } + const_iterator end() const { return const_cast(this)->end(); } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } - // This overload kicks in when the argument is a bitfield or an lvalue of - // insertable and decomposable type. - // - // union { int n : 1; }; - // flat_hash_set s; - // s.insert(n); - // - // flat_hash_set s; - // const char* p = "hello"; - // s.insert(p); - // - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable with RequiresInsertable. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - // -------------------------------------------------------------------- - template < - class T, RequiresInsertable = 0, - typename std::enable_if::value, int>::type = 0> - std::pair insert(const T& value) { - return emplace(value); - } + bool empty() const { return !size(); } - // This overload kicks in when the argument is an rvalue of init_type. Its - // purpose is to handle brace-init-list arguments. - // - // flat_hash_set> s; - // s.insert({"abc", 42}); - // -------------------------------------------------------------------- - std::pair insert(init_type&& value) { - return emplace(std::move(value)); - } + size_t size() const { + size_t sz = 0; + for (const auto& inner : sets_) sz += inner.set_.size(); + return sz; + } - template = 0, - typename std::enable_if::value, int>::type = 0, - T* = nullptr> - iterator insert(const_iterator, T&& value) { - return insert(std::forward(value)).first; - } + size_t capacity() const { + size_t c = 0; + for (const auto& inner : sets_) c += inner.set_.capacity(); + return c; + } - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable with RequiresInsertable. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - // -------------------------------------------------------------------- - template < - class T, RequiresInsertable = 0, - typename std::enable_if::value, int>::type = 0> - iterator insert(const_iterator, const T& value) { - return insert(value).first; - } + size_t max_size() const { return (std::numeric_limits::max)(); } - iterator insert(const_iterator, init_type&& value) { - return insert(std::move(value)).first; + PHMAP_ATTRIBUTE_REINITIALIZES void clear() { + for (auto& inner : sets_) { + typename Lockable::UniqueLock m(inner); + inner.set_.clear(); } + } - template - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) insert(*first); - } + // extension - clears only soecified submap + // ---------------------------------------- + void clear(std::size_t submap_index) { + Inner& inner = sets_[submap_index]; + typename Lockable::UniqueLock m(inner); + inner.set_.clear(); + } - template = 0, RequiresInsertable = 0> - void insert(std::initializer_list ilist) { - insert(ilist.begin(), ilist.end()); - } + // This overload kicks in when the argument is an rvalue of insertable and + // decomposable type other than init_type. + // + // flat_hash_map m; + // m.insert(std::make_pair("abc", 42)); + // -------------------------------------------------------------------- + template = 0, + typename std::enable_if::value, int>::type = 0, + T* = nullptr> + std::pair insert(T&& value) { + return emplace(std::forward(value)); + } - void insert(std::initializer_list ilist) { - insert(ilist.begin(), ilist.end()); - } + // This overload kicks in when the argument is a bitfield or an lvalue of + // insertable and decomposable type. + // + // union { int n : 1; }; + // flat_hash_set s; + // s.insert(n); + // + // flat_hash_set s; + // const char* p = "hello"; + // s.insert(p); + // + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable with RequiresInsertable. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + // -------------------------------------------------------------------- + template = 0, + typename std::enable_if::value, int>::type = 0> + std::pair insert(const T& value) { + return emplace(value); + } - insert_return_type insert(node_type&& node) { - if (!node) - return {end(), false, node_type()}; - auto& key = node.key(); - size_t hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - - typename Lockable::UniqueLock m(inner); - auto res = set.insert(std::move(node), hashval); - return { make_iterator(&inner, res.position), - res.inserted, - res.inserted ? node_type() : std::move(res.node) }; - } + // This overload kicks in when the argument is an rvalue of init_type. Its + // purpose is to handle brace-init-list arguments. + // + // flat_hash_set> s; + // s.insert({"abc", 42}); + // -------------------------------------------------------------------- + std::pair insert(init_type&& value) { + return emplace(std::move(value)); + } - iterator insert(const_iterator, node_type&& node) { - return insert(std::move(node)).first; - } + template = 0, + typename std::enable_if::value, int>::type = 0, + T* = nullptr> + iterator insert(const_iterator, T&& value) { + return insert(std::forward(value)).first; + } - struct ReturnKey_ - { - template - Key operator()(Key&& k, const Args&...) const { - return std::forward(k); - } - }; + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable with RequiresInsertable. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + // -------------------------------------------------------------------- + template = 0, + typename std::enable_if::value, int>::type = 0> + iterator insert(const_iterator, const T& value) { + return insert(value).first; + } - // -------------------------------------------------------------------- - // phmap expension: emplace_with_hash - // ---------------------------------- - // same as emplace, but hashval is provided - // -------------------------------------------------------------------- - template - std::pair emplace_decomposable_with_hash(const K& key, size_t hashval, Args&&... args) - { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward(args)...)); - } + iterator insert(const_iterator, init_type&& value) { + return insert(std::move(value)).first; + } - struct EmplaceDecomposableHashval - { - template - std::pair operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable_with_hash(key, hashval, std::forward(args)...); - } - parallel_hash_set& s; - size_t hashval; - }; + template + void insert(InputIt first, InputIt last) { + for (; first != last; ++first) insert(*first); + } - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - // -------------------------------------------------------------------- - template ::value, int>::type = 0> - std::pair emplace_with_hash(size_t hashval, Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, - std::forward(args)...); - } + template = 0, RequiresInsertable = 0> + void insert(std::initializer_list ilist) { + insert(ilist.begin(), ilist.end()); + } - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - // -------------------------------------------------------------------- - template ::value, int>::type = 0> - std::pair emplace_with_hash(size_t hashval, Args&&... args) { - typename std::aligned_storage::type raw; - slot_type* slot = reinterpret_cast(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); - const auto& elem = PolicyTraits::element(slot); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - typename EmbeddedSet::template InsertSlotWithHash f { - inner, std::move(*slot), hashval}; - return make_rv(PolicyTraits::apply(f, elem)); - } + void insert(std::initializer_list ilist) { + insert(ilist.begin(), ilist.end()); + } - template - iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { - return emplace_with_hash(hashval, std::forward(args)...).first; - } + insert_return_type insert(node_type&& node) { + if (!node) return {end(), false, node_type()}; + auto& key = node.key(); + size_t hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + + typename Lockable::UniqueLock m(inner); + auto res = set.insert(std::move(node), hashval); + return {make_iterator(&inner, res.position), res.inserted, + res.inserted ? node_type() : std::move(res.node)}; + } - template - iterator lazy_emplace_with_hash(size_t hashval, const key_arg& key, F&& f) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward(f))); - } + iterator insert(const_iterator, node_type&& node) { + return insert(std::move(node)).first; + } - // -------------------------------------------------------------------- - // end of phmap expension - // -------------------------------------------------------------------- + struct ReturnKey_ { + template + Key operator()(Key&& k, const Args&...) const { + return std::forward(k); + } + }; + + // -------------------------------------------------------------------- + // phmap expension: emplace_with_hash + // ---------------------------------- + // same as emplace, but hashval is provided + // -------------------------------------------------------------------- + template + std::pair emplace_decomposable_with_hash(const K& key, size_t hashval, + Args&&... args) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_rv(&inner, + set.emplace_decomposable(key, hashval, std::forward(args)...)); + } + struct EmplaceDecomposableHashval { template - std::pair emplace_decomposable(const K& key, Args&&... args) - { - size_t hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward(args)...)); - } + std::pair operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable_with_hash(key, hashval, std::forward(args)...); + } + parallel_hash_set& s; + size_t hashval; + }; + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + // -------------------------------------------------------------------- + template ::value, int>::type = 0> + std::pair emplace_with_hash(size_t hashval, Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, + std::forward(args)...); + } - struct EmplaceDecomposable - { - template - std::pair operator()(const K& key, Args&&... args) const { - return s.emplace_decomposable(key, std::forward(args)...); - } - parallel_hash_set& s; - }; + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + // -------------------------------------------------------------------- + template ::value, int>::type = 0> + std::pair emplace_with_hash(size_t hashval, Args&&... args) { + typename std::aligned_storage::type raw; + slot_type* slot = reinterpret_cast(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); + const auto& elem = PolicyTraits::element(slot); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + typename EmbeddedSet::template InsertSlotWithHash f{inner, std::move(*slot), + hashval}; + return make_rv(PolicyTraits::apply(f, elem)); + } - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - // -------------------------------------------------------------------- - template ::value, int>::type = 0> - std::pair emplace(Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposable{*this}, - std::forward(args)...); - } + template + iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { + return emplace_with_hash(hashval, std::forward(args)...).first; + } - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - // -------------------------------------------------------------------- - template ::value, int>::type = 0> - std::pair emplace(Args&&... args) { - typename std::aligned_storage::type raw; - slot_type* slot = reinterpret_cast(&raw); - size_t hashval = this->hash(PolicyTraits::key(slot)); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); - const auto& elem = PolicyTraits::element(slot); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - typename EmbeddedSet::template InsertSlotWithHash f { - inner, std::move(*slot), hashval}; - return make_rv(PolicyTraits::apply(f, elem)); - } + template + iterator lazy_emplace_with_hash(size_t hashval, const key_arg& key, F&& f) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_iterator(&inner, + set.lazy_emplace_with_hash(key, hashval, std::forward(f))); + } - template - iterator emplace_hint(const_iterator, Args&&... args) { - return emplace(std::forward(args)...).first; - } + // -------------------------------------------------------------------- + // end of phmap expension + // -------------------------------------------------------------------- + + template + std::pair emplace_decomposable(const K& key, Args&&... args) { + size_t hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_rv(&inner, + set.emplace_decomposable(key, hashval, std::forward(args)...)); + } - iterator make_iterator(Inner* inner, const EmbeddedIterator it) - { - if (it == inner->set_.end()) - return iterator(); - return iterator(inner, &sets_[0] + num_tables, it); - } + struct EmplaceDecomposable { + template + std::pair operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable(key, std::forward(args)...); + } + parallel_hash_set& s; + }; + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + // -------------------------------------------------------------------- + template ::value, int>::type = 0> + std::pair emplace(Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposable{*this}, std::forward(args)...); + } - std::pair make_rv(Inner* inner, - const std::pair& res) - { - return {iterator(inner, &sets_[0] + num_tables, res.first), res.second}; - } + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + // -------------------------------------------------------------------- + template ::value, int>::type = 0> + std::pair emplace(Args&&... args) { + typename std::aligned_storage::type raw; + slot_type* slot = reinterpret_cast(&raw); + size_t hashval = this->hash(PolicyTraits::key(slot)); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); + const auto& elem = PolicyTraits::element(slot); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + typename EmbeddedSet::template InsertSlotWithHash f{inner, std::move(*slot), + hashval}; + return make_rv(PolicyTraits::apply(f, elem)); + } - template - iterator lazy_emplace(const key_arg& key, F&& f) { - auto hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UniqueLock m(inner); - return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward(f))); - } + template + iterator emplace_hint(const_iterator, Args&&... args) { + return emplace(std::forward(args)...).first; + } - template - bool lazy_emplace_l(const key_arg& key, FExists&& fExists, FEmplace&& fEmplace) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(key, m); - Inner* inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.lazy_emplace_at(std::get<1>(res), std::forward(fEmplace)); - else { - auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); - std::forward(fExists)(Policy::value(&*it)); - } - return std::get<2>(res); - } + iterator make_iterator(Inner* inner, const EmbeddedIterator it) { + if (it == inner->set_.end()) return iterator(); + return iterator(inner, &sets_[0] + num_tables, it); + } - // Extension API: support iterating over all values - // - // flat_hash_set s; - // s.insert(...); - // s.for_each([](auto const & key) { - // // Safely iterates over all the keys - // }); - template - void for_each(F&& fCallback) const { - for (auto const& inner : sets_) { - typename Lockable::SharedLock m(const_cast(inner)); - std::for_each(inner.set_.begin(), inner.set_.end(), fCallback); - } - } + std::pair make_rv(Inner* inner, + const std::pair& res) { + return {iterator(inner, &sets_[0] + num_tables, res.first), res.second}; + } - // this version allows to modify the values - void for_each_m(std::function && fCallback) { - for (auto& inner : sets_) { - typename Lockable::UniqueLock m(const_cast(inner)); - std::for_each(inner.set_.begin(), inner.set_.end(), fCallback); - } - } + template + iterator lazy_emplace(const key_arg& key, F&& f) { + auto hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_iterator(&inner, + set.lazy_emplace_with_hash(key, hashval, std::forward(f))); + } - // Extension API: support for heterogeneous keys. - // - // std::unordered_set s; - // // Turns "abc" into std::string. - // s.erase("abc"); - // - // flat_hash_set s; - // // Uses "abc" directly without copying it into std::string. - // s.erase("abc"); - // -------------------------------------------------------------------- - template - size_type erase(const key_arg& key) { - auto hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::UpgradeLock m(inner); - auto it = set.find(key, hashval); - if (it == set.end()) - return 0; - - typename Lockable::UpgradeToUnique unique(m); - set._erase(it); - return 1; - } + template + bool lazy_emplace_l(const key_arg& key, FExists&& fExists, FEmplace&& fEmplace) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(key, m); + Inner* inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.lazy_emplace_at(std::get<1>(res), std::forward(fEmplace)); + else { + auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); + std::forward(fExists)(Policy::value(&*it)); + } + return std::get<2>(res); + } - // -------------------------------------------------------------------- - iterator erase(const_iterator cit) { return erase(cit.iter_); } + // Extension API: support iterating over all values + // + // flat_hash_set s; + // s.insert(...); + // s.for_each([](auto const & key) { + // // Safely iterates over all the keys + // }); + template + void for_each(F&& fCallback) const { + for (auto const& inner : sets_) { + typename Lockable::SharedLock m(const_cast(inner)); + std::for_each(inner.set_.begin(), inner.set_.end(), fCallback); + } + } - // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, - // this method returns void to reduce algorithmic complexity to O(1). In - // order to erase while iterating across a map, use the following idiom (which - // also works for standard containers): - // - // for (auto it = m.begin(), end = m.end(); it != end;) { - // if () { - // m._erase(it++); - // } else { - // ++it; - // } - // } - // -------------------------------------------------------------------- - void _erase(iterator it) { - assert(it.inner_ != nullptr); - it.inner_->set_._erase(it.it_); + // this version allows to modify the values + void for_each_m(std::function&& fCallback) { + for (auto& inner : sets_) { + typename Lockable::UniqueLock m(const_cast(inner)); + std::for_each(inner.set_.begin(), inner.set_.end(), fCallback); } - void _erase(const_iterator cit) { _erase(cit.iter_); } + } - // This overload is necessary because otherwise erase(const K&) would be - // a better match if non-const iterator is passed as an argument. - // -------------------------------------------------------------------- - iterator erase(iterator it) { _erase(it++); return it; } + // Extension API: support for heterogeneous keys. + // + // std::unordered_set s; + // // Turns "abc" into std::string. + // s.erase("abc"); + // + // flat_hash_set s; + // // Uses "abc" directly without copying it into std::string. + // s.erase("abc"); + // -------------------------------------------------------------------- + template + size_type erase(const key_arg& key) { + auto hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UpgradeLock m(inner); + auto it = set.find(key, hashval); + if (it == set.end()) return 0; + + typename Lockable::UpgradeToUnique unique(m); + set._erase(it); + return 1; + } + + // -------------------------------------------------------------------- + iterator erase(const_iterator cit) { return erase(cit.iter_); } + + // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, + // this method returns void to reduce algorithmic complexity to O(1). In + // order to erase while iterating across a map, use the following idiom (which + // also works for standard containers): + // + // for (auto it = m.begin(), end = m.end(); it != end;) { + // if () { + // m._erase(it++); + // } else { + // ++it; + // } + // } + // -------------------------------------------------------------------- + void _erase(iterator it) { + assert(it.inner_ != nullptr); + it.inner_->set_._erase(it.it_); + } + void _erase(const_iterator cit) { _erase(cit.iter_); } + + // This overload is necessary because otherwise erase(const K&) would be + // a better match if non-const iterator is passed as an argument. + // -------------------------------------------------------------------- + iterator erase(iterator it) { + _erase(it++); + return it; + } - iterator erase(const_iterator first, const_iterator last) { - while (first != last) { - _erase(first++); - } - return last.iter_; + iterator erase(const_iterator first, const_iterator last) { + while (first != last) { + _erase(first++); } + return last.iter_; + } - // Moves elements from `src` into `this`. - // If the element already exists in `this`, it is left unmodified in `src`. - // -------------------------------------------------------------------- - template - void merge(parallel_hash_set& src) { // NOLINT - assert(this != &src); - if (this != &src) - { - for (size_t i=0; i + void merge(parallel_hash_set& src) { // NOLINT + assert(this != &src); + if (this != &src) { + for (size_t i = 0; i < num_tables; ++i) { + typename Lockable::UniqueLocks l(sets_[i], src.sets_[i]); + sets_[i].set_.merge(src.sets_[i].set_); + } } + } - template - void merge(parallel_hash_set&& src) { - merge(src); - } + template + void merge(parallel_hash_set&& src) { + merge(src); + } - node_type extract(const_iterator position) { - return position.iter_.inner_->set_.extract(EmbeddedConstIterator(position.iter_.it_)); - } + node_type extract(const_iterator position) { + return position.iter_.inner_->set_.extract(EmbeddedConstIterator(position.iter_.it_)); + } - template < - class K = key_type, - typename std::enable_if::value, int>::type = 0> - node_type extract(const key_arg& key) { - auto it = find(key); - return it == end() ? node_type() : extract(const_iterator{it}); - } + template ::value, int>::type = 0> + node_type extract(const key_arg& key) { + auto it = find(key); + return it == end() ? node_type() : extract(const_iterator{it}); + } - void swap(parallel_hash_set& that) noexcept( - IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - using std::swap; - for (size_t i=0; i() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + using std::swap; + for (size_t i = 0; i < num_tables; ++i) { + typename Lockable::UniqueLocks l(sets_[i], that.sets_[i]); + swap(sets_[i].set_, that.sets_[i].set_); } + } - void rehash(size_t n) { - size_t nn = n / num_tables; - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.rehash(nn); - } + void rehash(size_t n) { + size_t nn = n / num_tables; + for (auto& inner : sets_) { + typename Lockable::UniqueLock m(inner); + inner.set_.rehash(nn); } + } - void reserve(size_t n) - { - size_t target = GrowthToLowerboundCapacity(n); - size_t normalized = 16 * NormalizeCapacity(n / num_tables); - rehash(normalized > target ? normalized : target); - } + void reserve(size_t n) { + size_t target = GrowthToLowerboundCapacity(n); + size_t normalized = 16 * NormalizeCapacity(n / num_tables); + rehash(normalized > target ? normalized : target); + } - // Extension API: support for heterogeneous keys. - // - // std::unordered_set s; - // // Turns "abc" into std::string. - // s.count("abc"); - // - // ch_set s; - // // Uses "abc" directly without copying it into std::string. - // s.count("abc"); - // -------------------------------------------------------------------- - template - size_t count(const key_arg& key) const { - return find(key) == end() ? 0 : 1; - } + // Extension API: support for heterogeneous keys. + // + // std::unordered_set s; + // // Turns "abc" into std::string. + // s.count("abc"); + // + // ch_set s; + // // Uses "abc" directly without copying it into std::string. + // s.count("abc"); + // -------------------------------------------------------------------- + template + size_t count(const key_arg& key) const { + return find(key) == end() ? 0 : 1; + } - // Issues CPU prefetch instructions for the memory needed to find or insert - // a key. Like all lookup functions, this support heterogeneous keys. - // - // NOTE: This is a very low level operation and should not be used without - // specific benchmarks indicating its importance. - // -------------------------------------------------------------------- - void prefetch_hash(size_t hashval) const { - const Inner& inner = sets_[subidx(hashval)]; - const auto& set = inner.set_; - typename Lockable::SharedLock m(const_cast(inner)); - set.prefetch_hash(hashval); - } + // Issues CPU prefetch instructions for the memory needed to find or insert + // a key. Like all lookup functions, this support heterogeneous keys. + // + // NOTE: This is a very low level operation and should not be used without + // specific benchmarks indicating its importance. + // -------------------------------------------------------------------- + void prefetch_hash(size_t hashval) const { + const Inner& inner = sets_[subidx(hashval)]; + const auto& set = inner.set_; + typename Lockable::SharedLock m(const_cast(inner)); + set.prefetch_hash(hashval); + } - template - void prefetch(const key_arg& key) const { - prefetch_hash(this->hash(key)); - } + template + void prefetch(const key_arg& key) const { + prefetch_hash(this->hash(key)); + } - // The API of find() has two extensions. - // - // 1. The hash can be passed by the user. It must be equal to the hash of the - // key. - // - // 2. The type of the key argument doesn't have to be key_type. This is so - // called heterogeneous key support. - // -------------------------------------------------------------------- - template - iterator find(const key_arg& key, size_t hashval) { - typename Lockable::SharedLock m; - return find(key, hashval, m); - } + // The API of find() has two extensions. + // + // 1. The hash can be passed by the user. It must be equal to the hash of the + // key. + // + // 2. The type of the key argument doesn't have to be key_type. This is so + // called heterogeneous key support. + // -------------------------------------------------------------------- + template + iterator find(const key_arg& key, size_t hashval) { + typename Lockable::SharedLock m; + return find(key, hashval, m); + } - template - iterator find(const key_arg& key) { - return find(key, this->hash(key)); - } + template + iterator find(const key_arg& key) { + return find(key, this->hash(key)); + } - template - const_iterator find(const key_arg& key, size_t hashval) const { - return const_cast(this)->find(key, hashval); - } + template + const_iterator find(const key_arg& key, size_t hashval) const { + return const_cast(this)->find(key, hashval); + } - template - const_iterator find(const key_arg& key) const { - return find(key, this->hash(key)); - } + template + const_iterator find(const key_arg& key) const { + return find(key, this->hash(key)); + } - template - bool contains(const key_arg& key) const { - return find(key) != end(); - } + template + bool contains(const key_arg& key) const { + return find(key) != end(); + } - template - bool contains(const key_arg& key, size_t hashval) const { - return find(key, hashval) != end(); - } + template + bool contains(const key_arg& key, size_t hashval) const { + return find(key, hashval) != end(); + } - template - std::pair equal_range(const key_arg& key) { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } + template + std::pair equal_range(const key_arg& key) { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } - template - std::pair equal_range( - const key_arg& key) const { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } + template + std::pair equal_range(const key_arg& key) const { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } - size_t bucket_count() const { - size_t sz = 0; - for (const auto& inner : sets_) - { - typename Lockable::SharedLock m(const_cast(inner)); - sz += inner.set_.bucket_count(); - } - return sz; + size_t bucket_count() const { + size_t sz = 0; + for (const auto& inner : sets_) { + typename Lockable::SharedLock m(const_cast(inner)); + sz += inner.set_.bucket_count(); } + return sz; + } - float load_factor() const { - size_t _capacity = bucket_count(); - return _capacity ? static_cast(static_cast(size()) / _capacity) : 0; - } + float load_factor() const { + size_t _capacity = bucket_count(); + return _capacity ? static_cast(static_cast(size()) / _capacity) : 0; + } - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { - // Does nothing. - } + float max_load_factor() const { return 1.0f; } + void max_load_factor(float) { + // Does nothing. + } - hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function - key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } + hasher hash_function() const { + return hash_ref(); + } // warning: doesn't match internal hash - use hash() member function + key_equal key_eq() const { return eq_ref(); } + allocator_type get_allocator() const { return alloc_ref(); } - friend bool operator==(const parallel_hash_set& a, const parallel_hash_set& b) { - return std::equal(a.sets_.begin(), a.sets_.end(), b.sets_.begin()); - } + friend bool operator==(const parallel_hash_set& a, const parallel_hash_set& b) { + return std::equal(a.sets_.begin(), a.sets_.end(), b.sets_.begin()); + } - friend bool operator!=(const parallel_hash_set& a, const parallel_hash_set& b) { - return !(a == b); - } + friend bool operator!=(const parallel_hash_set& a, const parallel_hash_set& b) { + return !(a == b); + } - friend void swap(parallel_hash_set& a, - parallel_hash_set& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); - } + friend void swap(parallel_hash_set& a, + parallel_hash_set& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); + } - template - size_t hash(const K& key) const { - return HashElement{hash_ref()}(key); - } + template + size_t hash(const K& key) const { + return HashElement{hash_ref()}(key); + } #if !defined(PHMAP_NON_DETERMINISTIC) - template - bool phmap_dump(OutputArchive& ar) const; + template + bool phmap_dump(OutputArchive& ar) const; - template - bool phmap_load(InputArchive& ar); + template + bool phmap_load(InputArchive& ar); #endif -private: - template - friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; - - struct FindElement - { - template - const_iterator operator()(const K& key, Args&&...) const { - return s.find(key); - } - const parallel_hash_set& s; - }; - - struct HashElement - { - template - size_t operator()(const K& key, Args&&...) const { - return phmap_mix()(h(key)); - } - const hasher& h; - }; - - template - struct EqualElement - { - template - bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); - } - const K1& rhs; - const key_equal& eq; - }; + private: + template + friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. - // -------------------------------------------------------------------- - void erase_meta_only(const_iterator cit) { - auto &it = cit.iter_; - assert(it.set_ != nullptr); - it.set_.erase_meta_only(const_iterator(it.it_)); + struct FindElement { + template + const_iterator operator()(const K& key, Args&&...) const { + return s.find(key); } + const parallel_hash_set& s; + }; - void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { - for (auto& inner : sets_) - { - typename Lockable::UniqueLock m(inner); - inner.set_.drop_deletes_without_resize(); - } - } + struct HashElement { + template + size_t operator()(const K& key, Args&&...) const { + return phmap_mix()(h(key)); + } + const hasher& h; + }; + + template + struct EqualElement { + template + bool operator()(const K2& lhs, Args&&...) const { + return eq(lhs, rhs); + } + const K1& rhs; + const key_equal& eq; + }; + + // "erases" the object from the container, except that it doesn't actually + // destroy the object. It only updates all the metadata of the class. + // This can be used in conjunction with Policy::transfer to move the object to + // another place. + // -------------------------------------------------------------------- + void erase_meta_only(const_iterator cit) { + auto& it = cit.iter_; + assert(it.set_ != nullptr); + it.set_.erase_meta_only(const_iterator(it.it_)); + } - bool has_element(const value_type& elem) const { - size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - typename Lockable::SharedLock m(const_cast(inner)); - return set.has_element(elem, hashval); + void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { + for (auto& inner : sets_) { + typename Lockable::UniqueLock m(inner); + inner.set_.drop_deletes_without_resize(); } + } - // TODO(alkis): Optimize this assuming *this and that don't overlap. - // -------------------------------------------------------------------- - parallel_hash_set& move_assign(parallel_hash_set&& that, std::true_type) { - parallel_hash_set tmp(std::move(that)); - swap(tmp); - return *this; - } + bool has_element(const value_type& elem) const { + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::SharedLock m(const_cast(inner)); + return set.has_element(elem, hashval); + } - parallel_hash_set& move_assign(parallel_hash_set&& that, std::false_type) { - parallel_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); - return *this; - } + // TODO(alkis): Optimize this assuming *this and that don't overlap. + // -------------------------------------------------------------------- + parallel_hash_set& move_assign(parallel_hash_set&& that, std::true_type) { + parallel_hash_set tmp(std::move(that)); + swap(tmp); + return *this; + } -protected: - template - pointer find_ptr(const key_arg& key, size_t hashval, L& mutexlock) - { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(L(inner)); - return set.find_ptr(key, hashval); - } + parallel_hash_set& move_assign(parallel_hash_set&& that, std::false_type) { + parallel_hash_set tmp(std::move(that), alloc_ref()); + swap(tmp); + return *this; + } - template - iterator find(const key_arg& key, size_t hashval, L& mutexlock) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(L(inner)); - return make_iterator(&inner, set.find(key, hashval)); - } + protected: + template + pointer find_ptr(const key_arg& key, size_t hashval, L& mutexlock) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(L(inner)); + return set.find_ptr(key, hashval); + } - template - std::tuple - find_or_prepare_insert_with_hash(size_t hashval, const K& key, typename Lockable::UniqueLock &mutexlock) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(typename Lockable::UniqueLock(inner)); - auto p = set.find_or_prepare_insert(key, hashval); // std::pair - return std::make_tuple(&inner, p.first, p.second); - } + template + iterator find(const key_arg& key, size_t hashval, L& mutexlock) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(L(inner)); + return make_iterator(&inner, set.find(key, hashval)); + } - template - std::tuple - find_or_prepare_insert(const K& key, typename Lockable::UniqueLock &mutexlock) { - return find_or_prepare_insert_with_hash(this->hash(key), key, mutexlock); - } + template + std::tuple find_or_prepare_insert_with_hash( + size_t hashval, const K& key, typename Lockable::UniqueLock& mutexlock) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(typename Lockable::UniqueLock(inner)); + auto p = set.find_or_prepare_insert(key, hashval); // std::pair + return std::make_tuple(&inner, p.first, p.second); + } - iterator iterator_at(Inner *inner, - const EmbeddedIterator& it) { - return {inner, &sets_[0] + num_tables, it}; - } - const_iterator iterator_at(Inner *inner, - const EmbeddedIterator& it) const { - return {inner, &sets_[0] + num_tables, it}; - } + template + std::tuple find_or_prepare_insert( + const K& key, typename Lockable::UniqueLock& mutexlock) { + return find_or_prepare_insert_with_hash(this->hash(key), key, mutexlock); + } - static size_t subidx(size_t hashval) { - return ((hashval >> 8) ^ (hashval >> 16) ^ (hashval >> 24)) & mask; - } + iterator iterator_at(Inner* inner, const EmbeddedIterator& it) { + return {inner, &sets_[0] + num_tables, it}; + } + const_iterator iterator_at(Inner* inner, const EmbeddedIterator& it) const { + return {inner, &sets_[0] + num_tables, it}; + } - static size_t subcnt() { - return num_tables; - } + static size_t subidx(size_t hashval) { + return ((hashval >> 8) ^ (hashval >> 16) ^ (hashval >> 24)) & mask; + } -private: - friend struct RawHashSetTestOnlyAccess; + static size_t subcnt() { return num_tables; } - size_t growth_left() { - size_t sz = 0; - for (const auto& set : sets_) - sz += set.growth_left(); - return sz; - } + private: + friend struct RawHashSetTestOnlyAccess; - hasher& hash_ref() { return sets_[0].set_.hash_ref(); } - const hasher& hash_ref() const { return sets_[0].set_.hash_ref(); } - key_equal& eq_ref() { return sets_[0].set_.eq_ref(); } - const key_equal& eq_ref() const { return sets_[0].set_.eq_ref(); } - allocator_type& alloc_ref() { return sets_[0].set_.alloc_ref(); } - const allocator_type& alloc_ref() const { - return sets_[0].set_.alloc_ref(); - } + size_t growth_left() { + size_t sz = 0; + for (const auto& set : sets_) sz += set.growth_left(); + return sz; + } -protected: // protected in case users want to derive fromm this - std::array sets_; + hasher& hash_ref() { return sets_[0].set_.hash_ref(); } + const hasher& hash_ref() const { return sets_[0].set_.hash_ref(); } + key_equal& eq_ref() { return sets_[0].set_.eq_ref(); } + const key_equal& eq_ref() const { return sets_[0].set_.eq_ref(); } + allocator_type& alloc_ref() { return sets_[0].set_.alloc_ref(); } + const allocator_type& alloc_ref() const { return sets_[0].set_.alloc_ref(); } + + protected: // protected in case users want to derive fromm this + std::array sets_; }; // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- -template class RefSet, - class Mtx_, +template class RefSet, class Mtx_, class Policy, class Hash, class Eq, class Alloc> -class parallel_hash_map : public parallel_hash_set -{ - // P is Policy. It's passed as a template argument to support maps that have - // incomplete types as values, as in unordered_map. - // MappedReference<> may be a non-reference type. - template - using MappedReference = decltype(P::value( - std::addressof(std::declval()))); - - // MappedConstReference<> may be a non-reference type. - template - using MappedConstReference = decltype(P::value( - std::addressof(std::declval()))); - - using KeyArgImpl = - KeyArg::value && IsTransparent::value>; - - using Base = typename parallel_hash_map::parallel_hash_set; - using Lockable = phmap::LockableImpl; - -public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - template - using key_arg = typename KeyArgImpl::template type; - - static_assert(!std::is_reference::value, ""); - // TODO(alkis): remove this assertion and verify that reference mapped_type is - // supported. - static_assert(!std::is_reference::value, ""); - - using iterator = typename parallel_hash_map::parallel_hash_set::iterator; - using const_iterator = typename parallel_hash_map::parallel_hash_set::const_iterator; - - parallel_hash_map() {} +class parallel_hash_map + : public parallel_hash_set { + // P is Policy. It's passed as a template argument to support maps that have + // incomplete types as values, as in unordered_map. + // MappedReference<> may be a non-reference type. + template + using MappedReference = decltype( + P::value(std::addressof(std::declval()))); + + // MappedConstReference<> may be a non-reference type. + template + using MappedConstReference = decltype(P::value( + std::addressof(std::declval()))); + + using KeyArgImpl = KeyArg::value && IsTransparent::value>; + + using Base = typename parallel_hash_map::parallel_hash_set; + using Lockable = phmap::LockableImpl; + + public: + using key_type = typename Policy::key_type; + using mapped_type = typename Policy::mapped_type; + template + using key_arg = typename KeyArgImpl::template type; + + static_assert(!std::is_reference::value, ""); + // TODO(alkis): remove this assertion and verify that reference mapped_type is + // supported. + static_assert(!std::is_reference::value, ""); + + using iterator = typename parallel_hash_map::parallel_hash_set::iterator; + using const_iterator = typename parallel_hash_map::parallel_hash_set::const_iterator; + + parallel_hash_map() {} #ifdef __INTEL_COMPILER - using Base::parallel_hash_set; + using Base::parallel_hash_set; #else - using parallel_hash_map::parallel_hash_set::parallel_hash_set; + using parallel_hash_map::parallel_hash_set::parallel_hash_set; #endif - // The last two template parameters ensure that both arguments are rvalues - // (lvalue arguments are handled by the overloads below). This is necessary - // for supporting bitfield arguments. - // - // union { int n : 1; }; - // flat_hash_map m; - // m.insert_or_assign(n, n); - template - std::pair insert_or_assign(key_arg&& k, V&& v) { - return insert_or_assign_impl(std::forward(k), std::forward(v)); - } - - template - std::pair insert_or_assign(key_arg&& k, const V& v) { - return insert_or_assign_impl(std::forward(k), v); - } + // The last two template parameters ensure that both arguments are rvalues + // (lvalue arguments are handled by the overloads below). This is necessary + // for supporting bitfield arguments. + // + // union { int n : 1; }; + // flat_hash_map m; + // m.insert_or_assign(n, n); + template + std::pair insert_or_assign(key_arg&& k, V&& v) { + return insert_or_assign_impl(std::forward(k), std::forward(v)); + } - template - std::pair insert_or_assign(const key_arg& k, V&& v) { - return insert_or_assign_impl(k, std::forward(v)); - } + template + std::pair insert_or_assign(key_arg&& k, const V& v) { + return insert_or_assign_impl(std::forward(k), v); + } - template - std::pair insert_or_assign(const key_arg& k, const V& v) { - return insert_or_assign_impl(k, v); - } + template + std::pair insert_or_assign(const key_arg& k, V&& v) { + return insert_or_assign_impl(k, std::forward(v)); + } - template - iterator insert_or_assign(const_iterator, key_arg&& k, V&& v) { - return insert_or_assign(std::forward(k), std::forward(v)).first; - } + template + std::pair insert_or_assign(const key_arg& k, const V& v) { + return insert_or_assign_impl(k, v); + } - template - iterator insert_or_assign(const_iterator, key_arg&& k, const V& v) { - return insert_or_assign(std::forward(k), v).first; - } + template + iterator insert_or_assign(const_iterator, key_arg&& k, V&& v) { + return insert_or_assign(std::forward(k), std::forward(v)).first; + } - template - iterator insert_or_assign(const_iterator, const key_arg& k, V&& v) { - return insert_or_assign(k, std::forward(v)).first; - } + template + iterator insert_or_assign(const_iterator, key_arg&& k, const V& v) { + return insert_or_assign(std::forward(k), v).first; + } - template - iterator insert_or_assign(const_iterator, const key_arg& k, const V& v) { - return insert_or_assign(k, v).first; - } + template + iterator insert_or_assign(const_iterator, const key_arg& k, V&& v) { + return insert_or_assign(k, std::forward(v)).first; + } - template ::value, int>::type = 0, - K* = nullptr> - std::pair try_emplace(key_arg&& k, Args&&... args) { - return try_emplace_impl(std::forward(k), std::forward(args)...); - } + template + iterator insert_or_assign(const_iterator, const key_arg& k, const V& v) { + return insert_or_assign(k, v).first; + } - template ::value, int>::type = 0> - std::pair try_emplace(const key_arg& k, Args&&... args) { - return try_emplace_impl(k, std::forward(args)...); - } + template ::value, + int>::type = 0, + K* = nullptr> + std::pair try_emplace(key_arg&& k, Args&&... args) { + return try_emplace_impl(std::forward(k), std::forward(args)...); + } - template - iterator try_emplace(const_iterator, key_arg&& k, Args&&... args) { - return try_emplace(std::forward(k), std::forward(args)...).first; - } + template ::value, + int>::type = 0> + std::pair try_emplace(const key_arg& k, Args&&... args) { + return try_emplace_impl(k, std::forward(args)...); + } - template - iterator try_emplace(const_iterator, const key_arg& k, Args&&... args) { - return try_emplace(k, std::forward(args)...).first; - } + template + iterator try_emplace(const_iterator, key_arg&& k, Args&&... args) { + return try_emplace(std::forward(k), std::forward(args)...).first; + } - template - MappedReference

at(const key_arg& key) { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } + template + iterator try_emplace(const_iterator, const key_arg& k, Args&&... args) { + return try_emplace(k, std::forward(args)...).first; + } - template - MappedConstReference

at(const key_arg& key) const { - auto it = this->find(key); - if (it == this->end()) - phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); - return Policy::value(&*it); - } + template + MappedReference

at(const key_arg& key) { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } - // ----------- phmap extensions -------------------------- + template + MappedConstReference

at(const key_arg& key) const { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } - template ::value, int>::type = 0, - K* = nullptr> - std::pair try_emplace_with_hash(size_t hashval, key_arg&& k, Args&&... args) { - return try_emplace_impl_with_hash(hashval, std::forward(k), std::forward(args)...); - } + // ----------- phmap extensions -------------------------- - template ::value, int>::type = 0> - std::pair try_emplace_with_hash(size_t hashval, const key_arg& k, Args&&... args) { - return try_emplace_impl_with_hash(hashval, k, std::forward(args)...); - } + template ::value, + int>::type = 0, + K* = nullptr> + std::pair try_emplace_with_hash(size_t hashval, key_arg&& k, + Args&&... args) { + return try_emplace_impl_with_hash(hashval, std::forward(k), + std::forward(args)...); + } - template - iterator try_emplace_with_hash(size_t hashval, const_iterator, key_arg&& k, Args&&... args) { - return try_emplace_with_hash(hashval, std::forward(k), std::forward(args)...).first; - } + template ::value, + int>::type = 0> + std::pair try_emplace_with_hash(size_t hashval, const key_arg& k, + Args&&... args) { + return try_emplace_impl_with_hash(hashval, k, std::forward(args)...); + } - template - iterator try_emplace_with_hash(size_t hashval, const_iterator, const key_arg& k, Args&&... args) { - return try_emplace_with_hash(hashval, k, std::forward(args)...).first; - } + template + iterator try_emplace_with_hash(size_t hashval, const_iterator, key_arg&& k, + Args&&... args) { + return try_emplace_with_hash(hashval, std::forward(k), std::forward(args)...) + .first; + } - // if map contains key, lambda is called with the mapped value (under read lock protection), - // and if_contains returns true. This is a const API and lambda should not modify the value - // ----------------------------------------------------------------------------------------- - template - bool if_contains(const key_arg& key, F&& f) const { - return const_cast(this)->template - modify_if_impl(key, std::forward(f)); - } + template + iterator try_emplace_with_hash(size_t hashval, const_iterator, const key_arg& k, + Args&&... args) { + return try_emplace_with_hash(hashval, k, std::forward(args)...).first; + } - // if map contains key, lambda is called with the mapped value without read lock protection, - // and if_contains_unsafe returns true. This is a const API and lambda should not modify the value - // This should be used only if we know that no other thread may be mutating the map at the time. - // ----------------------------------------------------------------------------------------- - template - bool if_contains_unsafe(const key_arg& key, F&& f) const { - return const_cast(this)->template - modify_if_impl::DoNothing>(key, std::forward(f)); - } + // if map contains key, lambda is called with the mapped value (under read lock + // protection), and if_contains returns true. This is a const API and lambda should not + // modify the value + // ----------------------------------------------------------------------------------------- + template + bool if_contains(const key_arg& key, F&& f) const { + return const_cast(this) + ->template modify_if_impl( + key, std::forward(f)); + } - // if map contains key, lambda is called with the mapped value (under write lock protection), - // and modify_if returns true. This is a non-const API and lambda is allowed to modify the mapped value - // ---------------------------------------------------------------------------------------------------- - template - bool modify_if(const key_arg& key, F&& f) { - return modify_if_impl(key, std::forward(f)); - } + // if map contains key, lambda is called with the mapped value without read lock + // protection, and if_contains_unsafe returns true. This is a const API and lambda + // should not modify the value This should be used only if we know that no other thread + // may be mutating the map at the time. + // ----------------------------------------------------------------------------------------- + template + bool if_contains_unsafe(const key_arg& key, F&& f) const { + return const_cast(this) + ->template modify_if_impl::DoNothing>( + key, std::forward(f)); + } + // if map contains key, lambda is called with the mapped value (under write lock + // protection), and modify_if returns true. This is a non-const API and lambda is + // allowed to modify the mapped value + // ---------------------------------------------------------------------------------------------------- + template + bool modify_if(const key_arg& key, F&& f) { + return modify_if_impl(key, std::forward(f)); + } - // if map contains key, lambda is called with the mapped value (under write lock protection). - // If the lambda returns true, the key is subsequently erased from the map (the write lock - // is only released after erase). - // returns true if key was erased, false otherwise. - // ---------------------------------------------------------------------------------------------------- - template - bool erase_if(const key_arg& key, F&& f) { - return erase_if_impl(key, std::forward(f)); - } + // if map contains key, lambda is called with the mapped value (under write lock + // protection). If the lambda returns true, the key is subsequently erased from the map + // (the write lock is only released after erase). returns true if key was erased, false + // otherwise. + // ---------------------------------------------------------------------------------------------------- + template + bool erase_if(const key_arg& key, F&& f) { + return erase_if_impl(key, std::forward(f)); + } - // if map does not contains key, it is inserted and the mapped value is value-constructed - // with the provided arguments (if any), as with try_emplace. - // if map already contains key, then the lambda is called with the mapped value (under - // write lock protection) and can update the mapped value. - // returns true if key was not already present, false otherwise. - // --------------------------------------------------------------------------------------- - template - bool try_emplace_l(K&& k, F&& f, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward(k)), - std::forward_as_tuple(std::forward(args)...)); - else { - auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); - std::forward(f)(Policy::value(&*it)); - } - return std::get<2>(res); + // if map does not contains key, it is inserted and the mapped value is + // value-constructed with the provided arguments (if any), as with try_emplace. if map + // already contains key, then the lambda is called with the mapped value (under write + // lock protection) and can update the mapped value. returns true if key was not already + // present, false otherwise. + // --------------------------------------------------------------------------------------- + template + bool try_emplace_l(K&& k, F&& f, Args&&... args) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); + typename Base::Inner* inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, + std::forward_as_tuple(std::forward(k)), + std::forward_as_tuple(std::forward(args)...)); + else { + auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); + std::forward(f)(Policy::value(&*it)); } + return std::get<2>(res); + } - // ----------- end of phmap extensions -------------------------- + // ----------- end of phmap extensions -------------------------- - template - MappedReference

operator[](key_arg&& key) { - return Policy::value(&*try_emplace(std::forward(key)).first); - } + template + MappedReference

operator[](key_arg&& key) { + return Policy::value(&*try_emplace(std::forward(key)).first); + } - template - MappedReference

operator[](const key_arg& key) { - return Policy::value(&*try_emplace(key).first); - } + template + MappedReference

operator[](const key_arg& key) { + return Policy::value(&*try_emplace(key).first); + } -private: - template - bool modify_if_impl(const key_arg& key, F&& f) { + private: + template + bool modify_if_impl(const key_arg& key, F&& f) { #if __cplusplus >= 201703L - static_assert(std::is_invocable::value); + static_assert(std::is_invocable::value); #endif - L m; - auto ptr = this->template find_ptr(key, this->hash(key), m); - if (ptr == nullptr) - return false; - std::forward(f)(Policy::value(ptr)); - return true; - } + L m; + auto ptr = this->template find_ptr(key, this->hash(key), m); + if (ptr == nullptr) return false; + std::forward(f)(Policy::value(ptr)); + return true; + } - template - bool erase_if_impl(const key_arg& key, F&& f) { + template + bool erase_if_impl(const key_arg& key, F&& f) { #if __cplusplus >= 201703L - static_assert(std::is_invocable::value); + static_assert(std::is_invocable::value); #endif - L m; - auto it = this->template find(key, this->hash(key), m); - if (it == this->end()) return false; - if (std::forward(f)(Policy::value(&*it))) - { - this->erase(it); - return true; - } - return false; - } - - - template - std::pair insert_or_assign_impl(K&& k, V&& v) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::forward(k), std::forward(v)); - else - Policy::value(&*inner->set_.iterator_at(std::get<1>(res))) = std::forward(v); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } + L m; + auto it = this->template find(key, this->hash(key), m); + if (it == this->end()) return false; + if (std::forward(f)(Policy::value(&*it))) { + this->erase(it); + return true; + } + return false; + } - template - std::pair try_emplace_impl(K&& k, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert(k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward(k)), - std::forward_as_tuple(std::forward(args)...)); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } + template + std::pair insert_or_assign_impl(K&& k, V&& v) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); + typename Base::Inner* inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::forward(k), std::forward(v)); + else + Policy::value(&*inner->set_.iterator_at(std::get<1>(res))) = std::forward(v); + return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), + std::get<2>(res)}; + } - template - std::pair try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) { - typename Lockable::UniqueLock m; - auto res = this->find_or_prepare_insert_with_hash(hashval, k, m); - typename Base::Inner *inner = std::get<0>(res); - if (std::get<2>(res)) - inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, - std::forward_as_tuple(std::forward(k)), - std::forward_as_tuple(std::forward(args)...)); - return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), - std::get<2>(res)}; - } + template + std::pair try_emplace_impl(K&& k, Args&&... args) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); + typename Base::Inner* inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, + std::forward_as_tuple(std::forward(k)), + std::forward_as_tuple(std::forward(args)...)); + return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), + std::get<2>(res)}; + } - + template + std::pair try_emplace_impl_with_hash(size_t hashval, K&& k, + Args&&... args) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert_with_hash(hashval, k, m); + typename Base::Inner* inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, + std::forward_as_tuple(std::forward(k)), + std::forward_as_tuple(std::forward(args)...)); + return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), + std::get<2>(res)}; + } }; - // Constructs T into uninitialized storage pointed by `ptr` using the args // specified in the tuple. // ---------------------------------------------------------------------------- template void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) { - memory_internal::ConstructFromTupleImpl( - alloc, ptr, std::forward(t), - phmap::make_index_sequence< - std::tuple_size::type>::value>()); + memory_internal::ConstructFromTupleImpl( + alloc, ptr, std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>()); } // Constructs T using the args specified in the tuple and calls F with the // constructed value. // ---------------------------------------------------------------------------- template -decltype(std::declval()(std::declval())) WithConstructed( - Tuple&& t, F&& f) { - return memory_internal::WithConstructedImpl( - std::forward(t), - phmap::make_index_sequence< - std::tuple_size::type>::value>(), - std::forward(f)); +decltype(std::declval()(std::declval())) WithConstructed(Tuple&& t, F&& f) { + return memory_internal::WithConstructedImpl( + std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>(), + std::forward(f)); } // ---------------------------------------------------------------------------- @@ -3880,32 +3755,30 @@ std::pair, std::tuple> PairArgs(F&& f, S&& s) { } template -std::pair, std::tuple> PairArgs( - const std::pair& p) { - return PairArgs(p.first, p.second); +std::pair, std::tuple> PairArgs(const std::pair& p) { + return PairArgs(p.first, p.second); } template std::pair, std::tuple> PairArgs(std::pair&& p) { - return PairArgs(std::forward(p.first), std::forward(p.second)); + return PairArgs(std::forward(p.first), std::forward(p.second)); } template auto PairArgs(std::piecewise_construct_t, F&& f, S&& s) -> decltype(std::make_pair(memory_internal::TupleRef(std::forward(f)), memory_internal::TupleRef(std::forward(s)))) { - return std::make_pair(memory_internal::TupleRef(std::forward(f)), - memory_internal::TupleRef(std::forward(s))); + return std::make_pair(memory_internal::TupleRef(std::forward(f)), + memory_internal::TupleRef(std::forward(s))); } // A helper function for implementing apply() in map policies. // ---------------------------------------------------------------------------- template -auto DecomposePair(F&& f, Args&&... args) - -> decltype(memory_internal::DecomposePairImpl( - std::forward(f), PairArgs(std::forward(args)...))) { - return memory_internal::DecomposePairImpl( - std::forward(f), PairArgs(std::forward(args)...)); +auto DecomposePair(F&& f, Args&&... args) -> decltype(memory_internal::DecomposePairImpl( + std::forward(f), PairArgs(std::forward(args)...))) { + return memory_internal::DecomposePairImpl(std::forward(f), + PairArgs(std::forward(args)...)); } // A helper function for implementing apply() in set policies. @@ -3913,11 +3786,10 @@ auto DecomposePair(F&& f, Args&&... args) template decltype(std::declval()(std::declval(), std::declval())) DecomposeValue(F&& f, Arg&& arg) { - const auto& key = arg; - return std::forward(f)(key, std::forward(arg)); + const auto& key = arg; + return std::forward(f)(key, std::forward(arg)); } - // -------------------------------------------------------------------------- // Policy: a policy defines how to perform different operations on // the slots of the hashtable (see hash_policy_traits.h for the full interface @@ -3938,220 +3810,201 @@ DecomposeValue(F&& f, Arg&& arg) { // constructed and destroyed. // -------------------------------------------------------------------------- template -struct FlatHashSetPolicy -{ - using slot_type = T; - using key_type = T; - using init_type = T; - using constant_iterators = std::true_type; - - template - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - phmap::allocator_traits::construct(*alloc, slot, - std::forward(args)...); - } +struct FlatHashSetPolicy { + using slot_type = T; + using key_type = T; + using init_type = T; + using constant_iterators = std::true_type; + + template + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + phmap::allocator_traits::construct(*alloc, slot, + std::forward(args)...); + } - template - static void destroy(Allocator* alloc, slot_type* slot) { - phmap::allocator_traits::destroy(*alloc, slot); - } + template + static void destroy(Allocator* alloc, slot_type* slot) { + phmap::allocator_traits::destroy(*alloc, slot); + } - template - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - construct(alloc, new_slot, std::move(*old_slot)); - destroy(alloc, old_slot); - } + template + static void transfer(Allocator* alloc, slot_type* new_slot, slot_type* old_slot) { + construct(alloc, new_slot, std::move(*old_slot)); + destroy(alloc, old_slot); + } - static T& element(slot_type* slot) { return *slot; } + static T& element(slot_type* slot) { return *slot; } - template - static decltype(phmap::priv::DecomposeValue( - std::declval(), std::declval()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposeValue( - std::forward(f), std::forward(args)...); - } + template + static decltype(phmap::priv::DecomposeValue(std::declval(), std::declval()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposeValue(std::forward(f), std::forward(args)...); + } - static size_t space_used(const T*) { return 0; } + static size_t space_used(const T*) { return 0; } }; // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- template -struct FlatHashMapPolicy -{ - using slot_policy = priv::map_slot_policy; - using slot_type = typename slot_policy::slot_type; - using key_type = K; - using mapped_type = V; - using init_type = std::pair; - - template - static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { - slot_policy::construct(alloc, slot, std::forward(args)...); - } +struct FlatHashMapPolicy { + using slot_policy = priv::map_slot_policy; + using slot_type = typename slot_policy::slot_type; + using key_type = K; + using mapped_type = V; + using init_type = std::pair; + + template + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + slot_policy::construct(alloc, slot, std::forward(args)...); + } - template - static void destroy(Allocator* alloc, slot_type* slot) { - slot_policy::destroy(alloc, slot); - } + template + static void destroy(Allocator* alloc, slot_type* slot) { + slot_policy::destroy(alloc, slot); + } - template - static void transfer(Allocator* alloc, slot_type* new_slot, - slot_type* old_slot) { - slot_policy::transfer(alloc, new_slot, old_slot); - } + template + static void transfer(Allocator* alloc, slot_type* new_slot, slot_type* old_slot) { + slot_policy::transfer(alloc, new_slot, old_slot); + } - template - static decltype(phmap::priv::DecomposePair( - std::declval(), std::declval()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposePair(std::forward(f), - std::forward(args)...); - } + template + static decltype(phmap::priv::DecomposePair(std::declval(), std::declval()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposePair(std::forward(f), std::forward(args)...); + } - static size_t space_used(const slot_type*) { return 0; } + static size_t space_used(const slot_type*) { return 0; } - static std::pair& element(slot_type* slot) { return slot->value; } + static std::pair& element(slot_type* slot) { return slot->value; } - static V& value(std::pair* kv) { return kv->second; } - static const V& value(const std::pair* kv) { return kv->second; } + static V& value(std::pair* kv) { return kv->second; } + static const V& value(const std::pair* kv) { return kv->second; } }; template struct node_hash_policy { - static_assert(std::is_lvalue_reference::value, ""); + static_assert(std::is_lvalue_reference::value, ""); - using slot_type = typename std::remove_cv< - typename std::remove_reference::type>::type*; + using slot_type = + typename std::remove_cv::type>::type*; - template - static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { - *slot = Policy::new_element(alloc, std::forward(args)...); - } + template + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + *slot = Policy::new_element(alloc, std::forward(args)...); + } - template - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::delete_element(alloc, *slot); - } + template + static void destroy(Alloc* alloc, slot_type* slot) { + Policy::delete_element(alloc, *slot); + } - template - static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) { - *new_slot = *old_slot; - } + template + static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) { + *new_slot = *old_slot; + } - static size_t space_used(const slot_type* slot) { - if (slot == nullptr) return Policy::element_space_used(nullptr); - return Policy::element_space_used(*slot); - } + static size_t space_used(const slot_type* slot) { + if (slot == nullptr) return Policy::element_space_used(nullptr); + return Policy::element_space_used(*slot); + } - static Reference element(slot_type* slot) { return **slot; } + static Reference element(slot_type* slot) { return **slot; } - template - static auto value(T* elem) -> decltype(P::value(elem)) { - return P::value(elem); - } + template + static auto value(T* elem) -> decltype(P::value(elem)) { + return P::value(elem); + } - template - static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward(ts)...)) { - return P::apply(std::forward(ts)...); - } + template + static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward(ts)...)) { + return P::apply(std::forward(ts)...); + } }; // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- template -struct NodeHashSetPolicy - : phmap::priv::node_hash_policy> -{ - using key_type = T; - using init_type = T; - using constant_iterators = std::true_type; - - template - static T* new_element(Allocator* alloc, Args&&... args) { - using ValueAlloc = - typename phmap::allocator_traits::template rebind_alloc; - ValueAlloc value_alloc(*alloc); - T* res = phmap::allocator_traits::allocate(value_alloc, 1); - phmap::allocator_traits::construct(value_alloc, res, - std::forward(args)...); - return res; - } +struct NodeHashSetPolicy : phmap::priv::node_hash_policy> { + using key_type = T; + using init_type = T; + using constant_iterators = std::true_type; + + template + static T* new_element(Allocator* alloc, Args&&... args) { + using ValueAlloc = + typename phmap::allocator_traits::template rebind_alloc; + ValueAlloc value_alloc(*alloc); + T* res = phmap::allocator_traits::allocate(value_alloc, 1); + phmap::allocator_traits::construct(value_alloc, res, + std::forward(args)...); + return res; + } - template - static void delete_element(Allocator* alloc, T* elem) { - using ValueAlloc = - typename phmap::allocator_traits::template rebind_alloc; - ValueAlloc value_alloc(*alloc); - phmap::allocator_traits::destroy(value_alloc, elem); - phmap::allocator_traits::deallocate(value_alloc, elem, 1); - } + template + static void delete_element(Allocator* alloc, T* elem) { + using ValueAlloc = + typename phmap::allocator_traits::template rebind_alloc; + ValueAlloc value_alloc(*alloc); + phmap::allocator_traits::destroy(value_alloc, elem); + phmap::allocator_traits::deallocate(value_alloc, elem, 1); + } - template - static decltype(phmap::priv::DecomposeValue( - std::declval(), std::declval()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposeValue( - std::forward(f), std::forward(args)...); - } + template + static decltype(phmap::priv::DecomposeValue(std::declval(), std::declval()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposeValue(std::forward(f), std::forward(args)...); + } - static size_t element_space_used(const T*) { return sizeof(T); } + static size_t element_space_used(const T*) { return sizeof(T); } }; // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- template class NodeHashMapPolicy - : public phmap::priv::node_hash_policy< - std::pair&, NodeHashMapPolicy> -{ - using value_type = std::pair; - -public: - using key_type = Key; - using mapped_type = Value; - using init_type = std::pair; - - template - static value_type* new_element(Allocator* alloc, Args&&... args) { - using PairAlloc = typename phmap::allocator_traits< - Allocator>::template rebind_alloc; - PairAlloc pair_alloc(*alloc); - value_type* res = - phmap::allocator_traits::allocate(pair_alloc, 1); - phmap::allocator_traits::construct(pair_alloc, res, - std::forward(args)...); - return res; - } + : public phmap::priv::node_hash_policy&, + NodeHashMapPolicy> { + using value_type = std::pair; + + public: + using key_type = Key; + using mapped_type = Value; + using init_type = std::pair; + + template + static value_type* new_element(Allocator* alloc, Args&&... args) { + using PairAlloc = + typename phmap::allocator_traits::template rebind_alloc; + PairAlloc pair_alloc(*alloc); + value_type* res = phmap::allocator_traits::allocate(pair_alloc, 1); + phmap::allocator_traits::construct(pair_alloc, res, + std::forward(args)...); + return res; + } - template - static void delete_element(Allocator* alloc, value_type* pair) { - using PairAlloc = typename phmap::allocator_traits< - Allocator>::template rebind_alloc; - PairAlloc pair_alloc(*alloc); - phmap::allocator_traits::destroy(pair_alloc, pair); - phmap::allocator_traits::deallocate(pair_alloc, pair, 1); - } + template + static void delete_element(Allocator* alloc, value_type* pair) { + using PairAlloc = + typename phmap::allocator_traits::template rebind_alloc; + PairAlloc pair_alloc(*alloc); + phmap::allocator_traits::destroy(pair_alloc, pair); + phmap::allocator_traits::deallocate(pair_alloc, pair, 1); + } - template - static decltype(phmap::priv::DecomposePair( - std::declval(), std::declval()...)) - apply(F&& f, Args&&... args) { - return phmap::priv::DecomposePair(std::forward(f), - std::forward(args)...); - } + template + static decltype(phmap::priv::DecomposePair(std::declval(), std::declval()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposePair(std::forward(f), std::forward(args)...); + } - static size_t element_space_used(const value_type*) { - return sizeof(value_type); - } + static size_t element_space_used(const value_type*) { return sizeof(value_type); } - static Value& value(value_type* elem) { return elem->second; } - static const Value& value(const value_type* elem) { return elem->second; } + static Value& value(value_type* elem) { return elem->second; } + static const Value& value(const value_type* elem) { return elem->second; } }; - // -------------------------------------------------------------------------- // hash_default // -------------------------------------------------------------------------- @@ -4159,30 +4012,30 @@ class NodeHashMapPolicy #if PHMAP_HAVE_STD_STRING_VIEW // support char16_t wchar_t .... -template -struct StringHashT -{ - using is_transparent = void; - - size_t operator()(std::basic_string_view v) const { - std::string_view bv{reinterpret_cast(v.data()), v.size() * sizeof(CharT)}; - return std::hash()(bv); - } +template +struct StringHashT { + using is_transparent = void; + + size_t operator()(std::basic_string_view v) const { + std::string_view bv{reinterpret_cast(v.data()), + v.size() * sizeof(CharT)}; + return std::hash()(bv); + } }; // Supports heterogeneous lookup for basic_string-like elements. -template -struct StringHashEqT -{ - using Hash = StringHashT; +template +struct StringHashEqT { + using Hash = StringHashT; - struct Eq { - using is_transparent = void; + struct Eq { + using is_transparent = void; - bool operator()(std::basic_string_view lhs, std::basic_string_view rhs) const { - return lhs == rhs; - } - }; + bool operator()(std::basic_string_view lhs, + std::basic_string_view rhs) const { + return lhs == rhs; + } + }; }; template <> @@ -4210,36 +4063,35 @@ struct HashEq : StringHashEqT {}; // Supports heterogeneous lookup for pointers and smart pointers. // ------------------------------------------------------------- template -struct HashEq -{ - struct Hash { - using is_transparent = void; - template - size_t operator()(const U& ptr) const { - return phmap::Hash{}(HashEq::ToPtr(ptr)); - } - }; +struct HashEq { + struct Hash { + using is_transparent = void; + template + size_t operator()(const U& ptr) const { + return phmap::Hash{}(HashEq::ToPtr(ptr)); + } + }; - struct Eq { - using is_transparent = void; - template - bool operator()(const A& a, const B& b) const { - return HashEq::ToPtr(a) == HashEq::ToPtr(b); - } - }; + struct Eq { + using is_transparent = void; + template + bool operator()(const A& a, const B& b) const { + return HashEq::ToPtr(a) == HashEq::ToPtr(b); + } + }; -private: - static const T* ToPtr(const T* ptr) { return ptr; } + private: + static const T* ToPtr(const T* ptr) { return ptr; } - template - static const T* ToPtr(const std::unique_ptr& ptr) { - return ptr.get(); - } + template + static const T* ToPtr(const std::unique_ptr& ptr) { + return ptr.get(); + } - template - static const T* ToPtr(const std::shared_ptr& ptr) { - return ptr.get(); - } + template + static const T* ToPtr(const std::shared_ptr& ptr) { + return ptr.get(); + } }; template @@ -4253,62 +4105,60 @@ namespace hashtable_debug_internal { // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- template -struct HashtableDebugAccess> -{ - using Traits = typename Set::PolicyTraits; - using Slot = typename Traits::slot_type; - - static size_t GetNumProbes(const Set& set, - const typename Set::key_type& key) { - size_t num_probes = 0; - size_t hashval = set.hash(key); - auto seq = set.probe(hashval); - while (true) { - priv::Group g{set.ctrl_ + seq.offset()}; - for (int i : g.Match(priv::H2(hashval))) { - if (Traits::apply( - typename Set::template EqualElement{ - key, set.eq_ref()}, - Traits::element(set.slots_ + seq.offset((size_t)i)))) - return num_probes; - ++num_probes; - } - if (g.MatchEmpty()) return num_probes; - seq.next(); - ++num_probes; - } +struct HashtableDebugAccess> { + using Traits = typename Set::PolicyTraits; + using Slot = typename Traits::slot_type; + + static size_t GetNumProbes(const Set& set, const typename Set::key_type& key) { + size_t num_probes = 0; + size_t hashval = set.hash(key); + auto seq = set.probe(hashval); + while (true) { + priv::Group g{set.ctrl_ + seq.offset()}; + for (int i : g.Match(priv::H2(hashval))) { + if (Traits::apply( + typename Set::template EqualElement{key, + set.eq_ref()}, + Traits::element(set.slots_ + seq.offset((size_t)i)))) + return num_probes; + ++num_probes; + } + if (g.MatchEmpty()) return num_probes; + seq.next(); + ++num_probes; } + } - static size_t AllocatedByteSize(const Set& c) { - size_t capacity = c.capacity_; - if (capacity == 0) return 0; - auto layout = Set::MakeLayout(capacity); - size_t m = layout.AllocSize(); - - size_t per_slot = Traits::space_used(static_cast(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * c.size(); - } else { - for (size_t i = 0; i != capacity; ++i) { - if (priv::IsFull(c.ctrl_[i])) { - m += Traits::space_used(c.slots_ + i); - } - } + static size_t AllocatedByteSize(const Set& c) { + size_t capacity = c.capacity_; + if (capacity == 0) return 0; + auto layout = Set::MakeLayout(capacity); + size_t m = layout.AllocSize(); + + size_t per_slot = Traits::space_used(static_cast(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * c.size(); + } else { + for (size_t i = 0; i != capacity; ++i) { + if (priv::IsFull(c.ctrl_[i])) { + m += Traits::space_used(c.slots_ + i); } - return m; + } } + return m; + } - static size_t LowerBoundAllocatedByteSize(size_t size) { - size_t capacity = GrowthToLowerboundCapacity(size); - if (capacity == 0) return 0; - auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); - size_t m = layout.AllocSize(); - size_t per_slot = Traits::space_used(static_cast(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * size; - } - return m; + static size_t LowerBoundAllocatedByteSize(size_t size) { + size_t capacity = GrowthToLowerboundCapacity(size); + if (capacity == 0) return 0; + auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); + size_t m = layout.AllocSize(); + size_t per_slot = Traits::space_used(static_cast(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * size; } + return m; + } }; } // namespace hashtable_debug_internal @@ -4331,49 +4181,48 @@ struct HashtableDebugAccess> // slots (open, deleted, and empty) within the hash set. // * Returns `void` from the `_erase(iterator)` overload. // ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h -class flat_hash_set - : public phmap::priv::raw_hash_set< - phmap::priv::FlatHashSetPolicy, Hash, Eq, Alloc> -{ - using Base = typename flat_hash_set::raw_hash_set; - -public: - flat_hash_set() {} +template // default values in phmap_fwd_decl.h +class flat_hash_set : public phmap::priv::raw_hash_set, + Hash, Eq, Alloc> { + using Base = typename flat_hash_set::raw_hash_set; + + public: + flat_hash_set() {} #ifdef __INTEL_COMPILER - using Base::raw_hash_set; + using Base::raw_hash_set; #else - using Base::Base; + using Base::Base; #endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; // may shrink - To avoid shrinking `erase(begin(), end())` - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; + using Base::begin; + using Base::bucket_count; + using Base::capacity; + using Base::cbegin; + using Base::cend; + using Base::clear; // may shrink - To avoid shrinking `erase(begin(), end())` + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::get_allocator; + using Base::hash; + using Base::hash_function; + using Base::insert; + using Base::key_eq; + using Base::load_factor; + using Base::max_load_factor; + using Base::max_size; + using Base::merge; + using Base::rehash; + using Base::reserve; + using Base::size; + using Base::swap; }; // ----------------------------------------------------------------------------- @@ -4394,52 +4243,53 @@ class flat_hash_set // slots (open, deleted, and empty) within the hash map. // * Returns `void` from the `_erase(iterator)` overload. // ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h -class flat_hash_map : public phmap::priv::raw_hash_map< - phmap::priv::FlatHashMapPolicy, - Hash, Eq, Alloc> { - using Base = typename flat_hash_map::raw_hash_map; - -public: - flat_hash_map() {} +template // default values in phmap_fwd_decl.h +class flat_hash_map + : public phmap::priv::raw_hash_map, Hash, Eq, + Alloc> { + using Base = typename flat_hash_map::raw_hash_map; + + public: + flat_hash_map() {} #ifdef __INTEL_COMPILER - using Base::raw_hash_map; + using Base::raw_hash_map; #else - using Base::Base; + using Base::Base; #endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; + using Base::at; + using Base::begin; + using Base::capacity; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::insert; + using Base::insert_or_assign; + using Base::max_size; + using Base::merge; + using Base::rehash; + using Base::reserve; + using Base::size; + using Base::swap; + using Base::try_emplace; + using Base::operator[]; + using Base::bucket_count; + using Base::get_allocator; + using Base::hash; + using Base::hash_function; + using Base::key_eq; + using Base::load_factor; + using Base::max_load_factor; }; // ----------------------------------------------------------------------------- @@ -4457,53 +4307,52 @@ class flat_hash_map : public phmap::priv::raw_hash_map< // slots (open, deleted, and empty) within the hash set. // * Returns `void` from the `erase(iterator)` overload. // ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h -class node_hash_set - : public phmap::priv::raw_hash_set< - phmap::priv::NodeHashSetPolicy, Hash, Eq, Alloc> -{ - using Base = typename node_hash_set::raw_hash_set; - -public: - node_hash_set() {} +template // default values in phmap_fwd_decl.h +class node_hash_set : public phmap::priv::raw_hash_set, + Hash, Eq, Alloc> { + using Base = typename node_hash_set::raw_hash_set; + + public: + node_hash_set() {} #ifdef __INTEL_COMPILER - using Base::raw_hash_set; + using Base::raw_hash_set; #else - using Base::Base; + using Base::Base; #endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } + using Base::begin; + using Base::bucket_count; + using Base::capacity; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_hint_with_hash; + using Base::emplace_with_hash; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::get_allocator; + using Base::hash; + using Base::hash_function; + using Base::insert; + using Base::key_eq; + using Base::load_factor; + using Base::max_load_factor; + using Base::max_size; + using Base::merge; + using Base::rehash; + using Base::reserve; + using Base::size; + using Base::swap; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } }; // ----------------------------------------------------------------------------- @@ -4522,168 +4371,165 @@ class node_hash_set // slots (open, deleted, and empty) within the hash map. // * Returns `void` from the `erase(iterator)` overload. // ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h +template // default values in phmap_fwd_decl.h class node_hash_map - : public phmap::priv::raw_hash_map< - phmap::priv::NodeHashMapPolicy, Hash, Eq, - Alloc> -{ - using Base = typename node_hash_map::raw_hash_map; - -public: - node_hash_map() {} + : public phmap::priv::raw_hash_map, Hash, + Eq, Alloc> { + using Base = typename node_hash_map::raw_hash_map; + + public: + node_hash_map() {} #ifdef __INTEL_COMPILER - using Base::raw_hash_map; + using Base::raw_hash_map; #else - using Base::Base; + using Base::Base; #endif - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::hash; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } + using Base::at; + using Base::begin; + using Base::capacity; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::insert; + using Base::insert_or_assign; + using Base::max_size; + using Base::merge; + using Base::rehash; + using Base::reserve; + using Base::size; + using Base::swap; + using Base::try_emplace; + using Base::operator[]; + using Base::bucket_count; + using Base::get_allocator; + using Base::hash; + using Base::hash_function; + using Base::key_eq; + using Base::load_factor; + using Base::max_load_factor; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } }; // ----------------------------------------------------------------------------- // phmap::parallel_flat_hash_set // ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h +template // default values in phmap_fwd_decl.h class parallel_flat_hash_set - : public phmap::priv::parallel_hash_set< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::FlatHashSetPolicy, - Hash, Eq, Alloc> -{ - using Base = typename parallel_flat_hash_set::parallel_hash_set; - -public: - parallel_flat_hash_set() {} + : public phmap::priv::parallel_hash_set, Hash, Eq, + Alloc> { + using Base = typename parallel_flat_hash_set::parallel_hash_set; + + public: + parallel_flat_hash_set() {} #ifdef __INTEL_COMPILER - using Base::parallel_hash_set; + using Base::parallel_hash_set; #else - using Base::Base; + using Base::Base; #endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; + using Base::begin; + using Base::bucket_count; + using Base::capacity; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_hint_with_hash; + using Base::emplace_with_hash; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::get_allocator; + using Base::hash; + using Base::hash_function; + using Base::insert; + using Base::key_eq; + using Base::load_factor; + using Base::max_load_factor; + using Base::max_size; + using Base::merge; + using Base::rehash; + using Base::reserve; + using Base::size; + using Base::subcnt; + using Base::subidx; + using Base::swap; }; // ----------------------------------------------------------------------------- // phmap::parallel_flat_hash_map - default values in phmap_fwd_decl.h // ----------------------------------------------------------------------------- template -class parallel_flat_hash_map : public phmap::priv::parallel_hash_map< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::FlatHashMapPolicy, - Hash, Eq, Alloc> -{ - using Base = typename parallel_flat_hash_map::parallel_hash_map; - -public: - parallel_flat_hash_map() {} +class parallel_flat_hash_map + : public phmap::priv::parallel_hash_map, Hash, + Eq, Alloc> { + using Base = typename parallel_flat_hash_map::parallel_hash_map; + + public: + parallel_flat_hash_map() {} #ifdef __INTEL_COMPILER - using Base::parallel_hash_map; + using Base::parallel_hash_map; #else - using Base::Base; + using Base::Base; #endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::try_emplace_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; + using Base::at; + using Base::begin; + using Base::capacity; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_hint_with_hash; + using Base::emplace_with_hash; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::hash; + using Base::insert; + using Base::insert_or_assign; + using Base::max_size; + using Base::merge; + using Base::rehash; + using Base::reserve; + using Base::size; + using Base::subcnt; + using Base::subidx; + using Base::swap; + using Base::try_emplace; + using Base::try_emplace_with_hash; + using Base::operator[]; + using Base::bucket_count; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; + using Base::load_factor; + using Base::max_load_factor; }; // ----------------------------------------------------------------------------- @@ -4691,54 +4537,53 @@ class parallel_flat_hash_map : public phmap::priv::parallel_hash_map< // ----------------------------------------------------------------------------- template class parallel_node_hash_set - : public phmap::priv::parallel_hash_set< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::NodeHashSetPolicy, Hash, Eq, Alloc> -{ - using Base = typename parallel_node_hash_set::parallel_hash_set; - -public: - parallel_node_hash_set() {} + : public phmap::priv::parallel_hash_set, Hash, Eq, + Alloc> { + using Base = typename parallel_node_hash_set::parallel_hash_set; + + public: + parallel_node_hash_set() {} #ifdef __INTEL_COMPILER - using Base::parallel_hash_set; + using Base::parallel_hash_set; #else - using Base::Base; + using Base::Base; #endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::emplace; - using Base::emplace_hint; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } + using Base::begin; + using Base::bucket_count; + using Base::capacity; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_hint_with_hash; + using Base::emplace_with_hash; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::get_allocator; + using Base::hash; + using Base::hash_function; + using Base::insert; + using Base::key_eq; + using Base::load_factor; + using Base::max_load_factor; + using Base::max_size; + using Base::merge; + using Base::rehash; + using Base::reserve; + using Base::size; + using Base::subcnt; + using Base::subidx; + using Base::swap; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } }; // ----------------------------------------------------------------------------- @@ -4746,67 +4591,64 @@ class parallel_node_hash_set // ----------------------------------------------------------------------------- template class parallel_node_hash_map - : public phmap::priv::parallel_hash_map< - N, phmap::priv::raw_hash_set, Mtx_, - phmap::priv::NodeHashMapPolicy, Hash, Eq, - Alloc> -{ - using Base = typename parallel_node_hash_map::parallel_hash_map; - -public: - parallel_node_hash_map() {} + : public phmap::priv::parallel_hash_map, + Hash, Eq, Alloc> { + using Base = typename parallel_node_hash_map::parallel_hash_map; + + public: + parallel_node_hash_map() {} #ifdef __INTEL_COMPILER - using Base::parallel_hash_map; + using Base::parallel_hash_map; #else - using Base::Base; + using Base::Base; #endif - using Base::hash; - using Base::subidx; - using Base::subcnt; - using Base::begin; - using Base::cbegin; - using Base::cend; - using Base::end; - using Base::capacity; - using Base::empty; - using Base::max_size; - using Base::size; - using Base::clear; - using Base::erase; - using Base::insert; - using Base::insert_or_assign; - using Base::emplace; - using Base::emplace_hint; - using Base::try_emplace; - using Base::emplace_with_hash; - using Base::emplace_hint_with_hash; - using Base::try_emplace_with_hash; - using Base::extract; - using Base::merge; - using Base::swap; - using Base::rehash; - using Base::reserve; - using Base::at; - using Base::contains; - using Base::count; - using Base::equal_range; - using Base::find; - using Base::operator[]; - using Base::bucket_count; - using Base::load_factor; - using Base::max_load_factor; - using Base::get_allocator; - using Base::hash_function; - using Base::key_eq; - typename Base::hasher hash_funct() { return this->hash_function(); } - void resize(typename Base::size_type hint) { this->rehash(hint); } + using Base::at; + using Base::begin; + using Base::capacity; + using Base::cbegin; + using Base::cend; + using Base::clear; + using Base::contains; + using Base::count; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_hint_with_hash; + using Base::emplace_with_hash; + using Base::empty; + using Base::end; + using Base::equal_range; + using Base::erase; + using Base::extract; + using Base::find; + using Base::hash; + using Base::insert; + using Base::insert_or_assign; + using Base::max_size; + using Base::merge; + using Base::rehash; + using Base::reserve; + using Base::size; + using Base::subcnt; + using Base::subidx; + using Base::swap; + using Base::try_emplace; + using Base::try_emplace_with_hash; + using Base::operator[]; + using Base::bucket_count; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; + using Base::load_factor; + using Base::max_load_factor; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } }; } // namespace phmap #ifdef _MSC_VER - #pragma warning(pop) +#pragma warning(pop) #endif - -#endif // phmap_h_guard_ +#endif // phmap_h_guard_ diff --git a/native-sql-engine/cpp/src/third_party/parallel_hashmap/phmap_base.h b/native-sql-engine/cpp/src/third_party/parallel_hashmap/phmap_base.h index 6b9ea9ee5..3b3b6b120 100644 --- a/native-sql-engine/cpp/src/third_party/parallel_hashmap/phmap_base.h +++ b/native-sql-engine/cpp/src/third_party/parallel_hashmap/phmap_base.h @@ -18,7 +18,7 @@ // // Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) // with modifications. -// +// // Copyright 2018 The Abseil Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,56 +37,51 @@ #include #include #include +#include #include #include +#include +#include // for std::lock #include -#include -#include -#include #include +#include #include -#include -#include // for std::lock #include "phmap_config.h" #ifdef PHMAP_HAVE_SHARED_MUTEX - #include // after "phmap_config.h" +#include // after "phmap_config.h" #endif #ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4582) // constructor is not implicitly called - #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted - #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted - #pragma warning(disable : 4710) // function not inlined - #pragma warning(disable : 4711) // selected for automatic inline expansion - #pragma warning(disable : 4820) // '6' bytes padding added after data member -#endif // _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4514) // unreferenced inline function has been removed +#pragma warning(disable : 4582) // constructor is not implicitly called +#pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted +#pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted +#pragma warning(disable : 4710) // function not inlined +#pragma warning(disable : 4711) // selected for automatic inline expansion +#pragma warning(disable : 4820) // '6' bytes padding added after data member +#endif // _MSC_VER namespace phmap { -template using Allocator = typename std::allocator; +template +using Allocator = typename std::allocator; -template using Pair = typename std::pair; +template +using Pair = typename std::pair; template -struct EqualTo -{ - inline bool operator()(const T& a, const T& b) const - { - return std::equal_to()(a, b); - } +struct EqualTo { + inline bool operator()(const T& a, const T& b) const { + return std::equal_to()(a, b); + } }; template -struct Less -{ - inline bool operator()(const T& a, const T& b) const - { - return std::less()(a, b); - } +struct Less { + inline bool operator()(const T& a, const T& b) const { return std::less()(a, b); } }; namespace type_traits_internal { @@ -104,8 +99,7 @@ template > struct default_alignment_of_aligned_storage; template -struct default_alignment_of_aligned_storage> { +struct default_alignment_of_aligned_storage> { static constexpr size_t value = Align; }; @@ -140,8 +134,8 @@ struct is_detected_convertible_impl { template class Op, class... Args> struct is_detected_convertible_impl< - typename std::enable_if, To>::value>::type, - To, Op, Args...> { + typename std::enable_if, To>::value>::type, To, Op, + Args...> { using type = std::true_type; }; @@ -150,8 +144,7 @@ struct is_detected_convertible : is_detected_convertible_impl::type {}; template -using IsCopyAssignableImpl = - decltype(std::declval() = std::declval()); +using IsCopyAssignableImpl = decltype(std::declval() = std::declval()); template using IsMoveAssignableImpl = decltype(std::declval() = std::declval()); @@ -159,14 +152,12 @@ using IsMoveAssignableImpl = decltype(std::declval() = std::declval()); } // namespace type_traits_internal template -struct is_copy_assignable : type_traits_internal::is_detected< - type_traits_internal::IsCopyAssignableImpl, T> { -}; +struct is_copy_assignable + : type_traits_internal::is_detected {}; template -struct is_move_assignable : type_traits_internal::is_detected< - type_traits_internal::IsMoveAssignableImpl, T> { -}; +struct is_move_assignable + : type_traits_internal::is_detected {}; // --------------------------------------------------------------------------- // void_t() @@ -200,8 +191,7 @@ template struct conjunction; template -struct conjunction - : std::conditional, T>::type {}; +struct conjunction : std::conditional, T>::type {}; template struct conjunction : T {}; @@ -224,8 +214,7 @@ template struct disjunction; template -struct disjunction : - std::conditional>::type {}; +struct disjunction : std::conditional>::type {}; template struct disjunction : T {}; @@ -239,25 +228,25 @@ struct negation : std::integral_constant {}; template struct is_trivially_destructible : std::integral_constant::value> {}; + std::is_destructible::value> {}; template struct is_trivially_default_constructible : std::integral_constant::value && - is_trivially_destructible::value> {}; + std::is_default_constructible::value && + is_trivially_destructible::value> {}; template struct is_trivially_copy_constructible : std::integral_constant::value && - is_trivially_destructible::value> {}; + std::is_copy_constructible::value && + is_trivially_destructible::value> {}; template struct is_trivially_copy_assignable - : std::integral_constant< - bool, __has_trivial_assign(typename std::remove_reference::type) && - phmap::is_copy_assignable::value> {}; + : std::integral_constant::type) && + phmap::is_copy_assignable::value> {}; // ----------------------------------------------------------------------------- // C++14 "_t" trait aliases @@ -308,8 +297,9 @@ using remove_extent_t = typename std::remove_extent::type; template using remove_all_extents_t = typename std::remove_all_extents::type; -template ::value> +template ::value> using aligned_storage_t = typename std::aligned_storage::type; template @@ -321,18 +311,17 @@ using enable_if_t = typename std::enable_if::type; template using conditional_t = typename std::conditional::type; - template using common_type_t = typename std::common_type::type; template using underlying_type_t = typename std::underlying_type::type; -template< class F, class... ArgTypes> +template #if PHMAP_HAVE_CC17 - using invoke_result_t = typename std::invoke_result_t; +using invoke_result_t = typename std::invoke_result_t; #else - using invoke_result_t = typename std::result_of::type; +using invoke_result_t = typename std::result_of::type; #endif namespace type_traits_internal { @@ -342,77 +331,71 @@ namespace type_traits_internal { // static_assert instead of failing substitution. Libc++ prior to 4.0 // also used a static_assert. // ---------------------------------------------------------------------- -#if defined(_MSC_VER) || (defined(_LIBCPP_VERSION) && \ - _LIBCPP_VERSION < 4000 && _LIBCPP_STD_VER > 11) - #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 0 +#if defined(_MSC_VER) || \ + (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 4000 && _LIBCPP_STD_VER > 11) +#define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 0 #else - #define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 1 +#define PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 1 #endif #if !PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ - template - struct IsHashable : std::true_type {}; -#else // PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ - template - struct IsHashable : std::false_type {}; - - template - struct IsHashable&>()(std::declval())), - std::size_t>::value>> : std::true_type {}; +template +struct IsHashable : std::true_type {}; +#else // PHMAP_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ +template +struct IsHashable : std::false_type {}; + +template +struct IsHashable< + Key, phmap::enable_if_t&>()( + std::declval())), + std::size_t>::value>> : std::true_type {}; #endif -struct AssertHashEnabledHelper -{ -private: - static void Sink(...) {} - struct NAT {}; - - template - static auto GetReturnType(int) - -> decltype(std::declval>()(std::declval())); - template - static NAT GetReturnType(...); - - template - static std::nullptr_t DoIt() { - static_assert(IsHashable::value, - "std::hash does not provide a call operator"); - static_assert( - std::is_default_constructible>::value, - "std::hash must be default constructible when it is enabled"); - static_assert( - std::is_copy_constructible>::value, - "std::hash must be copy constructible when it is enabled"); - static_assert(phmap::is_copy_assignable>::value, - "std::hash must be copy assignable when it is enabled"); - // is_destructible is unchecked as it's implied by each of the - // is_constructible checks. - using ReturnType = decltype(GetReturnType(0)); - static_assert(std::is_same::value || - std::is_same::value, - "std::hash must return size_t"); - return nullptr; - } +struct AssertHashEnabledHelper { + private: + static void Sink(...) {} + struct NAT {}; + + template + static auto GetReturnType(int) + -> decltype(std::declval>()(std::declval())); + template + static NAT GetReturnType(...); + + template + static std::nullptr_t DoIt() { + static_assert(IsHashable::value, + "std::hash does not provide a call operator"); + static_assert(std::is_default_constructible>::value, + "std::hash must be default constructible when it is enabled"); + static_assert(std::is_copy_constructible>::value, + "std::hash must be copy constructible when it is enabled"); + static_assert(phmap::is_copy_assignable>::value, + "std::hash must be copy assignable when it is enabled"); + // is_destructible is unchecked as it's implied by each of the + // is_constructible checks. + using ReturnType = decltype(GetReturnType(0)); + static_assert( + std::is_same::value || std::is_same::value, + "std::hash must return size_t"); + return nullptr; + } - template - friend void AssertHashEnabled(); + template + friend void AssertHashEnabled(); }; template -inline void AssertHashEnabled -() -{ - using Helper = AssertHashEnabledHelper; - Helper::Sink(Helper::DoIt()...); +inline void AssertHashEnabled() { + using Helper = AssertHashEnabledHelper; + Helper::Sink(Helper::DoIt()...); } } // namespace type_traits_internal } // namespace phmap - // ----------------------------------------------------------------------------- // hash_policy_traits // ----------------------------------------------------------------------------- @@ -421,164 +404,159 @@ namespace priv { // Defines how slots are initialized/destroyed/moved. template -struct hash_policy_traits -{ -private: - struct ReturnKey - { - // We return `Key` here. - // When Key=T&, we forward the lvalue reference. - // When Key=T, we return by value to avoid a dangling reference. - // eg, for string_hash_map. - template - Key operator()(Key&& k, const Args&...) const { - return std::forward(k); - } - }; - - template - struct ConstantIteratorsImpl : std::false_type {}; - - template - struct ConstantIteratorsImpl> - : P::constant_iterators {}; - -public: - // The actual object stored in the hash table. - using slot_type = typename Policy::slot_type; - - // The type of the keys stored in the hashtable. - using key_type = typename Policy::key_type; - - // The argument type for insertions into the hashtable. This is different - // from value_type for increased performance. See initializer_list constructor - // and insert() member functions for more details. - using init_type = typename Policy::init_type; - - using reference = decltype(Policy::element(std::declval())); - using pointer = typename std::remove_reference::type*; - using value_type = typename std::remove_reference::type; - - // Policies can set this variable to tell raw_hash_set that all iterators - // should be constant, even `iterator`. This is useful for set-like - // containers. - // Defaults to false if not provided by the policy. - using constant_iterators = ConstantIteratorsImpl<>; - - // PRECONDITION: `slot` is UNINITIALIZED - // POSTCONDITION: `slot` is INITIALIZED - template - static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { - Policy::construct(alloc, slot, std::forward(args)...); - } - - // PRECONDITION: `slot` is INITIALIZED - // POSTCONDITION: `slot` is UNINITIALIZED - template - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::destroy(alloc, slot); - } - - // Transfers the `old_slot` to `new_slot`. Any memory allocated by the - // allocator inside `old_slot` to `new_slot` can be transferred. - // - // OPTIONAL: defaults to: - // - // clone(new_slot, std::move(*old_slot)); - // destroy(old_slot); - // - // PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED - // POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is - // UNINITIALIZED - template - static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { - transfer_impl(alloc, new_slot, old_slot, 0); - } - - // PRECONDITION: `slot` is INITIALIZED - // POSTCONDITION: `slot` is INITIALIZED - template - static auto element(slot_type* slot) -> decltype(P::element(slot)) { - return P::element(slot); - } - - // Returns the amount of memory owned by `slot`, exclusive of `sizeof(*slot)`. - // - // If `slot` is nullptr, returns the constant amount of memory owned by any - // full slot or -1 if slots own variable amounts of memory. - // - // PRECONDITION: `slot` is INITIALIZED or nullptr - template - static size_t space_used(const slot_type* slot) { - return P::space_used(slot); - } - - // Provides generalized access to the key for elements, both for elements in - // the table and for elements that have not yet been inserted (or even - // constructed). We would like an API that allows us to say: `key(args...)` - // but we cannot do that for all cases, so we use this more general API that - // can be used for many things, including the following: - // - // - Given an element in a table, get its key. - // - Given an element initializer, get its key. - // - Given `emplace()` arguments, get the element key. - // - // Implementations of this must adhere to a very strict technical - // specification around aliasing and consuming arguments: - // - // Let `value_type` be the result type of `element()` without ref- and - // cv-qualifiers. The first argument is a functor, the rest are constructor - // arguments for `value_type`. Returns `std::forward(f)(k, xs...)`, where - // `k` is the element key, and `xs...` are the new constructor arguments for - // `value_type`. It's allowed for `k` to alias `xs...`, and for both to alias - // `ts...`. The key won't be touched once `xs...` are used to construct an - // element; `ts...` won't be touched at all, which allows `apply()` to consume - // any rvalues among them. - // - // If `value_type` is constructible from `Ts&&...`, `Policy::apply()` must not - // trigger a hard compile error unless it originates from `f`. In other words, - // `Policy::apply()` must be SFINAE-friendly. If `value_type` is not - // constructible from `Ts&&...`, either SFINAE or a hard compile error is OK. - // - // If `Ts...` is `[cv] value_type[&]` or `[cv] init_type[&]`, - // `Policy::apply()` must work. A compile error is not allowed, SFINAE or not. - template - static auto apply(F&& f, Ts&&... ts) - -> decltype(P::apply(std::forward(f), std::forward(ts)...)) { - return P::apply(std::forward(f), std::forward(ts)...); - } - - // Returns the "key" portion of the slot. - // Used for node handle manipulation. - template - static auto key(slot_type* slot) - -> decltype(P::apply(ReturnKey(), element(slot))) { - return P::apply(ReturnKey(), element(slot)); - } - - // Returns the "value" (as opposed to the "key") portion of the element. Used - // by maps to implement `operator[]`, `at()` and `insert_or_assign()`. - template - static auto value(T* elem) -> decltype(P::value(elem)) { - return P::value(elem); - } - -private: - - // Use auto -> decltype as an enabler. - template - static auto transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, int) - -> decltype((void)P::transfer(alloc, new_slot, old_slot)) { - P::transfer(alloc, new_slot, old_slot); - } - - template - static void transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, char) { - construct(alloc, new_slot, std::move(element(old_slot))); - destroy(alloc, old_slot); - } +struct hash_policy_traits { + private: + struct ReturnKey { + // We return `Key` here. + // When Key=T&, we forward the lvalue reference. + // When Key=T, we return by value to avoid a dangling reference. + // eg, for string_hash_map. + template + Key operator()(Key&& k, const Args&...) const { + return std::forward(k); + } + }; + + template + struct ConstantIteratorsImpl : std::false_type {}; + + template + struct ConstantIteratorsImpl> + : P::constant_iterators {}; + + public: + // The actual object stored in the hash table. + using slot_type = typename Policy::slot_type; + + // The type of the keys stored in the hashtable. + using key_type = typename Policy::key_type; + + // The argument type for insertions into the hashtable. This is different + // from value_type for increased performance. See initializer_list constructor + // and insert() member functions for more details. + using init_type = typename Policy::init_type; + + using reference = decltype(Policy::element(std::declval())); + using pointer = typename std::remove_reference::type*; + using value_type = typename std::remove_reference::type; + + // Policies can set this variable to tell raw_hash_set that all iterators + // should be constant, even `iterator`. This is useful for set-like + // containers. + // Defaults to false if not provided by the policy. + using constant_iterators = ConstantIteratorsImpl<>; + + // PRECONDITION: `slot` is UNINITIALIZED + // POSTCONDITION: `slot` is INITIALIZED + template + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + Policy::construct(alloc, slot, std::forward(args)...); + } + + // PRECONDITION: `slot` is INITIALIZED + // POSTCONDITION: `slot` is UNINITIALIZED + template + static void destroy(Alloc* alloc, slot_type* slot) { + Policy::destroy(alloc, slot); + } + + // Transfers the `old_slot` to `new_slot`. Any memory allocated by the + // allocator inside `old_slot` to `new_slot` can be transferred. + // + // OPTIONAL: defaults to: + // + // clone(new_slot, std::move(*old_slot)); + // destroy(old_slot); + // + // PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED + // POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is + // UNINITIALIZED + template + static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { + transfer_impl(alloc, new_slot, old_slot, 0); + } + + // PRECONDITION: `slot` is INITIALIZED + // POSTCONDITION: `slot` is INITIALIZED + template + static auto element(slot_type* slot) -> decltype(P::element(slot)) { + return P::element(slot); + } + + // Returns the amount of memory owned by `slot`, exclusive of `sizeof(*slot)`. + // + // If `slot` is nullptr, returns the constant amount of memory owned by any + // full slot or -1 if slots own variable amounts of memory. + // + // PRECONDITION: `slot` is INITIALIZED or nullptr + template + static size_t space_used(const slot_type* slot) { + return P::space_used(slot); + } + + // Provides generalized access to the key for elements, both for elements in + // the table and for elements that have not yet been inserted (or even + // constructed). We would like an API that allows us to say: `key(args...)` + // but we cannot do that for all cases, so we use this more general API that + // can be used for many things, including the following: + // + // - Given an element in a table, get its key. + // - Given an element initializer, get its key. + // - Given `emplace()` arguments, get the element key. + // + // Implementations of this must adhere to a very strict technical + // specification around aliasing and consuming arguments: + // + // Let `value_type` be the result type of `element()` without ref- and + // cv-qualifiers. The first argument is a functor, the rest are constructor + // arguments for `value_type`. Returns `std::forward(f)(k, xs...)`, where + // `k` is the element key, and `xs...` are the new constructor arguments for + // `value_type`. It's allowed for `k` to alias `xs...`, and for both to alias + // `ts...`. The key won't be touched once `xs...` are used to construct an + // element; `ts...` won't be touched at all, which allows `apply()` to consume + // any rvalues among them. + // + // If `value_type` is constructible from `Ts&&...`, `Policy::apply()` must not + // trigger a hard compile error unless it originates from `f`. In other words, + // `Policy::apply()` must be SFINAE-friendly. If `value_type` is not + // constructible from `Ts&&...`, either SFINAE or a hard compile error is OK. + // + // If `Ts...` is `[cv] value_type[&]` or `[cv] init_type[&]`, + // `Policy::apply()` must work. A compile error is not allowed, SFINAE or not. + template + static auto apply(F&& f, Ts&&... ts) + -> decltype(P::apply(std::forward(f), std::forward(ts)...)) { + return P::apply(std::forward(f), std::forward(ts)...); + } + + // Returns the "key" portion of the slot. + // Used for node handle manipulation. + template + static auto key(slot_type* slot) -> decltype(P::apply(ReturnKey(), element(slot))) { + return P::apply(ReturnKey(), element(slot)); + } + + // Returns the "value" (as opposed to the "key") portion of the element. Used + // by maps to implement `operator[]`, `at()` and `insert_or_assign()`. + template + static auto value(T* elem) -> decltype(P::value(elem)) { + return P::value(elem); + } + + private: + // Use auto -> decltype as an enabler. + template + static auto transfer_impl(Alloc* alloc, slot_type* new_slot, slot_type* old_slot, int) + -> decltype((void)P::transfer(alloc, new_slot, old_slot)) { + P::transfer(alloc, new_slot, old_slot); + } + + template + static void transfer_impl(Alloc* alloc, slot_type* new_slot, slot_type* old_slot, + char) { + construct(alloc, new_slot, std::move(element(old_slot))); + destroy(alloc, old_slot); + } }; } // namespace priv @@ -594,7 +572,7 @@ namespace internal { template struct identity { - typedef T type; + typedef T type; }; template @@ -603,16 +581,15 @@ using identity_t = typename identity::type; } // namespace internal } // namespace phmap - // --------- inline_variable.h #ifdef __cpp_inline_variables #if defined(__clang__) - #define PHMAP_INTERNAL_EXTERN_DECL(type, name) \ - extern const ::phmap::internal::identity_t name; +#define PHMAP_INTERNAL_EXTERN_DECL(type, name) \ + extern const ::phmap::internal::identity_t name; #else // Otherwise, just define the macro to do nothing. - #define PHMAP_INTERNAL_EXTERN_DECL(type, name) +#define PHMAP_INTERNAL_EXTERN_DECL(type, name) #endif // defined(__clang__) // See above comment at top of file for details. @@ -628,20 +605,19 @@ using identity_t = typename identity::type; // identity_t is used here so that the const and name are in the // appropriate place for pointer types, reference types, function pointer // types, etc.. -#define PHMAP_INTERNAL_INLINE_CONSTEXPR(var_type, name, init) \ - template \ - struct PhmapInternalInlineVariableHolder##name { \ - static constexpr ::phmap::internal::identity_t kInstance = init; \ - }; \ - \ - template \ - constexpr ::phmap::internal::identity_t \ - PhmapInternalInlineVariableHolder##name::kInstance; \ - \ - static constexpr const ::phmap::internal::identity_t& \ - name = /* NOLINT */ \ - PhmapInternalInlineVariableHolder##name<>::kInstance; \ - static_assert(sizeof(void (*)(decltype(name))) != 0, \ +#define PHMAP_INTERNAL_INLINE_CONSTEXPR(var_type, name, init) \ + template \ + struct PhmapInternalInlineVariableHolder##name { \ + static constexpr ::phmap::internal::identity_t kInstance = init; \ + }; \ + \ + template \ + constexpr ::phmap::internal::identity_t \ + PhmapInternalInlineVariableHolder##name::kInstance; \ + \ + static constexpr const ::phmap::internal::identity_t& name = /* NOLINT */ \ + PhmapInternalInlineVariableHolder##name<>::kInstance; \ + static_assert(sizeof(void (*)(decltype(name))) != 0, \ "Silence unused variable warnings.") #endif // __cpp_inline_variables @@ -655,11 +631,11 @@ namespace { template #ifdef PHMAP_HAVE_EXCEPTIONS [[noreturn]] void Throw(const T& error) { - throw error; + throw error; } #else [[noreturn]] void Throw(const T&) { - std::abort(); + std::abort(); } #endif } // namespace @@ -739,133 +715,119 @@ namespace phmap { namespace base_internal { template -struct StrippedAccept -{ - template - struct Accept : Derived::template AcceptImpl::type>::type...> {}; +struct StrippedAccept { + template + struct Accept : Derived::template AcceptImpl::type>::type...> {}; }; // (t1.*f)(t2, ..., tN) when f is a pointer to a member function of a class T // and t1 is an object of type T or a reference to an object of type T or a // reference to an object of a type derived from T. -struct MemFunAndRef : StrippedAccept -{ - template - struct AcceptImpl : std::false_type {}; - - template - struct AcceptImpl - : std::is_base_of {}; - - template - struct AcceptImpl - : std::is_base_of {}; - - template - static decltype((std::declval().* - std::declval())(std::declval()...)) - Invoke(MemFun&& mem_fun, Obj&& obj, Args&&... args) { - return (std::forward(obj).* - std::forward(mem_fun))(std::forward(args)...); - } +struct MemFunAndRef : StrippedAccept { + template + struct AcceptImpl : std::false_type {}; + + template + struct AcceptImpl : std::is_base_of {}; + + template + struct AcceptImpl : std::is_base_of {}; + + template + static decltype((std::declval().*std::declval())(std::declval()...)) + Invoke(MemFun&& mem_fun, Obj&& obj, Args&&... args) { + return (std::forward(obj).* + std::forward(mem_fun))(std::forward(args)...); + } }; // ((*t1).*f)(t2, ..., tN) when f is a pointer to a member function of a // class T and t1 is not one of the types described in the previous item. -struct MemFunAndPtr : StrippedAccept -{ - template - struct AcceptImpl : std::false_type {}; - - template - struct AcceptImpl - : std::integral_constant::value> {}; - - template - struct AcceptImpl - : std::integral_constant::value> {}; - - template - static decltype(((*std::declval()).* - std::declval())(std::declval()...)) - Invoke(MemFun&& mem_fun, Ptr&& ptr, Args&&... args) { - return ((*std::forward(ptr)).* - std::forward(mem_fun))(std::forward(args)...); - } +struct MemFunAndPtr : StrippedAccept { + template + struct AcceptImpl : std::false_type {}; + + template + struct AcceptImpl + : std::integral_constant::value> {}; + + template + struct AcceptImpl + : std::integral_constant::value> {}; + + template + static decltype(((*std::declval()).* + std::declval())(std::declval()...)) + Invoke(MemFun&& mem_fun, Ptr&& ptr, Args&&... args) { + return ((*std::forward(ptr)).* + std::forward(mem_fun))(std::forward(args)...); + } }; // t1.*f when N == 1 and f is a pointer to member data of a class T and t1 is // an object of type T or a reference to an object of type T or a reference // to an object of a type derived from T. -struct DataMemAndRef : StrippedAccept -{ - template - struct AcceptImpl : std::false_type {}; - - template - struct AcceptImpl : std::is_base_of {}; - - template - static decltype(std::declval().*std::declval()) Invoke( - DataMem&& data_mem, Ref&& ref) { - return std::forward(ref).*std::forward(data_mem); - } +struct DataMemAndRef : StrippedAccept { + template + struct AcceptImpl : std::false_type {}; + + template + struct AcceptImpl : std::is_base_of {}; + + template + static decltype(std::declval().*std::declval()) Invoke(DataMem&& data_mem, + Ref&& ref) { + return std::forward(ref).*std::forward(data_mem); + } }; // (*t1).*f when N == 1 and f is a pointer to member data of a class T and t1 // is not one of the types described in the previous item. -struct DataMemAndPtr : StrippedAccept -{ - template - struct AcceptImpl : std::false_type {}; - - template - struct AcceptImpl - : std::integral_constant::value> {}; - - template - static decltype((*std::declval()).*std::declval()) Invoke( - DataMem&& data_mem, Ptr&& ptr) { - return (*std::forward(ptr)).*std::forward(data_mem); - } +struct DataMemAndPtr : StrippedAccept { + template + struct AcceptImpl : std::false_type {}; + + template + struct AcceptImpl + : std::integral_constant::value> {}; + + template + static decltype((*std::declval()).*std::declval()) Invoke( + DataMem&& data_mem, Ptr&& ptr) { + return (*std::forward(ptr)).*std::forward(data_mem); + } }; // f(t1, t2, ..., tN) in all other cases. -struct Callable -{ - // Callable doesn't have Accept because it's the last clause that gets picked - // when none of the previous clauses are applicable. - template - static decltype(std::declval()(std::declval()...)) Invoke( - F&& f, Args&&... args) { - return std::forward(f)(std::forward(args)...); - } +struct Callable { + // Callable doesn't have Accept because it's the last clause that gets picked + // when none of the previous clauses are applicable. + template + static decltype(std::declval()(std::declval()...)) Invoke(F&& f, + Args&&... args) { + return std::forward(f)(std::forward(args)...); + } }; // Resolves to the first matching clause. template -struct Invoker -{ - typedef typename std::conditional< - MemFunAndRef::Accept::value, MemFunAndRef, - typename std::conditional< - MemFunAndPtr::Accept::value, MemFunAndPtr, - typename std::conditional< - DataMemAndRef::Accept::value, DataMemAndRef, - typename std::conditional::value, - DataMemAndPtr, Callable>::type>::type>:: - type>::type type; +struct Invoker { + typedef typename std::conditional< + MemFunAndRef::Accept::value, MemFunAndRef, + typename std::conditional< + MemFunAndPtr::Accept::value, MemFunAndPtr, + typename std::conditional< + DataMemAndRef::Accept::value, DataMemAndRef, + typename std::conditional::value, + DataMemAndPtr, Callable>::type>::type>::type>:: + type type; }; // The result type of Invoke. template -using InvokeT = decltype(Invoker::type::Invoke( - std::declval(), std::declval()...)); +using InvokeT = decltype( + Invoker::type::Invoke(std::declval(), std::declval()...)); // Invoke(f, args...) is an implementation of INVOKE(f, args...) from section // [func.require] of the C++ standard. @@ -877,7 +839,6 @@ InvokeT Invoke(F&& f, Args&&... args) { } // namespace base_internal } // namespace phmap - // ----------- utility.h namespace phmap { @@ -902,10 +863,9 @@ namespace phmap { // user_function(make_integer_sequence()); // } template -struct integer_sequence -{ - using value_type = T; - static constexpr size_t size() noexcept { return sizeof...(Ints); } +struct integer_sequence { + using value_type = T; + static constexpr size_t size() noexcept { return sizeof...(Ints); } }; // index_sequence @@ -936,8 +896,7 @@ struct Extend, SeqSize, 1> { // 'Gen::type' is an alias for 'integer_sequence'. template struct Gen { - using type = - typename Extend::type, N / 2, N % 2>::type; + using type = typename Extend::type, N / 2, N % 2>::type; }; template @@ -977,8 +936,8 @@ using index_sequence_for = make_index_sequence; #ifdef PHMAP_HAVE_STD_OPTIONAL -using std::in_place_t; using std::in_place; +using std::in_place_t; #else // PHMAP_HAVE_STD_OPTIONAL @@ -1043,13 +1002,11 @@ constexpr T&& forward( namespace utility_internal { // Helper method for expanding tuple into a called method. template -auto apply_helper(Functor&& functor, Tuple&& t, index_sequence) - -> decltype(phmap::base_internal::Invoke( - phmap::forward(functor), - std::get(phmap::forward(t))...)) { - return phmap::base_internal::Invoke( - phmap::forward(functor), - std::get(phmap::forward(t))...); +auto apply_helper(Functor&& functor, Tuple&& t, index_sequence) -> decltype( + phmap::base_internal::Invoke(phmap::forward(functor), + std::get(phmap::forward(t))...)) { + return phmap::base_internal::Invoke(phmap::forward(functor), + std::get(phmap::forward(t))...); } } // namespace utility_internal @@ -1094,21 +1051,21 @@ auto apply_helper(Functor&& functor, Tuple&& t, index_sequence) // phmap::apply(user_lambda, tuple4); // } template -auto apply(Functor&& functor, Tuple&& t) - -> decltype(utility_internal::apply_helper( - phmap::forward(functor), phmap::forward(t), - phmap::make_index_sequence::type>::value>{})) { +auto apply(Functor&& functor, Tuple&& t) -> decltype(utility_internal::apply_helper( + phmap::forward(functor), phmap::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>{})) { return utility_internal::apply_helper( phmap::forward(functor), phmap::forward(t), - phmap::make_index_sequence::type>::value>{}); + phmap::make_index_sequence< + std::tuple_size::type>::value>{}); } #ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4365) // '=': conversion from 'T' to 'T', signed/unsigned mismatch -#endif // _MSC_VER +#pragma warning(push) +#pragma warning( \ + disable : 4365) // '=': conversion from 'T' to 'T', signed/unsigned mismatch +#endif // _MSC_VER // exchange // @@ -1124,18 +1081,16 @@ auto apply(Functor&& functor, Tuple&& t) // return *this; // } template -T exchange(T& obj, U&& new_value) -{ - T old_value = phmap::move(obj); - obj = phmap::forward(new_value); - return old_value; +T exchange(T& obj, U&& new_value) { + T old_value = phmap::move(obj); + obj = phmap::forward(new_value); + return old_value; } #ifdef _MSC_VER - #pragma warning(pop) +#pragma warning(pop) #endif // _MSC_VER - } // namespace phmap // ----------------------------------------------------------------------------- @@ -1145,11 +1100,10 @@ T exchange(T& obj, U&& new_value) namespace phmap { template -std::unique_ptr WrapUnique(T* ptr) -{ - static_assert(!std::is_array::value, "array types are unsupported"); - static_assert(std::is_object::value, "non-object types are unsupported"); - return std::unique_ptr(ptr); +std::unique_ptr WrapUnique(T* ptr) { + static_assert(!std::is_array::value, "array types are unsupported"); + static_assert(std::is_object::value, "non-object types are unsupported"); + return std::unique_ptr(ptr); } namespace memory_internal { @@ -1157,71 +1111,68 @@ namespace memory_internal { // Traits to select proper overload and return type for `phmap::make_unique<>`. template struct MakeUniqueResult { - using scalar = std::unique_ptr; + using scalar = std::unique_ptr; }; template struct MakeUniqueResult { - using array = std::unique_ptr; + using array = std::unique_ptr; }; template struct MakeUniqueResult { - using invalid = void; + using invalid = void; }; } // namespace memory_internal #if (__cplusplus > 201103L || defined(_MSC_VER)) && \ - !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8) - using std::make_unique; + !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8) +using std::make_unique; #else - template - typename memory_internal::MakeUniqueResult::scalar make_unique( - Args&&... args) { - return std::unique_ptr(new T(std::forward(args)...)); - } - - template - typename memory_internal::MakeUniqueResult::array make_unique(size_t n) { - return std::unique_ptr(new typename phmap::remove_extent_t[n]()); - } - - template - typename memory_internal::MakeUniqueResult::invalid make_unique( - Args&&... /* args */) = delete; +template +typename memory_internal::MakeUniqueResult::scalar make_unique(Args&&... args) { + return std::unique_ptr(new T(std::forward(args)...)); +} + +template +typename memory_internal::MakeUniqueResult::array make_unique(size_t n) { + return std::unique_ptr(new typename phmap::remove_extent_t[n]()); +} + +template +typename memory_internal::MakeUniqueResult::invalid make_unique(Args&&... /* args */) = + delete; #endif template -auto RawPtr(T&& ptr) -> decltype(std::addressof(*ptr)) -{ - // ptr is a forwarding reference to support Ts with non-const operators. - return (ptr != nullptr) ? std::addressof(*ptr) : nullptr; +auto RawPtr(T&& ptr) -> decltype(std::addressof(*ptr)) { + // ptr is a forwarding reference to support Ts with non-const operators. + return (ptr != nullptr) ? std::addressof(*ptr) : nullptr; } inline std::nullptr_t RawPtr(std::nullptr_t) { return nullptr; } template std::shared_ptr ShareUniquePtr(std::unique_ptr&& ptr) { - return ptr ? std::shared_ptr(std::move(ptr)) : std::shared_ptr(); + return ptr ? std::shared_ptr(std::move(ptr)) : std::shared_ptr(); } template std::weak_ptr WeakenPtr(const std::shared_ptr& ptr) { - return std::weak_ptr(ptr); + return std::weak_ptr(ptr); } namespace memory_internal { // ExtractOr::type evaluates to E if possible. Otherwise, D. -template