[libc++] Replace __compressed_pair with [[no_unique_address]] (#7…

…6756)

This significantly simplifies the code, improves compile times and
improves the object layout of types using `__compressed_pair` in the
unstable ABI. The only downside is that this is extremely ABI sensitive
and pedantically breaks the ABI for empty final types, since the address
of the subobject may change. The ABI of the whole object should not be
affected.

Fixes #91266
Fixes #93069

libcxx/docs/ReleaseNotes/20.rst

@@ -43,7 +43,6 @@ Implemented Papers
 - P2985R0: A type trait for detecting virtual base classes (`Github <https://github.com/llvm/llvm-project/issues/105432>`__)
 - ``std::jthread`` and ``<stop_token>`` are not guarded behind ``-fexperimental-library`` anymore
 
-
 Improvements and New Features
 -----------------------------
 
@@ -53,6 +52,9 @@ Improvements and New Features
 - The ``_LIBCPP_ENABLE_CXX20_REMOVED_UNCAUGHT_EXCEPTION`` macro has been added to make ``std::uncaught_exception``
   available in C++20 and later modes.
 
+- The internal structure ``__compressed_pair`` has been replaced with ``[[no_unique_address]]``, resulting in reduced
+  compile times and smaller debug information as well as better code generation if optimizations are disabled.
+  The Chromium project measured a 5% reduction in object file and debug information size.
 
 Deprecations and Removals
 -------------------------
@@ -97,8 +99,16 @@ LLVM 21
 ABI Affecting Changes
 ---------------------
 
-- TODO
+- The internal structure ``__compressed_pair`` has been replaced with ``[[no_unique_address]]``. The ABI impact is:
 
+  - When using the Itanium ABI (most non-MSVC platforms), empty types are now placed at the beginning of the enclosing
+    object instead of where the beginning of the ``__compressed_pair`` subobject was. This is only observable by
+    checking the address of the empty allocator, equality comparator or hasher.
+  - Additionally, using an overaligned empty type as an allocator, comparator or hasher in the associative containers
+    (and only those containers) may result in the container's object object size and data layout changing beyond only
+    the address of the empty member.
+  - When using the MSVC ABI, this change results in some classes having a completely different memory layout, so this is
+    a genuine ABI break. However, the library does not currently guarantee ABI stability on MSVC platforms.
 
 Build System Changes
 --------------------

libcxx/include/__config

@@ -31,6 +31,7 @@
 
 #  define _LIBCPP_CONCAT_IMPL(_X, _Y) _X##_Y
 #  define _LIBCPP_CONCAT(_X, _Y) _LIBCPP_CONCAT_IMPL(_X, _Y)
+#  define _LIBCPP_CONCAT3(X, Y, Z) _LIBCPP_CONCAT(X, _LIBCPP_CONCAT(Y, Z))
 
 #  if __STDC_HOSTED__ == 0
 #    define _LIBCPP_FREESTANDING
@@ -191,25 +192,6 @@ _LIBCPP_HARDENING_MODE_DEBUG
 #    error "libc++ only supports C++03 with Clang-based compilers. Please enable C++11"
 #  endif
 
-// FIXME: ABI detection should be done via compiler builtin macros. This
-// is just a placeholder until Clang implements such macros. For now assume
-// that Windows compilers pretending to be MSVC++ target the Microsoft ABI,
-// and allow the user to explicitly specify the ABI to handle cases where this
-// heuristic falls short.
-#  if defined(_LIBCPP_ABI_FORCE_ITANIUM) && defined(_LIBCPP_ABI_FORCE_MICROSOFT)
-#    error "Only one of _LIBCPP_ABI_FORCE_ITANIUM and _LIBCPP_ABI_FORCE_MICROSOFT can be defined"
-#  elif defined(_LIBCPP_ABI_FORCE_ITANIUM)
-#    define _LIBCPP_ABI_ITANIUM
-#  elif defined(_LIBCPP_ABI_FORCE_MICROSOFT)
-#    define _LIBCPP_ABI_MICROSOFT
-#  else
-#    if defined(_WIN32) && defined(_MSC_VER)
-#      define _LIBCPP_ABI_MICROSOFT
-#    else
-#      define _LIBCPP_ABI_ITANIUM
-#    endif
-#  endif
-
 #  if defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_NO_VCRUNTIME)
 #    define _LIBCPP_ABI_VCRUNTIME
 #  endif

libcxx/include/__configuration/abi.h

@@ -18,6 +18,25 @@
 #  pragma GCC system_header
 #endif
 
+// FIXME: ABI detection should be done via compiler builtin macros. This
+// is just a placeholder until Clang implements such macros. For now assume
+// that Windows compilers pretending to be MSVC++ target the Microsoft ABI,
+// and allow the user to explicitly specify the ABI to handle cases where this
+// heuristic falls short.
+#if defined(_LIBCPP_ABI_FORCE_ITANIUM) && defined(_LIBCPP_ABI_FORCE_MICROSOFT)
+#  error "Only one of _LIBCPP_ABI_FORCE_ITANIUM and _LIBCPP_ABI_FORCE_MICROSOFT can be defined"
+#elif defined(_LIBCPP_ABI_FORCE_ITANIUM)
+#  define _LIBCPP_ABI_ITANIUM
+#elif defined(_LIBCPP_ABI_FORCE_MICROSOFT)
+#  define _LIBCPP_ABI_MICROSOFT
+#else
+#  if defined(_WIN32) && defined(_MSC_VER)
+#    define _LIBCPP_ABI_MICROSOFT
+#  else
+#    define _LIBCPP_ABI_ITANIUM
+#  endif
+#endif
+
 #if _LIBCPP_ABI_VERSION >= 2
 // Change short string representation so that string data starts at offset 0,
 // improving its alignment in some cases.
@@ -98,6 +117,13 @@
 // and WCHAR_MAX. This ABI setting determines whether we should instead track whether the fill
 // value has been initialized using a separate boolean, which changes the ABI.
 #  define _LIBCPP_ABI_IOS_ALLOW_ARBITRARY_FILL_VALUE
+// Historically, libc++ used a type called `__compressed_pair` to reduce storage needs in cases of empty types (e.g. an
+// empty allocator in std::vector). We switched to using `[[no_unique_address]]`. However, for ABI compatibility reasons
+// we had to add artificial padding in a few places.
+//
+// This setting disables the addition of such artificial padding, leading to a more optimal
+// representation for several types.
+#  define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
 #elif _LIBCPP_ABI_VERSION == 1
 #  if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
 // Enable compiling copies of now inline methods into the dylib to support
@@ -150,6 +176,11 @@
 // ABI impact: changes the iterator type of `vector` (except `vector<bool>`).
 // #define _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
 
+// [[msvc::no_unique_address]] seems to mostly affect empty classes, so the padding scheme for Itanium doesn't work.
+#if defined(_LIBCPP_ABI_MICROSOFT) && !defined(_LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING)
+#  define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
+#endif
+
 #if defined(_LIBCPP_COMPILER_CLANG_BASED)
 #  if defined(__APPLE__)
 #    if defined(__i386__) || defined(__x86_64__)

libcxx/include/__functional/function.h

@@ -143,45 +143,46 @@ class __default_alloc_func;
 
 template <class _Fp, class _Ap, class _Rp, class... _ArgTypes>
 class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> {
-  __compressed_pair<_Fp, _Ap> __f_;
+  _LIBCPP_COMPRESSED_PAIR(_Fp, __func_, _Ap, __alloc_);
 
 public:
   typedef _LIBCPP_NODEBUG _Fp _Target;
   typedef _LIBCPP_NODEBUG _Ap _Alloc;
 
-  _LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __f_.first(); }
+  _LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __func_; }
 
   // WIN32 APIs may define __allocator, so use __get_allocator instead.
-  _LIBCPP_HIDE_FROM_ABI const _Alloc& __get_allocator() const { return __f_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const _Alloc& __get_allocator() const { return __alloc_; }
 
-  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f)
-      : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple()) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f) : __func_(std::move(__f)), __alloc_() {}
 
-  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, const _Alloc& __a)
-      : __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(__a)) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, const _Alloc& __a) : __func_(__f), __alloc_(__a) {}
 
   _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, _Alloc&& __a)
-      : __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(std::move(__a))) {}
+      : __func_(__f), __alloc_(std::move(__a)) {}
 
   _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f, _Alloc&& __a)
-      : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple(std::move(__a))) {}
+      : __func_(std::move(__f)), __alloc_(std::move(__a)) {}
 
   _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __arg) {
     typedef __invoke_void_return_wrapper<_Rp> _Invoker;
-    return _Invoker::__call(__f_.first(), std::forward<_ArgTypes>(__arg)...);
+    return _Invoker::__call(__func_, std::forward<_ArgTypes>(__arg)...);
   }
 
   _LIBCPP_HIDE_FROM_ABI __alloc_func* __clone() const {
     typedef allocator_traits<_Alloc> __alloc_traits;
     typedef __rebind_alloc<__alloc_traits, __alloc_func> _AA;
-    _AA __a(__f_.second());
+    _AA __a(__alloc_);
     typedef __allocator_destructor<_AA> _Dp;
     unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1));
-    ::new ((void*)__hold.get()) __alloc_func(__f_.first(), _Alloc(__a));
+    ::new ((void*)__hold.get()) __alloc_func(__func_, _Alloc(__a));
     return __hold.release();
   }
 
-  _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT { __f_.~__compressed_pair<_Target, _Alloc>(); }
+  _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT {
+    __func_.~_Fp();
+    __alloc_.~_Alloc();
+  }
 
   _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__alloc_func* __f) {
     typedef allocator_traits<_Alloc> __alloc_traits;