diff --git a/library/alloc/src/boxed/thin.rs b/library/alloc/src/boxed/thin.rs index 0421a12b3a952..48a6058f1febb 100644 --- a/library/alloc/src/boxed/thin.rs +++ b/library/alloc/src/boxed/thin.rs @@ -4,10 +4,13 @@ use crate::alloc::{self, Layout, LayoutError}; use core::error::Error; use core::fmt::{self, Debug, Display, Formatter}; +use core::intrinsics::const_allocate; use core::marker::PhantomData; #[cfg(not(no_global_oom_handling))] use core::marker::Unsize; -use core::mem::{self, SizedTypeProperties}; +use core::mem; +#[cfg(not(no_global_oom_handling))] +use core::mem::{MaybeUninit, SizedTypeProperties}; use core::ops::{Deref, DerefMut}; use core::ptr::Pointee; use core::ptr::{self, NonNull}; @@ -91,6 +94,79 @@ impl ThinBox { #[unstable(feature = "thin_box", issue = "92791")] impl ThinBox { + #[cfg(not(no_global_oom_handling))] + fn new_unsize_zst(value: T) -> Self + where + T: Unsize, + { + assert!(mem::size_of::() == 0); + + // This is a helper struct to allocate the header with proper alignment: + // Header **end** must be aligned to the value. + // We insert the padding not after the `header` field, but before the `header` field. + // Allocation layout is be: + // ``` + // [ padding | metadata | ZST value ] + // ``` + #[repr(C)] + struct DynZstAlloc { + header: MaybeUninit<::Metadata>, + value: MaybeUninit, + } + + impl<'a, T: 'a, Dyn: ?Sized + 'a> DynZstAlloc + where + T: Unsize, + { + const fn value_offset() -> usize { + mem::offset_of!(DynZstAlloc, value) + } + + const ALLOC: &'a DynZstAlloc = { + // SAFETY: this is compile time evaluation. + unsafe { + let size = mem::size_of::>(); + let alloc: *mut u8 = + const_allocate(size, mem::align_of::>()); + + // Zerofill to be safe. + ptr::write_bytes(alloc, 0, size); + + // Here we pick proper placement of the metadata, + // which is not where the metadata field is declared. + let metadata_ptr = alloc.add( + Self::value_offset() + .checked_sub(mem::size_of::<::Metadata>()) + .unwrap(), + ); + ptr::write( + metadata_ptr as *mut ::Metadata, + ptr::metadata::(ptr::dangling::() as *const Dyn), + ); + + &*(alloc as *const DynZstAlloc) + } + }; + } + + let alloc: &DynZstAlloc = DynZstAlloc::::ALLOC; + + let value_offset = DynZstAlloc::::value_offset(); + + let ptr = WithOpaqueHeader( + NonNull::new( + // SAFETY: there's no overflow here because we add field offset. + unsafe { + (alloc as *const DynZstAlloc as *mut u8).add(value_offset) as *mut _ + }, + ) + .unwrap(), + ); + // Forget the value to avoid double drop. + mem::forget(value); + ThinBox:: { ptr, _marker: PhantomData } + } + /// Moves a type to the heap with its [`Metadata`] stored in the heap allocation instead of on /// the stack. /// @@ -109,9 +185,13 @@ impl ThinBox { where T: Unsize, { - let meta = ptr::metadata(&value as &Dyn); - let ptr = WithOpaqueHeader::new(meta, value); - ThinBox { ptr, _marker: PhantomData } + if mem::size_of::() == 0 { + Self::new_unsize_zst(value) + } else { + let meta = ptr::metadata(&value as &Dyn); + let ptr = WithOpaqueHeader::new(meta, value); + ThinBox { ptr, _marker: PhantomData } + } } } @@ -300,20 +380,19 @@ impl WithHeader { impl Drop for DropGuard { fn drop(&mut self) { + // All ZST are allocated statically. + if self.value_layout.size() == 0 { + return; + } + unsafe { // SAFETY: Layout must have been computable if we're in drop let (layout, value_offset) = WithHeader::::alloc_layout(self.value_layout).unwrap_unchecked(); - // Note: Don't deallocate if the layout size is zero, because the pointer - // didn't come from the allocator. - if layout.size() != 0 { - alloc::dealloc(self.ptr.as_ptr().sub(value_offset), layout); - } else { - debug_assert!( - value_offset == 0 && H::IS_ZST && self.value_layout.size() == 0 - ); - } + // Since we only allocate for non-ZSTs, the layout size cannot be zero. + debug_assert!(layout.size() != 0); + alloc::dealloc(self.ptr.as_ptr().sub(value_offset), layout); } } } diff --git a/library/alloc/src/lib.rs b/library/alloc/src/lib.rs index cafd59cb0d954..34146918288fa 100644 --- a/library/alloc/src/lib.rs +++ b/library/alloc/src/lib.rs @@ -114,8 +114,10 @@ #![feature(const_box)] #![feature(const_cow_is_borrowed)] #![feature(const_eval_select)] +#![feature(const_heap)] #![feature(const_maybe_uninit_as_mut_ptr)] #![feature(const_maybe_uninit_write)] +#![feature(const_option)] #![feature(const_pin)] #![feature(const_refs_to_cell)] #![feature(const_size_of_val)]